Compare commits

...

4 Commits

Author SHA1 Message Date
Vitaliy Filippov ee09b27875 Change git dependency URLs 2024-07-21 15:27:51 +03:00
Vitaliy Filippov 85a73e3fdd Make sproxydclient and hdclient dependencies optional 2024-07-21 15:26:49 +03:00
Maha Benzekri 817bb836ec
ARSN-420: bump arsenal version 2024-07-15 15:20:08 +02:00
Maha Benzekri e3e4b2aea7
ARSN-420: putObjectNoVar function update with hack
We agreed on Introducing the same “hack” as in internalDelete function,
so write the MD twice in the oplog: one "deleted: true" copy of the previous MD,
followed by the expected update with the new metadata
2024-07-15 15:19:06 +02:00
3 changed files with 121 additions and 29 deletions

View File

@ -1,8 +1,6 @@
const { http, https } = require('httpagent');
const url = require('url');
const AWS = require('aws-sdk');
const Sproxy = require('sproxydclient');
const Hyperdrive = require('hdclient');
const HttpsProxyAgent = require('https-proxy-agent');
const constants = require('../../constants');
@ -27,6 +25,7 @@ function parseLC(config, vault) {
}
if (locationObj.type === 'scality') {
if (locationObj.details.connector.sproxyd) {
const Sproxy = require('sproxydclient');
clients[location] = new Sproxy({
bootstrap: locationObj.details.connector
.sproxyd.bootstrap,
@ -41,6 +40,7 @@ function parseLC(config, vault) {
});
clients[location].clientType = 'scality';
} else if (locationObj.details.connector.hdclient) {
const Hyperdrive = require('hdclient');
clients[location] = new Hyperdrive.hdcontroller.HDProxydClient(
locationObj.details.connector.hdclient);
clients[location].clientType = 'scality';

View File

@ -899,35 +899,130 @@ class MongoClientInterface {
return cb(errors.InternalError);
});
}
/**
* Put object when versioning is not enabled
* @param {Object} c bucket collection
* @param {String} bucketName bucket name
* @param {String} objName object name
* @param {Object} objVal object metadata
* @param {Object} params params
* @param {Object} log logger
* @param {Function} cb callback
* @return {undefined}
* Puts an object into a MongoDB collection.
* Depending on the parameters, the object is either directly put into the collection
* or the existing object is marked as deleted and a new object is inserted.
*
* @param {Object} collection - The MongoDB collection to put the object into.
* @param {string} bucketName - The name of the bucket the object belongs to.
* @param {string} objName - The name of the object.
* @param {Object} value - The value of the object.
* @param {Object} params - Additional parameters.
* @param {string} params.vFormat - object key format.
* @param {boolean} params.needOplogUpdate - If true, the object is directly put into the collection
* with updating the operation log.
* @param {Object} log - The logger to use.
* @param {Function} cb - The callback function to call when the operation is complete. It is called with an error
* if there is an issue with the operation.
* @returns {Promise} A promise that resolves when the operation is complete. The promise is rejected with an error
* if there is an issue with the operation.
*/
putObjectNoVer(c, bucketName, objName, objVal, params, log, cb) {
const masterKey = formatMasterKey(objName, params.vFormat);
c.updateOne({
_id: masterKey,
}, {
putObjectNoVer(collection, bucketName, objName, value, params, log, cb) {
if (params?.needOplogUpdate) {
return this.putObjectNoVerWithOplogUpdate(collection, bucketName, objName, value, params, log, cb);
}
const key = formatMasterKey(objName, params.vFormat);
const putFilter = { _id: key };
return collection.updateOne(putFilter, {
$set: {
_id: masterKey,
value: objVal,
_id: key,
value,
},
}, {
upsert: true,
}).then(() => cb()).catch((err) => {
}).then(() => cb()).catch(err => {
log.error('putObjectNoVer: error putting obect with no versioning', { error: err.message });
return cb(errors.InternalError);
});
}
/**
* Updates an object in a MongoDB collection without changing its version.
* If the object doesn't exist, it will be created (upsert is true for the second update operation).
* The operation is logged in the oplog.
*
* @param {Object} collection - The MongoDB collection to update the object in.
* @param {string} bucketName - The name of the bucket the object belongs to.
* @param {string} objName - The name of the object.
* @param {Object} value - The new value of the object.
* @param {Object} params - Additional parameters.
* @param {string} params.vFormat - object key format
* @param {string} params.originOp - origin operation
* @param {Object} log - The logger to use.
* @param {Function} cb - The callback function to call when the operation is complete.
* It is called with an error if there is an issue with the operation.
* @returns {void}
*/
putObjectNoVerWithOplogUpdate(collection, bucketName, objName, value, params, log, cb) {
const key = formatMasterKey(objName, params.vFormat);
const putFilter = { _id: key };
// filter used when finding and updating object
const findFilter = {
...putFilter,
$or: [
{ 'value.deleted': { $exists: false } },
{ 'value.deleted': { $eq: false } },
],
};
const updateDeleteFilter = {
...putFilter,
'value.deleted': true,
};
return async.waterfall([
// Adding delete flag when getting the object
// to avoid having race conditions.
next => collection.findOneAndUpdate(findFilter, {
$set: updateDeleteFilter,
}, {
upsert: false,
}).then(doc => {
if (!doc.value) {
log.error('internalPutObject: unable to find target object to update',
{ bucket: bucketName, object: key });
return next(errors.NoSuchKey);
}
const obj = doc.value;
const objMetadata = new ObjectMD(obj.value);
objMetadata.setOriginOp(params.originOp);
objMetadata.setDeleted(true);
return next(null, objMetadata.getValue());
}).catch(err => {
log.error('internalPutObject: error getting object',
{ bucket: bucketName, object: key, error: err.message });
return next(errors.InternalError);
}),
// We update the full object to get the whole object metadata
// in the oplog update event
(objMetadata, next) => collection.bulkWrite([
{
updateOne: {
filter: updateDeleteFilter,
update: {
$set: { _id: key, value: objMetadata },
},
upsert: false,
},
},
{
updateOne: {
filter: putFilter,
update: {
$set: { _id: key, value },
},
upsert: true,
},
},
], { ordered: true }).then(() => next(null)).catch(next),
], (err) => {
if (err) {
log.error('internalPutObject: error updating object',
{ bucket: bucketName, object: key, error: err.message });
return cb(errors.InternalError);
}
return cb();
});
}
/**
* Returns the putObjectVerCase function to use
* depending on params
@ -973,8 +1068,7 @@ class MongoClientInterface {
return putObjectVer(c, bucketName, objName, objVal, _params, log,
cb);
}
return this.putObjectNoVer(c, bucketName, objName, objVal,
_params, log, cb);
return this.putObjectNoVer(c, bucketName, objName, objVal, _params, log, cb);
});
}

View File

@ -3,7 +3,7 @@
"engines": {
"node": ">=16"
},
"version": "8.1.132",
"version": "8.1.133",
"description": "Common utilities for the S3 project components",
"main": "build/index.js",
"repository": {
@ -33,9 +33,8 @@
"bson": "4.0.0",
"debug": "~4.1.0",
"diskusage": "^1.1.1",
"fcntl": "github:scality/node-fcntl#0.2.2",
"hdclient": "scality/hdclient#1.1.7",
"httpagent": "scality/httpagent#1.0.6",
"fcntl": "git:https://git.yourcmc.ru/vitalif/zenko-fcntl.git#0.2.2",
"httpagent": "git:https://git.yourcmc.ru/vitalif/zenko-httpagent.git#1.0.6",
"https-proxy-agent": "^2.2.0",
"ioredis": "^4.28.5",
"ipaddr.js": "1.9.1",
@ -48,10 +47,9 @@
"simple-glob": "^0.2.0",
"socket.io": "~4.6.1",
"socket.io-client": "~4.6.1",
"sproxydclient": "git+https://github.com/scality/sproxydclient#8.0.10",
"utf8": "3.0.0",
"uuid": "^3.0.1",
"werelogs": "scality/werelogs#8.1.4",
"werelogs": "git:https://git.yourcmc.ru/vitalif/zenko-werelogs.git#8.1.4",
"xml2js": "~0.4.23"
},
"optionalDependencies": {
@ -67,7 +65,7 @@
"@types/xml2js": "^0.4.11",
"eslint": "^8.14.0",
"eslint-config-airbnb": "6.2.0",
"eslint-config-scality": "scality/Guidelines#ec33dfb",
"eslint-config-scality": "git+https://git.yourcmc.ru/vitalif/zenko-eslint-config-scality.git#8.2.0",
"eslint-plugin-react": "^4.3.0",
"jest": "^27.5.1",
"mongodb-memory-server": "^8.12.2",