Compare commits

...

4 Commits

Author SHA1 Message Date
Vitaliy Filippov ee09b27875 Change git dependency URLs 2024-07-21 15:27:51 +03:00
Vitaliy Filippov 85a73e3fdd Make sproxydclient and hdclient dependencies optional 2024-07-21 15:26:49 +03:00
Maha Benzekri 817bb836ec
ARSN-420: bump arsenal version 2024-07-15 15:20:08 +02:00
Maha Benzekri e3e4b2aea7
ARSN-420: putObjectNoVar function update with hack
We agreed on Introducing the same “hack” as in internalDelete function,
so write the MD twice in the oplog: one "deleted: true" copy of the previous MD,
followed by the expected update with the new metadata
2024-07-15 15:19:06 +02:00
3 changed files with 121 additions and 29 deletions

View File

@ -1,8 +1,6 @@
const { http, https } = require('httpagent'); const { http, https } = require('httpagent');
const url = require('url'); const url = require('url');
const AWS = require('aws-sdk'); const AWS = require('aws-sdk');
const Sproxy = require('sproxydclient');
const Hyperdrive = require('hdclient');
const HttpsProxyAgent = require('https-proxy-agent'); const HttpsProxyAgent = require('https-proxy-agent');
const constants = require('../../constants'); const constants = require('../../constants');
@ -27,6 +25,7 @@ function parseLC(config, vault) {
} }
if (locationObj.type === 'scality') { if (locationObj.type === 'scality') {
if (locationObj.details.connector.sproxyd) { if (locationObj.details.connector.sproxyd) {
const Sproxy = require('sproxydclient');
clients[location] = new Sproxy({ clients[location] = new Sproxy({
bootstrap: locationObj.details.connector bootstrap: locationObj.details.connector
.sproxyd.bootstrap, .sproxyd.bootstrap,
@ -41,6 +40,7 @@ function parseLC(config, vault) {
}); });
clients[location].clientType = 'scality'; clients[location].clientType = 'scality';
} else if (locationObj.details.connector.hdclient) { } else if (locationObj.details.connector.hdclient) {
const Hyperdrive = require('hdclient');
clients[location] = new Hyperdrive.hdcontroller.HDProxydClient( clients[location] = new Hyperdrive.hdcontroller.HDProxydClient(
locationObj.details.connector.hdclient); locationObj.details.connector.hdclient);
clients[location].clientType = 'scality'; clients[location].clientType = 'scality';

View File

@ -899,35 +899,130 @@ class MongoClientInterface {
return cb(errors.InternalError); return cb(errors.InternalError);
}); });
} }
/** /**
* Put object when versioning is not enabled * Puts an object into a MongoDB collection.
* @param {Object} c bucket collection * Depending on the parameters, the object is either directly put into the collection
* @param {String} bucketName bucket name * or the existing object is marked as deleted and a new object is inserted.
* @param {String} objName object name *
* @param {Object} objVal object metadata * @param {Object} collection - The MongoDB collection to put the object into.
* @param {Object} params params * @param {string} bucketName - The name of the bucket the object belongs to.
* @param {Object} log logger * @param {string} objName - The name of the object.
* @param {Function} cb callback * @param {Object} value - The value of the object.
* @return {undefined} * @param {Object} params - Additional parameters.
* @param {string} params.vFormat - object key format.
* @param {boolean} params.needOplogUpdate - If true, the object is directly put into the collection
* with updating the operation log.
* @param {Object} log - The logger to use.
* @param {Function} cb - The callback function to call when the operation is complete. It is called with an error
* if there is an issue with the operation.
* @returns {Promise} A promise that resolves when the operation is complete. The promise is rejected with an error
* if there is an issue with the operation.
*/ */
putObjectNoVer(c, bucketName, objName, objVal, params, log, cb) { putObjectNoVer(collection, bucketName, objName, value, params, log, cb) {
const masterKey = formatMasterKey(objName, params.vFormat); if (params?.needOplogUpdate) {
c.updateOne({ return this.putObjectNoVerWithOplogUpdate(collection, bucketName, objName, value, params, log, cb);
_id: masterKey, }
}, { const key = formatMasterKey(objName, params.vFormat);
const putFilter = { _id: key };
return collection.updateOne(putFilter, {
$set: { $set: {
_id: masterKey, _id: key,
value: objVal, value,
}, },
}, { }, {
upsert: true, upsert: true,
}).then(() => cb()).catch((err) => { }).then(() => cb()).catch(err => {
log.error('putObjectNoVer: error putting obect with no versioning', { error: err.message }); log.error('putObjectNoVer: error putting obect with no versioning', { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
}); });
} }
/**
* Updates an object in a MongoDB collection without changing its version.
* If the object doesn't exist, it will be created (upsert is true for the second update operation).
* The operation is logged in the oplog.
*
* @param {Object} collection - The MongoDB collection to update the object in.
* @param {string} bucketName - The name of the bucket the object belongs to.
* @param {string} objName - The name of the object.
* @param {Object} value - The new value of the object.
* @param {Object} params - Additional parameters.
* @param {string} params.vFormat - object key format
* @param {string} params.originOp - origin operation
* @param {Object} log - The logger to use.
* @param {Function} cb - The callback function to call when the operation is complete.
* It is called with an error if there is an issue with the operation.
* @returns {void}
*/
putObjectNoVerWithOplogUpdate(collection, bucketName, objName, value, params, log, cb) {
const key = formatMasterKey(objName, params.vFormat);
const putFilter = { _id: key };
// filter used when finding and updating object
const findFilter = {
...putFilter,
$or: [
{ 'value.deleted': { $exists: false } },
{ 'value.deleted': { $eq: false } },
],
};
const updateDeleteFilter = {
...putFilter,
'value.deleted': true,
};
return async.waterfall([
// Adding delete flag when getting the object
// to avoid having race conditions.
next => collection.findOneAndUpdate(findFilter, {
$set: updateDeleteFilter,
}, {
upsert: false,
}).then(doc => {
if (!doc.value) {
log.error('internalPutObject: unable to find target object to update',
{ bucket: bucketName, object: key });
return next(errors.NoSuchKey);
}
const obj = doc.value;
const objMetadata = new ObjectMD(obj.value);
objMetadata.setOriginOp(params.originOp);
objMetadata.setDeleted(true);
return next(null, objMetadata.getValue());
}).catch(err => {
log.error('internalPutObject: error getting object',
{ bucket: bucketName, object: key, error: err.message });
return next(errors.InternalError);
}),
// We update the full object to get the whole object metadata
// in the oplog update event
(objMetadata, next) => collection.bulkWrite([
{
updateOne: {
filter: updateDeleteFilter,
update: {
$set: { _id: key, value: objMetadata },
},
upsert: false,
},
},
{
updateOne: {
filter: putFilter,
update: {
$set: { _id: key, value },
},
upsert: true,
},
},
], { ordered: true }).then(() => next(null)).catch(next),
], (err) => {
if (err) {
log.error('internalPutObject: error updating object',
{ bucket: bucketName, object: key, error: err.message });
return cb(errors.InternalError);
}
return cb();
});
}
/** /**
* Returns the putObjectVerCase function to use * Returns the putObjectVerCase function to use
* depending on params * depending on params
@ -973,8 +1068,7 @@ class MongoClientInterface {
return putObjectVer(c, bucketName, objName, objVal, _params, log, return putObjectVer(c, bucketName, objName, objVal, _params, log,
cb); cb);
} }
return this.putObjectNoVer(c, bucketName, objName, objVal, return this.putObjectNoVer(c, bucketName, objName, objVal, _params, log, cb);
_params, log, cb);
}); });
} }

View File

@ -3,7 +3,7 @@
"engines": { "engines": {
"node": ">=16" "node": ">=16"
}, },
"version": "8.1.132", "version": "8.1.133",
"description": "Common utilities for the S3 project components", "description": "Common utilities for the S3 project components",
"main": "build/index.js", "main": "build/index.js",
"repository": { "repository": {
@ -33,9 +33,8 @@
"bson": "4.0.0", "bson": "4.0.0",
"debug": "~4.1.0", "debug": "~4.1.0",
"diskusage": "^1.1.1", "diskusage": "^1.1.1",
"fcntl": "github:scality/node-fcntl#0.2.2", "fcntl": "git:https://git.yourcmc.ru/vitalif/zenko-fcntl.git#0.2.2",
"hdclient": "scality/hdclient#1.1.7", "httpagent": "git:https://git.yourcmc.ru/vitalif/zenko-httpagent.git#1.0.6",
"httpagent": "scality/httpagent#1.0.6",
"https-proxy-agent": "^2.2.0", "https-proxy-agent": "^2.2.0",
"ioredis": "^4.28.5", "ioredis": "^4.28.5",
"ipaddr.js": "1.9.1", "ipaddr.js": "1.9.1",
@ -48,10 +47,9 @@
"simple-glob": "^0.2.0", "simple-glob": "^0.2.0",
"socket.io": "~4.6.1", "socket.io": "~4.6.1",
"socket.io-client": "~4.6.1", "socket.io-client": "~4.6.1",
"sproxydclient": "git+https://github.com/scality/sproxydclient#8.0.10",
"utf8": "3.0.0", "utf8": "3.0.0",
"uuid": "^3.0.1", "uuid": "^3.0.1",
"werelogs": "scality/werelogs#8.1.4", "werelogs": "git:https://git.yourcmc.ru/vitalif/zenko-werelogs.git#8.1.4",
"xml2js": "~0.4.23" "xml2js": "~0.4.23"
}, },
"optionalDependencies": { "optionalDependencies": {
@ -67,7 +65,7 @@
"@types/xml2js": "^0.4.11", "@types/xml2js": "^0.4.11",
"eslint": "^8.14.0", "eslint": "^8.14.0",
"eslint-config-airbnb": "6.2.0", "eslint-config-airbnb": "6.2.0",
"eslint-config-scality": "scality/Guidelines#ec33dfb", "eslint-config-scality": "git+https://git.yourcmc.ru/vitalif/zenko-eslint-config-scality.git#8.2.0",
"eslint-plugin-react": "^4.3.0", "eslint-plugin-react": "^4.3.0",
"jest": "^27.5.1", "jest": "^27.5.1",
"mongodb-memory-server": "^8.12.2", "mongodb-memory-server": "^8.12.2",