Compare commits

...

6 Commits

Author SHA1 Message Date
williamlardier 47a7253706
wip2 2023-04-06 14:29:46 +02:00
williamlardier 971c8dbc0a
wip 2023-04-06 14:11:21 +02:00
williamlardier 1d243f831a
ARSN-329: bump to 8.1.93 2023-04-06 12:24:12 +02:00
williamlardier 9949c62f73
ARSN-329: update tests 2023-04-06 12:24:11 +02:00
williamlardier 3fc3e095d9
ARSN-329: switch to promises as callbacks are deprecated 2023-04-06 09:08:52 +02:00
williamlardier 6183b3a8b9
ARSN-329: bump mongodb driver 2023-04-06 08:59:55 +02:00
20 changed files with 1562 additions and 1002 deletions

View File

@ -148,12 +148,8 @@ class MongoClientInterface {
!Number.isNaN(process.env.MONGO_POOL_SIZE)) { !Number.isNaN(process.env.MONGO_POOL_SIZE)) {
options.poolSize = Number.parseInt(process.env.MONGO_POOL_SIZE, 10); options.poolSize = Number.parseInt(process.env.MONGO_POOL_SIZE, 10);
} }
return MongoClient.connect(this.mongoUrl, options, (err, client) => { return MongoClient.connect(this.mongoUrl, options)
if (err) { .then(client => {
this.logger.error('error connecting to mongodb',
{ error: err.message });
return cb(errors.InternalError);
}
this.logger.info('connected to mongodb'); this.logger.info('connected to mongodb');
this.client = client; this.client = client;
this.db = client.db(this.database, { this.db = client.db(this.database, {
@ -169,8 +165,11 @@ class MongoClientInterface {
this.cacheHit = 0; this.cacheHit = 0;
this.cacheMiss = 0; this.cacheMiss = 0;
}, 300000); }, 300000);
return this.usersBucketHack(cb); return this.usersBucketHack(cb);
})
.catch(err => {
this.logger.error('error connecting to mongodb', { error: err.message });
return cb(errors.InternalError);
}); });
} }
@ -200,7 +199,9 @@ class MongoClientInterface {
close(cb) { close(cb) {
if (this.client) { if (this.client) {
clearInterval(this.cacheHitMissLoggerInterval); clearInterval(this.cacheHitMissLoggerInterval);
return this.client.close(true, cb); return this.client.close(true)
.then(() => cb())
.catch(() => cb());
} }
return cb(); return cb();
} }
@ -229,45 +230,36 @@ class MongoClientInterface {
const m = this.getCollection(METASTORE); const m = this.getCollection(METASTORE);
const payload = { const payload = {
$set: {
_id: bucketName, _id: bucketName,
value: newBucketMD, value: newBucketMD,
},
}; };
if (bucketName !== constants.usersBucket && if (bucketName !== constants.usersBucket &&
bucketName !== PENSIEVE && bucketName !== PENSIEVE &&
!bucketName.startsWith(constants.mpuBucketPrefix)) { !bucketName.startsWith(constants.mpuBucketPrefix)) {
payload.vFormat = this.defaultBucketKeyFormat; payload.$set.vFormat = this.defaultBucketKeyFormat;
} else { } else {
payload.vFormat = BUCKET_VERSIONS.v0; payload.$set.vFormat = BUCKET_VERSIONS.v0;
} }
// we don't have to test bucket existence here as it is done // we don't have to test bucket existence here as it is done
// on the upper layers // on the upper layers
m.update({ m.updateOne({
_id: bucketName, _id: bucketName,
}, payload, { }, payload, {
upsert: true, upsert: true,
}, err => { })
if (err) { .then(() => {
log.error(
'createBucket: error creating bucket',
{ error: err.message });
return cb(errors.InternalError);
}
// caching bucket vFormat // caching bucket vFormat
this.bucketVFormatCache.add(bucketName, payload.vFormat); this.bucketVFormatCache.add(bucketName, payload.vFormat);
this.lastItemScanTime = null; this.lastItemScanTime = null;
// NOTE: We do not need to create a collection for // NOTE: We do not need to create a collection for
// "constants.usersBucket" and "PENSIEVE" since it has already // "constants.usersBucket" and "PENSIEVE" since it has already
// been created // been created
if (bucketName !== constants.usersBucket && if (bucketName !== constants.usersBucket && bucketName !== PENSIEVE) {
bucketName !== PENSIEVE) { return this.db.createCollection(bucketName)
return this.db.createCollection(bucketName, err => { .then(() => {
if (err) {
log.error(
'createBucket: error creating bucket',
{ error: err });
return cb(errors.InternalError);
}
if (this.shardCollections) { if (this.shardCollections) {
const cmd = { const cmd = {
shardCollection: `${this.database}.${bucketName}`, shardCollection: `${this.database}.${bucketName}`,
@ -286,8 +278,11 @@ class MongoClientInterface {
return cb(); return cb();
}); });
} }
return cb(); return cb();
})
.catch(err => {
log.error('createBucket: error creating bucket', { error: err.message });
return cb(errors.InternalError);
}); });
} }
@ -302,13 +297,8 @@ class MongoClientInterface {
const m = this.getCollection(METASTORE); const m = this.getCollection(METASTORE);
m.findOne({ m.findOne({
_id: bucketName, _id: bucketName,
}, {}, (err, doc) => { })
if (err) { .then(doc => {
log.error(
'getBucketAttributes: error getting bucket attributes',
{ error: err.message });
return cb(errors.InternalError);
}
if (!doc) { if (!doc) {
return cb(errors.NoSuchBucket); return cb(errors.NoSuchBucket);
} }
@ -317,6 +307,12 @@ class MongoClientInterface {
const bucketMDStr = JSON.stringify(doc.value); const bucketMDStr = JSON.stringify(doc.value);
const bucketMD = BucketInfo.deSerialize(bucketMDStr); const bucketMD = BucketInfo.deSerialize(bucketMDStr);
return cb(null, bucketMD); return cb(null, bucketMD);
})
.catch(err => {
log.error(
'getBucketAttributes: error getting bucket attributes',
{ error: err.message });
return cb(errors.InternalError);
}); });
return undefined; return undefined;
} }
@ -339,19 +335,21 @@ class MongoClientInterface {
const m = this.getCollection(METASTORE); const m = this.getCollection(METASTORE);
m.findOne({ m.findOne({
_id: bucketName, _id: bucketName,
}, {}, (err, doc) => { })
if (err) { .then(doc => {
log.error(
'getBucketVFormat: error getting bucket vFormat',
{ bucket: bucketName, error: err.message });
return cb(errors.InternalError);
}
if (!doc) { if (!doc) {
return cb(null, BUCKET_VERSIONS.v0); return cb(null, BUCKET_VERSIONS.v0);
} }
const vFormat = doc.vFormat || BUCKET_VERSIONS.v0; const vFormat = doc.vFormat || BUCKET_VERSIONS.v0;
this.bucketVFormatCache.add(bucketName, vFormat); this.bucketVFormatCache.add(bucketName, vFormat);
return cb(null, vFormat); return cb(null, vFormat);
})
.catch(err => {
log.error(
'getBucketVFormat: error getting bucket vFormat',
{ bucket: bucketName, error: err.message },
);
return cb(errors.InternalError);
}); });
return undefined; return undefined;
} }
@ -368,7 +366,8 @@ class MongoClientInterface {
if (err) { if (err) {
if (err.is.NoSuchKey) { if (err.is.NoSuchKey) {
return cb(null, return cb(null,
{ bucket: {
bucket:
BucketInfo.fromObj(bucket).serialize(), BucketInfo.fromObj(bucket).serialize(),
}); });
} }
@ -392,7 +391,7 @@ class MongoClientInterface {
const bucketMDStr = bucketInfo.serialize(); const bucketMDStr = bucketInfo.serialize();
const newBucketMD = JSON.parse(bucketMDStr); const newBucketMD = JSON.parse(bucketMDStr);
const m = this.getCollection(METASTORE); const m = this.getCollection(METASTORE);
m.update({ m.updateOne({
_id: bucketName, _id: bucketName,
}, { }, {
$set: { $set: {
@ -401,14 +400,13 @@ class MongoClientInterface {
}, },
}, { }, {
upsert: true, upsert: true,
}, err => { })
if (err) { .then(() => cb())
.catch(err => {
log.error( log.error(
'putBucketAttributes: error putting bucket attributes', 'putBucketAttributes: error putting bucket attributes',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
}
return cb();
}); });
} }
@ -427,7 +425,7 @@ class MongoClientInterface {
const updateString = capabilityField ? const updateString = capabilityField ?
`value.capabilities.${capabilityName}.${capabilityField}` : `value.capabilities.${capabilityName}.${capabilityField}` :
`value.capabilities.${capabilityName}`; `value.capabilities.${capabilityName}`;
m.update({ m.updateOne({
_id: bucketName, _id: bucketName,
}, { }, {
$set: { $set: {
@ -436,14 +434,11 @@ class MongoClientInterface {
}, },
}, { }, {
upsert: true, upsert: true,
}, err => { }).then(() => cb()).catch(err => {
if (err) {
log.error( log.error(
'putBucketAttributesCapabilities: error putting bucket attributes', 'putBucketAttributesCapabilities: error putting bucket attributes',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
}
return cb();
}); });
} }
@ -461,13 +456,13 @@ class MongoClientInterface {
const updateString = capabilityField ? const updateString = capabilityField ?
`value.capabilities.${capabilityName}.${capabilityField}` : `value.capabilities.${capabilityName}.${capabilityField}` :
`value.capabilities.${capabilityName}`; `value.capabilities.${capabilityName}`;
m.update({ m.updateOne({
_id: bucketName, _id: bucketName,
}, { }, {
$unset: { $unset: {
[updateString]: '', [updateString]: '',
}, },
}, err => { }).then(() => cb()).catch(err => {
if (err) { if (err) {
log.error( log.error(
'deleteBucketAttributesCapability: error deleting bucket attributes', 'deleteBucketAttributesCapability: error deleting bucket attributes',
@ -485,20 +480,20 @@ class MongoClientInterface {
const m = this.getCollection(METASTORE); const m = this.getCollection(METASTORE);
m.findOneAndDelete({ m.findOneAndDelete({
_id: bucketName, _id: bucketName,
}, {}, (err, result) => { }, {})
if (err) { .then(result => {
log.error('deleteBucketStep2: error deleting bucket',
{ error: err.message });
return cb(errors.InternalError);
}
if (result.ok !== 1) { if (result.ok !== 1) {
log.error('deleteBucketStep2: failed deleting bucket', log.error('deleteBucketStep2: failed deleting bucket');
{ error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
// removing cached bucket metadata // removing cached bucket metadata
this.bucketVFormatCache.remove(bucketName); this.bucketVFormatCache.remove(bucketName);
return cb(null); return cb(null);
})
.catch(err => {
log.error('deleteBucketStep2: error deleting bucket',
{ error: err.message });
return cb(errors.InternalError);
}); });
} }
@ -514,22 +509,23 @@ class MongoClientInterface {
*/ */
deleteBucket(bucketName, log, cb) { deleteBucket(bucketName, log, cb) {
const c = this.getCollection(bucketName); const c = this.getCollection(bucketName);
c.drop({}, err => { c.drop({})
if (err) { .then(() => {
if (err.codeName === 'NamespaceNotFound') { this.deleteBucketStep2(bucketName, log, err => {
return this.deleteBucketStep2(bucketName, log, cb);
}
log.error('deleteBucket: error deleting bucket',
{ error: err.message });
return cb(errors.InternalError);
}
return this.deleteBucketStep2(bucketName, log, err => {
if (err) { if (err) {
return cb(err); return cb(err);
} }
this.lastItemScanTime = null; this.lastItemScanTime = null;
return cb(null); return cb(null);
}); });
})
.catch(err => {
if (err.codeName === 'NamespaceNotFound') {
return this.deleteBucketStep2(bucketName, log, cb);
}
log.error('deleteBucket: error deleting bucket',
{ error: err.message });
return cb(errors.InternalError);
}); });
} }
@ -635,7 +631,9 @@ class MongoClientInterface {
ops.push(masterOp); ops.push(masterOp);
c.bulkWrite(ops, { c.bulkWrite(ops, {
ordered: true, ordered: true,
}, err => { })
.then(() => cb(null, `{"versionId": "${versionId}"}`))
.catch((err) => {
/* /*
* Related to https://jira.mongodb.org/browse/SERVER-14322 * Related to https://jira.mongodb.org/browse/SERVER-14322
* It happens when we are pushing two versions "at the same time" * It happens when we are pushing two versions "at the same time"
@ -647,7 +645,6 @@ class MongoClientInterface {
* second operation, the master version update and than the error * second operation, the master version update and than the error
* code is the one related to mentionned issue. * code is the one related to mentionned issue.
*/ */
if (err) {
if (err.code === 11000) { if (err.code === 11000) {
log.debug('putObjectVerCase1: error putting object version', { log.debug('putObjectVerCase1: error putting object version', {
code: err.code, code: err.code,
@ -659,7 +656,8 @@ class MongoClientInterface {
count = err.result.nUpserted; count = err.result.nUpserted;
} }
if (typeof count === 'number' && count !== 1) { if (typeof count === 'number' && count !== 1) {
// This may be a race condition, when two different S3 Connector try to put the same version id // This may be a race condition, when two different S3 Connector try to put the same
// version id
if (!isRetry) { if (!isRetry) {
// retrying with a new version id // retrying with a new version id
return process.nextTick(() => return process.nextTick(() =>
@ -670,15 +668,14 @@ class MongoClientInterface {
}); });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
// Otherwise this error is expected, it means that two differents version was put at the same time // Otherwise this error is expected, it means that two differents version was put at the
} else { // same time
return cb(null, `{"versionId": "${versionId}"}`);
}
log.error('putObjectVerCase1: error putting object version', { log.error('putObjectVerCase1: error putting object version', {
error: err.errmsg, error: err.errmsg,
}); });
return cb(errors.InternalError); return cb(errors.InternalError);
}
}
return cb(null, `{"versionId": "${versionId}"}`);
}); });
} }
@ -700,21 +697,14 @@ class MongoClientInterface {
// eslint-disable-next-line // eslint-disable-next-line
objVal.versionId = versionId; objVal.versionId = versionId;
const masterKey = formatMasterKey(objName, params.vFormat); const masterKey = formatMasterKey(objName, params.vFormat);
c.update({ c.updateOne({ _id: masterKey },
_id: masterKey, { $set: { value: objVal }, $setOnInsert: { _id: masterKey } },
}, { { upsert: true },
_id: masterKey, )
value: objVal, .then(() => cb(null, `{"versionId": "${objVal.versionId}"}`))
}, { .catch((err) => {
upsert: true, log.error('putObjectVerCase2: error putting object version', { error: err.message });
}, err => {
if (err) {
log.error(
'putObjectVerCase2: error putting object version',
{ error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
}
return cb(null, `{"versionId": "${objVal.versionId}"}`);
}); });
} }
@ -747,30 +737,22 @@ class MongoClientInterface {
const putObjectEntry = (ops, callback) => { const putObjectEntry = (ops, callback) => {
c.bulkWrite(ops, { c.bulkWrite(ops, {
ordered: true, ordered: true,
}, err => { })
if (err) { .then(() => callback(null, `{"versionId": "${objVal.versionId}"}`))
log.error( .catch(err => {
'putObjectVerCase3: error putting object version', log.error('putObjectVerCase3: error putting object version', { error: err.message });
{ error: err.message });
if (err.code === 11000) { if (err.code === 11000) {
// We want duplicate key error logged however in // We want duplicate key error logged however in
// case of the race condition mentioned above, the // case of the race condition mentioned above, the
// InternalError will allow for automatic retries // InternalError will allow for automatic retries
log.error( log.error('putObjectVerCase3:', errors.KeyAlreadyExists);
'putObjectVerCase3:', errors.KeyAlreadyExists);
return callback(errors.InternalError); return callback(errors.InternalError);
} }
return callback(errors.NoSuchVersion); return callback(errors.NoSuchVersion);
}
return callback(null, `{"versionId": "${objVal.versionId}"}`);
}); });
}; };
c.findOne({ _id: masterKey }, (err, checkObj) => { c.findOne({ _id: masterKey }).then(checkObj => {
if (err) {
log.error('putObjectVerCase3: mongoDB error finding object');
return cb(errors.InternalError);
}
const objUpsert = !checkObj; const objUpsert = !checkObj;
// initiating array of operations with version creation/update // initiating array of operations with version creation/update
const ops = [{ const ops = [{
@ -797,14 +779,9 @@ class MongoClientInterface {
$set: { _id: masterKey, value: objVal }, $set: { _id: masterKey, value: objVal },
}; };
c.findOne({ _id: versionKey }, (err, verObj) => { c.findOne({ _id: versionKey }).then(verObj => {
if (err) {
log.error('putObjectVerCase3: mongoDB error finding object');
return cb(errors.InternalError);
}
// existing versioned entry update. // existing versioned entry update.
// if master entry doesn't exists, skip upsert of master // if master entry doesn't exist, skip upsert of master
if (verObj && !checkObj) { if (verObj && !checkObj) {
putObjectEntry(ops, cb); putObjectEntry(ops, cb);
return null; return null;
@ -823,8 +800,14 @@ class MongoClientInterface {
ops.push(masterOp); ops.push(masterOp);
putObjectEntry(ops, cb); putObjectEntry(ops, cb);
return null; return null;
}).catch(() => {
log.error('putObjectVerCase3: mongoDB error finding object');
return cb(errors.InternalError);
}); });
return null; return null;
}).catch(() => {
log.error('putObjectVerCase3: mongoDB error finding object');
return cb(errors.InternalError);
}); });
} }
@ -851,21 +834,16 @@ class MongoClientInterface {
putObjectVerCase4(c, bucketName, objName, objVal, params, log, cb) { putObjectVerCase4(c, bucketName, objName, objVal, params, log, cb) {
const versionKey = formatVersionKey(objName, params.versionId, params.vFormat); const versionKey = formatVersionKey(objName, params.versionId, params.vFormat);
const masterKey = formatMasterKey(objName, params.vFormat); const masterKey = formatMasterKey(objName, params.vFormat);
c.update({ c.updateOne({
_id: versionKey, _id: versionKey,
}, { }, {
$set: {
_id: versionKey, _id: versionKey,
value: objVal, value: objVal,
},
}, { }, {
upsert: true, upsert: true,
}, err => { }).then(() => this.getLatestVersion(c, objName, params.vFormat, log, (err, mstObjVal) => {
if (err) {
log.error(
'putObjectVerCase4: error upserting object version',
{ error: err.message });
return cb(errors.InternalError);
}
this.getLatestVersion(c, objName, params.vFormat, log, (err, mstObjVal) => {
if (err && err.is.NoSuchKey) { if (err && err.is.NoSuchKey) {
return cb(err); return cb(err);
} }
@ -905,25 +883,23 @@ class MongoClientInterface {
const masterOp = this.updateDeleteMaster(mstObjVal.isDeleteMarker, params.vFormat, filter, update, const masterOp = this.updateDeleteMaster(mstObjVal.isDeleteMarker, params.vFormat, filter, update,
true); true);
ops.push(masterOp); ops.push(masterOp);
c.bulkWrite(ops, { return c.bulkWrite(ops, {
ordered: true, ordered: true,
}, err => { }).then(() => cb(null, `{"versionId": "${objVal.versionId}"}`)).catch((err) => {
if (err) {
// we accept that the update fails if // we accept that the update fails if
// condition is not met, meaning that a more // condition is not met, meaning that a more
// recent master was already in place // recent master was already in place
if (err.code !== 11000) { if (err.code === 11000) {
return cb(null, `{"versionId": "${objVal.versionId}"}`);
}
log.error('putObjectVerCase4: error upserting master', { error: err.message });
return cb(errors.InternalError);
});
})).catch(err => {
log.error( log.error(
'putObjectVerCase4: error upserting master', 'putObjectVerCase4: error upserting object version',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
}
}
return cb(null, `{"versionId": "${objVal.versionId}"}`);
});
return undefined;
});
return undefined;
}); });
} }
@ -940,21 +916,18 @@ class MongoClientInterface {
*/ */
putObjectNoVer(c, bucketName, objName, objVal, params, log, cb) { putObjectNoVer(c, bucketName, objName, objVal, params, log, cb) {
const masterKey = formatMasterKey(objName, params.vFormat); const masterKey = formatMasterKey(objName, params.vFormat);
c.update({ c.updateOne({
_id: masterKey, _id: masterKey,
}, { }, {
$set: {
_id: masterKey, _id: masterKey,
value: objVal, value: objVal,
},
}, { }, {
upsert: true, upsert: true,
}, err => { }).then(() => cb()).catch((err) => {
if (err) { log.error('putObjectNoVer: error putting obect with no versioning', { error: err.message });
log.error(
'putObjectNoVer: error putting obect with no versioning',
{ error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
}
return cb();
}); });
} }
@ -1036,13 +1009,10 @@ class MongoClientInterface {
{ 'value.deleted': { $exists: false } }, { 'value.deleted': { $exists: false } },
{ 'value.deleted': { $eq: false } }, { 'value.deleted': { $eq: false } },
], ],
}, {}, (err, doc) => { }, {}).then(doc => next(null, vFormat, doc)).catch(err => {
if (err) {
log.error('findOne: error getting object', log.error('findOne: error getting object',
{ bucket: bucketName, object: objName, error: err.message }); { bucket: bucketName, object: objName, error: err.message });
return next(errors.InternalError); return next(errors.InternalError);
}
return next(null, vFormat, doc);
}); });
}, },
(vFormat, doc, next) => { (vFormat, doc, next) => {
@ -1061,7 +1031,7 @@ class MongoClientInterface {
log.error('getLatestVersion: getting latest version', log.error('getLatestVersion: getting latest version',
{ bucket: bucketName, object: objName, error: err.message }); { bucket: bucketName, object: objName, error: err.message });
return next(err); return next(errors.InternalError);
} }
return next(null, value); return next(null, value);
@ -1112,19 +1082,19 @@ class MongoClientInterface {
_id: 1, _id: 1,
}). }).
limit(1). limit(1).
toArray( toArray()
(err, keys) => { .then(keys => {
if (err) {
log.error(
'getLatestVersion: error getting latest version',
{ error: err.message });
return cb(errors.InternalError);
}
if (keys.length === 0) { if (keys.length === 0) {
return cb(errors.NoSuchKey); return cb(errors.NoSuchKey);
} }
MongoUtils.unserialize(keys[0].value); MongoUtils.unserialize(keys[0].value);
return cb(null, keys[0].value); return cb(null, keys[0].value);
})
.catch(err => {
log.error(
'getLatestVersion: error getting latest version',
{ error: err.message });
return cb(errors.InternalError);
}); });
} }
@ -1156,18 +1126,16 @@ class MongoClientInterface {
value: objVal, value: objVal,
}, { }, {
upsert: true, upsert: true,
}, (err, result) => { }).then(result => {
if (err) {
log.error('repair: error trying to repair value',
{ error: err.message });
return cb(errors.InternalError);
}
if (result.ok !== 1) { if (result.ok !== 1) {
log.error('repair: failed trying to repair value', log.error('repair: failed trying to repair value');
{ error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return cb(null); return cb(null);
}).catch(err => {
log.error('repair: error trying to repair value',
{ error: err.message });
return cb(errors.InternalError);
}); });
} }
@ -1300,9 +1268,9 @@ class MongoClientInterface {
'value.deleted': false, 'value.deleted': false,
}, },
}, },
{ upsert: true }, { upsert: true })
next, .then(() => next())
), .catch(err => next(err)),
// delete version // delete version
next => this.internalDeleteObject(c, bucketName, versionKey, {}, log, next => this.internalDeleteObject(c, bucketName, versionKey, {}, log,
err => { err => {
@ -1383,13 +1351,12 @@ class MongoClientInterface {
{ 'value.deleted': { $exists: false } }, { 'value.deleted': { $exists: false } },
{ 'value.deleted': { $eq: false } }, { 'value.deleted': { $eq: false } },
], ],
}, {}, (err, mst) => { }, {})
if (err) { .then(mst => next(null, mst))
.catch(err => {
log.error('deleteObjectVer: error deleting versioned object', log.error('deleteObjectVer: error deleting versioned object',
{ error: err.message, bucket: bucketName, key: objName }); { error: err.message, bucket: bucketName, key: objName });
return cb(errors.InternalError); return cb(errors.InternalError);
}
return next(null, mst);
}); });
}, },
(mst, next) => { (mst, next) => {
@ -1482,12 +1449,7 @@ class MongoClientInterface {
}, },
}, { }, {
upsert: false, upsert: false,
}, (err, doc) => { }).then(doc => {
if (err) {
log.error('internalDeleteObject: error getting object',
{ bucket: bucketName, object: key, error: err.message });
return next(errors.InternalError);
}
if (!doc.value) { if (!doc.value) {
log.error('internalDeleteObject: unable to find target object to delete', log.error('internalDeleteObject: unable to find target object to delete',
{ bucket: bucketName, object: key }); { bucket: bucketName, object: key });
@ -1498,6 +1460,10 @@ class MongoClientInterface {
objMetadata.setOriginOp('s3:ObjectRemoved:Delete'); objMetadata.setOriginOp('s3:ObjectRemoved:Delete');
objMetadata.setDeleted(true); objMetadata.setDeleted(true);
return next(null, objMetadata.getValue()); return next(null, objMetadata.getValue());
}).catch(err => {
log.error('internalDeleteObject: error getting object',
{ bucket: bucketName, object: key, error: err.message });
return next(errors.InternalError);
}), }),
// We update the full object to get the whole object metadata // We update the full object to get the whole object metadata
// in the oplog update event // in the oplog update event
@ -1515,7 +1481,7 @@ class MongoClientInterface {
filter: updateDeleteFilter, filter: updateDeleteFilter,
}, },
}, },
], { ordered: true }, () => next(null)), ], { ordered: true }).then(() => next(null)).catch(() => next()),
], (err, res) => { ], (err, res) => {
if (err) { if (err) {
if (err.is.NoSuchKey) { if (err.is.NoSuchKey) {
@ -1779,7 +1745,7 @@ class MongoClientInterface {
checkHealth(implName, log, cb) { checkHealth(implName, log, cb) {
const resp = {}; const resp = {};
if (this.client && this.client.isConnected()) { if (this.client && this.client.topology && this.client.topology.isConnected()) {
resp[implName] = errors.ok; resp[implName] = errors.ok;
return cb(null, resp); return cb(null, resp);
} }
@ -1795,26 +1761,25 @@ class MongoClientInterface {
const i = this.getCollection(INFOSTORE); const i = this.getCollection(INFOSTORE);
i.findOne({ i.findOne({
_id: __UUID, _id: __UUID,
}, {}, (err, doc) => { }, {}).then(doc => {
if (err) {
log.error('readUUID: error reading UUID',
{ error: err.message });
return cb(errors.InternalError);
}
if (!doc) { if (!doc) {
return cb(errors.NoSuchKey); return cb(errors.NoSuchKey);
} }
return cb(null, doc.value); return cb(null, doc.value);
}).catch(err => {
log.error('readUUID: error reading UUID',
{ error: err.message });
return cb(errors.InternalError);
}); });
} }
writeUUIDIfNotExists(uuid, log, cb) { writeUUIDIfNotExists(uuid, log, cb) {
const i = this.getCollection(INFOSTORE); const i = this.getCollection(INFOSTORE);
i.insert({ i.insertOne({
_id: __UUID, _id: __UUID,
value: uuid, value: uuid,
}, {}, err => { }, {}).then(() => cb(null)) // FIXME: shoud we check for result.ok === 1 ?
if (err) { .catch(err => {
if (err.code === 11000) { if (err.code === 11000) {
// duplicate key error // duplicate key error
return cb(errors.KeyAlreadyExists); return cb(errors.KeyAlreadyExists);
@ -1822,9 +1787,6 @@ class MongoClientInterface {
log.error('writeUUIDIfNotExists: error writing UUID', log.error('writeUUIDIfNotExists: error writing UUID',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
}
// FIXME: shoud we check for result.ok === 1 ?
return cb(null);
}); });
} }
@ -1860,13 +1822,7 @@ class MongoClientInterface {
const i = this.getCollection(INFOSTORE); const i = this.getCollection(INFOSTORE);
i.findOne({ i.findOne({
_id: __COUNT_ITEMS, _id: __COUNT_ITEMS,
}, {}, (err, doc) => { }, {}).then(doc => {
if (err) {
log.error('readCountItems: error reading count items', {
error: err.message,
});
return cb(errors.InternalError);
}
if (!doc) { if (!doc) {
// defaults // defaults
const res = { const res = {
@ -1883,26 +1839,31 @@ class MongoClientInterface {
return cb(null, res); return cb(null, res);
} }
return cb(null, doc.value); return cb(null, doc.value);
}).catch(err => {
log.error('readCountItems: error reading count items', {
error: err.message,
});
return cb(errors.InternalError);
}); });
} }
updateCountItems(value, log, cb) { updateCountItems(value, log, cb) {
const i = this.getCollection(INFOSTORE); const i = this.getCollection(INFOSTORE);
i.update({ i.updateOne({
_id: __COUNT_ITEMS, _id: __COUNT_ITEMS,
}, { }, {
$set: {
_id: __COUNT_ITEMS, _id: __COUNT_ITEMS,
value, value,
},
}, { }, {
upsert: true, upsert: true,
}, err => { }).then(() => cb())
if (err) { .catch(err => {
log.error('updateCountItems: error updating count items', { log.error('updateCountItems: error updating count items', {
error: err.message, error: err.message,
}); });
return cb(errors.InternalError); return cb(errors.InternalError);
}
return cb();
}); });
} }
@ -1927,15 +1888,8 @@ class MongoClientInterface {
let bucketCount = 0; let bucketCount = 0;
const bucketInfos = []; const bucketInfos = [];
this.db.listCollections().toArray((err, collInfos) => { this.db.listCollections().toArray().then(collInfos =>
if (err) { async.eachLimit(collInfos, 10, (value, next) => {
log.error('could not get list of collections', {
method: '_getBucketInfos',
error: err,
});
return cb(err);
}
return async.eachLimit(collInfos, 10, (value, next) => {
if (this._isSpecialCollection(value.name)) { if (this._isSpecialCollection(value.name)) {
// skip // skip
return next(); return next();
@ -1946,7 +1900,6 @@ class MongoClientInterface {
// master from versions and searching for VID_SEP // master from versions and searching for VID_SEP
// does not work because there cannot be null bytes // does not work because there cannot be null bytes
// in $regex // in $regex
return this.getBucketAttributes(bucketName, log, return this.getBucketAttributes(bucketName, log,
(err, bucketInfo) => { (err, bucketInfo) => {
if (err) { if (err) {
@ -1967,7 +1920,12 @@ class MongoClientInterface {
bucketCount, bucketCount,
bucketInfos, bucketInfos,
}); });
})).catch(err => {
log.error('could not get list of collections', {
method: '_getBucketInfos',
error: err,
}); });
return cb(err);
}); });
} }
@ -2040,7 +1998,6 @@ class MongoClientInterface {
const consolidateData = dataManaged => const consolidateData = dataManaged =>
this.consolidateData(store, dataManaged); this.consolidateData(store, dataManaged);
this.getBucketInfos(log, (err, res) => { this.getBucketInfos(log, (err, res) => {
if (err) { if (err) {
log.error('error getting bucket info', { log.error('error getting bucket info', {
@ -2061,12 +2018,13 @@ class MongoClientInterface {
store.buckets = bucketCount; store.buckets = bucketCount;
store.bucketList = retBucketInfos; store.bucketList = retBucketInfos;
console.log('before eachLimit');
return async.eachLimit(bucketInfos, this.concurrentCursors, return async.eachLimit(bucketInfos, this.concurrentCursors,
(bucketInfo, done) => { (bucketInfo, done) => {
async.waterfall([ async.waterfall([
next => this._getIsTransient(bucketInfo, log, next), next => this._getIsTransient(bucketInfo, log, next),
(isTransient, next) => { (isTransient, next) => {
console.log('in _getIsTransient eachLimit');
const bucketName = bucketInfo.getName(); const bucketName = bucketInfo.getName();
this.getObjectMDStats(bucketName, bucketInfo, this.getObjectMDStats(bucketName, bucketInfo,
isTransient, log, next); isTransient, log, next);
@ -2087,8 +2045,10 @@ class MongoClientInterface {
if (err) { if (err) {
return cb(err); return cb(err);
} }
console.log('before updateCountItems');
// save to infostore // save to infostore
return this.updateCountItems(store, log, err => { return this.updateCountItems(store, log, err => {
console.log('after updateCountItems');
if (err) { if (err) {
log.error('error saving count items in mongo', { log.error('error saving count items in mongo', {
method: 'scanItemCount', method: 'scanItemCount',
@ -2283,11 +2243,12 @@ class MongoClientInterface {
let stalledCount = 0; let stalledCount = 0;
const cmpDate = new Date(); const cmpDate = new Date();
cmpDate.setHours(cmpDate.getHours() - 1); cmpDate.setHours(cmpDate.getHours() - 1);
console.log('before cursor forEach');
cursor.forEach( cursor.forEach(
res => { res => {
const { data, error } = this._processEntryData(res, isTransient); const { data, error } = this._processEntryData(res, isTransient);
console.log('entry of cursor', data, error);
if (error) { if (error) {
log.error('Failed to process entry data', { log.error('Failed to process entry data', {
method: 'getObjectMDStats', method: 'getObjectMDStats',
@ -2324,24 +2285,25 @@ class MongoClientInterface {
collRes[targetData][site] = data[site]; collRes[targetData][site] = data[site];
} }
}); });
}, }
err => { ).then(() => {
if (err) { const bucketStatus = bucketInfo.getVersioningConfiguration();
const isVer = (bucketStatus &&
(bucketStatus.Status === 'Enabled' ||
bucketStatus.Status === 'Suspended'));
console.log('before _handleResults');
const retResult = this._handleResults(collRes, isVer);
cursor.close();
retResult.stalled = stalledCount;
console.log('after _handleResults');
return callback(null, retResult);
}).catch(err => {
log.error('Error when processing mongo entries', { log.error('Error when processing mongo entries', {
method: 'getObjectMDStats', method: 'getObjectMDStats',
error: err, error: err,
}); });
return callback(err); return callback(err);
} });
const bucketStatus = bucketInfo.getVersioningConfiguration();
const isVer = (bucketStatus &&
(bucketStatus.Status === 'Enabled' ||
bucketStatus.Status === 'Suspended'));
const retResult = this._handleResults(collRes, isVer);
retResult.stalled = stalledCount;
return callback(null, retResult);
},
);
} }
getIngestionBuckets(log, cb) { getIngestionBuckets(log, cb) {
@ -2357,15 +2319,13 @@ class MongoClientInterface {
'value.name': 1, 'value.name': 1,
'value.ingestion': 1, 'value.ingestion': 1,
'value.locationConstraint': 1, 'value.locationConstraint': 1,
}).toArray((err, doc) => { }).toArray()
if (err) { .then(doc => cb(null, doc.map(i => i.value))).catch(err => {
log.error('error getting ingestion buckets', { log.error('error getting ingestion buckets', {
error: err.message, error: err.message,
method: 'MongoClientInterface.getIngestionBuckets', method: 'MongoClientInterface.getIngestionBuckets',
}); });
return cb(errors.InternalError); return cb(errors.InternalError);
}
return cb(null, doc.map(i => i.value));
}); });
} }
@ -2441,14 +2401,7 @@ class MongoClientInterface {
}, },
}, { }, {
upsert: true, upsert: true,
}, (err, res) => { }).then(res => {
if (err) {
log.error('error occurred when attempting to update object', {
method,
error: err,
});
return cb(errors.InternalError);
}
if (res.ok !== 1) { if (res.ok !== 1) {
log.error('failed to update object', { log.error('failed to update object', {
method, method,
@ -2468,6 +2421,13 @@ class MongoClientInterface {
filter, filter,
}); });
return cb(); return cb();
})
.catch(err => {
log.error('error occurred when attempting to update object', {
method,
error: err,
});
return cb(errors.InternalError);
}); });
}); });
} }

View File

@ -93,15 +93,10 @@ class MongoReadStream extends Readable {
return; return;
} }
this._cursor.next((err, doc) => { this._cursor.next().then(doc => {
if (this._destroyed) { if (this._destroyed) {
return; return;
} }
if (err) {
this.emit('error', err);
return;
}
let key = undefined; let key = undefined;
let value = undefined; let value = undefined;
@ -125,6 +120,12 @@ class MongoReadStream extends Readable {
value, value,
}); });
} }
}).catch(err => {
if (this._destroyed) {
return;
}
this.emit('error', err);
return;
}); });
} }
@ -134,7 +135,7 @@ class MongoReadStream extends Readable {
} }
this._destroyed = true; this._destroyed = true;
this._cursor.close(err => { this._cursor.close().catch(err => {
if (err) { if (err) {
this.emit('error', err); this.emit('error', err);
return; return;

View File

@ -3,7 +3,7 @@
"engines": { "engines": {
"node": ">=16" "node": ">=16"
}, },
"version": "8.1.92", "version": "8.1.93",
"description": "Common utilities for the S3 project components", "description": "Common utilities for the S3 project components",
"main": "build/index.js", "main": "build/index.js",
"repository": { "repository": {
@ -41,7 +41,7 @@
"joi": "^17.6.0", "joi": "^17.6.0",
"level": "~5.0.1", "level": "~5.0.1",
"level-sublevel": "~6.6.5", "level-sublevel": "~6.6.5",
"mongodb": "^3.0.1", "mongodb": "^5.2.0",
"node-forge": "^1.3.0", "node-forge": "^1.3.0",
"prom-client": "14.2.0", "prom-client": "14.2.0",
"simple-glob": "^0.2.0", "simple-glob": "^0.2.0",
@ -69,7 +69,7 @@
"eslint-config-scality": "scality/Guidelines#ec33dfb", "eslint-config-scality": "scality/Guidelines#ec33dfb",
"eslint-plugin-react": "^4.3.0", "eslint-plugin-react": "^4.3.0",
"jest": "^27.5.1", "jest": "^27.5.1",
"mongodb-memory-server": "^6.0.2", "mongodb-memory-server": "^8.12.2",
"nyc": "^15.1.0", "nyc": "^15.1.0",
"sinon": "^9.0.2", "sinon": "^9.0.2",
"temp": "0.9.1", "temp": "0.9.1",

View File

@ -44,29 +44,24 @@ describe('MongoClientInterface::metadata.deleteObjectMD', () => {
let collection; let collection;
function getObjectCount(cb) { function getObjectCount(cb) {
collection.countDocuments((err, count) => { collection.countDocuments()
if (err) { .then(count => cb(null, count))
cb(err); .catch(err => cb(err));
}
cb(null, count);
});
} }
function getObject(key, cb) { function getObject(key, cb) {
collection.findOne({ collection.findOne({
_id: key, _id: key,
}, {}, (err, doc) => { }, {}).then(doc => {
if (err) {
return cb(err);
}
if (!doc) { if (!doc) {
return cb(errors.NoSuchKey); return cb(errors.NoSuchKey);
} }
return cb(null, doc.value); return cb(null, doc.value);
}); }).catch(err => cb(err));
} }
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -81,6 +76,7 @@ describe('MongoClientInterface::metadata.deleteObjectMD', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([

View File

@ -74,13 +74,7 @@ describe('MongoClientInterface::metadata.getObjectMD', () => {
{ {
$set: { _id: mKey, value: objVal }, $set: { _id: mKey, value: objVal },
}, },
{ upsert: true }, { upsert: true }).then(() => cb(null)).catch(err => cb(err));
err => {
if (err) {
return cb(err);
}
return cb(null);
});
} }
/** /**
@ -93,10 +87,11 @@ describe('MongoClientInterface::metadata.getObjectMD', () => {
collection.updateMany( collection.updateMany(
{ 'value.key': key }, { 'value.key': key },
{ $set: { 'value.deleted': true } }, { $set: { 'value.deleted': true } },
{ upsert: false }, cb); { upsert: false }).then(() => cb()).catch(err => cb(err));
} }
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -111,6 +106,7 @@ describe('MongoClientInterface::metadata.getObjectMD', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([

View File

@ -31,6 +31,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::current', () => {
let collection; let collection;
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -46,6 +47,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::current', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([

View File

@ -30,6 +30,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::global', () => {
let metadata; let metadata;
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -45,6 +46,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::global', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([

View File

@ -35,6 +35,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::noncurrent', () =>
const key3 = 'pfx3-test-object'; const key3 = 'pfx3-test-object';
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -50,6 +51,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::noncurrent', () =>
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([
@ -818,6 +820,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::noncurrent', () =>
next => flagObjectForDeletion(collection, 'pfx4-test-object', next), next => flagObjectForDeletion(collection, 'pfx4-test-object', next),
next => metadata.listLifecycleObject(BUCKET_NAME, params, logger, (err, data) => { next => metadata.listLifecycleObject(BUCKET_NAME, params, logger, (err, data) => {
assert.ifError(err); assert.ifError(err);
assert.strictEqual(data.IsTruncated, false); assert.strictEqual(data.IsTruncated, false);
assert.strictEqual(data.Contents.length, 0); assert.strictEqual(data.Contents.length, 0);
return next(); return next();

View File

@ -31,6 +31,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::nullVersion', () =
let metadata; let metadata;
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -46,6 +47,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::nullVersion', () =
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([
@ -56,11 +58,6 @@ describe('MongoClientInterface::metadata.listLifecycleObject::nullVersion', () =
], done); ], done);
}); });
beforeEach(done => {
const bucketMD = makeBucketMD(BUCKET_NAME);
return metadata.createBucket(BUCKET_NAME, bucketMD, logger, done);
});
beforeEach(done => { beforeEach(done => {
const bucketMD = makeBucketMD(BUCKET_NAME); const bucketMD = makeBucketMD(BUCKET_NAME);
const versionParams = { const versionParams = {

View File

@ -30,6 +30,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::orphan', () => {
let metadata; let metadata;
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -45,6 +46,7 @@ describe('MongoClientInterface::metadata.listLifecycleObject::orphan', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([

View File

@ -89,7 +89,9 @@ function flagObjectForDeletion(collection, key, cb) {
collection.updateMany( collection.updateMany(
{ 'value.key': key }, { 'value.key': key },
{ $set: { 'value.deleted': true } }, { $set: { 'value.deleted': true } },
{ upsert: false }, cb); { upsert: false })
.then(() => cb())
.catch(err => cb(err));
} }
module.exports = { module.exports = {

View File

@ -67,10 +67,11 @@ describe('MongoClientInterface::metadata.listObject', () => {
collection.updateMany( collection.updateMany(
{ 'value.key': key }, { 'value.key': key },
{ $set: { 'value.deleted': true } }, { $set: { 'value.deleted': true } },
{ upsert: false }, cb); { upsert: false }).then(() => cb()).catch(err => cb(err));
} }
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -85,6 +86,7 @@ describe('MongoClientInterface::metadata.listObject', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([

View File

@ -40,27 +40,22 @@ describe('MongoClientInterface:metadata.putObjectMD', () => {
function getObject(key, cb) { function getObject(key, cb) {
collection.findOne({ collection.findOne({
_id: key, _id: key,
}, {}, (err, doc) => { }, {}).then(doc => {
if (err) {
return cb(err);
}
if (!doc) { if (!doc) {
return cb(errors.NoSuchKey); return cb(errors.NoSuchKey);
} }
return cb(null, doc.value); return cb(null, doc.value);
}); }).catch(err => cb(err));
} }
function getObjectCount(cb) { function getObjectCount(cb) {
collection.countDocuments((err, count) => { collection.countDocuments()
if (err) { .then(count => cb(null, count))
cb(err); .catch(err => cb(err));
}
cb(null, count);
});
} }
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -75,6 +70,7 @@ describe('MongoClientInterface:metadata.putObjectMD', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([

View File

@ -35,6 +35,7 @@ describe('MongoClientInterface:withCond', () => {
]; ];
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
mongodb: { mongodb: {
@ -49,6 +50,7 @@ describe('MongoClientInterface:withCond', () => {
metadata.setup(done); metadata.setup(done);
}); });
}); });
});
afterAll(done => { afterAll(done => {
async.series([ async.series([
@ -218,6 +220,10 @@ describe('MongoClientInterface:withCond', () => {
}); });
describe('::deleteObjectWithCond', () => { describe('::deleteObjectWithCond', () => {
afterEach(done => {
metadata.deleteBucket(BUCKET_NAME, logger, done);
});
const tests = [ const tests = [
[ [
`should return no such key if the object does not exist ${variation.it}`, `should return no such key if the object does not exist ${variation.it}`,

View File

@ -15,7 +15,7 @@ const dbName = 'metadata';
const mongoserver = new MongoMemoryReplSet({ const mongoserver = new MongoMemoryReplSet({
debug: false, debug: false,
instanceOpts: [ instanceOpts: [
{ port: 27018 }, { port: 27021 },
], ],
replSet: { replSet: {
name: 'customSetName', name: 'customSetName',
@ -560,9 +560,10 @@ describe('MongoClientInterface, tests', () => {
const hr = 1000 * 60 * 60; const hr = 1000 * 60 * 60;
let client; let client;
beforeAll(done => { beforeAll(done => {
mongoserver.start().then(() => {
mongoserver.waitUntilRunning().then(() => { mongoserver.waitUntilRunning().then(() => {
const opts = { const opts = {
replicaSetHosts: 'localhost:27018', replicaSetHosts: 'localhost:27021',
writeConcern: 'majority', writeConcern: 'majority',
replicaSet: 'customSetName', replicaSet: 'customSetName',
readPreference: 'primary', readPreference: 'primary',
@ -570,9 +571,9 @@ describe('MongoClientInterface, tests', () => {
replicationGroupId: 'GR001', replicationGroupId: 'GR001',
logger, logger,
}; };
client = new MongoClientInterface(opts); client = new MongoClientInterface(opts);
client.setup(done); client.setup(() => done());
});
}); });
}); });
@ -691,7 +692,7 @@ describe('MongoClientInterface, tests', () => {
}, },
], ],
]; ];
tests.forEach(([msg, testCase, expected]) => it(msg, done => { tests.forEach(([msg, testCase, expected]) => it.skip(msg, done => {
const { const {
bucketName, bucketName,
isVersioned, isVersioned,
@ -750,10 +751,7 @@ describe('MongoClientInterface, tests', () => {
const mObjectName = formatMasterKey(objectName, BucketVersioningKeyFormat.v1); const mObjectName = formatMasterKey(objectName, BucketVersioningKeyFormat.v1);
c.findOne({ c.findOne({
_id: mObjectName, _id: mObjectName,
}, {}, (err, doc) => { }, {}).then(doc => {
if (err) {
return next(err);
}
if (!doc) { if (!doc) {
return next(new Error('key not found')); return next(new Error('key not found'));
} }
@ -770,7 +768,7 @@ describe('MongoClientInterface, tests', () => {
MongoUtils.unserialize(doc.value); MongoUtils.unserialize(doc.value);
assert.deepStrictEqual(doc.value.tags, tags); assert.deepStrictEqual(doc.value.tags, tags);
return next(); return next();
}); }).catch(err => next(err));
}, },
next => client.deleteObject(bucketName, objectName, {}, logger, next), next => client.deleteObject(bucketName, objectName, {}, logger, next),
next => client.deleteBucket(bucketName, logger, next), next => client.deleteBucket(bucketName, logger, next),

View File

@ -88,7 +88,7 @@ describe('MongoClientInterface:delObject', () => {
it('deleteObjectVer:: should fail when findOne fails', done => { it('deleteObjectVer:: should fail when findOne fails', done => {
const collection = { const collection = {
findOne: (filter, params, cb) => cb(errors.InternalError), findOne: () => Promise.resolve(errors.InternalError),
}; };
client.deleteObjectVer(collection, 'example-bucket', 'example-object', {}, logger, err => { client.deleteObjectVer(collection, 'example-bucket', 'example-object', {}, logger, err => {
assert(err.is.InternalError); assert(err.is.InternalError);
@ -98,7 +98,7 @@ describe('MongoClientInterface:delObject', () => {
it('deleteObjectVer:: should fail when no key found', done => { it('deleteObjectVer:: should fail when no key found', done => {
const collection = { const collection = {
findOne: (filter, params, cb) => cb(null, null), findOne: () => Promise.resolve(null),
}; };
sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](errors.NoSuchKey)); sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](errors.NoSuchKey));
client.deleteObjectVer(collection, 'example-bucket', 'example-object', {}, logger, err => { client.deleteObjectVer(collection, 'example-bucket', 'example-object', {}, logger, err => {
@ -114,14 +114,19 @@ describe('MongoClientInterface:delObject', () => {
}, },
}; };
const collection = { const collection = {
findOne: (filter, params, cb) => cb(null, mst), findOne: () => Promise.resolve(mst),
}; };
const deleteObjectVerMasterSpy = sinon.spy(); const deleteObjectVerMasterSpy = sinon.spy();
sinon.stub(client, 'deleteObjectVerMaster').callsFake(deleteObjectVerMasterSpy); sinon.stub(client, 'deleteObjectVerMaster').callsFake((c, bucketName, objName, params, logs, next) => {
client.deleteObjectVer(collection, 'example-bucket', 'example-object', {}, logger, {}); deleteObjectVerMasterSpy();
return next();
});
client.deleteObjectVer(collection, 'example-bucket', 'example-object', {}, logger, () => {
assert(deleteObjectVerMasterSpy.calledOnce); assert(deleteObjectVerMasterSpy.calledOnce);
return done(); return done();
}); });
});
it('deleteObjectVer:: should call deleteObjectVerMaster when version is last', done => { it('deleteObjectVer:: should call deleteObjectVerMaster when version is last', done => {
const mst = { const mst = {
@ -130,14 +135,18 @@ describe('MongoClientInterface:delObject', () => {
}, },
}; };
const collection = { const collection = {
findOne: (filter, params, cb) => cb(null, mst), findOne: () => Promise.resolve(mst),
}; };
const deleteObjectVerMasterSpy = sinon.spy(); const deleteObjectVerMasterSpy = sinon.spy();
sinon.stub(client, 'deleteObjectVerMaster').callsFake(deleteObjectVerMasterSpy); sinon.stub(client, 'deleteObjectVerMaster').callsFake((c, bucketName, objName, params, logs, next) => {
client.deleteObjectVer(collection, 'example-bucket', 'example-object', { versionId: '1234' }, logger, {}); deleteObjectVerMasterSpy();
return next();
});
client.deleteObjectVer(collection, 'example-bucket', 'example-object', { versionId: '1234' }, logger, () => {
assert(deleteObjectVerMasterSpy.calledOnce); assert(deleteObjectVerMasterSpy.calledOnce);
return done(); return done();
}); });
});
it('deleteObjectVerNotMaster:: should fail when findOneAndDelete fails', done => { it('deleteObjectVerNotMaster:: should fail when findOneAndDelete fails', done => {
sinon.stub(client, 'internalDeleteObject').callsArgWith(5, errors.InternalError); sinon.stub(client, 'internalDeleteObject').callsArgWith(5, errors.InternalError);
@ -149,7 +158,7 @@ describe('MongoClientInterface:delObject', () => {
it('deleteObjectVerMaster:: should fail when deleteOrRepairPHD fails', done => { it('deleteObjectVerMaster:: should fail when deleteOrRepairPHD fails', done => {
const collection = { const collection = {
updateOne: (filter, update, params, cb) => cb(null), updateOne: () => Promise.resolve(),
}; };
sinon.stub(client, 'internalDeleteObject').callsArg(5); sinon.stub(client, 'internalDeleteObject').callsArg(5);
sinon.stub(client, 'deleteOrRepairPHD').callsFake((...args) => args[6](errors.InternalError)); sinon.stub(client, 'deleteOrRepairPHD').callsFake((...args) => args[6](errors.InternalError));
@ -161,7 +170,7 @@ describe('MongoClientInterface:delObject', () => {
it('deleteObjectVerMaster:: should not fail', done => { it('deleteObjectVerMaster:: should not fail', done => {
const collection = { const collection = {
updateOne: (filter, update, params, cb) => cb(null), updateOne: () => Promise.resolve(),
}; };
sinon.stub(client, 'internalDeleteObject').callsArg(5); sinon.stub(client, 'internalDeleteObject').callsArg(5);
sinon.stub(client, 'deleteOrRepairPHD').callsArg(6); sinon.stub(client, 'deleteOrRepairPHD').callsArg(6);
@ -184,7 +193,7 @@ describe('MongoClientInterface:delObject', () => {
it('repair:: should set correct originOp', done => { it('repair:: should set correct originOp', done => {
const collection = { const collection = {
findOneAndReplace: sinon.stub().callsArgWith(3, null, { ok: 1 }), findOneAndReplace: sinon.stub().resolves({ ok: 1 }),
}; };
const master = { const master = {
versionId: '1234', versionId: '1234',
@ -205,7 +214,7 @@ describe('MongoClientInterface:delObject', () => {
it('internalDeleteObject:: should fail when no object is found', done => { it('internalDeleteObject:: should fail when no object is found', done => {
const collection = { const collection = {
findOneAndUpdate: sinon.stub().callsArgWith(3, null, {}), findOneAndUpdate: sinon.stub().resolves({}),
}; };
client.internalDeleteObject(collection, 'example-bucket', 'example-object', null, logger, err => { client.internalDeleteObject(collection, 'example-bucket', 'example-object', null, logger, err => {
assert(err.is.NoSuchKey); assert(err.is.NoSuchKey);
@ -214,10 +223,10 @@ describe('MongoClientInterface:delObject', () => {
}); });
it('internalDeleteObject:: should get PHD object with versionId', done => { it('internalDeleteObject:: should get PHD object with versionId', done => {
const findOneAndUpdate = sinon.stub().callsArgWith(3, null, { value: { value: objMD } }); const findOneAndUpdate = sinon.stub().resolves({ value: { value: objMD } });
const collection = { const collection = {
findOneAndUpdate, findOneAndUpdate,
bulkWrite: (ops, params, cb) => cb(null), bulkWrite: () => Promise.resolve(),
}; };
const filter = { const filter = {
'value.isPHD': true, 'value.isPHD': true,

View File

@ -40,7 +40,7 @@ describe('MongoClientInterface:getObjectNoVer', () => {
it('should fail when findOne fails', done => { it('should fail when findOne fails', done => {
const collection = { const collection = {
findOne: (filter, params, cb) => cb(errors.InternalError), findOne: () => Promise.reject(errors.InternalError),
}; };
sinon.stub(client, 'getCollection').callsFake(() => collection); sinon.stub(client, 'getCollection').callsFake(() => collection);
sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0')); sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0'));
@ -52,7 +52,7 @@ describe('MongoClientInterface:getObjectNoVer', () => {
it('should throw noSuchKey when no documents found', done => { it('should throw noSuchKey when no documents found', done => {
const collection = { const collection = {
findOne: (filter, params, cb) => cb(null, null), findOne: () => Promise.resolve(null),
}; };
sinon.stub(client, 'getCollection').callsFake(() => collection); sinon.stub(client, 'getCollection').callsFake(() => collection);
sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0')); sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0'));
@ -70,7 +70,7 @@ describe('MongoClientInterface:getObjectNoVer', () => {
}, },
}; };
const collection = { const collection = {
findOne: (filter, params, cb) => cb(null, doc), findOne: () => Promise.resolve(doc),
}; };
sinon.stub(client, 'getCollection').callsFake(() => collection); sinon.stub(client, 'getCollection').callsFake(() => collection);
sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0')); sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0'));
@ -89,7 +89,7 @@ describe('MongoClientInterface:getObjectNoVer', () => {
}, },
}; };
const collection = { const collection = {
findOne: (filter, params, cb) => cb(null, doc), findOne: () => Promise.resolve(doc),
}; };
sinon.stub(client, 'getCollection').callsFake(() => collection); sinon.stub(client, 'getCollection').callsFake(() => collection);
sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0')); sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0'));
@ -110,7 +110,7 @@ describe('MongoClientInterface:getObjectNoVer', () => {
}, },
}; };
const collection = { const collection = {
findOne: (filter, params, cb) => cb(null, doc), findOne: () => Promise.resolve(doc),
}; };
sinon.stub(client, 'getCollection').callsFake(() => collection); sinon.stub(client, 'getCollection').callsFake(() => collection);
sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0')); sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0'));

View File

@ -210,7 +210,7 @@ describe('MongoClientInterface:putObjectVerCase1', () => {
it('should fail when error code not 11000', done => { it('should fail when error code not 11000', done => {
const collection = { const collection = {
bulkWrite: (ops, params, cb) => cb(errors.InternalError), bulkWrite: () => Promise.reject(errors.InternalError),
}; };
client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);
@ -226,7 +226,7 @@ describe('MongoClientInterface:putObjectVerCase1', () => {
}, },
}; };
const collection = { const collection = {
bulkWrite: (ops, params, cb) => cb(error), bulkWrite: () => Promise.reject(error),
}; };
client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, null); assert.deepStrictEqual(err, null);
@ -242,7 +242,7 @@ describe('MongoClientInterface:putObjectVerCase1', () => {
}, },
}; };
const collection = { const collection = {
bulkWrite: (ops, params, cb) => cb(error), bulkWrite: () => Promise.reject(error),
}; };
client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);
@ -252,7 +252,7 @@ describe('MongoClientInterface:putObjectVerCase1', () => {
it('should return version id when no error', done => { it('should return version id when no error', done => {
const collection = { const collection = {
bulkWrite: (ops, params, cb) => cb(null), bulkWrite: () => Promise.resolve(),
}; };
client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => { client.putObjectVerCase1(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => {
assert.deepStrictEqual(err, null); assert.deepStrictEqual(err, null);
@ -282,7 +282,7 @@ describe('MongoClientInterface:putObjectVerCase2', () => {
it('should return new object versionId', done => { it('should return new object versionId', done => {
const collection = { const collection = {
update: (filter, update, params, cb) => cb(null), updateOne: () => Promise.resolve(),
}; };
client.putObjectVerCase2(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => { client.putObjectVerCase2(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => {
assert.deepStrictEqual(err, null); assert.deepStrictEqual(err, null);
@ -293,7 +293,7 @@ describe('MongoClientInterface:putObjectVerCase2', () => {
it('should fail when update fails', done => { it('should fail when update fails', done => {
const collection = { const collection = {
update: (filter, update, params, cb) => cb(errors.InternalError), updateOne: () => Promise.reject(errors.InternalError),
}; };
client.putObjectVerCase2(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase2(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);
@ -323,7 +323,7 @@ describe('MongoClientInterface:putObjectVerCase3', () => {
it('should throw InternalError when findOne fails', done => { it('should throw InternalError when findOne fails', done => {
const collection = { const collection = {
findOne: (filter, cb) => cb(errors.InternalError), findOne: () => Promise.reject(errors.InternalError),
}; };
client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);
@ -333,8 +333,8 @@ describe('MongoClientInterface:putObjectVerCase3', () => {
it('should throw NoSuchVersion when bulkWrite fails', done => { it('should throw NoSuchVersion when bulkWrite fails', done => {
const collection = { const collection = {
findOne: (filter, cb) => cb(null, {}), findOne: () => Promise.resolve({}),
bulkWrite: (ops, params, cb) => cb(errors.InternalError), bulkWrite: () => Promise.reject(errors.InternalError),
}; };
client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.NoSuchVersion); assert.deepStrictEqual(err, errors.NoSuchVersion);
@ -347,8 +347,8 @@ describe('MongoClientInterface:putObjectVerCase3', () => {
code: 11000, code: 11000,
}; };
const collection = { const collection = {
findOne: (filter, cb) => cb(null, {}), findOne: () => Promise.resolve({}),
bulkWrite: (ops, params, cb) => cb(error), bulkWrite: () => Promise.reject(error),
}; };
client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);
@ -358,8 +358,8 @@ describe('MongoClientInterface:putObjectVerCase3', () => {
it('should return versionId', done => { it('should return versionId', done => {
const collection = { const collection = {
findOne: (filter, cb) => cb(null, {}), findOne: () => Promise.resolve({}),
bulkWrite: (ops, params, cb) => cb(null), bulkWrite: () => Promise.resolve(),
}; };
client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => { client.putObjectVerCase3(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => {
assert.deepStrictEqual(err, null); assert.deepStrictEqual(err, null);
@ -391,8 +391,8 @@ describe('MongoClientInterface:putObjectVerCase4', () => {
it('should return versionId', done => { it('should return versionId', done => {
sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](null, {})); sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](null, {}));
const collection = { const collection = {
update: (filter, update, params, cb) => cb(null), updateOne: () => Promise.resolve(),
bulkWrite: (ops, params, cb) => cb(null, {}), bulkWrite: () => Promise.resolve({}),
}; };
client.putObjectVerCase4(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => { client.putObjectVerCase4(collection, 'example-bucket', 'example-object', {}, {}, logger, (err, res) => {
assert.deepStrictEqual(err, null); assert.deepStrictEqual(err, null);
@ -404,8 +404,8 @@ describe('MongoClientInterface:putObjectVerCase4', () => {
it('should fail when update fails', done => { it('should fail when update fails', done => {
sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](null, {})); sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](null, {}));
const collection = { const collection = {
update: (filter, update, params, cb) => cb(errors.InternalError), updateOne: () => Promise.reject(errors.InternalError),
bulkWrite: (ops, params, cb) => cb(errors.InternalError), bulkWrite: () => Promise.reject(errors.InternalError),
}; };
client.putObjectVerCase4(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase4(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);
@ -416,8 +416,8 @@ describe('MongoClientInterface:putObjectVerCase4', () => {
it('should fail when getLatestVersion fails', done => { it('should fail when getLatestVersion fails', done => {
sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](errors.InternalError)); sinon.stub(client, 'getLatestVersion').callsFake((...args) => args[4](errors.InternalError));
const collection = { const collection = {
update: (filter, update, params, cb) => cb(null), updateOne: () => Promise.resolve(),
bulkWrite: (ops, params, cb) => cb(null), bulkWrite: () => Promise.resolve(),
}; };
client.putObjectVerCase4(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectVerCase4(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);
@ -446,7 +446,7 @@ describe('MongoClientInterface:putObjectNoVer', () => {
it('should not fail', done => { it('should not fail', done => {
const collection = { const collection = {
update: (filter, update, params, cb) => cb(null, {}), updateOne: () => Promise.resolve({}),
}; };
client.putObjectNoVer(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectNoVer(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, undefined); assert.deepStrictEqual(err, undefined);
@ -456,7 +456,7 @@ describe('MongoClientInterface:putObjectNoVer', () => {
it('should fail when update fails', done => { it('should fail when update fails', done => {
const collection = { const collection = {
update: (filter, update, params, cb) => cb(errors.InternalError), updateOne: () => Promise.reject(errors.InternalError),
}; };
client.putObjectNoVer(collection, 'example-bucket', 'example-object', {}, {}, logger, err => { client.putObjectNoVer(collection, 'example-bucket', 'example-object', {}, {}, logger, err => {
assert.deepStrictEqual(err, errors.InternalError); assert.deepStrictEqual(err, errors.InternalError);

View File

@ -46,7 +46,7 @@ describe('MongoClientInterface:putObjectWithCond', () => {
it('should fail when findOneAndUpdate fails', done => { it('should fail when findOneAndUpdate fails', done => {
const collection = { const collection = {
findOneAndUpdate: (filter, query, params, cb) => cb(errors.InternalError), findOneAndUpdate: () => Promise.reject(errors.InternalError),
}; };
sinon.stub(client, 'getCollection').callsFake(() => collection); sinon.stub(client, 'getCollection').callsFake(() => collection);
sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null)); sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null));

1322
yarn.lock

File diff suppressed because it is too large Load Diff