Compare commits
5 Commits
developmen
...
exp/perfMe
| Author | SHA1 | Date |
|---|---|---|
Anurag Mittal
|
fff956b02b | |
|
Anurag Mittal
|
2c0d6c9c13 | |
|
Anurag Mittal
|
bf84e8b525 | |
|
Anurag Mittal
|
a8e585c2e6 | |
|
Anurag Mittal
|
1bcf735e3d |
|
|
@ -50,7 +50,7 @@ const VID_SEP = require('../../../versioning/constants')
|
|||
function generateVersionId(replicationGroupId) {
|
||||
// generate a unique number for each member of the nodejs cluster
|
||||
return genVID(`${process.pid}.${uidCounter++}`,
|
||||
replicationGroupId);
|
||||
replicationGroupId);
|
||||
}
|
||||
|
||||
function formatVersionKey(key, versionId) {
|
||||
|
|
@ -59,7 +59,7 @@ function formatVersionKey(key, versionId) {
|
|||
|
||||
function inc(str) {
|
||||
return str ? (str.slice(0, str.length - 1) +
|
||||
String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str;
|
||||
String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str;
|
||||
}
|
||||
|
||||
const VID_SEPPLUS = inc(VID_SEP);
|
||||
|
|
@ -116,11 +116,11 @@ class MongoClientInterface {
|
|||
// should be an async_init(cb) method in the wrapper to
|
||||
// initialize this backend
|
||||
if ((process.env.MONGO_CONNECT_TIMEOUT_MS &&
|
||||
Number.isNaN(process.env.MONGO_CONNECT_TIMEOUT_MS)) ||
|
||||
(process.env.MONGO_SOCKET_TIMEOUT_MS &&
|
||||
Number.isNaN(process.env.MONGO_SOCKET_TIMEOUT_MS))) {
|
||||
Number.isNaN(process.env.MONGO_CONNECT_TIMEOUT_MS)) ||
|
||||
(process.env.MONGO_SOCKET_TIMEOUT_MS &&
|
||||
Number.isNaN(process.env.MONGO_SOCKET_TIMEOUT_MS))) {
|
||||
this.logger.error('MongoDB connect and socket timeouts must be a ' +
|
||||
'number. Using default value(s).');
|
||||
'number. Using default value(s).');
|
||||
}
|
||||
const connectTimeoutMS = Number.parseInt(
|
||||
process.env.MONGO_CONNECT_TIMEOUT_MS, 10) || CONNECT_TIMEOUT_MS;
|
||||
|
|
@ -161,8 +161,8 @@ class MongoClientInterface {
|
|||
err => {
|
||||
if (err) {
|
||||
this.logger.fatal('error writing usersBucket ' +
|
||||
'attributes to metastore',
|
||||
{ error: err });
|
||||
'attributes to metastore',
|
||||
{ error: err });
|
||||
throw (errors.InternalError);
|
||||
}
|
||||
return cb();
|
||||
|
|
@ -179,7 +179,7 @@ class MongoClientInterface {
|
|||
getCollection(name) {
|
||||
/* mongo has a problem with .. in collection names */
|
||||
const newName = (name === constants.usersBucket) ?
|
||||
USERSBUCKET : name;
|
||||
USERSBUCKET : name;
|
||||
return this.db.collection(newName);
|
||||
}
|
||||
|
||||
|
|
@ -203,7 +203,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'createBucket: error creating bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
this.lastItemScanTime = null;
|
||||
|
|
@ -216,7 +216,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'createBucket: error creating bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb();
|
||||
|
|
@ -228,6 +228,8 @@ class MongoClientInterface {
|
|||
}
|
||||
|
||||
getBucketAttributes(bucketName, log, cb) {
|
||||
log.debug('entered getBucketAttributes',
|
||||
{ database: bucketName });
|
||||
const m = this.getCollection(METASTORE);
|
||||
m.findOne({
|
||||
_id: bucketName,
|
||||
|
|
@ -245,11 +247,15 @@ class MongoClientInterface {
|
|||
// that properly inits w/o JSON.parse()
|
||||
const bucketMDStr = JSON.stringify(doc.value);
|
||||
const bucketMD = BucketInfo.deSerialize(bucketMDStr);
|
||||
log.debug('exiting getBucketAttributes',
|
||||
{ database: bucketName });
|
||||
return cb(null, bucketMD);
|
||||
});
|
||||
}
|
||||
|
||||
getBucketAndObject(bucketName, objName, params, log, cb) {
|
||||
log.debug('entered getBucketAndObject',
|
||||
{ database: bucketName, object: objName });
|
||||
this.getBucketAttributes(bucketName, log, (err, bucket) => {
|
||||
if (err) {
|
||||
log.error(
|
||||
|
|
@ -260,15 +266,20 @@ class MongoClientInterface {
|
|||
this.getObject(bucketName, objName, params, log, (err, obj) => {
|
||||
if (err) {
|
||||
if (err === errors.NoSuchKey) {
|
||||
log.debug('exiting getBucketAndObject with no such key',
|
||||
{ database: bucketName, object: objName });
|
||||
return cb(null,
|
||||
{ bucket:
|
||||
{
|
||||
bucket:
|
||||
BucketInfo.fromObj(bucket).serialize(),
|
||||
});
|
||||
});
|
||||
}
|
||||
log.error('getObject: error getting object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
log.debug('exiting getBucketAndObject',
|
||||
{ database: bucketName, object: objName });
|
||||
return cb(null, {
|
||||
bucket: BucketInfo.fromObj(bucket).serialize(),
|
||||
obj: JSON.stringify(obj),
|
||||
|
|
@ -296,7 +307,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'putBucketAttributes: error putting bucket attributes',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb();
|
||||
|
|
@ -313,12 +324,12 @@ class MongoClientInterface {
|
|||
}, {}, (err, result) => {
|
||||
if (err) {
|
||||
log.error('deleteBucketStep2: error deleting bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (result.ok !== 1) {
|
||||
log.error('deleteBucketStep2: failed deleting bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null);
|
||||
|
|
@ -343,7 +354,7 @@ class MongoClientInterface {
|
|||
return this.deleteBucketStep2(bucketName, log, cb);
|
||||
}
|
||||
log.error('deleteBucket: error deleting bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return this.deleteBucketStep2(bucketName, log, err => {
|
||||
|
|
@ -398,7 +409,7 @@ class MongoClientInterface {
|
|||
$gt: objVal.versionId,
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
update: {
|
||||
_id: objName, value: objVal,
|
||||
|
|
@ -421,10 +432,10 @@ class MongoClientInterface {
|
|||
*/
|
||||
if (err) {
|
||||
if (!err.index || err.index !== 1
|
||||
|| !err.code || err.code !== 11000) {
|
||||
|| !err.code || err.code !== 11000) {
|
||||
log.error(
|
||||
'putObjectVerCase1: error putting object version',
|
||||
{ error: err.errmsg });
|
||||
{ error: err.errmsg });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
log.debug('putObjectVerCase1: error putting object version',
|
||||
|
|
@ -453,7 +464,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'putObjectVerCase2: error putting object version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null, `{"versionId": "${objVal.versionId}"}`);
|
||||
|
|
@ -514,7 +525,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'putObjectVerCase3: error putting object version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
if (err.code === 11000) {
|
||||
// We want duplicate key error logged however in
|
||||
// case of the race condition mentioned above, the
|
||||
|
|
@ -560,7 +571,7 @@ class MongoClientInterface {
|
|||
this.getLatestVersion(c, objName, log, (err, mstObjVal) => {
|
||||
if (err) {
|
||||
log.error('getLatestVersion: getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
// eslint-disable-next-line
|
||||
|
|
@ -618,26 +629,44 @@ class MongoClientInterface {
|
|||
|
||||
putObject(bucketName, objName, objVal, params, log, cb) {
|
||||
MongoUtils.serialize(objVal);
|
||||
log.debug('entered putObject',
|
||||
{ database: bucketName, object: objName });
|
||||
const c = this.getCollection(bucketName);
|
||||
log.debug('inside putObject, got the collection name',
|
||||
{ database: bucketName, object: objName });
|
||||
if (params && params.versioning && !params.versionId) {
|
||||
log.debug('inside putObject, case: putObjectVerCase1',
|
||||
{ database: bucketName, object: objName });
|
||||
return this.putObjectVerCase1(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
} else if (params && params.versionId === '') {
|
||||
log.debug('inside putObject, case: putObjectVerCase2',
|
||||
{ database: bucketName, object: objName });
|
||||
return this.putObjectVerCase2(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
} else if (params && params.versionId && !params.repairMaster) {
|
||||
log.debug('inside putObject, case: putObjectVerCase3',
|
||||
{ database: bucketName, object: objName });
|
||||
return this.putObjectVerCase3(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
} else if (params && params.versionId && params.repairMaster) {
|
||||
log.debug('inside putObject, case: putObjectVerCase4',
|
||||
{ database: bucketName, object: objName });
|
||||
return this.putObjectVerCase4(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
}
|
||||
log.debug('inside putObject, case: noversion',
|
||||
{ database: bucketName, object: objName });
|
||||
return this.putObjectNoVer(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
}
|
||||
|
||||
getObject(bucketName, objName, params, log, cb) {
|
||||
log.debug('entered getObject',
|
||||
{ database: bucketName, object: objName });
|
||||
const c = this.getCollection(bucketName);
|
||||
log.debug('inside getObject, got the collection',
|
||||
{ database: bucketName, object: objName });
|
||||
if (params && params.versionId) {
|
||||
// eslint-disable-next-line
|
||||
objName = formatVersionKey(objName, params.versionId);
|
||||
|
|
@ -645,26 +674,35 @@ class MongoClientInterface {
|
|||
c.findOne({
|
||||
_id: objName,
|
||||
}, {}, (err, doc) => {
|
||||
log.debug('inside getObject, inside the cb for findOne query',
|
||||
{ database: bucketName, object: objName });
|
||||
if (err) {
|
||||
log.error('findOne: error getting object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
|
||||
if (!doc) {
|
||||
log.debug('inside getObject, got reply from database, obj not found',
|
||||
{ database: bucketName, object: objName });
|
||||
return cb(errors.NoSuchKey);
|
||||
}
|
||||
if (doc.value.isPHD) {
|
||||
this.getLatestVersion(c, objName, log, (err, value) => {
|
||||
if (err) {
|
||||
log.error('getLatestVersion: getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
log.debug('exited in PHD getObject',
|
||||
{ database: bucketName, object: objName });
|
||||
return cb(null, value);
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
MongoUtils.unserialize(doc.value);
|
||||
log.debug('exited getObject',
|
||||
{ database: bucketName, object: objName });
|
||||
return cb(null, doc.value);
|
||||
});
|
||||
}
|
||||
|
|
@ -719,12 +757,12 @@ class MongoClientInterface {
|
|||
}, (err, result) => {
|
||||
if (err) {
|
||||
log.error('repair: error trying to repair value',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (result.ok !== 1) {
|
||||
log.error('repair: failed trying to repair value',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null);
|
||||
|
|
@ -739,7 +777,7 @@ class MongoClientInterface {
|
|||
this.getLatestVersion(c, objName, log, (err, value) => {
|
||||
if (err) {
|
||||
log.error('async-repair: getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return undefined;
|
||||
}
|
||||
this.repair(c, bucketName, objName, value, mst, log, err => {
|
||||
|
|
@ -787,7 +825,7 @@ class MongoClientInterface {
|
|||
return undefined;
|
||||
}
|
||||
log.error('getLatestVersion: error getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
// We have other versions available so repair:
|
||||
|
|
@ -831,7 +869,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'deleteObjectVerMaster: error deleting object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return this.deleteOrRepairPHD(c, bucketName, objName, mst, log, cb);
|
||||
|
|
@ -875,7 +913,7 @@ class MongoClientInterface {
|
|||
}, {}, (err, mst) => {
|
||||
if (err) {
|
||||
log.error('deleteObjectVer: error deleting versioned object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (!mst) {
|
||||
|
|
@ -884,15 +922,15 @@ class MongoClientInterface {
|
|||
if (mst.value.isPHD ||
|
||||
mst.value.versionId === params.versionId) {
|
||||
return this.deleteObjectVerMaster(c, bucketName, objName,
|
||||
params, log, err => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
return cb();
|
||||
});
|
||||
params, log, err => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
return this.deleteObjectVerNotMaster(c, bucketName, objName,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -923,10 +961,10 @@ class MongoClientInterface {
|
|||
const c = this.getCollection(bucketName);
|
||||
if (params && params.versionId) {
|
||||
return this.deleteObjectVer(c, bucketName, objName,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
}
|
||||
return this.deleteObjectNoVer(c, bucketName, objName,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
}
|
||||
|
||||
internalListObject(bucketName, params, extension, log, cb) {
|
||||
|
|
@ -989,7 +1027,7 @@ class MongoClientInterface {
|
|||
const internalParams = extension.genMDParams();
|
||||
internalParams.mongifiedSearch = params.mongifiedSearch;
|
||||
return this.internalListObject(bucketName, internalParams, extension,
|
||||
log, cb);
|
||||
log, cb);
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, params, log, cb) {
|
||||
|
|
@ -998,7 +1036,7 @@ class MongoClientInterface {
|
|||
const internalParams = extension.genMDParams();
|
||||
internalParams.mongifiedSearch = params.mongifiedSearch;
|
||||
return this.internalListObject(bucketName, internalParams, extension,
|
||||
log, cb);
|
||||
log, cb);
|
||||
}
|
||||
|
||||
checkHealth(implName, log, cb) {
|
||||
|
|
@ -1022,7 +1060,7 @@ class MongoClientInterface {
|
|||
}, {}, (err, doc) => {
|
||||
if (err) {
|
||||
log.error('readUUID: error reading UUID',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (!doc) {
|
||||
|
|
@ -1044,7 +1082,7 @@ class MongoClientInterface {
|
|||
return cb(errors.KeyAlreadyExists);
|
||||
}
|
||||
log.error('writeUUIDIfNotExists: error writing UUID',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
// FIXME: shoud we check for result.ok === 1 ?
|
||||
|
|
@ -1062,7 +1100,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
if (err === errors.InternalError) {
|
||||
log.error('getUUID: error getting UUID',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
return this.readUUID(log, cb);
|
||||
|
|
@ -1077,7 +1115,7 @@ class MongoClientInterface {
|
|||
// For Kub/cluster deployments there should be a more sophisticated
|
||||
// way for guessing free space.
|
||||
diskusage.check(this.path !== undefined ?
|
||||
this.path : '/', cb);
|
||||
this.path : '/', cb);
|
||||
}
|
||||
|
||||
readCountItems(log, cb) {
|
||||
|
|
@ -1541,7 +1579,7 @@ class MongoClientInterface {
|
|||
const bucketStatus = bucketInfo.getVersioningConfiguration();
|
||||
const isVer = (bucketStatus &&
|
||||
(bucketStatus.Status === 'Enabled' ||
|
||||
bucketStatus.Status === 'Suspended'));
|
||||
bucketStatus.Status === 'Suspended'));
|
||||
const retResult = this._handleResults(collRes, isVer);
|
||||
retResult.stalled = stalledCount;
|
||||
return callback(null, retResult);
|
||||
|
|
@ -1583,7 +1621,7 @@ class MongoClientInterface {
|
|||
const filter = { _id: objName };
|
||||
try {
|
||||
MongoUtils.translateConditions(0, 'value', filter,
|
||||
params.conditions);
|
||||
params.conditions);
|
||||
} catch (err) {
|
||||
log.error('error creating mongodb filter', {
|
||||
error: reshapeExceptionError(err),
|
||||
|
|
@ -1629,7 +1667,7 @@ class MongoClientInterface {
|
|||
const filter = { _id: objName };
|
||||
try {
|
||||
MongoUtils.translateConditions(0, 'value', filter,
|
||||
params.conditions);
|
||||
params.conditions);
|
||||
} catch (err) {
|
||||
log.error('error creating mongodb filter', {
|
||||
error: reshapeExceptionError(err),
|
||||
|
|
|
|||
Loading…
Reference in New Issue