Compare commits

...

5 Commits

Author SHA1 Message Date
Anurag Mittal fff956b02b
added console log statements for putObject 2020-01-21 15:19:32 +01:00
Anurag Mittal 2c0d6c9c13
added get object console.log 2020-01-20 18:53:40 +01:00
Anurag Mittal bf84e8b525
updated arsenal 2020-01-20 18:08:20 +01:00
Anurag Mittal a8e585c2e6
udpdated console.log 2020-01-20 17:53:59 +01:00
Anurag Mittal 1bcf735e3d
added debug statements 2020-01-20 17:33:12 +01:00
1 changed files with 92 additions and 54 deletions

View File

@ -50,7 +50,7 @@ const VID_SEP = require('../../../versioning/constants')
function generateVersionId(replicationGroupId) { function generateVersionId(replicationGroupId) {
// generate a unique number for each member of the nodejs cluster // generate a unique number for each member of the nodejs cluster
return genVID(`${process.pid}.${uidCounter++}`, return genVID(`${process.pid}.${uidCounter++}`,
replicationGroupId); replicationGroupId);
} }
function formatVersionKey(key, versionId) { function formatVersionKey(key, versionId) {
@ -59,7 +59,7 @@ function formatVersionKey(key, versionId) {
function inc(str) { function inc(str) {
return str ? (str.slice(0, str.length - 1) + return str ? (str.slice(0, str.length - 1) +
String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str; String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str;
} }
const VID_SEPPLUS = inc(VID_SEP); const VID_SEPPLUS = inc(VID_SEP);
@ -116,11 +116,11 @@ class MongoClientInterface {
// should be an async_init(cb) method in the wrapper to // should be an async_init(cb) method in the wrapper to
// initialize this backend // initialize this backend
if ((process.env.MONGO_CONNECT_TIMEOUT_MS && if ((process.env.MONGO_CONNECT_TIMEOUT_MS &&
Number.isNaN(process.env.MONGO_CONNECT_TIMEOUT_MS)) || Number.isNaN(process.env.MONGO_CONNECT_TIMEOUT_MS)) ||
(process.env.MONGO_SOCKET_TIMEOUT_MS && (process.env.MONGO_SOCKET_TIMEOUT_MS &&
Number.isNaN(process.env.MONGO_SOCKET_TIMEOUT_MS))) { Number.isNaN(process.env.MONGO_SOCKET_TIMEOUT_MS))) {
this.logger.error('MongoDB connect and socket timeouts must be a ' + this.logger.error('MongoDB connect and socket timeouts must be a ' +
'number. Using default value(s).'); 'number. Using default value(s).');
} }
const connectTimeoutMS = Number.parseInt( const connectTimeoutMS = Number.parseInt(
process.env.MONGO_CONNECT_TIMEOUT_MS, 10) || CONNECT_TIMEOUT_MS; process.env.MONGO_CONNECT_TIMEOUT_MS, 10) || CONNECT_TIMEOUT_MS;
@ -161,8 +161,8 @@ class MongoClientInterface {
err => { err => {
if (err) { if (err) {
this.logger.fatal('error writing usersBucket ' + this.logger.fatal('error writing usersBucket ' +
'attributes to metastore', 'attributes to metastore',
{ error: err }); { error: err });
throw (errors.InternalError); throw (errors.InternalError);
} }
return cb(); return cb();
@ -179,7 +179,7 @@ class MongoClientInterface {
getCollection(name) { getCollection(name) {
/* mongo has a problem with .. in collection names */ /* mongo has a problem with .. in collection names */
const newName = (name === constants.usersBucket) ? const newName = (name === constants.usersBucket) ?
USERSBUCKET : name; USERSBUCKET : name;
return this.db.collection(newName); return this.db.collection(newName);
} }
@ -203,7 +203,7 @@ class MongoClientInterface {
if (err) { if (err) {
log.error( log.error(
'createBucket: error creating bucket', 'createBucket: error creating bucket',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
this.lastItemScanTime = null; this.lastItemScanTime = null;
@ -216,7 +216,7 @@ class MongoClientInterface {
if (err) { if (err) {
log.error( log.error(
'createBucket: error creating bucket', 'createBucket: error creating bucket',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return cb(); return cb();
@ -228,6 +228,8 @@ class MongoClientInterface {
} }
getBucketAttributes(bucketName, log, cb) { getBucketAttributes(bucketName, log, cb) {
log.debug('entered getBucketAttributes',
{ database: bucketName });
const m = this.getCollection(METASTORE); const m = this.getCollection(METASTORE);
m.findOne({ m.findOne({
_id: bucketName, _id: bucketName,
@ -245,11 +247,15 @@ class MongoClientInterface {
// that properly inits w/o JSON.parse() // that properly inits w/o JSON.parse()
const bucketMDStr = JSON.stringify(doc.value); const bucketMDStr = JSON.stringify(doc.value);
const bucketMD = BucketInfo.deSerialize(bucketMDStr); const bucketMD = BucketInfo.deSerialize(bucketMDStr);
log.debug('exiting getBucketAttributes',
{ database: bucketName });
return cb(null, bucketMD); return cb(null, bucketMD);
}); });
} }
getBucketAndObject(bucketName, objName, params, log, cb) { getBucketAndObject(bucketName, objName, params, log, cb) {
log.debug('entered getBucketAndObject',
{ database: bucketName, object: objName });
this.getBucketAttributes(bucketName, log, (err, bucket) => { this.getBucketAttributes(bucketName, log, (err, bucket) => {
if (err) { if (err) {
log.error( log.error(
@ -260,15 +266,20 @@ class MongoClientInterface {
this.getObject(bucketName, objName, params, log, (err, obj) => { this.getObject(bucketName, objName, params, log, (err, obj) => {
if (err) { if (err) {
if (err === errors.NoSuchKey) { if (err === errors.NoSuchKey) {
log.debug('exiting getBucketAndObject with no such key',
{ database: bucketName, object: objName });
return cb(null, return cb(null,
{ bucket: {
bucket:
BucketInfo.fromObj(bucket).serialize(), BucketInfo.fromObj(bucket).serialize(),
}); });
} }
log.error('getObject: error getting object', log.error('getObject: error getting object',
{ error: err.message }); { error: err.message });
return cb(err); return cb(err);
} }
log.debug('exiting getBucketAndObject',
{ database: bucketName, object: objName });
return cb(null, { return cb(null, {
bucket: BucketInfo.fromObj(bucket).serialize(), bucket: BucketInfo.fromObj(bucket).serialize(),
obj: JSON.stringify(obj), obj: JSON.stringify(obj),
@ -296,7 +307,7 @@ class MongoClientInterface {
if (err) { if (err) {
log.error( log.error(
'putBucketAttributes: error putting bucket attributes', 'putBucketAttributes: error putting bucket attributes',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return cb(); return cb();
@ -313,12 +324,12 @@ class MongoClientInterface {
}, {}, (err, result) => { }, {}, (err, result) => {
if (err) { if (err) {
log.error('deleteBucketStep2: error deleting bucket', log.error('deleteBucketStep2: error deleting bucket',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
if (result.ok !== 1) { if (result.ok !== 1) {
log.error('deleteBucketStep2: failed deleting bucket', log.error('deleteBucketStep2: failed deleting bucket',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return cb(null); return cb(null);
@ -343,7 +354,7 @@ class MongoClientInterface {
return this.deleteBucketStep2(bucketName, log, cb); return this.deleteBucketStep2(bucketName, log, cb);
} }
log.error('deleteBucket: error deleting bucket', log.error('deleteBucket: error deleting bucket',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return this.deleteBucketStep2(bucketName, log, err => { return this.deleteBucketStep2(bucketName, log, err => {
@ -398,7 +409,7 @@ class MongoClientInterface {
$gt: objVal.versionId, $gt: objVal.versionId,
}, },
}, },
], ],
}, },
update: { update: {
_id: objName, value: objVal, _id: objName, value: objVal,
@ -421,10 +432,10 @@ class MongoClientInterface {
*/ */
if (err) { if (err) {
if (!err.index || err.index !== 1 if (!err.index || err.index !== 1
|| !err.code || err.code !== 11000) { || !err.code || err.code !== 11000) {
log.error( log.error(
'putObjectVerCase1: error putting object version', 'putObjectVerCase1: error putting object version',
{ error: err.errmsg }); { error: err.errmsg });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
log.debug('putObjectVerCase1: error putting object version', log.debug('putObjectVerCase1: error putting object version',
@ -453,7 +464,7 @@ class MongoClientInterface {
if (err) { if (err) {
log.error( log.error(
'putObjectVerCase2: error putting object version', 'putObjectVerCase2: error putting object version',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return cb(null, `{"versionId": "${objVal.versionId}"}`); return cb(null, `{"versionId": "${objVal.versionId}"}`);
@ -514,7 +525,7 @@ class MongoClientInterface {
if (err) { if (err) {
log.error( log.error(
'putObjectVerCase3: error putting object version', 'putObjectVerCase3: error putting object version',
{ error: err.message }); { error: err.message });
if (err.code === 11000) { if (err.code === 11000) {
// We want duplicate key error logged however in // We want duplicate key error logged however in
// case of the race condition mentioned above, the // case of the race condition mentioned above, the
@ -560,7 +571,7 @@ class MongoClientInterface {
this.getLatestVersion(c, objName, log, (err, mstObjVal) => { this.getLatestVersion(c, objName, log, (err, mstObjVal) => {
if (err) { if (err) {
log.error('getLatestVersion: getting latest version', log.error('getLatestVersion: getting latest version',
{ error: err.message }); { error: err.message });
return cb(err); return cb(err);
} }
// eslint-disable-next-line // eslint-disable-next-line
@ -618,26 +629,44 @@ class MongoClientInterface {
putObject(bucketName, objName, objVal, params, log, cb) { putObject(bucketName, objName, objVal, params, log, cb) {
MongoUtils.serialize(objVal); MongoUtils.serialize(objVal);
log.debug('entered putObject',
{ database: bucketName, object: objName });
const c = this.getCollection(bucketName); const c = this.getCollection(bucketName);
log.debug('inside putObject, got the collection name',
{ database: bucketName, object: objName });
if (params && params.versioning && !params.versionId) { if (params && params.versioning && !params.versionId) {
log.debug('inside putObject, case: putObjectVerCase1',
{ database: bucketName, object: objName });
return this.putObjectVerCase1(c, bucketName, objName, objVal, return this.putObjectVerCase1(c, bucketName, objName, objVal,
params, log, cb); params, log, cb);
} else if (params && params.versionId === '') { } else if (params && params.versionId === '') {
log.debug('inside putObject, case: putObjectVerCase2',
{ database: bucketName, object: objName });
return this.putObjectVerCase2(c, bucketName, objName, objVal, return this.putObjectVerCase2(c, bucketName, objName, objVal,
params, log, cb); params, log, cb);
} else if (params && params.versionId && !params.repairMaster) { } else if (params && params.versionId && !params.repairMaster) {
log.debug('inside putObject, case: putObjectVerCase3',
{ database: bucketName, object: objName });
return this.putObjectVerCase3(c, bucketName, objName, objVal, return this.putObjectVerCase3(c, bucketName, objName, objVal,
params, log, cb); params, log, cb);
} else if (params && params.versionId && params.repairMaster) { } else if (params && params.versionId && params.repairMaster) {
log.debug('inside putObject, case: putObjectVerCase4',
{ database: bucketName, object: objName });
return this.putObjectVerCase4(c, bucketName, objName, objVal, return this.putObjectVerCase4(c, bucketName, objName, objVal,
params, log, cb); params, log, cb);
} }
log.debug('inside putObject, case: noversion',
{ database: bucketName, object: objName });
return this.putObjectNoVer(c, bucketName, objName, objVal, return this.putObjectNoVer(c, bucketName, objName, objVal,
params, log, cb); params, log, cb);
} }
getObject(bucketName, objName, params, log, cb) { getObject(bucketName, objName, params, log, cb) {
log.debug('entered getObject',
{ database: bucketName, object: objName });
const c = this.getCollection(bucketName); const c = this.getCollection(bucketName);
log.debug('inside getObject, got the collection',
{ database: bucketName, object: objName });
if (params && params.versionId) { if (params && params.versionId) {
// eslint-disable-next-line // eslint-disable-next-line
objName = formatVersionKey(objName, params.versionId); objName = formatVersionKey(objName, params.versionId);
@ -645,26 +674,35 @@ class MongoClientInterface {
c.findOne({ c.findOne({
_id: objName, _id: objName,
}, {}, (err, doc) => { }, {}, (err, doc) => {
log.debug('inside getObject, inside the cb for findOne query',
{ database: bucketName, object: objName });
if (err) { if (err) {
log.error('findOne: error getting object', log.error('findOne: error getting object',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
if (!doc) { if (!doc) {
log.debug('inside getObject, got reply from database, obj not found',
{ database: bucketName, object: objName });
return cb(errors.NoSuchKey); return cb(errors.NoSuchKey);
} }
if (doc.value.isPHD) { if (doc.value.isPHD) {
this.getLatestVersion(c, objName, log, (err, value) => { this.getLatestVersion(c, objName, log, (err, value) => {
if (err) { if (err) {
log.error('getLatestVersion: getting latest version', log.error('getLatestVersion: getting latest version',
{ error: err.message }); { error: err.message });
return cb(err); return cb(err);
} }
log.debug('exited in PHD getObject',
{ database: bucketName, object: objName });
return cb(null, value); return cb(null, value);
}); });
return undefined; return undefined;
} }
MongoUtils.unserialize(doc.value); MongoUtils.unserialize(doc.value);
log.debug('exited getObject',
{ database: bucketName, object: objName });
return cb(null, doc.value); return cb(null, doc.value);
}); });
} }
@ -719,12 +757,12 @@ class MongoClientInterface {
}, (err, result) => { }, (err, result) => {
if (err) { if (err) {
log.error('repair: error trying to repair value', log.error('repair: error trying to repair value',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
if (result.ok !== 1) { if (result.ok !== 1) {
log.error('repair: failed trying to repair value', log.error('repair: failed trying to repair value',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return cb(null); return cb(null);
@ -739,7 +777,7 @@ class MongoClientInterface {
this.getLatestVersion(c, objName, log, (err, value) => { this.getLatestVersion(c, objName, log, (err, value) => {
if (err) { if (err) {
log.error('async-repair: getting latest version', log.error('async-repair: getting latest version',
{ error: err.message }); { error: err.message });
return undefined; return undefined;
} }
this.repair(c, bucketName, objName, value, mst, log, err => { this.repair(c, bucketName, objName, value, mst, log, err => {
@ -787,7 +825,7 @@ class MongoClientInterface {
return undefined; return undefined;
} }
log.error('getLatestVersion: error getting latest version', log.error('getLatestVersion: error getting latest version',
{ error: err.message }); { error: err.message });
return cb(err); return cb(err);
} }
// We have other versions available so repair: // We have other versions available so repair:
@ -831,7 +869,7 @@ class MongoClientInterface {
if (err) { if (err) {
log.error( log.error(
'deleteObjectVerMaster: error deleting object', 'deleteObjectVerMaster: error deleting object',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
return this.deleteOrRepairPHD(c, bucketName, objName, mst, log, cb); return this.deleteOrRepairPHD(c, bucketName, objName, mst, log, cb);
@ -875,7 +913,7 @@ class MongoClientInterface {
}, {}, (err, mst) => { }, {}, (err, mst) => {
if (err) { if (err) {
log.error('deleteObjectVer: error deleting versioned object', log.error('deleteObjectVer: error deleting versioned object',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
if (!mst) { if (!mst) {
@ -884,15 +922,15 @@ class MongoClientInterface {
if (mst.value.isPHD || if (mst.value.isPHD ||
mst.value.versionId === params.versionId) { mst.value.versionId === params.versionId) {
return this.deleteObjectVerMaster(c, bucketName, objName, return this.deleteObjectVerMaster(c, bucketName, objName,
params, log, err => { params, log, err => {
if (err) { if (err) {
return cb(err); return cb(err);
} }
return cb(); return cb();
}); });
} }
return this.deleteObjectVerNotMaster(c, bucketName, objName, return this.deleteObjectVerNotMaster(c, bucketName, objName,
params, log, cb); params, log, cb);
}); });
} }
@ -923,10 +961,10 @@ class MongoClientInterface {
const c = this.getCollection(bucketName); const c = this.getCollection(bucketName);
if (params && params.versionId) { if (params && params.versionId) {
return this.deleteObjectVer(c, bucketName, objName, return this.deleteObjectVer(c, bucketName, objName,
params, log, cb); params, log, cb);
} }
return this.deleteObjectNoVer(c, bucketName, objName, return this.deleteObjectNoVer(c, bucketName, objName,
params, log, cb); params, log, cb);
} }
internalListObject(bucketName, params, extension, log, cb) { internalListObject(bucketName, params, extension, log, cb) {
@ -989,7 +1027,7 @@ class MongoClientInterface {
const internalParams = extension.genMDParams(); const internalParams = extension.genMDParams();
internalParams.mongifiedSearch = params.mongifiedSearch; internalParams.mongifiedSearch = params.mongifiedSearch;
return this.internalListObject(bucketName, internalParams, extension, return this.internalListObject(bucketName, internalParams, extension,
log, cb); log, cb);
} }
listMultipartUploads(bucketName, params, log, cb) { listMultipartUploads(bucketName, params, log, cb) {
@ -998,7 +1036,7 @@ class MongoClientInterface {
const internalParams = extension.genMDParams(); const internalParams = extension.genMDParams();
internalParams.mongifiedSearch = params.mongifiedSearch; internalParams.mongifiedSearch = params.mongifiedSearch;
return this.internalListObject(bucketName, internalParams, extension, return this.internalListObject(bucketName, internalParams, extension,
log, cb); log, cb);
} }
checkHealth(implName, log, cb) { checkHealth(implName, log, cb) {
@ -1022,7 +1060,7 @@ class MongoClientInterface {
}, {}, (err, doc) => { }, {}, (err, doc) => {
if (err) { if (err) {
log.error('readUUID: error reading UUID', log.error('readUUID: error reading UUID',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
if (!doc) { if (!doc) {
@ -1044,7 +1082,7 @@ class MongoClientInterface {
return cb(errors.KeyAlreadyExists); return cb(errors.KeyAlreadyExists);
} }
log.error('writeUUIDIfNotExists: error writing UUID', log.error('writeUUIDIfNotExists: error writing UUID',
{ error: err.message }); { error: err.message });
return cb(errors.InternalError); return cb(errors.InternalError);
} }
// FIXME: shoud we check for result.ok === 1 ? // FIXME: shoud we check for result.ok === 1 ?
@ -1062,7 +1100,7 @@ class MongoClientInterface {
if (err) { if (err) {
if (err === errors.InternalError) { if (err === errors.InternalError) {
log.error('getUUID: error getting UUID', log.error('getUUID: error getting UUID',
{ error: err.message }); { error: err.message });
return cb(err); return cb(err);
} }
return this.readUUID(log, cb); return this.readUUID(log, cb);
@ -1077,7 +1115,7 @@ class MongoClientInterface {
// For Kub/cluster deployments there should be a more sophisticated // For Kub/cluster deployments there should be a more sophisticated
// way for guessing free space. // way for guessing free space.
diskusage.check(this.path !== undefined ? diskusage.check(this.path !== undefined ?
this.path : '/', cb); this.path : '/', cb);
} }
readCountItems(log, cb) { readCountItems(log, cb) {
@ -1541,7 +1579,7 @@ class MongoClientInterface {
const bucketStatus = bucketInfo.getVersioningConfiguration(); const bucketStatus = bucketInfo.getVersioningConfiguration();
const isVer = (bucketStatus && const isVer = (bucketStatus &&
(bucketStatus.Status === 'Enabled' || (bucketStatus.Status === 'Enabled' ||
bucketStatus.Status === 'Suspended')); bucketStatus.Status === 'Suspended'));
const retResult = this._handleResults(collRes, isVer); const retResult = this._handleResults(collRes, isVer);
retResult.stalled = stalledCount; retResult.stalled = stalledCount;
return callback(null, retResult); return callback(null, retResult);
@ -1583,7 +1621,7 @@ class MongoClientInterface {
const filter = { _id: objName }; const filter = { _id: objName };
try { try {
MongoUtils.translateConditions(0, 'value', filter, MongoUtils.translateConditions(0, 'value', filter,
params.conditions); params.conditions);
} catch (err) { } catch (err) {
log.error('error creating mongodb filter', { log.error('error creating mongodb filter', {
error: reshapeExceptionError(err), error: reshapeExceptionError(err),
@ -1629,7 +1667,7 @@ class MongoClientInterface {
const filter = { _id: objName }; const filter = { _id: objName };
try { try {
MongoUtils.translateConditions(0, 'value', filter, MongoUtils.translateConditions(0, 'value', filter,
params.conditions); params.conditions);
} catch (err) { } catch (err) {
log.error('error creating mongodb filter', { log.error('error creating mongodb filter', {
error: reshapeExceptionError(err), error: reshapeExceptionError(err),