Compare commits

...

1 Commits

Author SHA1 Message Date
Alexander Chan 6ddb776004 rf: use mongodb aggregate method for item count 2018-06-04 15:53:53 -07:00
1 changed files with 108 additions and 212 deletions

View File

@ -965,12 +965,11 @@ class MongoClientInterface {
if (err) { if (err) {
return next(errors.InternalError); return next(errors.InternalError);
} }
if (results.bucketList && results.counter && if (results.bucketList && results.dataManaged) {
results.counter.dataManaged) {
res.bucketList.push(results.bucketList); res.bucketList.push(results.bucketList);
res.objects += results.counter.objects; res.objects += results.objects;
res.versions += results.counter.versions; res.versions += results.versions;
consolidateData(results.counter.dataManaged); consolidateData(results.dataManaged);
} }
return next(); return next();
}); });
@ -986,166 +985,45 @@ class MongoClientInterface {
return undefined; return undefined;
} }
getDataManaged(c, query, log, cb) { _handleResults(res, isVersioned) {
const dataManaged = { const total = { curr: 0, prev: 0 };
total: 0, const locations = {};
locations: {},
};
c.find(query).toArray((err, entries) => {
if (err) {
log.error('error occured in mongo client', {
method: 'getDataManaged',
error: err,
});
return cb(err);
}
entries.forEach(entry => {
const { backends } = entry.value.replicationInfo;
const objMD = entry.value;
dataManaged.total += objMD['content-length'];
if (!dataManaged.locations[objMD.dataStoreName]) {
dataManaged.locations[objMD.dataStoreName] =
objMD['content-length'];
} else {
dataManaged.locations[objMD.dataStoreName] +=
objMD['content-length'];
}
// replication list
backends.forEach(location => {
const { site } = location;
if (!dataManaged.locations[site]) {
dataManaged.locations[site] = objMD['content-length'];
} else {
dataManaged.locations[site] += objMD['content-length'];
}
});
});
return cb(null, {
objCount: entries.length,
dataManaged,
});
});
}
getDataInfoNoVer(c, log, callback) { if (isVersioned) {
return this.getDataManaged(c, {}, log, (err, res) => { Object.keys(res.versionData).forEach(loc => {
if (err) { const bytes = res.versionData[loc];
log.error('error occured in mongo client', { const locName = loc === 'mem' || loc === 'file' ?
method: 'getDataInfoNoVer', 'us-east-1' : loc;
error: err, if (!locations[locName]) {
}); locations[locName] = { curr: 0, prev: 0 };
return callback(err);
} }
const dataManaged = { total.prev += bytes;
total: { curr: 0, prev: 0 }, locations[locName].prev += bytes;
locations: {},
};
const data = res.dataManaged.locations;
// add total
dataManaged.total.curr += res.dataManaged.total;
Object.keys(data).forEach(loc => {
if (!dataManaged.locations[loc]) {
dataManaged.locations[loc] = { prev: 0, curr: 0 };
}
dataManaged.locations[loc].curr = data[loc];
});
return callback(null, {
objects: res.objCount,
versions: 0,
dataManaged,
});
}); });
} }
Object.keys(res.masterData).forEach(loc => {
getDataInfoVer(c, mstVerIds, log, callback) { const bytes = res.masterData[loc];
// query for removing placeholder entries const locName = loc === 'mem' || loc === 'file' ?
/* eslint-disable quote-props */ 'us-east-1' : loc;
const queryFilter = { if (!locations[locName]) {
'$or': [ locations[locName] = { curr: 0, prev: 0 };
{ '_id': { '$regex': /\0.*$/g } }, }
{ 'value.isNull': { '$exists': true } }, total.curr += bytes;
{ locations[locName].curr += bytes;
'value.nullVersionId': { '$exists': false }, if (isVersioned) {
'value.versionId': { '$exists': false }, total.prev -= bytes;
locations[locName].prev -= bytes;
}
});
return {
versions: isVersioned ?
res.versionCount - res.masterCount : 0,
objects: res.masterCount,
dataManaged: {
total,
locations,
}, },
],
}; };
/* eslint-enable quote-props */
return async.series({
master: done => {
// query for getting only master version entries
const mstFilter = [];
/* eslint-disable quote-props */
mstVerIds.forEach(id => {
if (id) {
mstFilter.push({ 'value.versionId': { '$eq': id } });
}
});
const query = {
'$and': [
queryFilter,
{ '$or': (mstFilter.length ? mstFilter : [{}]) },
],
};
/* eslint-enable quote-props */
return this.getDataManaged(c, query, log, done);
},
archived: done => {
// query for getting only versioned entries
const mstFilter = [];
/* eslint-disable quote-props */
mstVerIds.forEach(id => {
if (id) {
mstFilter.push({ 'value.versionId': { '$ne': id } });
}
});
const query = {
'$and': [
queryFilter,
{ '$and': (mstFilter.length ? mstFilter : [{}]) },
],
};
/* eslint-enable quote-props */
return this.getDataManaged(c, query, log, done);
},
}, (err, res) => {
if (err) {
log.error('error occured in mongo client', {
method: 'getDataInfoVer',
error: err,
});
return callback(err);
}
const dataManaged = {
total: { curr: 0, prev: 0 },
locations: {},
};
const mstData = res.master.dataManaged.locations;
const verData = res.archived.dataManaged.locations;
// add total
dataManaged.total.curr += res.master.dataManaged.total;
dataManaged.total.prev += res.archived.dataManaged.total;
Object.keys(mstData).forEach(loc => {
if (!dataManaged.locations[loc]) {
dataManaged.locations[loc] = { prev: 0, curr: 0 };
}
dataManaged.locations[loc].curr = mstData[loc];
});
Object.keys(verData).forEach(loc => {
if (!dataManaged.locations[loc]) {
dataManaged.locations[loc] = { prev: 0, curr: 0 };
}
dataManaged.locations[loc].prev = verData[loc];
});
return callback(null, {
objects: res.master.objCount,
versions: res.archived.objCount,
dataManaged,
});
});
} }
getObjectMDStats(bucketName, bucketInfo, log, callback) { getObjectMDStats(bucketName, bucketInfo, log, callback) {
@ -1154,60 +1032,78 @@ class MongoClientInterface {
name: bucketName, name: bucketName,
location: bucketInfo.getLocationConstraint(), location: bucketInfo.getLocationConstraint(),
}; };
if (bucketInfo && bucketInfo._versioningConfiguration &&
(bucketInfo._versioningConfiguration.Status === 'Suspended' || const mstFilter = { _id: /^[^\0]+$/ };
bucketInfo._versioningConfiguration.Status === 'Enabled')) { const verFilter = { _id: /\0/ };
// if versioning is enabled const grpObj = {
c.distinct('_id').then(keys => { _id: '$value.location.dataStoreName',
const trimmedKeys = keys.map(key => key.replace(/\0.*$/g, '')); bytes: { $sum: '$value.content-length' },
const uniqKeys = trimmedKeys.filter( };
(key, index, self) => self.indexOf(key) === index);
// for each uniqKey get master version id const _handleCount = (err, entries, cb) => {
return async.map(uniqKeys, (key, done) => {
this.getLatestVersion(c, key, log, (err, mst) => {
if (err) { if (err) {
if (err.NoSuchKey) { log.error('Error when processing master docs', {
log.debug('NoSuchKey master info', {
method: 'getObjectMDStats', method: 'getObjectMDStats',
error: err, error: err,
}); });
return done(); return cb(err);
} }
log.error('unable to retrieve master info', { return cb(null,
entries && entries.length > 0 ? entries.length : 0);
};
const _handleEntries = (err, entries, cb) => {
if (err) {
log.error('Error when processing master docs', {
method: 'getObjectMDStats', method: 'getObjectMDStats',
error: err, error: err,
}); });
return done(err); return cb(err);
} }
return done(null, mst.versionId); const results = {};
if (entries) {
entries.forEach(entry => {
results[entry._id] = entry.bytes;
}); });
}, (err, mstVerIds) => { }
return cb(null, results);
};
const mstCursor = c.aggregate().match(mstFilter);
const verCursor = c.aggregate().match(verFilter);
async.parallel({
versionCount: done => {
const tmpCursor = verCursor.clone();
tmpCursor.toArray(
(err, res) => _handleCount(err, res, done));
},
versionData: done => {
const tmpCursor = verCursor.clone();
tmpCursor.unwind('$value.location')
.group(grpObj)
.toArray((err, res) => _handleEntries(err, res, done));
},
masterCount: done => {
const tmpCursor = mstCursor.clone();
tmpCursor.toArray(
(err, res) => _handleCount(err, res, done));
},
masterData: done => {
const tmpCursor = mstCursor.clone();
tmpCursor.unwind('$value.location')
.group(grpObj)
.toArray((err, res) => _handleEntries(err, res, done));
},
}, (err, res) => {
if (err) { if (err) {
return callback(err); return callback(err);
} }
return this.getDataInfoVer(c, mstVerIds, log, const bucketStatus = bucketInfo.getVersioningConfiguration();
(err, res) => { const isVer = (bucketStatus && (bucketStatus.Status === 'Enabled' ||
if (err) { bucketStatus.Status === 'Suspended'));
return callback(err); const retResult = this._handleResults(res, isVer);
} retResult.bucketList = retBucketInfo;
return callback(null, { return callback(null, retResult);
bucketList: retBucketInfo,
counter: res,
}); });
});
});
}).catch(callback);
} else {
this.getDataInfoNoVer(c, log, (err, res) => {
if (err) {
return callback(err);
}
return callback(null, {
bucketList: retBucketInfo,
counter: res,
});
});
}
} }
} }