Compare commits

...

4 Commits

5 changed files with 1242 additions and 390 deletions

View File

@ -1 +1,6 @@
{ "extends": "scality" } {
"extends": "scality",
"parserOptions": {
"ecmaVersion": 2020
}
}

View File

@ -167,3 +167,5 @@ export const maxCachedBuckets = process.env.METADATA_MAX_CACHED_BUCKETS ?
Number(process.env.METADATA_MAX_CACHED_BUCKETS) : 1000; Number(process.env.METADATA_MAX_CACHED_BUCKETS) : 1000;
export const validRestoreObjectTiers = new Set(['Expedited', 'Standard', 'Bulk']); export const validRestoreObjectTiers = new Set(['Expedited', 'Standard', 'Bulk']);
export const validStorageMetricLevels = new Set(['bucket', 'location', 'account']);

View File

@ -35,6 +35,7 @@ const { Transform } = require('stream');
const { Version } = require('../../../versioning/Version'); const { Version } = require('../../../versioning/Version');
const { formatMasterKey, formatVersionKey } = require('./utils'); const { formatMasterKey, formatVersionKey } = require('./utils');
const { validStorageMetricLevels } = require('../../../constants');
const VID_NONE = ''; const VID_NONE = '';
@ -1773,6 +1774,11 @@ class MongoClientInterface {
byLocation: {}, byLocation: {},
}, },
stalled: 0, stalled: 0,
dataMetrics: {
bucket: {},
location: {},
account: {},
},
}; };
return cb(null, res); return cb(null, res);
} }
@ -1898,105 +1904,6 @@ class MongoClientInterface {
}); });
} }
consolidateData(store, dataManaged) {
/* eslint-disable */
if (dataManaged && dataManaged.locations && dataManaged.total) {
const locations = dataManaged.locations;
store.dataManaged.total.curr += dataManaged.total.curr;
store.dataManaged.total.prev += dataManaged.total.prev;
Object.keys(locations).forEach(site => {
if (!store.dataManaged.byLocation[site]) {
store.dataManaged.byLocation[site] =
Object.assign({}, locations[site]);
} else {
store.dataManaged.byLocation[site].curr +=
locations[site].curr;
store.dataManaged.byLocation[site].prev +=
locations[site].prev;
}
});
}
/* eslint-enable */
}
scanItemCount(log, cb) {
const store = {
objects: 0,
versions: 0,
buckets: 0,
bucketList: [],
dataManaged: {
total: { curr: 0, prev: 0 },
byLocation: {},
},
stalled: 0,
};
const consolidateData = dataManaged =>
this.consolidateData(store, dataManaged);
this.getBucketInfos(log, (err, res) => {
if (err) {
log.error('error getting bucket info', {
method: 'scanItemCount',
error: err,
});
return cb(err);
}
const { bucketCount, bucketInfos } = res;
const retBucketInfos = bucketInfos.map(bucket => ({
name: bucket.getName(),
location: bucket.getLocationConstraint(),
isVersioned: !!bucket.getVersioningConfiguration(),
ownerCanonicalId: bucket.getOwner(),
ingestion: bucket.isIngestionBucket(),
}));
store.buckets = bucketCount;
store.bucketList = retBucketInfos;
return async.eachLimit(bucketInfos, this.concurrentCursors,
(bucketInfo, done) => {
async.waterfall([
next => this._getIsTransient(bucketInfo, log, next),
(isTransient, next) => {
const bucketName = bucketInfo.getName();
this.getObjectMDStats(bucketName, bucketInfo,
isTransient, log, next);
},
], (err, results) => {
if (err) {
return done(err);
}
if (results.dataManaged) {
store.objects += results.objects;
store.versions += results.versions;
store.stalled += results.stalled;
consolidateData(results.dataManaged);
}
return done();
});
}, err => {
if (err) {
return cb(err);
}
// save to infostore
return this.updateCountItems(store, log, err => {
if (err) {
log.error('error saving count items in mongo', {
method: 'scanItemCount',
error: err,
});
return cb(err);
}
return cb(null, store);
});
});
});
return undefined;
}
_getIsTransient(bucketInfo, log, cb) { _getIsTransient(bucketInfo, log, cb) {
const locConstraint = bucketInfo.getLocationConstraint(); const locConstraint = bucketInfo.getLocationConstraint();
@ -2036,67 +1943,107 @@ class MongoClientInterface {
} }
_handleResults(res, isVersioned) { _handleResults(res, isVersioned) {
const total = { curr: 0, prev: 0 }; let totalNonCurrentCount = 0;
const locations = {}; let totalCurrentCount = 0;
const totalBytes = { curr: 0, prev: 0 };
const locationBytes = {};
const dataMetrics = {
bucket: {},
location: {},
account: {},
};
Object.keys(res.nullData).forEach(loc => { Object.keys(res).forEach(metricLevel => {
const bytes = res.nullData[loc]; // metricLevel can only be 'bucket', 'location' or 'account'
const locName = this._getLocName(loc); if (validStorageMetricLevels.has(metricLevel)) {
if (!locations[locName]) { Object.keys(res[metricLevel]).forEach(resource => {
locations[locName] = { curr: 0, prev: 0 }; // resource can be the name of bucket, location or account
} const resourceName = metricLevel === 'location' ? this._getLocName(resource) : resource;
total.curr += bytes; if (!dataMetrics[metricLevel][resourceName]) {
locations[locName].curr += bytes; dataMetrics[metricLevel][resourceName] = {
}); usedCapacity: {
if (isVersioned) { current: 0,
Object.keys(res.versionData).forEach(loc => { nonCurrent: 0,
const bytes = res.versionData[loc];
const locName = this._getLocName(loc);
if (!locations[locName]) {
locations[locName] = { curr: 0, prev: 0 };
}
total.prev += bytes;
locations[locName].prev += bytes;
});
}
Object.keys(res.masterData).forEach(loc => {
const bytes = res.masterData[loc];
const locName = this._getLocName(loc);
if (!locations[locName]) {
locations[locName] = { curr: 0, prev: 0 };
}
total.curr += bytes;
locations[locName].curr += bytes;
if (isVersioned) {
total.prev -= bytes;
total.prev = Math.max(0, total.prev);
locations[locName].prev -= bytes;
locations[locName].prev =
Math.max(0, locations[locName].prev);
}
});
let versionCount = isVersioned ?
res.versionCount - res.masterCount : 0;
versionCount = Math.max(0, versionCount);
return {
versions: versionCount,
objects: res.masterCount + res.nullCount,
dataManaged: {
total,
locations,
}, },
objectCount: {
current: 0,
nonCurrent: 0,
deleteMarker: 0,
},
};
}
const {
masterCount,
masterData,
nullCount,
nullData,
versionCount,
versionData,
deleteMarkerCount,
} = res[metricLevel][resourceName];
dataMetrics[metricLevel][resourceName].usedCapacity.current += nullData + masterData;
dataMetrics[metricLevel][resourceName].objectCount.current += nullCount + masterCount;
if (isVersioned) {
dataMetrics[metricLevel][resourceName].usedCapacity.nonCurrent += versionData - masterData;
dataMetrics[metricLevel][resourceName].usedCapacity.nonCurrent
= Math.max(dataMetrics[metricLevel][resourceName].usedCapacity.nonCurrent, 0);
dataMetrics[metricLevel][resourceName].objectCount.nonCurrent
+= versionCount - masterCount - deleteMarkerCount;
dataMetrics[metricLevel][resourceName].objectCount.nonCurrent
= Math.max(dataMetrics[metricLevel][resourceName].objectCount.nonCurrent, 0);
dataMetrics[metricLevel][resourceName].objectCount.deleteMarker += deleteMarkerCount;
}
if (metricLevel === 'location') { // calculate usedCapacity metrics at global and location level
totalBytes.curr += nullData + masterData;
if (!locationBytes[resourceName]) {
locationBytes[resourceName] = { curr: 0, prev: 0 };
}
locationBytes[resourceName].curr += nullData + masterData;
if (isVersioned) {
totalBytes.prev += versionData;
totalBytes.prev -= masterData;
totalBytes.prev = Math.max(0, totalBytes.prev);
locationBytes[resourceName].prev += versionData;
locationBytes[resourceName].prev -= masterData;
locationBytes[resourceName].prev = Math.max(0, locationBytes[resourceName].prev);
}
}
if (metricLevel === 'bucket') { // count objects up of all buckets
totalCurrentCount += (masterCount + nullCount);
totalNonCurrentCount += isVersioned ? (versionCount - masterCount - deleteMarkerCount) : 0;
}
});
}
});
return {
versions: Math.max(0, totalNonCurrentCount),
objects: totalCurrentCount,
dataManaged: {
total: totalBytes,
locations: locationBytes,
},
dataMetrics,
}; };
} }
/** /**
* @param{string} bucketName -
* @param{object} entry - * @param{object} entry -
* @param{string} entry._id - * @param{string} entry._id -
* @param{object} entry.value - * @param{object} entry.value -
* @param{boolean} isTransient - * @param{boolean} isTransient -
* @returns{object.<string, number>} results - * @returns{object.<string, number>} results -
*/ */
_processEntryData(entry, isTransient) { _processEntryData(bucketName, entry, isTransient) {
const results = {}; if (!bucketName) {
return {
data: {},
error: new Error('no bucket name provided'),
};
}
const size = Number.parseInt(entry.value['content-length'], 10); const size = Number.parseInt(entry.value['content-length'], 10);
if (Number.isNaN(size)) { if (Number.isNaN(size)) {
@ -2106,24 +2053,31 @@ class MongoClientInterface {
}; };
} }
const results = {
// there will be only one bucket for an object entry
bucket: { [bucketName]: size },
// there can be multiple locations for an object entry
location: {},
// there will be only one account for an object entry
account: { [entry.value['owner-display-name']]: size },
};
if (!isTransient || if (!isTransient ||
entry.value.replicationInfo.status !== 'COMPLETED') { entry.value.replicationInfo.status !== 'COMPLETED') {
if (results[entry.value.dataStoreName]) { // only count it in current dataStore if object is not in transient or replication not completed
results[entry.value.dataStoreName] += size; if (results.location[entry.value.dataStoreName]) {
results.location[entry.value.dataStoreName] += size;
} else { } else {
results[entry.value.dataStoreName] = size; results.location[entry.value.dataStoreName] = size;
}
} else {
if (!results[entry.value.dataStoreName]) {
results[entry.value.dataStoreName] = 0;
} }
} }
entry.value.replicationInfo.backends.forEach(rep => { entry.value.replicationInfo.backends.forEach(rep => {
// count it in the replication destination location if replication compeleted
if (rep.status === 'COMPLETED') { if (rep.status === 'COMPLETED') {
if (results[rep.site]) { if (results.location[rep.site]) {
results[rep.site] += size; results.location[rep.site] += size;
} else { } else {
results[rep.site] = size; results.location[rep.site] = size;
} }
} }
}); });
@ -2164,15 +2118,15 @@ class MongoClientInterface {
'value.dataStoreName': 1, 'value.dataStoreName': 1,
'value.content-length': 1, 'value.content-length': 1,
'value.versionId': 1, 'value.versionId': 1,
'value.owner-display-name': 1,
'value.isDeleteMarker': 1,
'value.isNull': 1,
}, },
}); });
const collRes = { const collRes = {
masterCount: 0, bucket: {}, // bucket level metrics
masterData: {}, location: {}, // location level metrics
nullCount: 0, account: {}, // account level metrics
nullData: {},
versionCount: 0,
versionData: {},
}; };
let stalledCount = 0; let stalledCount = 0;
const cmpDate = new Date(); const cmpDate = new Date();
@ -2180,7 +2134,7 @@ class MongoClientInterface {
cursor.forEach( cursor.forEach(
res => { res => {
const { data, error } = this._processEntryData(res, isTransient); const { data, error } = this._processEntryData(bucketName, res, isTransient);
if (error) { if (error) {
log.error('Failed to process entry data', { log.error('Failed to process entry data', {
@ -2201,7 +2155,7 @@ class MongoClientInterface {
this._isReplicationEntryStalled(res, cmpDate)) { this._isReplicationEntryStalled(res, cmpDate)) {
stalledCount++; stalledCount++;
} }
} else if (!!res.value.versionId) { } else if (!!res.value.versionId && !res.value.isNull) {
// master version // master version
targetCount = 'masterCount'; targetCount = 'masterCount';
targetData = 'masterData'; targetData = 'masterData';
@ -2210,12 +2164,26 @@ class MongoClientInterface {
targetCount = 'nullCount'; targetCount = 'nullCount';
targetData = 'nullData'; targetData = 'nullData';
} }
collRes[targetCount]++; Object.keys(data).forEach(metricLevel => {
Object.keys(data).forEach(site => { // metricLevel can only be 'bucket', 'location' or 'account'
if (collRes[targetData][site]) { if (validStorageMetricLevels.has(metricLevel)) {
collRes[targetData][site] += data[site]; Object.keys(data[metricLevel]).forEach(resourceName => {
} else { // resourceName can be the name of bucket, location or account
collRes[targetData][site] = data[site]; if (!collRes[metricLevel][resourceName]) {
collRes[metricLevel][resourceName] = {
masterCount: 0,
masterData: 0,
nullCount: 0,
nullData: 0,
versionCount: 0,
versionData: 0,
deleteMarkerCount: 0,
};
}
collRes[metricLevel][resourceName][targetData] += data[metricLevel][resourceName];
collRes[metricLevel][resourceName][targetCount]++;
collRes[metricLevel][resourceName].deleteMarkerCount += res.value.isDeleteMarker ? 1 : 0;
});
} }
}); });
}, },

View File

@ -0,0 +1,572 @@
const async = require('async');
const assert = require('assert');
const werelogs = require('werelogs');
const { MongoMemoryReplSet } = require('mongodb-memory-server');
const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug');
const BucketInfo = require('../../../../lib/models/BucketInfo').default;
const MetadataWrapper =
require('../../../../lib/storage/metadata/MetadataWrapper');
const { versioning } = require('../../../../index');
const { BucketVersioningKeyFormat } = versioning.VersioningConstants;
const IMPL_NAME = 'mongodb';
const DB_NAME = 'metadata';
const BUCKET_NAME = 'test-bucket';
const ACCOUNT_NAME = 'test-account';
const mongoserver = new MongoMemoryReplSet({
debug: false,
instanceOpts: [
{ port: 27018 },
],
replSet: {
name: 'rs0',
count: 1,
DB_NAME,
storageEngine: 'ephemeralForTest',
},
});
const variations = [
{ it: '(v0)', vFormat: BucketVersioningKeyFormat.v0 },
{ it: '(v1)', vFormat: BucketVersioningKeyFormat.v1 },
];
describe('MongoClientInterface::metadata.getObjectMDStats', () => {
let metadata;
beforeAll(done => {
mongoserver.waitUntilRunning().then(() => {
const opts = {
mongodb: {
replicaSetHosts: 'localhost:27018',
writeConcern: 'majority',
replicaSet: 'rs0',
readPreference: 'primary',
database: DB_NAME,
},
};
metadata = new MetadataWrapper(IMPL_NAME, opts, null, logger);
metadata.setup(done);
});
});
afterAll(done => {
async.series([
next => metadata.close(next),
next => mongoserver.stop()
.then(() => next())
.catch(next),
], done);
});
const bucketMD = BucketInfo.fromObj({
_name: BUCKET_NAME,
_owner: 'testowner',
_ownerDisplayName: ACCOUNT_NAME,
_creationDate: new Date().toJSON(),
_acl: {
Canned: 'private',
FULL_CONTROL: [],
WRITE: [],
WRITE_ACP: [],
READ: [],
READ_ACP: [],
},
_mdBucketModelVersion: 10,
_transient: false,
_deleted: false,
_serverSideEncryption: null,
_versioningConfiguration: null,
_locationConstraint: 'us-east-1',
_readLocationConstraint: null,
_cors: null,
_replicationConfiguration: null,
_lifecycleConfiguration: null,
_uid: '',
_isNFS: null,
ingestion: null,
});
const versionedBucketMD = {
...bucketMD,
_versioningConfiguration: {
Status: 'Enabled',
},
};
const suspendedBucketMD = {
...bucketMD,
_versioningConfiguration: {
Status: 'Suspended',
},
};
describe('Should get correct results for versioning disabled bucket', () => {
const versionParams = {
versioning: false,
versionId: null,
};
const object1Params = {
'key': 'non-versioned-test-object1',
'content-length': 10,
'dataStoreName': 'us-east-1',
'owner-display-name': ACCOUNT_NAME,
'replicationInfo': {
backends: [],
},
};
const object2Params = {
...object1Params,
key: 'non-versioned-test-object2',
};
variations.forEach(variation => {
describe(variation.it, () => {
beforeEach(done => {
async.series([
next => {
metadata.client.defaultBucketKeyFormat = variation.vFormat;
return next();
},
next => metadata.createBucket(BUCKET_NAME, bucketMD, logger, next),
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
object1Params, versionParams, logger, next), // put object1
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
object1Params, versionParams, logger, next), // put object1 again
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
object2Params, versionParams, logger, next), // put object2
], done);
});
afterEach(done => metadata.deleteBucket(BUCKET_NAME, logger, done));
it(`Should get correct results ${variation.it}`, done => {
const expected = {
dataManaged: {
locations: { 'us-east-1': { curr: 20, prev: 0 } },
total: { curr: 20, prev: 0 },
},
objects: 2, stalled: 0, versions: 0,
dataMetrics: {
account: {
[ACCOUNT_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
bucket: {
[BUCKET_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
location: {
'us-east-1': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
},
};
return metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
assert.deepStrictEqual(err, null);
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger, (err, data) => {
assert.deepStrictEqual(err, null);
assert.deepStrictEqual(data, expected);
return done();
});
});
});
});
});
});
describe('Should get correct results for versioning enabled bucket', () => {
const versionParams = {
versioning: true,
versionId: null,
};
const object1Params = {
'key': 'versioned-test-object1',
'content-length': 10,
'dataStoreName': 'us-east-1',
'owner-display-name': ACCOUNT_NAME,
'replicationInfo': {
backends: [],
},
};
const object2Params = {
...object1Params,
key: 'versioned-test-object2',
};
variations.forEach(variation => {
const itOnlyInV1 = variation.vFormat === 'v1' ? it : it.skip;
describe(variation.it, () => {
beforeEach(done => {
async.series([
next => {
metadata.client.defaultBucketKeyFormat = variation.vFormat;
return next();
},
next => metadata.createBucket(BUCKET_NAME, versionedBucketMD, logger, next),
], done);
});
afterEach(done => metadata.deleteBucket(BUCKET_NAME, logger, done));
it(`Should get correct results ${variation.it}`, done => {
const expected = {
dataManaged: {
locations: { 'us-east-1': { curr: 20, prev: 10 } },
total: { curr: 20, prev: 10 },
},
objects: 2, stalled: 0, versions: 1,
dataMetrics: {
account: {
[ACCOUNT_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
},
bucket: {
[BUCKET_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
},
location: {
'us-east-1': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
},
},
};
return async.series([
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
object1Params, versionParams, logger, next), // put object1
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
object1Params, versionParams, logger, next), // put object1 again
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
object2Params, versionParams, logger, next), // put object2
], () =>
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
assert.deepStrictEqual(err, null);
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
(err, data) => {
assert.deepStrictEqual(err, null);
assert.deepStrictEqual(data, expected);
return done();
});
}),
);
});
itOnlyInV1(`Should get correct results with deleteMarker ${variation.it}`, done => {
const expected = {
dataManaged: {
locations: { 'us-east-1': { curr: 0, prev: 20 } },
total: { curr: 0, prev: 20 },
},
objects: 0, stalled: 0, versions: 2,
dataMetrics: {
account: {
[ACCOUNT_NAME]: {
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
usedCapacity: { current: 0, nonCurrent: 20 },
},
},
bucket: {
[BUCKET_NAME]: {
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
usedCapacity: { current: 0, nonCurrent: 20 },
},
},
location: {
'us-east-1': {
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
usedCapacity: { current: 0, nonCurrent: 20 },
},
},
},
};
return async.series([
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
object1Params, versionParams, logger, next), // put object1
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
object1Params, versionParams, logger, next), // put object1 again
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
{
...object1Params,
'isDeleteMarker': true,
'content-length': 0,
}, versionParams, logger, next), // delete object1
], () =>
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
assert.deepStrictEqual(err, null);
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
(err, data) => {
assert.deepStrictEqual(err, null);
assert.deepStrictEqual(data, expected);
return done();
});
}),
);
});
it('should get correct results with lifecycle replication enabled ' +
`and location transient is true ${variation.it}`, done => {
const expected = {
dataManaged: {
locations: {
'us-east-1': { curr: 10, prev: 0 },
'completed': { curr: 10, prev: 0 },
},
total: { curr: 20, prev: 0 },
},
objects: 2, stalled: 0, versions: 0,
dataMetrics: {
account: {
[ACCOUNT_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
bucket: {
[BUCKET_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
location: {
'us-east-1': {
objectCount: { current: 1, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 10, nonCurrent: 0 },
},
'completed': {
objectCount: { current: 1, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 10, nonCurrent: 0 },
},
},
},
};
return async.series([
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
{
...object1Params,
replicationInfo: {
status: 'PENDING',
backends: [
{
status: 'PENDING',
site: 'not-completed',
},
{
status: 'COMPLETED',
site: 'completed',
},
],
},
}, versionParams, logger, next), // object1 with one site pending and one site complete
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
{
...object2Params,
replicationInfo: {
status: 'COMPLETED',
backends: [
{
status: 'COMPLETE',
site: 'completed',
},
],
},
}, versionParams, logger, next), // object2 with one site complete
], () =>
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
assert.deepStrictEqual(err, null);
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, true, logger,
(err, data) => {
assert.deepStrictEqual(err, null);
assert.deepStrictEqual(data, expected);
return done();
});
}),
);
});
it('should get correct results with lifecycle replication enabled' +
`and location transient is false ${variation.it}`, done => {
const expected = {
dataManaged: {
locations: {
'us-east-1': { curr: 20, prev: 0 },
'completed': { curr: 10, prev: 0 },
},
total: { curr: 30, prev: 0 },
},
objects: 2, stalled: 0, versions: 0,
dataMetrics: {
account: {
[ACCOUNT_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
bucket: {
[BUCKET_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
location: {
'us-east-1': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
'completed': {
objectCount: { current: 1, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 10, nonCurrent: 0 },
},
},
},
};
return async.series([
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
{
...object1Params,
replicationInfo: {
status: 'PENDING',
backends: [
{
status: 'PENDING',
site: 'not-completed',
},
{
status: 'COMPLETED',
site: 'completed',
},
],
},
}, versionParams, logger, next), // object1 with one site pending and one site complete
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
{
...object2Params,
replicationInfo: {
status: 'COMPLETED',
backends: [
{
status: 'COMPLETE',
site: 'completed',
},
],
},
}, versionParams, logger, next), // object2 with one site complete
], () =>
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
assert.deepStrictEqual(err, null);
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
(err, data) => {
assert.deepStrictEqual(err, null);
assert.deepStrictEqual(data, expected);
return done();
});
}),
);
});
});
});
});
describe('Should get correct results for versioning suspended bucket', () => {
const object1Params = {
'key': 'test-object1',
'content-length': 10,
'dataStoreName': 'us-east-1',
'owner-display-name': ACCOUNT_NAME,
'replicationInfo': {
backends: [],
},
};
const object2Params = {
...object1Params,
key: 'test-object2',
};
variations.forEach(variation => {
describe(variation.it, () => {
beforeEach(done => {
async.series([
next => {
metadata.client.defaultBucketKeyFormat = variation.vFormat;
return next();
},
next => metadata.createBucket(BUCKET_NAME, suspendedBucketMD, logger, next),
], done);
});
afterEach(done => metadata.deleteBucket(BUCKET_NAME, logger, done));
it(`Should get correct results ${variation.it}`, done => {
const expected = {
dataManaged: {
locations: { 'us-east-1': { curr: 20, prev: 10 } },
total: { curr: 20, prev: 10 },
},
objects: 2, stalled: 0, versions: 1,
dataMetrics: {
account: {
[ACCOUNT_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
},
bucket: {
[BUCKET_NAME]: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
},
location: {
'us-east-1': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
},
},
};
return async.series([
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
object1Params, {
versionId: null,
versioning: true,
}, logger, next), // versioned object1 put before suspend
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
{
...object1Params,
isNull: true,
},
{
versionId: null,
}, logger, next), // null versioned object1
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
{
...object2Params,
isNull: true,
},
{
versionId: null,
}, logger, next), // null versioned object2
], () =>
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
assert.deepStrictEqual(err, null);
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
(err, data) => {
assert.deepStrictEqual(err, null);
assert.deepStrictEqual(data, expected);
return done();
});
}),
);
});
});
});
});
});

View File

@ -32,37 +32,127 @@ const DummyConfigObject = require('./utils/DummyConfigObject');
const mongoTestClient = new MongoClientInterface({}); const mongoTestClient = new MongoClientInterface({});
describe('MongoClientInterface::_handleResults', () => { describe('MongoClientInterface::_handleResults', () => {
it('should return zero-result', () => {
const testInput = { const testInput = {
masterCount: 0, masterData: {}, bucket: {
nullCount: 0, nullData: {}, bucket1: {
versionCount: 0, versionData: {}, masterCount: 2,
masterData: 20,
nullCount: 2,
nullData: 20,
versionCount: 4,
versionData: 40,
deleteMarkerCount: 2,
},
},
location: {
location1: {
masterCount: 1,
masterData: 10,
nullCount: 1,
nullData: 10,
versionCount: 2,
versionData: 20,
deleteMarkerCount: 1,
},
location2: {
masterCount: 1,
masterData: 10,
nullCount: 1,
nullData: 10,
versionCount: 2,
versionData: 20,
deleteMarkerCount: 1,
},
},
account: {
account1: {
masterCount: 2,
masterData: 20,
nullCount: 2,
nullData: 20,
versionCount: 4,
versionData: 40,
deleteMarkerCount: 2,
},
},
}; };
const testResults = mongoTestClient._handleResults(testInput, true); it('should return zero-result when input is empty', () => {
const testInputEmpty = {
bucket: {},
location: {},
account: {},
};
const testResults = mongoTestClient._handleResults(testInputEmpty, true);
const expectedRes = { const expectedRes = {
versions: 0, objects: 0, versions: 0, objects: 0,
dataManaged: { dataManaged: {
total: { curr: 0, prev: 0 }, total: { curr: 0, prev: 0 },
locations: {}, locations: {},
}, },
dataMetrics: {
bucket: {},
location: {},
account: {},
},
};
assert.deepStrictEqual(testResults, expectedRes);
});
it('should return zero-result when input metric keys are not valid', () => {
const testInputWithInvalidMetricKeys = {
InvalidMetric0: testInput.bucket,
InvalidMetric1: testInput.location,
InvalidMetric2: testInput.account,
};
const testResults = mongoTestClient._handleResults(testInputWithInvalidMetricKeys, true);
const expectedRes = {
versions: 0, objects: 0,
dataManaged: {
total: { curr: 0, prev: 0 },
locations: {},
},
dataMetrics: {
bucket: {},
location: {},
account: {},
},
}; };
assert.deepStrictEqual(testResults, expectedRes); assert.deepStrictEqual(testResults, expectedRes);
}); });
it('should return correct value if isVer is false', () => { it('should return correct value if isVer is false', () => {
const testInput = {
masterCount: 2, masterData: { test1: 10, test2: 10 },
nullCount: 2, nullData: { test1: 10, test2: 10 },
versionCount: 2, versionData: { test1: 20, test2: 20 },
};
const testResults = mongoTestClient._handleResults(testInput, false); const testResults = mongoTestClient._handleResults(testInput, false);
const expectedRes = { const expectedRes = {
versions: 0, objects: 4, versions: 0, objects: 4,
dataManaged: { dataManaged: {
total: { curr: 40, prev: 0 }, total: { curr: 40, prev: 0 },
locations: { locations: {
test1: { curr: 20, prev: 0 }, location1: { curr: 20, prev: 0 },
test2: { curr: 20, prev: 0 }, location2: { curr: 20, prev: 0 },
},
},
dataMetrics: {
bucket: {
bucket1: {
objectCount: { current: 4, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 40, nonCurrent: 0 },
},
},
location: {
location1: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
location2: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 0 },
},
},
account: {
account1: {
objectCount: { current: 4, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 40, nonCurrent: 0 },
},
}, },
}, },
}; };
@ -70,24 +160,76 @@ describe('MongoClientInterface::_handleResults', () => {
}); });
it('should return correct value if isVer is true', () => { it('should return correct value if isVer is true', () => {
const testInput = {
masterCount: 2, masterData: { test1: 10, test2: 10 },
nullCount: 2, nullData: { test1: 10, test2: 10 },
versionCount: 4, versionData: { test1: 20, test2: 20 },
};
const testResults = mongoTestClient._handleResults(testInput, true); const testResults = mongoTestClient._handleResults(testInput, true);
const expectedRes = { const expectedRes = {
versions: 2, objects: 4, versions: 0, objects: 4,
dataManaged: { dataManaged: {
total: { curr: 40, prev: 20 }, total: { curr: 40, prev: 20 },
locations: { locations: {
test1: { curr: 20, prev: 10 }, location1: { curr: 20, prev: 10 },
test2: { curr: 20, prev: 10 }, location2: { curr: 20, prev: 10 },
},
},
dataMetrics: {
bucket: {
bucket1: {
objectCount: { current: 4, deleteMarker: 2, nonCurrent: 0 },
usedCapacity: { current: 40, nonCurrent: 20 },
},
},
location: {
location1: {
objectCount: { current: 2, deleteMarker: 1, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
location2: {
objectCount: { current: 2, deleteMarker: 1, nonCurrent: 0 },
usedCapacity: { current: 20, nonCurrent: 10 },
},
},
account: {
account1: {
objectCount: { current: 4, deleteMarker: 2, nonCurrent: 0 },
usedCapacity: { current: 40, nonCurrent: 20 },
},
}, },
}, },
}; };
assert.deepStrictEqual(testResults, expectedRes); assert.deepStrictEqual(testResults, expectedRes);
}); });
it('should calculate dataManaged based on input location metrics', () => {
const testInputOnlyContainsLocation = {
bucket: {},
location: testInput.location,
account: {},
};
const testResults = mongoTestClient._handleResults(testInputOnlyContainsLocation, true);
const expectedRes = {
dataManaged: {
total: { curr: 40, prev: 20 },
locations: {
location1: { curr: 20, prev: 10 },
location2: { curr: 20, prev: 10 },
},
},
};
assert.deepStrictEqual(testResults.dataManaged, expectedRes.dataManaged);
});
it('should calculate total current and nonCurrent counts based on input bucket metrics', () => {
const testInputOnlyContainsLocation = {
bucket: testInput.bucket,
location: {},
account: {},
};
const testResults = mongoTestClient._handleResults(testInputOnlyContainsLocation, true);
const expectedRes = {
versions: 0, objects: 4,
};
assert.deepStrictEqual(testResults.versions, expectedRes.versions);
assert.deepStrictEqual(testResults.objects, expectedRes.objects);
});
}); });
describe('MongoClientInterface, misc', () => { describe('MongoClientInterface, misc', () => {
@ -105,140 +247,150 @@ describe('MongoClientInterface, misc', () => {
}); });
describe('MongoClientInterface::_processEntryData', () => { describe('MongoClientInterface::_processEntryData', () => {
const testBucketName = 'testBucket';
const objectMdTemp = {
'last-modified': new Date(),
'replicationInfo': {
status: 'PENDING',
backends: [],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
},
'transient': false,
'dataStoreName': 'us-east-1',
'content-length': 42,
'versionId': '0123456789abcdefg',
'owner-display-name': 'account1',
};
const tests = [ const tests = [
[ [
'should add content-length to total if replication status != ' + 'should add content-length to current dataStore but not replication destination ' +
'COMPLETED and transient == true', 'if replication status != COMPLETED and transient == true',
testBucketName,
true, true,
{ {
_id: 'testkey', _id: 'testkey0',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': { replicationInfo: {
...objectMdTemp.replicationInfo,
backends: [{
site: 'not-completed',
status: 'PENDING',
}],
status: 'PENDING', status: 'PENDING',
backends: [],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
}, },
'dataStoreName': 'us-east-1',
'content-length': 42,
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
data: { data: {
'us-east-1': 42, account: { account1: 42 },
bucket: { [testBucketName]: 42 },
location: { 'us-east-1': 42 },
}, },
error: null, error: null,
}, },
], ],
[ [
'should not add content-length to total if replication ' + 'should not add content-length to replication destination but not in current dataStore ' +
'status == COMPLETED and transient == true', 'if replication status == COMPLETED and transient == true',
testBucketName,
true, true,
{ {
_id: 'testkey', _id: 'testkey1',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': { replicationInfo: {
...objectMdTemp.replicationInfo,
backends: [{
site: 'completed',
status: 'COMPLETED',
}],
status: 'COMPLETED', status: 'COMPLETED',
backends: [],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
}, },
'dataStoreName': 'us-east-1',
'content-length': 42,
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
data: { data: {
'us-east-1': 0, account: { account1: 42 },
bucket: { [testBucketName]: 42 },
location: { completed: 42 },
}, },
error: null, error: null,
}, },
], ],
[ [
'should add content-length to total if replication status != ' + 'should add content-length to current dataStore but not replication destination ' +
'COMPLETED and transient == false', 'if replication status != COMPLETED and transient == false',
testBucketName,
false, false,
{ {
_id: 'testkey', _id: 'testkey2',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': { replicationInfo: {
...objectMdTemp.replicationInfo,
backends: [{
site: 'not-completed',
status: 'PENDING',
}],
status: 'PENDING', status: 'PENDING',
backends: [],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
}, },
'dataStoreName': 'us-east-1',
'content-length': 42,
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
data: { data: {
'us-east-1': 42, account: { account1: 42 },
bucket: { [testBucketName]: 42 },
location: { 'us-east-1': 42 },
}, },
error: null, error: null,
}, },
], ],
[ [
'should add content-length to total if replication ' + 'should add content-length to current dataStore and replication destination ' +
'status == COMPLETED and transient == false', 'if replication status == COMPLETED and transient == false',
testBucketName,
false, false,
{ {
_id: 'testkey', _id: 'testkey3',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': { replicationInfo: {
...objectMdTemp.replicationInfo,
backends: [{
site: 'completed',
status: 'COMPLETED',
}],
status: 'COMPLETED', status: 'COMPLETED',
backends: [],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
}, },
'dataStoreName': 'us-east-1',
'content-length': 42,
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
data: { data: {
'us-east-1': 42, account: { account1: 42 },
bucket: { [testBucketName]: 42 },
location: { 'us-east-1': 42, 'completed': 42 },
}, },
error: null, error: null,
}, },
], ],
[ [
'should add content-length to total for each COMPLETED backends ' + 'should add content-length to each COMPLETED replication destination but not current dataStore ' +
'(replication status: COMPLETED)', '(object replication status: COMPLETED)',
testBucketName,
true, true,
{ {
_id: 'testkey', _id: 'testkey4',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': { replicationInfo: {
...objectMdTemp.replicationInfo,
status: 'COMPLETED', status: 'COMPLETED',
backends: [ backends: [
{ {
@ -254,38 +406,33 @@ describe('MongoClientInterface::_processEntryData', () => {
site: 'completed-3', site: 'completed-3',
}, },
], ],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
}, },
'dataStoreName': 'us-east-1',
'content-length': 42,
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
data: { data: {
'us-east-1': 0, account: { account1: 42 },
bucket: { [testBucketName]: 42 },
location: {
'completed-1': 42, 'completed-1': 42,
'completed-2': 42, 'completed-2': 42,
'completed-3': 42, 'completed-3': 42,
}, },
},
error: null, error: null,
}, },
], ],
[ [
'should add content-length to total for each COMPLETED backends ' + 'should add content-length to each COMPLETED replications destination and current dataStore ' +
'(replication status: PENDING)', '(object replication status: PENDING)',
testBucketName,
true, true,
{ {
_id: 'testkey', _id: 'testkey5',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': { replicationInfo: {
...objectMdTemp.replicationInfo,
status: 'PENDING', status: 'PENDING',
backends: [ backends: [
{ {
@ -301,62 +448,32 @@ describe('MongoClientInterface::_processEntryData', () => {
site: 'completed-2', site: 'completed-2',
}, },
], ],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
}, },
'dataStoreName': 'us-east-1', transient: true,
'content-length': 42,
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
data: { data: {
account: { account1: 42 },
bucket: { [testBucketName]: 42 },
location: {
'us-east-1': 42, 'us-east-1': 42,
'completed-1': 42, 'completed-1': 42,
'completed-2': 42, 'completed-2': 42,
}, },
},
error: null, error: null,
}, },
], ],
[ [
'should error if content-length is invalid', 'should return error if content-length is invalid',
testBucketName,
true, true,
{ {
_id: 'testkey', _id: 'testkey6',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': {
status: 'PENDING',
backends: [
{
status: 'PENDING',
site: 'not-completed',
},
{
status: 'COMPLETED',
site: 'completed-1',
},
{
status: 'COMPLETED',
site: 'completed-2',
},
],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
},
'dataStoreName': 'us-east-1',
'content-length': 'not-a-number', 'content-length': 'not-a-number',
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
@ -366,38 +483,41 @@ describe('MongoClientInterface::_processEntryData', () => {
], ],
[ [
'should correctly process entry with string typed content-length', 'should correctly process entry with string typed content-length',
testBucketName,
true, true,
{ {
_id: 'testkey', _id: 'testkey7',
value: { value: {
'last-modified': new Date(), ...objectMdTemp,
'replicationInfo': {
status: 'PENDING',
backends: [],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
},
'dataStoreName': 'us-east-1',
'content-length': '42', 'content-length': '42',
'versionId': '0123456789abcdefg',
}, },
}, },
{ {
data: { data: {
'us-east-1': 42, account: { account1: 42 },
bucket: { [testBucketName]: 42 },
location: { 'us-east-1': 42 },
}, },
error: null, error: null,
}, },
], ],
[
'should return error if bucketName is empty',
undefined,
true,
{
_id: 'testkey8',
value: objectMdTemp,
},
{
data: {},
error: new Error('no bucket name provided'),
},
],
]; ];
tests.forEach(([msg, isTransient, params, expected]) => it(msg, () => { tests.forEach(([msg, bucketName, isTransient, params, expected]) => it(msg, () => {
assert.deepStrictEqual( assert.deepStrictEqual(
mongoTestClient._processEntryData(params, isTransient), mongoTestClient._processEntryData(bucketName, params, isTransient),
expected, expected,
); );
})); }));
@ -545,7 +665,10 @@ function uploadObjects(client, bucketName, objectList, callback) {
.setKey(obj.name) .setKey(obj.name)
.setDataStoreName('us-east-1') .setDataStoreName('us-east-1')
.setContentLength(100) .setContentLength(100)
.setLastModified(obj.lastModified); .setLastModified(obj.lastModified)
.setOwnerDisplayName(obj.ownerDisplayName)
.setIsNull(obj.isNull)
.setIsDeleteMarker(obj.isDeleteMarker);
if (obj.repInfo) { if (obj.repInfo) {
objMD.setReplicationInfo(obj.repInfo); objMD.setReplicationInfo(obj.repInfo);
} }
@ -585,7 +708,60 @@ describe('MongoClientInterface, tests', () => {
], done); ], done);
}); });
const nonVersionedObjectMdTemp = {
name: 'testkey',
versioning: false,
versionId: null,
lastModified: new Date(Date.now()),
ownerDisplayName: 'testAccount',
};
const objectMdTemp = {
name: 'testkey',
versioning: true,
versionId: null,
lastModified: new Date(Date.now()),
ownerDisplayName: 'testAccount',
repInfo: {
status: 'COMPLETED',
backends: [
{
status: 'COMPLETED',
site: 'rep-loc-1',
},
],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
},
};
const tests = [ const tests = [
[
'getObjectMDStats() should return zero-result when no objects in the bucket',
{
bucketName: 'test-bucket',
isVersioned: false,
objectList: [],
},
{
dataManaged: {
locations: {},
total: { curr: 0, prev: 0 },
},
objects: 0,
stalled: 0,
versions: 0,
dataMetrics: {
bucket: {},
location: {},
account: {},
},
},
],
[ [
'getObjectMDStats() should return correct results', 'getObjectMDStats() should return correct results',
{ {
@ -594,57 +770,21 @@ describe('MongoClientInterface, tests', () => {
objectList: [ objectList: [
// versioned object 1, // versioned object 1,
{ {
name: 'testkey', ...objectMdTemp,
versioning: true, versioning: true,
versionId: null,
lastModified: new Date(Date.now()),
repInfo: {
status: 'COMPLETED',
backends: [
{
status: 'COMPLETED',
site: 'rep-loc-1',
},
],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
},
}, },
// versioned object 2, // versioned object 2,
{ {
name: 'testkey', ...objectMdTemp,
versioning: true, versioning: true,
versionId: null,
lastModified: new Date(Date.now()),
repInfo: {
status: 'COMPLETED',
backends: [
{
status: 'COMPLETED',
site: 'rep-loc-1',
},
],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
},
}, },
// stalled object 1 // stalled object 1
{ {
name: 'testkey', ...objectMdTemp,
versioning: true, versioning: true,
versionId: null,
lastModified: new Date(Date.now() - hr), lastModified: new Date(Date.now() - hr),
repInfo: { repInfo: {
...objectMdTemp.repInfo,
status: 'PENDING', status: 'PENDING',
backends: [ backends: [
{ {
@ -652,18 +792,13 @@ describe('MongoClientInterface, tests', () => {
site: 'rep-loc-1', site: 'rep-loc-1',
}, },
], ],
content: [],
destination: '',
storageClass: '',
role: '',
storageType: '',
dataStoreVersionId: '',
isNFS: null,
}, },
}, },
// null versioned object // null versioned object
{ {
name: 'nullkey', name: 'nullkey',
isNull: true,
ownerDisplayName: 'testAccount',
lastModified: new Date(Date.now() - hr), lastModified: new Date(Date.now() - hr),
}, },
], ],
@ -688,6 +823,176 @@ describe('MongoClientInterface, tests', () => {
objects: 2, objects: 2,
stalled: 1, stalled: 1,
versions: 2, versions: 2,
dataMetrics: {
account: {
testAccount: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 2 },
usedCapacity: { current: 200, nonCurrent: 200 },
},
},
bucket: {
'test-bucket': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 2 },
usedCapacity: { current: 200, nonCurrent: 200 },
},
},
location: {
'rep-loc-1': {
objectCount: { current: 0, deleteMarker: 0, nonCurrent: 2 },
usedCapacity: { current: 0, nonCurrent: 200 },
},
'us-east-1': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 2 },
usedCapacity: { current: 200, nonCurrent: 200 },
},
},
},
},
],
[
'getObjectMDStats() should return correct results for non versioned bucket',
{
bucketName: 'test-bucket',
isVersioned: false,
objectList: [
// non versioned object 1,
{
...nonVersionedObjectMdTemp,
name: 'testkey1',
},
// non versioned object 1,
{
...nonVersionedObjectMdTemp,
name: 'testkey1',
},
// non versioned object 2
{
...nonVersionedObjectMdTemp,
name: 'testkey2',
},
],
},
{
dataManaged: {
locations: {
'us-east-1': {
curr: 200,
prev: 0,
},
},
total: {
curr: 200,
prev: 0,
},
},
objects: 2,
stalled: 0,
versions: 0,
dataMetrics: {
account: {
testAccount: {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 200, nonCurrent: 0 },
},
},
bucket: {
'test-bucket': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 200, nonCurrent: 0 },
},
},
location: {
'us-east-1': {
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
usedCapacity: { current: 200, nonCurrent: 0 },
},
},
},
},
],
[
'getObjectMDStats() should return correct results for versioned bucket',
{
bucketName: 'test-bucket',
isVersioned: true,
objectList: [
// a version of object 1,
{
...objectMdTemp,
versioning: true,
},
// a version of object 1,
{
...objectMdTemp,
versioning: true,
},
// deleteMarker of object 1
{
...objectMdTemp,
versioning: true,
isDeleteMarker: true,
},
// a version of object 1,
{
...objectMdTemp,
versioning: true,
repInfo: {
...objectMdTemp.repInfo,
status: 'PENDING',
backends: [
{
status: 'PENDING',
site: 'rep-loc-1',
},
],
},
},
],
},
{
dataManaged: {
locations: {
'rep-loc-1': {
curr: 0,
prev: 300,
},
'us-east-1': {
curr: 100,
prev: 300,
},
},
total: {
curr: 100,
prev: 600,
},
},
objects: 1,
stalled: 0,
versions: 2,
dataMetrics: {
account: {
testAccount: {
objectCount: { current: 1, deleteMarker: 1, nonCurrent: 2 },
usedCapacity: { current: 100, nonCurrent: 300 },
},
},
bucket: {
'test-bucket': {
objectCount: { current: 1, deleteMarker: 1, nonCurrent: 2 },
usedCapacity: { current: 100, nonCurrent: 300 },
},
},
location: {
'rep-loc-1': {
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
usedCapacity: { current: 0, nonCurrent: 300 },
},
'us-east-1': {
objectCount: { current: 1, deleteMarker: 1, nonCurrent: 2 },
usedCapacity: { current: 100, nonCurrent: 300 },
},
},
},
}, },
], ],
]; ];