Compare commits
4 Commits
7a88a54918
...
df839bdfd4
| Author | SHA1 | Date |
|---|---|---|
Xin LI
|
df839bdfd4 | |
|
Xin LI
|
018f247a7a | |
|
Xin LI
|
64e4ff0f70 | |
|
Xin LI
|
59b2439f03 |
|
|
@ -1 +1,6 @@
|
|||
{ "extends": "scality" }
|
||||
{
|
||||
"extends": "scality",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2020
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -167,3 +167,5 @@ export const maxCachedBuckets = process.env.METADATA_MAX_CACHED_BUCKETS ?
|
|||
Number(process.env.METADATA_MAX_CACHED_BUCKETS) : 1000;
|
||||
|
||||
export const validRestoreObjectTiers = new Set(['Expedited', 'Standard', 'Bulk']);
|
||||
|
||||
export const validStorageMetricLevels = new Set(['bucket', 'location', 'account']);
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ const { Transform } = require('stream');
|
|||
const { Version } = require('../../../versioning/Version');
|
||||
|
||||
const { formatMasterKey, formatVersionKey } = require('./utils');
|
||||
const { validStorageMetricLevels } = require('../../../constants');
|
||||
|
||||
const VID_NONE = '';
|
||||
|
||||
|
|
@ -1773,6 +1774,11 @@ class MongoClientInterface {
|
|||
byLocation: {},
|
||||
},
|
||||
stalled: 0,
|
||||
dataMetrics: {
|
||||
bucket: {},
|
||||
location: {},
|
||||
account: {},
|
||||
},
|
||||
};
|
||||
return cb(null, res);
|
||||
}
|
||||
|
|
@ -1898,105 +1904,6 @@ class MongoClientInterface {
|
|||
});
|
||||
}
|
||||
|
||||
consolidateData(store, dataManaged) {
|
||||
/* eslint-disable */
|
||||
if (dataManaged && dataManaged.locations && dataManaged.total) {
|
||||
const locations = dataManaged.locations;
|
||||
store.dataManaged.total.curr += dataManaged.total.curr;
|
||||
store.dataManaged.total.prev += dataManaged.total.prev;
|
||||
Object.keys(locations).forEach(site => {
|
||||
if (!store.dataManaged.byLocation[site]) {
|
||||
store.dataManaged.byLocation[site] =
|
||||
Object.assign({}, locations[site]);
|
||||
} else {
|
||||
store.dataManaged.byLocation[site].curr +=
|
||||
locations[site].curr;
|
||||
store.dataManaged.byLocation[site].prev +=
|
||||
locations[site].prev;
|
||||
}
|
||||
});
|
||||
}
|
||||
/* eslint-enable */
|
||||
}
|
||||
|
||||
scanItemCount(log, cb) {
|
||||
const store = {
|
||||
objects: 0,
|
||||
versions: 0,
|
||||
buckets: 0,
|
||||
bucketList: [],
|
||||
dataManaged: {
|
||||
total: { curr: 0, prev: 0 },
|
||||
byLocation: {},
|
||||
},
|
||||
stalled: 0,
|
||||
};
|
||||
|
||||
const consolidateData = dataManaged =>
|
||||
this.consolidateData(store, dataManaged);
|
||||
|
||||
this.getBucketInfos(log, (err, res) => {
|
||||
if (err) {
|
||||
log.error('error getting bucket info', {
|
||||
method: 'scanItemCount',
|
||||
error: err,
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
const { bucketCount, bucketInfos } = res;
|
||||
const retBucketInfos = bucketInfos.map(bucket => ({
|
||||
name: bucket.getName(),
|
||||
location: bucket.getLocationConstraint(),
|
||||
isVersioned: !!bucket.getVersioningConfiguration(),
|
||||
ownerCanonicalId: bucket.getOwner(),
|
||||
ingestion: bucket.isIngestionBucket(),
|
||||
}));
|
||||
|
||||
store.buckets = bucketCount;
|
||||
store.bucketList = retBucketInfos;
|
||||
|
||||
return async.eachLimit(bucketInfos, this.concurrentCursors,
|
||||
(bucketInfo, done) => {
|
||||
async.waterfall([
|
||||
next => this._getIsTransient(bucketInfo, log, next),
|
||||
(isTransient, next) => {
|
||||
const bucketName = bucketInfo.getName();
|
||||
this.getObjectMDStats(bucketName, bucketInfo,
|
||||
isTransient, log, next);
|
||||
},
|
||||
], (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
if (results.dataManaged) {
|
||||
store.objects += results.objects;
|
||||
store.versions += results.versions;
|
||||
store.stalled += results.stalled;
|
||||
consolidateData(results.dataManaged);
|
||||
}
|
||||
return done();
|
||||
});
|
||||
}, err => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
// save to infostore
|
||||
return this.updateCountItems(store, log, err => {
|
||||
if (err) {
|
||||
log.error('error saving count items in mongo', {
|
||||
method: 'scanItemCount',
|
||||
error: err,
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, store);
|
||||
});
|
||||
});
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
|
||||
_getIsTransient(bucketInfo, log, cb) {
|
||||
const locConstraint = bucketInfo.getLocationConstraint();
|
||||
|
||||
|
|
@ -2036,67 +1943,107 @@ class MongoClientInterface {
|
|||
}
|
||||
|
||||
_handleResults(res, isVersioned) {
|
||||
const total = { curr: 0, prev: 0 };
|
||||
const locations = {};
|
||||
let totalNonCurrentCount = 0;
|
||||
let totalCurrentCount = 0;
|
||||
const totalBytes = { curr: 0, prev: 0 };
|
||||
const locationBytes = {};
|
||||
const dataMetrics = {
|
||||
bucket: {},
|
||||
location: {},
|
||||
account: {},
|
||||
};
|
||||
|
||||
Object.keys(res.nullData).forEach(loc => {
|
||||
const bytes = res.nullData[loc];
|
||||
const locName = this._getLocName(loc);
|
||||
if (!locations[locName]) {
|
||||
locations[locName] = { curr: 0, prev: 0 };
|
||||
}
|
||||
total.curr += bytes;
|
||||
locations[locName].curr += bytes;
|
||||
});
|
||||
if (isVersioned) {
|
||||
Object.keys(res.versionData).forEach(loc => {
|
||||
const bytes = res.versionData[loc];
|
||||
const locName = this._getLocName(loc);
|
||||
if (!locations[locName]) {
|
||||
locations[locName] = { curr: 0, prev: 0 };
|
||||
}
|
||||
total.prev += bytes;
|
||||
locations[locName].prev += bytes;
|
||||
});
|
||||
}
|
||||
Object.keys(res.masterData).forEach(loc => {
|
||||
const bytes = res.masterData[loc];
|
||||
const locName = this._getLocName(loc);
|
||||
if (!locations[locName]) {
|
||||
locations[locName] = { curr: 0, prev: 0 };
|
||||
}
|
||||
total.curr += bytes;
|
||||
locations[locName].curr += bytes;
|
||||
if (isVersioned) {
|
||||
total.prev -= bytes;
|
||||
total.prev = Math.max(0, total.prev);
|
||||
locations[locName].prev -= bytes;
|
||||
locations[locName].prev =
|
||||
Math.max(0, locations[locName].prev);
|
||||
}
|
||||
});
|
||||
let versionCount = isVersioned ?
|
||||
res.versionCount - res.masterCount : 0;
|
||||
versionCount = Math.max(0, versionCount);
|
||||
return {
|
||||
versions: versionCount,
|
||||
objects: res.masterCount + res.nullCount,
|
||||
dataManaged: {
|
||||
total,
|
||||
locations,
|
||||
Object.keys(res).forEach(metricLevel => {
|
||||
// metricLevel can only be 'bucket', 'location' or 'account'
|
||||
if (validStorageMetricLevels.has(metricLevel)) {
|
||||
Object.keys(res[metricLevel]).forEach(resource => {
|
||||
// resource can be the name of bucket, location or account
|
||||
const resourceName = metricLevel === 'location' ? this._getLocName(resource) : resource;
|
||||
if (!dataMetrics[metricLevel][resourceName]) {
|
||||
dataMetrics[metricLevel][resourceName] = {
|
||||
usedCapacity: {
|
||||
current: 0,
|
||||
nonCurrent: 0,
|
||||
},
|
||||
objectCount: {
|
||||
current: 0,
|
||||
nonCurrent: 0,
|
||||
deleteMarker: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
const {
|
||||
masterCount,
|
||||
masterData,
|
||||
nullCount,
|
||||
nullData,
|
||||
versionCount,
|
||||
versionData,
|
||||
deleteMarkerCount,
|
||||
} = res[metricLevel][resourceName];
|
||||
|
||||
dataMetrics[metricLevel][resourceName].usedCapacity.current += nullData + masterData;
|
||||
dataMetrics[metricLevel][resourceName].objectCount.current += nullCount + masterCount;
|
||||
|
||||
if (isVersioned) {
|
||||
dataMetrics[metricLevel][resourceName].usedCapacity.nonCurrent += versionData - masterData;
|
||||
dataMetrics[metricLevel][resourceName].usedCapacity.nonCurrent
|
||||
= Math.max(dataMetrics[metricLevel][resourceName].usedCapacity.nonCurrent, 0);
|
||||
dataMetrics[metricLevel][resourceName].objectCount.nonCurrent
|
||||
+= versionCount - masterCount - deleteMarkerCount;
|
||||
dataMetrics[metricLevel][resourceName].objectCount.nonCurrent
|
||||
= Math.max(dataMetrics[metricLevel][resourceName].objectCount.nonCurrent, 0);
|
||||
dataMetrics[metricLevel][resourceName].objectCount.deleteMarker += deleteMarkerCount;
|
||||
}
|
||||
|
||||
if (metricLevel === 'location') { // calculate usedCapacity metrics at global and location level
|
||||
totalBytes.curr += nullData + masterData;
|
||||
if (!locationBytes[resourceName]) {
|
||||
locationBytes[resourceName] = { curr: 0, prev: 0 };
|
||||
}
|
||||
locationBytes[resourceName].curr += nullData + masterData;
|
||||
if (isVersioned) {
|
||||
totalBytes.prev += versionData;
|
||||
totalBytes.prev -= masterData;
|
||||
totalBytes.prev = Math.max(0, totalBytes.prev);
|
||||
locationBytes[resourceName].prev += versionData;
|
||||
locationBytes[resourceName].prev -= masterData;
|
||||
locationBytes[resourceName].prev = Math.max(0, locationBytes[resourceName].prev);
|
||||
}
|
||||
}
|
||||
if (metricLevel === 'bucket') { // count objects up of all buckets
|
||||
totalCurrentCount += (masterCount + nullCount);
|
||||
totalNonCurrentCount += isVersioned ? (versionCount - masterCount - deleteMarkerCount) : 0;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
return {
|
||||
versions: Math.max(0, totalNonCurrentCount),
|
||||
objects: totalCurrentCount,
|
||||
dataManaged: {
|
||||
total: totalBytes,
|
||||
locations: locationBytes,
|
||||
},
|
||||
dataMetrics,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param{string} bucketName -
|
||||
* @param{object} entry -
|
||||
* @param{string} entry._id -
|
||||
* @param{object} entry.value -
|
||||
* @param{boolean} isTransient -
|
||||
* @returns{object.<string, number>} results -
|
||||
*/
|
||||
_processEntryData(entry, isTransient) {
|
||||
const results = {};
|
||||
_processEntryData(bucketName, entry, isTransient) {
|
||||
if (!bucketName) {
|
||||
return {
|
||||
data: {},
|
||||
error: new Error('no bucket name provided'),
|
||||
};
|
||||
}
|
||||
|
||||
const size = Number.parseInt(entry.value['content-length'], 10);
|
||||
if (Number.isNaN(size)) {
|
||||
|
|
@ -2106,24 +2053,31 @@ class MongoClientInterface {
|
|||
};
|
||||
}
|
||||
|
||||
const results = {
|
||||
// there will be only one bucket for an object entry
|
||||
bucket: { [bucketName]: size },
|
||||
// there can be multiple locations for an object entry
|
||||
location: {},
|
||||
// there will be only one account for an object entry
|
||||
account: { [entry.value['owner-display-name']]: size },
|
||||
};
|
||||
|
||||
if (!isTransient ||
|
||||
entry.value.replicationInfo.status !== 'COMPLETED') {
|
||||
if (results[entry.value.dataStoreName]) {
|
||||
results[entry.value.dataStoreName] += size;
|
||||
// only count it in current dataStore if object is not in transient or replication not completed
|
||||
if (results.location[entry.value.dataStoreName]) {
|
||||
results.location[entry.value.dataStoreName] += size;
|
||||
} else {
|
||||
results[entry.value.dataStoreName] = size;
|
||||
}
|
||||
} else {
|
||||
if (!results[entry.value.dataStoreName]) {
|
||||
results[entry.value.dataStoreName] = 0;
|
||||
results.location[entry.value.dataStoreName] = size;
|
||||
}
|
||||
}
|
||||
entry.value.replicationInfo.backends.forEach(rep => {
|
||||
// count it in the replication destination location if replication compeleted
|
||||
if (rep.status === 'COMPLETED') {
|
||||
if (results[rep.site]) {
|
||||
results[rep.site] += size;
|
||||
if (results.location[rep.site]) {
|
||||
results.location[rep.site] += size;
|
||||
} else {
|
||||
results[rep.site] = size;
|
||||
results.location[rep.site] = size;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -2164,15 +2118,15 @@ class MongoClientInterface {
|
|||
'value.dataStoreName': 1,
|
||||
'value.content-length': 1,
|
||||
'value.versionId': 1,
|
||||
'value.owner-display-name': 1,
|
||||
'value.isDeleteMarker': 1,
|
||||
'value.isNull': 1,
|
||||
},
|
||||
});
|
||||
const collRes = {
|
||||
masterCount: 0,
|
||||
masterData: {},
|
||||
nullCount: 0,
|
||||
nullData: {},
|
||||
versionCount: 0,
|
||||
versionData: {},
|
||||
bucket: {}, // bucket level metrics
|
||||
location: {}, // location level metrics
|
||||
account: {}, // account level metrics
|
||||
};
|
||||
let stalledCount = 0;
|
||||
const cmpDate = new Date();
|
||||
|
|
@ -2180,7 +2134,7 @@ class MongoClientInterface {
|
|||
|
||||
cursor.forEach(
|
||||
res => {
|
||||
const { data, error } = this._processEntryData(res, isTransient);
|
||||
const { data, error } = this._processEntryData(bucketName, res, isTransient);
|
||||
|
||||
if (error) {
|
||||
log.error('Failed to process entry data', {
|
||||
|
|
@ -2201,7 +2155,7 @@ class MongoClientInterface {
|
|||
this._isReplicationEntryStalled(res, cmpDate)) {
|
||||
stalledCount++;
|
||||
}
|
||||
} else if (!!res.value.versionId) {
|
||||
} else if (!!res.value.versionId && !res.value.isNull) {
|
||||
// master version
|
||||
targetCount = 'masterCount';
|
||||
targetData = 'masterData';
|
||||
|
|
@ -2210,12 +2164,26 @@ class MongoClientInterface {
|
|||
targetCount = 'nullCount';
|
||||
targetData = 'nullData';
|
||||
}
|
||||
collRes[targetCount]++;
|
||||
Object.keys(data).forEach(site => {
|
||||
if (collRes[targetData][site]) {
|
||||
collRes[targetData][site] += data[site];
|
||||
} else {
|
||||
collRes[targetData][site] = data[site];
|
||||
Object.keys(data).forEach(metricLevel => {
|
||||
// metricLevel can only be 'bucket', 'location' or 'account'
|
||||
if (validStorageMetricLevels.has(metricLevel)) {
|
||||
Object.keys(data[metricLevel]).forEach(resourceName => {
|
||||
// resourceName can be the name of bucket, location or account
|
||||
if (!collRes[metricLevel][resourceName]) {
|
||||
collRes[metricLevel][resourceName] = {
|
||||
masterCount: 0,
|
||||
masterData: 0,
|
||||
nullCount: 0,
|
||||
nullData: 0,
|
||||
versionCount: 0,
|
||||
versionData: 0,
|
||||
deleteMarkerCount: 0,
|
||||
};
|
||||
}
|
||||
collRes[metricLevel][resourceName][targetData] += data[metricLevel][resourceName];
|
||||
collRes[metricLevel][resourceName][targetCount]++;
|
||||
collRes[metricLevel][resourceName].deleteMarkerCount += res.value.isDeleteMarker ? 1 : 0;
|
||||
});
|
||||
}
|
||||
});
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,572 @@
|
|||
const async = require('async');
|
||||
const assert = require('assert');
|
||||
const werelogs = require('werelogs');
|
||||
const { MongoMemoryReplSet } = require('mongodb-memory-server');
|
||||
const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug');
|
||||
const BucketInfo = require('../../../../lib/models/BucketInfo').default;
|
||||
const MetadataWrapper =
|
||||
require('../../../../lib/storage/metadata/MetadataWrapper');
|
||||
const { versioning } = require('../../../../index');
|
||||
const { BucketVersioningKeyFormat } = versioning.VersioningConstants;
|
||||
|
||||
const IMPL_NAME = 'mongodb';
|
||||
const DB_NAME = 'metadata';
|
||||
const BUCKET_NAME = 'test-bucket';
|
||||
const ACCOUNT_NAME = 'test-account';
|
||||
|
||||
const mongoserver = new MongoMemoryReplSet({
|
||||
debug: false,
|
||||
instanceOpts: [
|
||||
{ port: 27018 },
|
||||
],
|
||||
replSet: {
|
||||
name: 'rs0',
|
||||
count: 1,
|
||||
DB_NAME,
|
||||
storageEngine: 'ephemeralForTest',
|
||||
},
|
||||
});
|
||||
|
||||
const variations = [
|
||||
{ it: '(v0)', vFormat: BucketVersioningKeyFormat.v0 },
|
||||
{ it: '(v1)', vFormat: BucketVersioningKeyFormat.v1 },
|
||||
];
|
||||
|
||||
describe('MongoClientInterface::metadata.getObjectMDStats', () => {
|
||||
let metadata;
|
||||
|
||||
beforeAll(done => {
|
||||
mongoserver.waitUntilRunning().then(() => {
|
||||
const opts = {
|
||||
mongodb: {
|
||||
replicaSetHosts: 'localhost:27018',
|
||||
writeConcern: 'majority',
|
||||
replicaSet: 'rs0',
|
||||
readPreference: 'primary',
|
||||
database: DB_NAME,
|
||||
},
|
||||
};
|
||||
metadata = new MetadataWrapper(IMPL_NAME, opts, null, logger);
|
||||
metadata.setup(done);
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(done => {
|
||||
async.series([
|
||||
next => metadata.close(next),
|
||||
next => mongoserver.stop()
|
||||
.then(() => next())
|
||||
.catch(next),
|
||||
], done);
|
||||
});
|
||||
|
||||
const bucketMD = BucketInfo.fromObj({
|
||||
_name: BUCKET_NAME,
|
||||
_owner: 'testowner',
|
||||
_ownerDisplayName: ACCOUNT_NAME,
|
||||
_creationDate: new Date().toJSON(),
|
||||
_acl: {
|
||||
Canned: 'private',
|
||||
FULL_CONTROL: [],
|
||||
WRITE: [],
|
||||
WRITE_ACP: [],
|
||||
READ: [],
|
||||
READ_ACP: [],
|
||||
},
|
||||
_mdBucketModelVersion: 10,
|
||||
_transient: false,
|
||||
_deleted: false,
|
||||
_serverSideEncryption: null,
|
||||
_versioningConfiguration: null,
|
||||
_locationConstraint: 'us-east-1',
|
||||
_readLocationConstraint: null,
|
||||
_cors: null,
|
||||
_replicationConfiguration: null,
|
||||
_lifecycleConfiguration: null,
|
||||
_uid: '',
|
||||
_isNFS: null,
|
||||
ingestion: null,
|
||||
});
|
||||
const versionedBucketMD = {
|
||||
...bucketMD,
|
||||
_versioningConfiguration: {
|
||||
Status: 'Enabled',
|
||||
},
|
||||
};
|
||||
const suspendedBucketMD = {
|
||||
...bucketMD,
|
||||
_versioningConfiguration: {
|
||||
Status: 'Suspended',
|
||||
},
|
||||
};
|
||||
|
||||
describe('Should get correct results for versioning disabled bucket', () => {
|
||||
const versionParams = {
|
||||
versioning: false,
|
||||
versionId: null,
|
||||
};
|
||||
const object1Params = {
|
||||
'key': 'non-versioned-test-object1',
|
||||
'content-length': 10,
|
||||
'dataStoreName': 'us-east-1',
|
||||
'owner-display-name': ACCOUNT_NAME,
|
||||
'replicationInfo': {
|
||||
backends: [],
|
||||
},
|
||||
};
|
||||
const object2Params = {
|
||||
...object1Params,
|
||||
key: 'non-versioned-test-object2',
|
||||
};
|
||||
variations.forEach(variation => {
|
||||
describe(variation.it, () => {
|
||||
beforeEach(done => {
|
||||
async.series([
|
||||
next => {
|
||||
metadata.client.defaultBucketKeyFormat = variation.vFormat;
|
||||
return next();
|
||||
},
|
||||
next => metadata.createBucket(BUCKET_NAME, bucketMD, logger, next),
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
object1Params, versionParams, logger, next), // put object1
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
object1Params, versionParams, logger, next), // put object1 again
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
|
||||
object2Params, versionParams, logger, next), // put object2
|
||||
], done);
|
||||
});
|
||||
|
||||
afterEach(done => metadata.deleteBucket(BUCKET_NAME, logger, done));
|
||||
|
||||
it(`Should get correct results ${variation.it}`, done => {
|
||||
const expected = {
|
||||
dataManaged: {
|
||||
locations: { 'us-east-1': { curr: 20, prev: 0 } },
|
||||
total: { curr: 20, prev: 0 },
|
||||
},
|
||||
objects: 2, stalled: 0, versions: 0,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
[ACCOUNT_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
[BUCKET_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'us-east-1': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger, (err, data) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
assert.deepStrictEqual(data, expected);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Should get correct results for versioning enabled bucket', () => {
|
||||
const versionParams = {
|
||||
versioning: true,
|
||||
versionId: null,
|
||||
};
|
||||
const object1Params = {
|
||||
'key': 'versioned-test-object1',
|
||||
'content-length': 10,
|
||||
'dataStoreName': 'us-east-1',
|
||||
'owner-display-name': ACCOUNT_NAME,
|
||||
'replicationInfo': {
|
||||
backends: [],
|
||||
},
|
||||
};
|
||||
const object2Params = {
|
||||
...object1Params,
|
||||
key: 'versioned-test-object2',
|
||||
};
|
||||
variations.forEach(variation => {
|
||||
const itOnlyInV1 = variation.vFormat === 'v1' ? it : it.skip;
|
||||
describe(variation.it, () => {
|
||||
beforeEach(done => {
|
||||
async.series([
|
||||
next => {
|
||||
metadata.client.defaultBucketKeyFormat = variation.vFormat;
|
||||
return next();
|
||||
},
|
||||
next => metadata.createBucket(BUCKET_NAME, versionedBucketMD, logger, next),
|
||||
], done);
|
||||
});
|
||||
|
||||
afterEach(done => metadata.deleteBucket(BUCKET_NAME, logger, done));
|
||||
|
||||
it(`Should get correct results ${variation.it}`, done => {
|
||||
const expected = {
|
||||
dataManaged: {
|
||||
locations: { 'us-east-1': { curr: 20, prev: 10 } },
|
||||
total: { curr: 20, prev: 10 },
|
||||
},
|
||||
objects: 2, stalled: 0, versions: 1,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
[ACCOUNT_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
[BUCKET_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'us-east-1': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return async.series([
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
object1Params, versionParams, logger, next), // put object1
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
object1Params, versionParams, logger, next), // put object1 again
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
|
||||
object2Params, versionParams, logger, next), // put object2
|
||||
], () =>
|
||||
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
|
||||
(err, data) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
assert.deepStrictEqual(data, expected);
|
||||
return done();
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
itOnlyInV1(`Should get correct results with deleteMarker ${variation.it}`, done => {
|
||||
const expected = {
|
||||
dataManaged: {
|
||||
locations: { 'us-east-1': { curr: 0, prev: 20 } },
|
||||
total: { curr: 0, prev: 20 },
|
||||
},
|
||||
objects: 0, stalled: 0, versions: 2,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
[ACCOUNT_NAME]: {
|
||||
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
|
||||
usedCapacity: { current: 0, nonCurrent: 20 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
[BUCKET_NAME]: {
|
||||
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
|
||||
usedCapacity: { current: 0, nonCurrent: 20 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'us-east-1': {
|
||||
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
|
||||
usedCapacity: { current: 0, nonCurrent: 20 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return async.series([
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
object1Params, versionParams, logger, next), // put object1
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
object1Params, versionParams, logger, next), // put object1 again
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
{
|
||||
...object1Params,
|
||||
'isDeleteMarker': true,
|
||||
'content-length': 0,
|
||||
}, versionParams, logger, next), // delete object1
|
||||
], () =>
|
||||
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
|
||||
(err, data) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
assert.deepStrictEqual(data, expected);
|
||||
return done();
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should get correct results with lifecycle replication enabled ' +
|
||||
`and location transient is true ${variation.it}`, done => {
|
||||
const expected = {
|
||||
dataManaged: {
|
||||
locations: {
|
||||
'us-east-1': { curr: 10, prev: 0 },
|
||||
'completed': { curr: 10, prev: 0 },
|
||||
},
|
||||
total: { curr: 20, prev: 0 },
|
||||
},
|
||||
objects: 2, stalled: 0, versions: 0,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
[ACCOUNT_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
[BUCKET_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'us-east-1': {
|
||||
objectCount: { current: 1, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 10, nonCurrent: 0 },
|
||||
},
|
||||
'completed': {
|
||||
objectCount: { current: 1, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 10, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return async.series([
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
{
|
||||
...object1Params,
|
||||
replicationInfo: {
|
||||
status: 'PENDING',
|
||||
backends: [
|
||||
{
|
||||
status: 'PENDING',
|
||||
site: 'not-completed',
|
||||
},
|
||||
{
|
||||
status: 'COMPLETED',
|
||||
site: 'completed',
|
||||
},
|
||||
],
|
||||
},
|
||||
}, versionParams, logger, next), // object1 with one site pending and one site complete
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
|
||||
{
|
||||
...object2Params,
|
||||
replicationInfo: {
|
||||
status: 'COMPLETED',
|
||||
backends: [
|
||||
{
|
||||
status: 'COMPLETE',
|
||||
site: 'completed',
|
||||
},
|
||||
],
|
||||
},
|
||||
}, versionParams, logger, next), // object2 with one site complete
|
||||
], () =>
|
||||
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, true, logger,
|
||||
(err, data) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
assert.deepStrictEqual(data, expected);
|
||||
return done();
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should get correct results with lifecycle replication enabled' +
|
||||
`and location transient is false ${variation.it}`, done => {
|
||||
const expected = {
|
||||
dataManaged: {
|
||||
locations: {
|
||||
'us-east-1': { curr: 20, prev: 0 },
|
||||
'completed': { curr: 10, prev: 0 },
|
||||
},
|
||||
total: { curr: 30, prev: 0 },
|
||||
},
|
||||
objects: 2, stalled: 0, versions: 0,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
[ACCOUNT_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
[BUCKET_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'us-east-1': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
'completed': {
|
||||
objectCount: { current: 1, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 10, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return async.series([
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
{
|
||||
...object1Params,
|
||||
replicationInfo: {
|
||||
status: 'PENDING',
|
||||
backends: [
|
||||
{
|
||||
status: 'PENDING',
|
||||
site: 'not-completed',
|
||||
},
|
||||
{
|
||||
status: 'COMPLETED',
|
||||
site: 'completed',
|
||||
},
|
||||
],
|
||||
},
|
||||
}, versionParams, logger, next), // object1 with one site pending and one site complete
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
|
||||
{
|
||||
...object2Params,
|
||||
replicationInfo: {
|
||||
status: 'COMPLETED',
|
||||
backends: [
|
||||
{
|
||||
status: 'COMPLETE',
|
||||
site: 'completed',
|
||||
},
|
||||
],
|
||||
},
|
||||
}, versionParams, logger, next), // object2 with one site complete
|
||||
], () =>
|
||||
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
|
||||
(err, data) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
assert.deepStrictEqual(data, expected);
|
||||
return done();
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Should get correct results for versioning suspended bucket', () => {
|
||||
const object1Params = {
|
||||
'key': 'test-object1',
|
||||
'content-length': 10,
|
||||
'dataStoreName': 'us-east-1',
|
||||
'owner-display-name': ACCOUNT_NAME,
|
||||
'replicationInfo': {
|
||||
backends: [],
|
||||
},
|
||||
};
|
||||
const object2Params = {
|
||||
...object1Params,
|
||||
key: 'test-object2',
|
||||
};
|
||||
variations.forEach(variation => {
|
||||
describe(variation.it, () => {
|
||||
beforeEach(done => {
|
||||
async.series([
|
||||
next => {
|
||||
metadata.client.defaultBucketKeyFormat = variation.vFormat;
|
||||
return next();
|
||||
},
|
||||
next => metadata.createBucket(BUCKET_NAME, suspendedBucketMD, logger, next),
|
||||
], done);
|
||||
});
|
||||
|
||||
afterEach(done => metadata.deleteBucket(BUCKET_NAME, logger, done));
|
||||
|
||||
it(`Should get correct results ${variation.it}`, done => {
|
||||
const expected = {
|
||||
dataManaged: {
|
||||
locations: { 'us-east-1': { curr: 20, prev: 10 } },
|
||||
total: { curr: 20, prev: 10 },
|
||||
},
|
||||
objects: 2, stalled: 0, versions: 1,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
[ACCOUNT_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
[BUCKET_NAME]: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'us-east-1': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 1 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return async.series([
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
object1Params, {
|
||||
versionId: null,
|
||||
versioning: true,
|
||||
}, logger, next), // versioned object1 put before suspend
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object1Params.key,
|
||||
{
|
||||
...object1Params,
|
||||
isNull: true,
|
||||
},
|
||||
{
|
||||
versionId: null,
|
||||
|
||||
}, logger, next), // null versioned object1
|
||||
next => metadata.putObjectMD(BUCKET_NAME, object2Params.key,
|
||||
{
|
||||
...object2Params,
|
||||
isNull: true,
|
||||
},
|
||||
{
|
||||
versionId: null,
|
||||
}, logger, next), // null versioned object2
|
||||
], () =>
|
||||
metadata.client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
return metadata.client.getObjectMDStats(BUCKET_NAME, bucketInfo, false, logger,
|
||||
(err, data) => {
|
||||
assert.deepStrictEqual(err, null);
|
||||
assert.deepStrictEqual(data, expected);
|
||||
return done();
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -32,37 +32,127 @@ const DummyConfigObject = require('./utils/DummyConfigObject');
|
|||
const mongoTestClient = new MongoClientInterface({});
|
||||
|
||||
describe('MongoClientInterface::_handleResults', () => {
|
||||
it('should return zero-result', () => {
|
||||
const testInput = {
|
||||
masterCount: 0, masterData: {},
|
||||
nullCount: 0, nullData: {},
|
||||
versionCount: 0, versionData: {},
|
||||
bucket: {
|
||||
bucket1: {
|
||||
masterCount: 2,
|
||||
masterData: 20,
|
||||
nullCount: 2,
|
||||
nullData: 20,
|
||||
versionCount: 4,
|
||||
versionData: 40,
|
||||
deleteMarkerCount: 2,
|
||||
},
|
||||
},
|
||||
location: {
|
||||
location1: {
|
||||
masterCount: 1,
|
||||
masterData: 10,
|
||||
nullCount: 1,
|
||||
nullData: 10,
|
||||
versionCount: 2,
|
||||
versionData: 20,
|
||||
deleteMarkerCount: 1,
|
||||
},
|
||||
location2: {
|
||||
masterCount: 1,
|
||||
masterData: 10,
|
||||
nullCount: 1,
|
||||
nullData: 10,
|
||||
versionCount: 2,
|
||||
versionData: 20,
|
||||
deleteMarkerCount: 1,
|
||||
},
|
||||
},
|
||||
account: {
|
||||
account1: {
|
||||
masterCount: 2,
|
||||
masterData: 20,
|
||||
nullCount: 2,
|
||||
nullData: 20,
|
||||
versionCount: 4,
|
||||
versionData: 40,
|
||||
deleteMarkerCount: 2,
|
||||
},
|
||||
},
|
||||
};
|
||||
const testResults = mongoTestClient._handleResults(testInput, true);
|
||||
it('should return zero-result when input is empty', () => {
|
||||
const testInputEmpty = {
|
||||
bucket: {},
|
||||
location: {},
|
||||
account: {},
|
||||
};
|
||||
const testResults = mongoTestClient._handleResults(testInputEmpty, true);
|
||||
const expectedRes = {
|
||||
versions: 0, objects: 0,
|
||||
dataManaged: {
|
||||
total: { curr: 0, prev: 0 },
|
||||
locations: {},
|
||||
},
|
||||
dataMetrics: {
|
||||
bucket: {},
|
||||
location: {},
|
||||
account: {},
|
||||
},
|
||||
};
|
||||
assert.deepStrictEqual(testResults, expectedRes);
|
||||
});
|
||||
|
||||
it('should return zero-result when input metric keys are not valid', () => {
|
||||
const testInputWithInvalidMetricKeys = {
|
||||
InvalidMetric0: testInput.bucket,
|
||||
InvalidMetric1: testInput.location,
|
||||
InvalidMetric2: testInput.account,
|
||||
};
|
||||
const testResults = mongoTestClient._handleResults(testInputWithInvalidMetricKeys, true);
|
||||
const expectedRes = {
|
||||
versions: 0, objects: 0,
|
||||
dataManaged: {
|
||||
total: { curr: 0, prev: 0 },
|
||||
locations: {},
|
||||
},
|
||||
dataMetrics: {
|
||||
bucket: {},
|
||||
location: {},
|
||||
account: {},
|
||||
},
|
||||
};
|
||||
assert.deepStrictEqual(testResults, expectedRes);
|
||||
});
|
||||
|
||||
it('should return correct value if isVer is false', () => {
|
||||
const testInput = {
|
||||
masterCount: 2, masterData: { test1: 10, test2: 10 },
|
||||
nullCount: 2, nullData: { test1: 10, test2: 10 },
|
||||
versionCount: 2, versionData: { test1: 20, test2: 20 },
|
||||
};
|
||||
const testResults = mongoTestClient._handleResults(testInput, false);
|
||||
const expectedRes = {
|
||||
versions: 0, objects: 4,
|
||||
dataManaged: {
|
||||
total: { curr: 40, prev: 0 },
|
||||
locations: {
|
||||
test1: { curr: 20, prev: 0 },
|
||||
test2: { curr: 20, prev: 0 },
|
||||
location1: { curr: 20, prev: 0 },
|
||||
location2: { curr: 20, prev: 0 },
|
||||
},
|
||||
},
|
||||
dataMetrics: {
|
||||
bucket: {
|
||||
bucket1: {
|
||||
objectCount: { current: 4, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 40, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
location1: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
location2: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
account: {
|
||||
account1: {
|
||||
objectCount: { current: 4, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 40, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -70,24 +160,76 @@ describe('MongoClientInterface::_handleResults', () => {
|
|||
});
|
||||
|
||||
it('should return correct value if isVer is true', () => {
|
||||
const testInput = {
|
||||
masterCount: 2, masterData: { test1: 10, test2: 10 },
|
||||
nullCount: 2, nullData: { test1: 10, test2: 10 },
|
||||
versionCount: 4, versionData: { test1: 20, test2: 20 },
|
||||
};
|
||||
const testResults = mongoTestClient._handleResults(testInput, true);
|
||||
const expectedRes = {
|
||||
versions: 2, objects: 4,
|
||||
versions: 0, objects: 4,
|
||||
dataManaged: {
|
||||
total: { curr: 40, prev: 20 },
|
||||
locations: {
|
||||
test1: { curr: 20, prev: 10 },
|
||||
test2: { curr: 20, prev: 10 },
|
||||
location1: { curr: 20, prev: 10 },
|
||||
location2: { curr: 20, prev: 10 },
|
||||
},
|
||||
},
|
||||
dataMetrics: {
|
||||
bucket: {
|
||||
bucket1: {
|
||||
objectCount: { current: 4, deleteMarker: 2, nonCurrent: 0 },
|
||||
usedCapacity: { current: 40, nonCurrent: 20 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
location1: {
|
||||
objectCount: { current: 2, deleteMarker: 1, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
location2: {
|
||||
objectCount: { current: 2, deleteMarker: 1, nonCurrent: 0 },
|
||||
usedCapacity: { current: 20, nonCurrent: 10 },
|
||||
},
|
||||
},
|
||||
account: {
|
||||
account1: {
|
||||
objectCount: { current: 4, deleteMarker: 2, nonCurrent: 0 },
|
||||
usedCapacity: { current: 40, nonCurrent: 20 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
assert.deepStrictEqual(testResults, expectedRes);
|
||||
});
|
||||
|
||||
it('should calculate dataManaged based on input location metrics', () => {
|
||||
const testInputOnlyContainsLocation = {
|
||||
bucket: {},
|
||||
location: testInput.location,
|
||||
account: {},
|
||||
};
|
||||
const testResults = mongoTestClient._handleResults(testInputOnlyContainsLocation, true);
|
||||
const expectedRes = {
|
||||
dataManaged: {
|
||||
total: { curr: 40, prev: 20 },
|
||||
locations: {
|
||||
location1: { curr: 20, prev: 10 },
|
||||
location2: { curr: 20, prev: 10 },
|
||||
},
|
||||
},
|
||||
};
|
||||
assert.deepStrictEqual(testResults.dataManaged, expectedRes.dataManaged);
|
||||
});
|
||||
|
||||
it('should calculate total current and nonCurrent counts based on input bucket metrics', () => {
|
||||
const testInputOnlyContainsLocation = {
|
||||
bucket: testInput.bucket,
|
||||
location: {},
|
||||
account: {},
|
||||
};
|
||||
const testResults = mongoTestClient._handleResults(testInputOnlyContainsLocation, true);
|
||||
const expectedRes = {
|
||||
versions: 0, objects: 4,
|
||||
};
|
||||
assert.deepStrictEqual(testResults.versions, expectedRes.versions);
|
||||
assert.deepStrictEqual(testResults.objects, expectedRes.objects);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MongoClientInterface, misc', () => {
|
||||
|
|
@ -105,140 +247,150 @@ describe('MongoClientInterface, misc', () => {
|
|||
});
|
||||
|
||||
describe('MongoClientInterface::_processEntryData', () => {
|
||||
const testBucketName = 'testBucket';
|
||||
const objectMdTemp = {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
status: 'PENDING',
|
||||
backends: [],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'transient': false,
|
||||
'dataStoreName': 'us-east-1',
|
||||
'content-length': 42,
|
||||
'versionId': '0123456789abcdefg',
|
||||
'owner-display-name': 'account1',
|
||||
};
|
||||
const tests = [
|
||||
[
|
||||
'should add content-length to total if replication status != ' +
|
||||
'COMPLETED and transient == true',
|
||||
'should add content-length to current dataStore but not replication destination ' +
|
||||
'if replication status != COMPLETED and transient == true',
|
||||
testBucketName,
|
||||
true,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey0',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
...objectMdTemp,
|
||||
replicationInfo: {
|
||||
...objectMdTemp.replicationInfo,
|
||||
backends: [{
|
||||
site: 'not-completed',
|
||||
status: 'PENDING',
|
||||
}],
|
||||
status: 'PENDING',
|
||||
backends: [],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
'content-length': 42,
|
||||
'versionId': '0123456789abcdefg',
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
'us-east-1': 42,
|
||||
account: { account1: 42 },
|
||||
bucket: { [testBucketName]: 42 },
|
||||
location: { 'us-east-1': 42 },
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
[
|
||||
'should not add content-length to total if replication ' +
|
||||
'status == COMPLETED and transient == true',
|
||||
'should not add content-length to replication destination but not in current dataStore ' +
|
||||
'if replication status == COMPLETED and transient == true',
|
||||
testBucketName,
|
||||
true,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey1',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
...objectMdTemp,
|
||||
replicationInfo: {
|
||||
...objectMdTemp.replicationInfo,
|
||||
backends: [{
|
||||
site: 'completed',
|
||||
status: 'COMPLETED',
|
||||
}],
|
||||
status: 'COMPLETED',
|
||||
backends: [],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
'content-length': 42,
|
||||
'versionId': '0123456789abcdefg',
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
'us-east-1': 0,
|
||||
account: { account1: 42 },
|
||||
bucket: { [testBucketName]: 42 },
|
||||
location: { completed: 42 },
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
[
|
||||
'should add content-length to total if replication status != ' +
|
||||
'COMPLETED and transient == false',
|
||||
'should add content-length to current dataStore but not replication destination ' +
|
||||
'if replication status != COMPLETED and transient == false',
|
||||
testBucketName,
|
||||
false,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey2',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
...objectMdTemp,
|
||||
replicationInfo: {
|
||||
...objectMdTemp.replicationInfo,
|
||||
backends: [{
|
||||
site: 'not-completed',
|
||||
status: 'PENDING',
|
||||
}],
|
||||
status: 'PENDING',
|
||||
backends: [],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
'content-length': 42,
|
||||
'versionId': '0123456789abcdefg',
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
'us-east-1': 42,
|
||||
account: { account1: 42 },
|
||||
bucket: { [testBucketName]: 42 },
|
||||
location: { 'us-east-1': 42 },
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
[
|
||||
'should add content-length to total if replication ' +
|
||||
'status == COMPLETED and transient == false',
|
||||
'should add content-length to current dataStore and replication destination ' +
|
||||
'if replication status == COMPLETED and transient == false',
|
||||
testBucketName,
|
||||
false,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey3',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
...objectMdTemp,
|
||||
replicationInfo: {
|
||||
...objectMdTemp.replicationInfo,
|
||||
backends: [{
|
||||
site: 'completed',
|
||||
status: 'COMPLETED',
|
||||
}],
|
||||
status: 'COMPLETED',
|
||||
backends: [],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
'content-length': 42,
|
||||
'versionId': '0123456789abcdefg',
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
'us-east-1': 42,
|
||||
account: { account1: 42 },
|
||||
bucket: { [testBucketName]: 42 },
|
||||
location: { 'us-east-1': 42, 'completed': 42 },
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
[
|
||||
'should add content-length to total for each COMPLETED backends ' +
|
||||
'(replication status: COMPLETED)',
|
||||
'should add content-length to each COMPLETED replication destination but not current dataStore ' +
|
||||
'(object replication status: COMPLETED)',
|
||||
testBucketName,
|
||||
true,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey4',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
...objectMdTemp,
|
||||
replicationInfo: {
|
||||
...objectMdTemp.replicationInfo,
|
||||
status: 'COMPLETED',
|
||||
backends: [
|
||||
{
|
||||
|
|
@ -254,38 +406,33 @@ describe('MongoClientInterface::_processEntryData', () => {
|
|||
site: 'completed-3',
|
||||
},
|
||||
],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
'content-length': 42,
|
||||
'versionId': '0123456789abcdefg',
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
'us-east-1': 0,
|
||||
account: { account1: 42 },
|
||||
bucket: { [testBucketName]: 42 },
|
||||
location: {
|
||||
'completed-1': 42,
|
||||
'completed-2': 42,
|
||||
'completed-3': 42,
|
||||
},
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
[
|
||||
'should add content-length to total for each COMPLETED backends ' +
|
||||
'(replication status: PENDING)',
|
||||
'should add content-length to each COMPLETED replications destination and current dataStore ' +
|
||||
'(object replication status: PENDING)',
|
||||
testBucketName,
|
||||
true,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey5',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
...objectMdTemp,
|
||||
replicationInfo: {
|
||||
...objectMdTemp.replicationInfo,
|
||||
status: 'PENDING',
|
||||
backends: [
|
||||
{
|
||||
|
|
@ -301,62 +448,32 @@ describe('MongoClientInterface::_processEntryData', () => {
|
|||
site: 'completed-2',
|
||||
},
|
||||
],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
'content-length': 42,
|
||||
'versionId': '0123456789abcdefg',
|
||||
transient: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
account: { account1: 42 },
|
||||
bucket: { [testBucketName]: 42 },
|
||||
location: {
|
||||
'us-east-1': 42,
|
||||
'completed-1': 42,
|
||||
'completed-2': 42,
|
||||
},
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
[
|
||||
'should error if content-length is invalid',
|
||||
'should return error if content-length is invalid',
|
||||
testBucketName,
|
||||
true,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey6',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
status: 'PENDING',
|
||||
backends: [
|
||||
{
|
||||
status: 'PENDING',
|
||||
site: 'not-completed',
|
||||
},
|
||||
{
|
||||
status: 'COMPLETED',
|
||||
site: 'completed-1',
|
||||
},
|
||||
{
|
||||
status: 'COMPLETED',
|
||||
site: 'completed-2',
|
||||
},
|
||||
],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
...objectMdTemp,
|
||||
'content-length': 'not-a-number',
|
||||
'versionId': '0123456789abcdefg',
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -366,38 +483,41 @@ describe('MongoClientInterface::_processEntryData', () => {
|
|||
],
|
||||
[
|
||||
'should correctly process entry with string typed content-length',
|
||||
testBucketName,
|
||||
true,
|
||||
{
|
||||
_id: 'testkey',
|
||||
_id: 'testkey7',
|
||||
value: {
|
||||
'last-modified': new Date(),
|
||||
'replicationInfo': {
|
||||
status: 'PENDING',
|
||||
backends: [],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
'dataStoreName': 'us-east-1',
|
||||
...objectMdTemp,
|
||||
'content-length': '42',
|
||||
'versionId': '0123456789abcdefg',
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
'us-east-1': 42,
|
||||
account: { account1: 42 },
|
||||
bucket: { [testBucketName]: 42 },
|
||||
location: { 'us-east-1': 42 },
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
[
|
||||
'should return error if bucketName is empty',
|
||||
undefined,
|
||||
true,
|
||||
{
|
||||
_id: 'testkey8',
|
||||
value: objectMdTemp,
|
||||
},
|
||||
{
|
||||
data: {},
|
||||
error: new Error('no bucket name provided'),
|
||||
},
|
||||
],
|
||||
];
|
||||
tests.forEach(([msg, isTransient, params, expected]) => it(msg, () => {
|
||||
tests.forEach(([msg, bucketName, isTransient, params, expected]) => it(msg, () => {
|
||||
assert.deepStrictEqual(
|
||||
mongoTestClient._processEntryData(params, isTransient),
|
||||
mongoTestClient._processEntryData(bucketName, params, isTransient),
|
||||
expected,
|
||||
);
|
||||
}));
|
||||
|
|
@ -545,7 +665,10 @@ function uploadObjects(client, bucketName, objectList, callback) {
|
|||
.setKey(obj.name)
|
||||
.setDataStoreName('us-east-1')
|
||||
.setContentLength(100)
|
||||
.setLastModified(obj.lastModified);
|
||||
.setLastModified(obj.lastModified)
|
||||
.setOwnerDisplayName(obj.ownerDisplayName)
|
||||
.setIsNull(obj.isNull)
|
||||
.setIsDeleteMarker(obj.isDeleteMarker);
|
||||
if (obj.repInfo) {
|
||||
objMD.setReplicationInfo(obj.repInfo);
|
||||
}
|
||||
|
|
@ -585,7 +708,60 @@ describe('MongoClientInterface, tests', () => {
|
|||
], done);
|
||||
});
|
||||
|
||||
const nonVersionedObjectMdTemp = {
|
||||
name: 'testkey',
|
||||
versioning: false,
|
||||
versionId: null,
|
||||
lastModified: new Date(Date.now()),
|
||||
ownerDisplayName: 'testAccount',
|
||||
};
|
||||
const objectMdTemp = {
|
||||
name: 'testkey',
|
||||
versioning: true,
|
||||
versionId: null,
|
||||
lastModified: new Date(Date.now()),
|
||||
ownerDisplayName: 'testAccount',
|
||||
repInfo: {
|
||||
status: 'COMPLETED',
|
||||
backends: [
|
||||
{
|
||||
status: 'COMPLETED',
|
||||
site: 'rep-loc-1',
|
||||
},
|
||||
],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
};
|
||||
|
||||
const tests = [
|
||||
[
|
||||
'getObjectMDStats() should return zero-result when no objects in the bucket',
|
||||
{
|
||||
bucketName: 'test-bucket',
|
||||
isVersioned: false,
|
||||
objectList: [],
|
||||
},
|
||||
{
|
||||
dataManaged: {
|
||||
locations: {},
|
||||
total: { curr: 0, prev: 0 },
|
||||
},
|
||||
objects: 0,
|
||||
stalled: 0,
|
||||
versions: 0,
|
||||
dataMetrics: {
|
||||
bucket: {},
|
||||
location: {},
|
||||
account: {},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
'getObjectMDStats() should return correct results',
|
||||
{
|
||||
|
|
@ -594,57 +770,21 @@ describe('MongoClientInterface, tests', () => {
|
|||
objectList: [
|
||||
// versioned object 1,
|
||||
{
|
||||
name: 'testkey',
|
||||
...objectMdTemp,
|
||||
versioning: true,
|
||||
versionId: null,
|
||||
lastModified: new Date(Date.now()),
|
||||
repInfo: {
|
||||
status: 'COMPLETED',
|
||||
backends: [
|
||||
{
|
||||
status: 'COMPLETED',
|
||||
site: 'rep-loc-1',
|
||||
},
|
||||
],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
},
|
||||
// versioned object 2,
|
||||
{
|
||||
name: 'testkey',
|
||||
...objectMdTemp,
|
||||
versioning: true,
|
||||
versionId: null,
|
||||
lastModified: new Date(Date.now()),
|
||||
repInfo: {
|
||||
status: 'COMPLETED',
|
||||
backends: [
|
||||
{
|
||||
status: 'COMPLETED',
|
||||
site: 'rep-loc-1',
|
||||
},
|
||||
],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
},
|
||||
// stalled object 1
|
||||
{
|
||||
name: 'testkey',
|
||||
...objectMdTemp,
|
||||
versioning: true,
|
||||
versionId: null,
|
||||
lastModified: new Date(Date.now() - hr),
|
||||
repInfo: {
|
||||
...objectMdTemp.repInfo,
|
||||
status: 'PENDING',
|
||||
backends: [
|
||||
{
|
||||
|
|
@ -652,18 +792,13 @@ describe('MongoClientInterface, tests', () => {
|
|||
site: 'rep-loc-1',
|
||||
},
|
||||
],
|
||||
content: [],
|
||||
destination: '',
|
||||
storageClass: '',
|
||||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
},
|
||||
// null versioned object
|
||||
{
|
||||
name: 'nullkey',
|
||||
isNull: true,
|
||||
ownerDisplayName: 'testAccount',
|
||||
lastModified: new Date(Date.now() - hr),
|
||||
},
|
||||
],
|
||||
|
|
@ -688,6 +823,176 @@ describe('MongoClientInterface, tests', () => {
|
|||
objects: 2,
|
||||
stalled: 1,
|
||||
versions: 2,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
testAccount: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 2 },
|
||||
usedCapacity: { current: 200, nonCurrent: 200 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
'test-bucket': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 2 },
|
||||
usedCapacity: { current: 200, nonCurrent: 200 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'rep-loc-1': {
|
||||
objectCount: { current: 0, deleteMarker: 0, nonCurrent: 2 },
|
||||
usedCapacity: { current: 0, nonCurrent: 200 },
|
||||
},
|
||||
'us-east-1': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 2 },
|
||||
usedCapacity: { current: 200, nonCurrent: 200 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
'getObjectMDStats() should return correct results for non versioned bucket',
|
||||
{
|
||||
bucketName: 'test-bucket',
|
||||
isVersioned: false,
|
||||
objectList: [
|
||||
// non versioned object 1,
|
||||
{
|
||||
...nonVersionedObjectMdTemp,
|
||||
name: 'testkey1',
|
||||
},
|
||||
// non versioned object 1,
|
||||
{
|
||||
...nonVersionedObjectMdTemp,
|
||||
name: 'testkey1',
|
||||
},
|
||||
// non versioned object 2
|
||||
{
|
||||
...nonVersionedObjectMdTemp,
|
||||
name: 'testkey2',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
dataManaged: {
|
||||
locations: {
|
||||
'us-east-1': {
|
||||
curr: 200,
|
||||
prev: 0,
|
||||
},
|
||||
},
|
||||
total: {
|
||||
curr: 200,
|
||||
prev: 0,
|
||||
},
|
||||
},
|
||||
objects: 2,
|
||||
stalled: 0,
|
||||
versions: 0,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
testAccount: {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 200, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
'test-bucket': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 200, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'us-east-1': {
|
||||
objectCount: { current: 2, deleteMarker: 0, nonCurrent: 0 },
|
||||
usedCapacity: { current: 200, nonCurrent: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
'getObjectMDStats() should return correct results for versioned bucket',
|
||||
{
|
||||
bucketName: 'test-bucket',
|
||||
isVersioned: true,
|
||||
objectList: [
|
||||
// a version of object 1,
|
||||
{
|
||||
...objectMdTemp,
|
||||
versioning: true,
|
||||
},
|
||||
// a version of object 1,
|
||||
{
|
||||
...objectMdTemp,
|
||||
versioning: true,
|
||||
},
|
||||
// deleteMarker of object 1
|
||||
{
|
||||
...objectMdTemp,
|
||||
versioning: true,
|
||||
isDeleteMarker: true,
|
||||
},
|
||||
// a version of object 1,
|
||||
{
|
||||
...objectMdTemp,
|
||||
versioning: true,
|
||||
repInfo: {
|
||||
...objectMdTemp.repInfo,
|
||||
status: 'PENDING',
|
||||
backends: [
|
||||
{
|
||||
status: 'PENDING',
|
||||
site: 'rep-loc-1',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
dataManaged: {
|
||||
locations: {
|
||||
'rep-loc-1': {
|
||||
curr: 0,
|
||||
prev: 300,
|
||||
},
|
||||
'us-east-1': {
|
||||
curr: 100,
|
||||
prev: 300,
|
||||
},
|
||||
},
|
||||
total: {
|
||||
curr: 100,
|
||||
prev: 600,
|
||||
},
|
||||
},
|
||||
objects: 1,
|
||||
stalled: 0,
|
||||
versions: 2,
|
||||
dataMetrics: {
|
||||
account: {
|
||||
testAccount: {
|
||||
objectCount: { current: 1, deleteMarker: 1, nonCurrent: 2 },
|
||||
usedCapacity: { current: 100, nonCurrent: 300 },
|
||||
},
|
||||
},
|
||||
bucket: {
|
||||
'test-bucket': {
|
||||
objectCount: { current: 1, deleteMarker: 1, nonCurrent: 2 },
|
||||
usedCapacity: { current: 100, nonCurrent: 300 },
|
||||
},
|
||||
},
|
||||
location: {
|
||||
'rep-loc-1': {
|
||||
objectCount: { current: 0, deleteMarker: 1, nonCurrent: 2 },
|
||||
usedCapacity: { current: 0, nonCurrent: 300 },
|
||||
},
|
||||
'us-east-1': {
|
||||
objectCount: { current: 1, deleteMarker: 1, nonCurrent: 2 },
|
||||
usedCapacity: { current: 100, nonCurrent: 300 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
];
|
||||
|
|
|
|||
Loading…
Reference in New Issue