Compare commits
No commits in common. "5772c37b00505ba99eb339c161f8299c2e60449a" and "67e5cc770d3897086c7baf2ba0cc8e353feac0a1" have entirely different histories.
5772c37b00
...
67e5cc770d
|
@ -32,7 +32,6 @@ class DelimiterMaster extends Delimiter {
|
||||||
// non-PHD master version or a version whose master is a PHD version
|
// non-PHD master version or a version whose master is a PHD version
|
||||||
this.prvKey = undefined;
|
this.prvKey = undefined;
|
||||||
this.prvPHDKey = undefined;
|
this.prvPHDKey = undefined;
|
||||||
this.inReplayPrefix = false;
|
|
||||||
|
|
||||||
Object.assign(this, {
|
Object.assign(this, {
|
||||||
[BucketVersioningKeyFormat.v0]: {
|
[BucketVersioningKeyFormat.v0]: {
|
||||||
|
@ -62,12 +61,6 @@ class DelimiterMaster extends Delimiter {
|
||||||
let key = obj.key;
|
let key = obj.key;
|
||||||
const value = obj.value;
|
const value = obj.value;
|
||||||
|
|
||||||
if (key.startsWith(DbPrefixes.Replay)) {
|
|
||||||
this.inReplayPrefix = true;
|
|
||||||
return FILTER_SKIP;
|
|
||||||
}
|
|
||||||
this.inReplayPrefix = false;
|
|
||||||
|
|
||||||
/* Skip keys not starting with the prefix or not alphabetically
|
/* Skip keys not starting with the prefix or not alphabetically
|
||||||
* ordered. */
|
* ordered. */
|
||||||
if ((this.prefix && !key.startsWith(this.prefix))
|
if ((this.prefix && !key.startsWith(this.prefix))
|
||||||
|
@ -162,7 +155,7 @@ class DelimiterMaster extends Delimiter {
|
||||||
return super.filter(obj);
|
return super.filter(obj);
|
||||||
}
|
}
|
||||||
|
|
||||||
skippingBase() {
|
skippingV0() {
|
||||||
if (this[this.nextContinueMarker]) {
|
if (this[this.nextContinueMarker]) {
|
||||||
// next marker or next continuation token:
|
// next marker or next continuation token:
|
||||||
// - foo/ : skipping foo/
|
// - foo/ : skipping foo/
|
||||||
|
@ -177,15 +170,8 @@ class DelimiterMaster extends Delimiter {
|
||||||
return SKIP_NONE;
|
return SKIP_NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
skippingV0() {
|
|
||||||
if (this.inReplayPrefix) {
|
|
||||||
return DbPrefixes.Replay;
|
|
||||||
}
|
|
||||||
return this.skippingBase();
|
|
||||||
}
|
|
||||||
|
|
||||||
skippingV1() {
|
skippingV1() {
|
||||||
const skipTo = this.skippingBase();
|
const skipTo = this.skippingV0();
|
||||||
if (skipTo === SKIP_NONE) {
|
if (skipTo === SKIP_NONE) {
|
||||||
return SKIP_NONE;
|
return SKIP_NONE;
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,7 +33,6 @@ class DelimiterVersions extends Delimiter {
|
||||||
// listing results
|
// listing results
|
||||||
this.NextMarker = parameters.keyMarker;
|
this.NextMarker = parameters.keyMarker;
|
||||||
this.NextVersionIdMarker = undefined;
|
this.NextVersionIdMarker = undefined;
|
||||||
this.inReplayPrefix = false;
|
|
||||||
|
|
||||||
Object.assign(this, {
|
Object.assign(this, {
|
||||||
[BucketVersioningKeyFormat.v0]: {
|
[BucketVersioningKeyFormat.v0]: {
|
||||||
|
@ -164,12 +163,6 @@ class DelimiterVersions extends Delimiter {
|
||||||
* @return {number} - indicates if iteration should continue
|
* @return {number} - indicates if iteration should continue
|
||||||
*/
|
*/
|
||||||
filterV0(obj) {
|
filterV0(obj) {
|
||||||
if (obj.key.startsWith(DbPrefixes.Replay)) {
|
|
||||||
this.inReplayPrefix = true;
|
|
||||||
return FILTER_SKIP;
|
|
||||||
}
|
|
||||||
this.inReplayPrefix = false;
|
|
||||||
|
|
||||||
if (Version.isPHD(obj.value)) {
|
if (Version.isPHD(obj.value)) {
|
||||||
// return accept to avoid skipping the next values in range
|
// return accept to avoid skipping the next values in range
|
||||||
return FILTER_ACCEPT;
|
return FILTER_ACCEPT;
|
||||||
|
@ -231,9 +224,6 @@ class DelimiterVersions extends Delimiter {
|
||||||
}
|
}
|
||||||
|
|
||||||
skippingV0() {
|
skippingV0() {
|
||||||
if (this.inReplayPrefix) {
|
|
||||||
return DbPrefixes.Replay;
|
|
||||||
}
|
|
||||||
if (this.NextMarker) {
|
if (this.NextMarker) {
|
||||||
const index = this.NextMarker.lastIndexOf(this.delimiter);
|
const index = this.NextMarker.lastIndexOf(this.delimiter);
|
||||||
if (index === this.NextMarker.length - 1) {
|
if (index === this.NextMarker.length - 1) {
|
||||||
|
|
|
@ -110,7 +110,6 @@ class ObjectMD {
|
||||||
// should be undefined when not set explicitly
|
// should be undefined when not set explicitly
|
||||||
'isNull': undefined,
|
'isNull': undefined,
|
||||||
'nullVersionId': undefined,
|
'nullVersionId': undefined,
|
||||||
'nullUploadId': undefined,
|
|
||||||
'isDeleteMarker': undefined,
|
'isDeleteMarker': undefined,
|
||||||
'versionId': undefined,
|
'versionId': undefined,
|
||||||
'uploadId': undefined,
|
'uploadId': undefined,
|
||||||
|
@ -633,27 +632,6 @@ class ObjectMD {
|
||||||
return this._data.nullVersionId;
|
return this._data.nullVersionId;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Set metadata nullUploadId value
|
|
||||||
*
|
|
||||||
* @param {string} nullUploadId - The upload ID used to complete
|
|
||||||
* the MPU of the null version
|
|
||||||
* @return {ObjectMD} itself
|
|
||||||
*/
|
|
||||||
setNullUploadId(nullUploadId) {
|
|
||||||
this._data.nullUploadId = nullUploadId;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get metadata nullUploadId value
|
|
||||||
*
|
|
||||||
* @return {string|undefined} The object nullUploadId
|
|
||||||
*/
|
|
||||||
getNullUploadId() {
|
|
||||||
return this._data.nullUploadId;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set metadata isDeleteMarker value
|
* Set metadata isDeleteMarker value
|
||||||
*
|
*
|
||||||
|
|
|
@ -5,7 +5,6 @@ module.exports.VersioningConstants = {
|
||||||
DbPrefixes: {
|
DbPrefixes: {
|
||||||
Master: '\x7fM',
|
Master: '\x7fM',
|
||||||
Version: '\x7fV',
|
Version: '\x7fV',
|
||||||
Replay: '\x7fR',
|
|
||||||
},
|
},
|
||||||
BucketVersioningKeyFormat: {
|
BucketVersioningKeyFormat: {
|
||||||
current: 'v1',
|
current: 'v1',
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=6.9.5"
|
"node": ">=6.9.5"
|
||||||
},
|
},
|
||||||
"version": "7.10.4",
|
"version": "7.10.2",
|
||||||
"description": "Common utilities for the S3 project components",
|
"description": "Common utilities for the S3 project components",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"repository": {
|
"repository": {
|
||||||
|
|
|
@ -8,14 +8,12 @@ const {
|
||||||
FILTER_ACCEPT,
|
FILTER_ACCEPT,
|
||||||
FILTER_SKIP,
|
FILTER_SKIP,
|
||||||
SKIP_NONE,
|
SKIP_NONE,
|
||||||
inc,
|
|
||||||
} = require('../../../../lib/algos/list/tools');
|
} = require('../../../../lib/algos/list/tools');
|
||||||
const VSConst =
|
const VSConst =
|
||||||
require('../../../../lib/versioning/constants').VersioningConstants;
|
require('../../../../lib/versioning/constants').VersioningConstants;
|
||||||
const Version = require('../../../../lib/versioning/Version').Version;
|
const Version = require('../../../../lib/versioning/Version').Version;
|
||||||
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
|
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
|
||||||
const { DbPrefixes } = VSConst;
|
const { DbPrefixes } = VSConst;
|
||||||
const zpad = require('../../helpers').zpad;
|
|
||||||
|
|
||||||
|
|
||||||
const VID_SEP = VSConst.VersionId.Separator;
|
const VID_SEP = VSConst.VersionId.Separator;
|
||||||
|
@ -455,39 +453,6 @@ function getListingKey(key, vFormat) {
|
||||||
|
|
||||||
assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
|
assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should return good skipping value for DelimiterMaster on replay keys', () => {
|
|
||||||
const delimiter = new DelimiterMaster(
|
|
||||||
{ delimiter: '/', v2: true },
|
|
||||||
fakeLogger, vFormat);
|
|
||||||
|
|
||||||
for (let i = 0; i < 10; i++) {
|
|
||||||
delimiter.filter({
|
|
||||||
key: `foo/${zpad(i)}`,
|
|
||||||
value: '{}',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// simulate a listing that goes through a replay key, ...
|
|
||||||
assert.strictEqual(
|
|
||||||
delimiter.filter({
|
|
||||||
key: `${DbPrefixes.Replay}xyz`,
|
|
||||||
value: 'abcdef',
|
|
||||||
}),
|
|
||||||
FILTER_SKIP);
|
|
||||||
// ...it should skip the whole replay prefix
|
|
||||||
assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
|
|
||||||
|
|
||||||
// simulate a listing that reaches regular object keys
|
|
||||||
// beyond the replay prefix, ...
|
|
||||||
assert.strictEqual(
|
|
||||||
delimiter.filter({
|
|
||||||
key: `${inc(DbPrefixes.Replay)}foo/bar`,
|
|
||||||
value: '{}',
|
|
||||||
}),
|
|
||||||
FILTER_ACCEPT);
|
|
||||||
// ...it should return to skipping by prefix as usual
|
|
||||||
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -7,12 +7,12 @@ const {
|
||||||
FILTER_ACCEPT,
|
FILTER_ACCEPT,
|
||||||
FILTER_SKIP,
|
FILTER_SKIP,
|
||||||
SKIP_NONE,
|
SKIP_NONE,
|
||||||
inc,
|
|
||||||
} = require('../../../../lib/algos/list/tools');
|
} = require('../../../../lib/algos/list/tools');
|
||||||
const Werelogs = require('werelogs').Logger;
|
const Werelogs = require('werelogs').Logger;
|
||||||
const logger = new Werelogs('listTest');
|
const logger = new Werelogs('listTest');
|
||||||
const performListing = require('../../../utils/performListing');
|
const performListing = require('../../../utils/performListing');
|
||||||
const zpad = require('../../helpers').zpad;
|
const zpad = require('../../helpers').zpad;
|
||||||
|
const { inc } = require('../../../../lib/algos/list/tools');
|
||||||
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
|
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
|
||||||
const Version = require('../../../../lib/versioning/Version').Version;
|
const Version = require('../../../../lib/versioning/Version').Version;
|
||||||
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
|
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
|
||||||
|
@ -520,55 +520,17 @@ function getTestListing(test, data, vFormat) {
|
||||||
['v0', 'v1'].forEach(vFormat => {
|
['v0', 'v1'].forEach(vFormat => {
|
||||||
describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
|
describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
|
||||||
it('Should return good skipping value for DelimiterVersions', () => {
|
it('Should return good skipping value for DelimiterVersions', () => {
|
||||||
const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
|
const delimiter = new DelimiterVersions({ delimiter: '/' });
|
||||||
for (let i = 0; i < 100; i++) {
|
for (let i = 0; i < 100; i++) {
|
||||||
delimiter.filter({
|
delimiter.filter({
|
||||||
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
|
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
|
||||||
value: '{}',
|
value: '{}',
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (vFormat === 'v1') {
|
assert.strictEqual(delimiter.skipping(),
|
||||||
assert.deepStrictEqual(delimiter.skipping(), [
|
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
|
||||||
`${DbPrefixes.Master}foo/`,
|
|
||||||
`${DbPrefixes.Version}foo/`,
|
|
||||||
]);
|
|
||||||
} else {
|
|
||||||
assert.strictEqual(delimiter.skipping(), 'foo/');
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (vFormat === 'v0') {
|
|
||||||
it('Should return good skipping value for DelimiterVersions on replay keys', () => {
|
|
||||||
const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
|
|
||||||
for (let i = 0; i < 10; i++) {
|
|
||||||
delimiter.filter({
|
|
||||||
key: `foo/${zpad(i)}`,
|
|
||||||
value: '{}',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// simulate a listing that goes through a replay key, ...
|
|
||||||
assert.strictEqual(
|
|
||||||
delimiter.filter({
|
|
||||||
key: `${DbPrefixes.Replay}xyz`,
|
|
||||||
value: 'abcdef',
|
|
||||||
}),
|
|
||||||
FILTER_SKIP);
|
|
||||||
// ...it should skip the whole replay prefix
|
|
||||||
assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
|
|
||||||
|
|
||||||
// simulate a listing that reaches regular object keys
|
|
||||||
// beyond the replay prefix, ...
|
|
||||||
assert.strictEqual(
|
|
||||||
delimiter.filter({
|
|
||||||
key: `${inc(DbPrefixes.Replay)}foo/bar`,
|
|
||||||
value: '{}',
|
|
||||||
}),
|
|
||||||
FILTER_ACCEPT);
|
|
||||||
// ...it should return to skipping by prefix as usual
|
|
||||||
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
tests.forEach(test => {
|
tests.forEach(test => {
|
||||||
it(`Should return metadata listing params to list ${test.name}`, () => {
|
it(`Should return metadata listing params to list ${test.name}`, () => {
|
||||||
const listing = new DelimiterVersions(test.input, logger, vFormat);
|
const listing = new DelimiterVersions(test.input, logger, vFormat);
|
||||||
|
|
|
@ -69,8 +69,6 @@ describe('ObjectMD class setters/getters', () => {
|
||||||
['IsNull', true],
|
['IsNull', true],
|
||||||
['NullVersionId', null, undefined],
|
['NullVersionId', null, undefined],
|
||||||
['NullVersionId', '111111'],
|
['NullVersionId', '111111'],
|
||||||
['NullUploadId', null, undefined],
|
|
||||||
['NullUploadId', 'abcdefghi'],
|
|
||||||
['IsDeleteMarker', null, false],
|
['IsDeleteMarker', null, false],
|
||||||
['IsDeleteMarker', true],
|
['IsDeleteMarker', true],
|
||||||
['VersionId', null, undefined],
|
['VersionId', null, undefined],
|
||||||
|
@ -336,7 +334,6 @@ describe('getAttributes static method', () => {
|
||||||
'location': true,
|
'location': true,
|
||||||
'isNull': true,
|
'isNull': true,
|
||||||
'nullVersionId': true,
|
'nullVersionId': true,
|
||||||
'nullUploadId': true,
|
|
||||||
'isDeleteMarker': true,
|
'isDeleteMarker': true,
|
||||||
'versionId': true,
|
'versionId': true,
|
||||||
'tags': true,
|
'tags': true,
|
||||||
|
|
Loading…
Reference in New Issue