Compare commits

...

1 Commits

Author SHA1 Message Date
Jonathan Gramain 87bdac0391 bugfix: S3C-2987 update listing algos for v0mig vformat migration state
Update the listing algos to work for buckets in v0mig state.

The main particularity of v0mig is to require a listing of two
disjoint key ranges: before and after the V1 prefix, to gather the
same listing than V0 without listing the V1 keys being migrated.

It will require a change in metadata RepdServer to allow for the
disjoint listing, triggered by the 'serial' attribute sent back in the
listing params.
2020-07-09 16:31:04 -07:00
11 changed files with 774 additions and 206 deletions

View File

@ -1,6 +1,6 @@
'use strict'; // eslint-disable-line strict
const { inc, checkLimit, listingParamsMasterKeysV0ToV1,
const { inc, checkLimit, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT } = require('./tools');
const DEFAULT_MAX_KEYS = 1000;
const VSConst = require('../../versioning/constants').VersioningConstants;
@ -44,6 +44,10 @@ class MultipartUploads {
genMDParams: this.genMDParamsV0,
getObjectKey: this.getObjectKeyV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
getObjectKey: this.getObjectKeyV0,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
@ -73,6 +77,11 @@ class MultipartUploads {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
const v0params = this.genMDParamsV0();
return listingParamsMasterKeysV0ToV1(v0params);

View File

@ -1,7 +1,7 @@
'use strict'; // eslint-disable-line strict
const Extension = require('./Extension').default;
const { inc, listingParamsMasterKeysV0ToV1,
const { inc, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
const VSConst = require('../../versioning/constants').VersioningConstants;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
@ -99,6 +99,11 @@ class Delimiter extends Extension {
getObjectKey: this.getObjectKeyV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
getObjectKey: this.getObjectKeyV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
@ -124,6 +129,11 @@ class Delimiter extends Extension {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
const params = this.genMDParamsV0();
return listingParamsMasterKeysV0ToV1(params);

View File

@ -38,6 +38,10 @@ class DelimiterMaster extends Delimiter {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v1]: {
filter: this.filterV1,
skipping: this.skippingV1,

View File

@ -3,8 +3,8 @@
const Delimiter = require('./delimiter').Delimiter;
const Version = require('../../versioning/Version').Version;
const VSConst = require('../../versioning/constants').VersioningConstants;
const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
require('./tools');
const { inc, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } = require('./tools');
const VID_SEP = VSConst.VersionId.Separator;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
@ -40,6 +40,11 @@ class DelimiterVersions extends Delimiter {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
filter: this.filterV1,
@ -72,6 +77,11 @@ class DelimiterVersions extends Delimiter {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
// return an array of two listing params sets to ask for
// synchronized listing of M and V ranges

View File

@ -59,10 +59,59 @@ function listingParamsMasterKeysV0ToV1(v0params) {
return v1params;
}
function listingParamsV0ToV0Mig(v0params) {
if ((v0params.gt !== undefined && v0params.gt >= inc(DbPrefixes.V1))
|| (v0params.gte !== undefined && v0params.gte >= inc(DbPrefixes.V1))
|| (v0params.lt !== undefined && v0params.lt <= DbPrefixes.V1)) {
return v0params;
}
if ((v0params.gt !== undefined && v0params.gt >= DbPrefixes.V1)
|| (v0params.gte !== undefined && v0params.gte >= DbPrefixes.V1)) {
const v0migParams = Object.assign({}, v0params);
let greaterParam;
if (v0params.gt !== undefined) {
v0migParams.gt = inc(DbPrefixes.V1);
greaterParam = v0migParams.gt;
}
if (v0params.gte !== undefined) {
v0migParams.gte = inc(DbPrefixes.V1);
greaterParam = v0migParams.gte;
}
if (v0params.lt !== undefined && greaterParam !== undefined
&& v0params.lt <= greaterParam) {
// we annihilated the valid range during our v0mig
// transform to skip V1 prefix: return an empty range with
// a trick instead of an invalid combo
return { lt: '' };
}
return v0migParams;
}
const rangeParams1 = {
lt: DbPrefixes.V1,
};
if (v0params.gt !== undefined) {
rangeParams1.gt = v0params.gt;
}
if (v0params.gte !== undefined) {
rangeParams1.gte = v0params.gte;
}
const rangeParams2 = {
gte: inc(DbPrefixes.V1),
// tell RepdServer._listObject() that the second listing is to
// be done after the first, not in parallel
serial: true,
};
if (v0params.lt !== undefined) {
rangeParams2.lt = v0params.lt;
}
return [rangeParams1, rangeParams2];
}
module.exports = {
checkLimit,
inc,
listingParamsMasterKeysV0ToV1,
listingParamsV0ToV0Mig,
SKIP_NONE,
FILTER_END,
FILTER_SKIP,

View File

@ -3,6 +3,7 @@ module.exports.VersioningConstants = {
Separator: '\0',
},
DbPrefixes: {
V1: '\x7f',
Master: '\x7fM',
Version: '\x7fV',
},

View File

@ -1,14 +1,26 @@
'use strict'; // eslint-disable-line strict
const assert = require('assert');
const MultipartUploads =
require('../../../../lib/algos/list/MPU').MultipartUploads;
const MultipartUploads = require('../../../../lib/algos/list/MPU').MultipartUploads;
const { inc } = require('../../../../lib/algos/list/tools');
const werelogs = require('werelogs').Logger;
// eslint-disable-next-line new-cap
const logger = new werelogs('listMpuTest');
const performListing = require('../../../utils/performListing');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
function getListingKey(key, vFormat) {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return key;
}
if (vFormat === BucketVersioningKeyFormat.v1) {
return `${DbPrefixes.Master}${key}`;
}
assert.fail(`bad vFormat ${vFormat}`);
return undefined;
}
describe('Multipart Uploads listing algorithm', () => {
const splitter = '**';
@ -16,22 +28,14 @@ describe('Multipart Uploads listing algorithm', () => {
const storageClass = 'STANDARD';
const initiator1 = { ID: '1', DisplayName: 'initiator1' };
const initiator2 = { ID: '2', DisplayName: 'initiator2' };
const keys = {
v0: [`${overviewPrefix}test/1${splitter}uploadId1`,
`${overviewPrefix}test/2${splitter}uploadId2`,
`${overviewPrefix}test/3${splitter}uploadId3`,
`${overviewPrefix}testMore/4${splitter}uploadId4`,
`${overviewPrefix}testMore/5${splitter}uploadId5`,
`${overviewPrefix}prefixTest/5${splitter}uploadId5`,
],
v1: [`${DbPrefixes.Master}${overviewPrefix}test/1${splitter}uploadId1`,
`${DbPrefixes.Master}${overviewPrefix}test/2${splitter}uploadId2`,
`${DbPrefixes.Master}${overviewPrefix}test/3${splitter}uploadId3`,
`${DbPrefixes.Master}${overviewPrefix}testMore/4${splitter}uploadId4`,
`${DbPrefixes.Master}${overviewPrefix}testMore/5${splitter}uploadId5`,
`${DbPrefixes.Master}${overviewPrefix}prefixTest/5${splitter}uploadId5`,
],
};
const v0keys = [
`${overviewPrefix}test/1${splitter}uploadId1`,
`${overviewPrefix}test/2${splitter}uploadId2`,
`${overviewPrefix}test/3${splitter}uploadId3`,
`${overviewPrefix}testMore/4${splitter}uploadId4`,
`${overviewPrefix}testMore/5${splitter}uploadId5`,
`${overviewPrefix}prefixTest/5${splitter}uploadId5`,
];
const values = [
JSON.stringify({
'key': 'test/1',
@ -128,9 +132,13 @@ describe('Multipart Uploads listing algorithm', () => {
done();
});
['v0', 'v1'].forEach(vFormat => {
const dbListing = keys[vFormat].map((key, i) => ({
key,
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
const dbListing = v0keys.map((key, i) => ({
key: getListingKey(key, vFormat),
value: values[i],
}));
it(`should perform a vFormat=${vFormat} listing of all keys`, () => {
@ -171,4 +179,76 @@ describe('Multipart Uploads listing algorithm', () => {
assert.deepStrictEqual(listingResult, expectedResult);
});
});
describe('MultipartUploads.genMDParams()', () => {
[{
listingParams: {
splitter,
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
},
}, {
listingParams: {
splitter,
prefix: 'foo/bar',
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {
gte: 'foo/bar',
lt: 'foo/bas',
},
[BucketVersioningKeyFormat.v0mig]: {
gte: 'foo/bar',
lt: 'foo/bas',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}foo/bar`,
lt: `${DbPrefixes.Master}foo/bas`,
},
},
}, {
listingParams: {
splitter,
keyMarker: 'marker',
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {
gt: `${overviewPrefix}marker${inc(splitter)}`,
},
[BucketVersioningKeyFormat.v0mig]: [{
gt: `${overviewPrefix}marker${inc(splitter)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${overviewPrefix}marker${inc(splitter)}`,
lt: inc(DbPrefixes.Master),
},
},
}].forEach(testCase => {
[BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v1].forEach(vFormat => {
it(`with vFormat=${vFormat}, listing params ${JSON.stringify(testCase.listingParams)}`, () => {
const delimiter = new MultipartUploads(
testCase.listingParams, logger, vFormat);
const mdParams = delimiter.genMDParams();
assert.deepStrictEqual(mdParams, testCase.mdParams[vFormat]);
});
});
});
});
});

View File

@ -11,7 +11,7 @@ const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
class Test {
constructor(name, input, genMDParams, output, filter) {
@ -90,8 +90,14 @@ const receivedNonAlphaData = nonAlphabeticalData.map(
const tests = [
new Test('all elements', {}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -105,10 +111,17 @@ const tests = [
new Test('with valid marker', {
marker: receivedData[4].key,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -129,10 +142,17 @@ const tests = [
marker: 'zzzz',
delimiter: '/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -146,8 +166,14 @@ const tests = [
new Test('with makKeys', {
maxKeys: 3,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -161,8 +187,14 @@ const tests = [
new Test('with big makKeys', {
maxKeys: 15000,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -176,8 +208,14 @@ const tests = [
new Test('with delimiter', {
delimiter: '/',
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -193,8 +231,14 @@ const tests = [
new Test('with long delimiter', {
delimiter: 'notes/summer',
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -218,11 +262,15 @@ const tests = [
prefix: 'notes/summer/',
marker: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer${inc('/')}`,
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer${inc('/')}`,
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -237,11 +285,15 @@ const tests = [
delimiter: '/',
prefix: 'notes/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -264,11 +316,15 @@ const tests = [
prefix: 'notes/',
marker: 'notes/year.txt',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -289,11 +345,15 @@ const tests = [
marker: 'notes/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -311,11 +371,15 @@ const tests = [
marker: 'notes/spring/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/spring/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -333,11 +397,15 @@ const tests = [
marker: 'notes/summer/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -357,11 +425,15 @@ const tests = [
marker: 'notes/year.txt',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -381,11 +453,15 @@ const tests = [
marker: 'notes/yore.rs',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/yore.rs`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -400,8 +476,14 @@ const tests = [
new Test('all elements v2', {
v2: true,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -416,10 +498,17 @@ const tests = [
startAfter: receivedData[4].key,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -441,10 +530,17 @@ const tests = [
delimiter: '/',
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -459,10 +555,17 @@ const tests = [
continuationToken: receivedData[4].key,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -484,10 +587,17 @@ const tests = [
delimiter: '/',
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -503,11 +613,15 @@ const tests = [
prefix: 'notes/summer/',
startAfter: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -523,11 +637,15 @@ const tests = [
prefix: 'notes/summer/',
continuationToken: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -544,10 +662,17 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'notes/year.txt',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: inc(DbPrefixes.Master),
},
@ -568,11 +693,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -591,11 +720,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/spring/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -614,11 +747,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -639,11 +776,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -664,11 +805,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/yore.rs`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -680,6 +825,109 @@ const tests = [
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with startAfter after vformat V1 prefix', {
delimiter: '/',
startAfter: 'éléphant pâle',
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gt: 'éléphant pâle',
},
[BucketVersioningKeyFormat.v0mig]: {
gt: 'éléphant pâle',
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}éléphant pâle`,
lt: inc(DbPrefixes.Master),
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with startAfter inside vformat V1 prefix', {
delimiter: '/',
startAfter: `${DbPrefixes.V1}foo`,
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gt: `${DbPrefixes.V1}foo`,
},
// v0mig skips all V1-prefixed keys to start at the beginning
// of the second v0 range (to skip V1 keys being migrated)
[BucketVersioningKeyFormat.v0mig]: {
gt: inc(DbPrefixes.V1),
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${DbPrefixes.V1}foo`,
lt: inc(DbPrefixes.Master),
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with prefix after vformat V1 prefix', {
delimiter: '/',
prefix: 'éléphant pâle/',
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gte: 'éléphant pâle/',
lt: 'éléphant pâle0',
},
[BucketVersioningKeyFormat.v0mig]: {
gte: 'éléphant pâle/',
lt: 'éléphant pâle0',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}éléphant pâle/`,
lt: `${DbPrefixes.Master}éléphant pâle0`,
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with prefix inside vformat V1 prefix', {
delimiter: '/',
prefix: `${DbPrefixes.V1}foo/`,
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gte: `${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.V1}foo0`,
},
// v0mig skips the V1 prefix altogether to avoid returning V1
// keys being migrated. It uses a trick: passing "lt: ''"
// (empty string) forces to list an empty range
[BucketVersioningKeyFormat.v0mig]: {
lt: '',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.Master}${DbPrefixes.V1}foo0`,
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
];
const alphabeticalOrderTests = [
@ -708,10 +956,11 @@ function getTestListing(test, data, vFormat) {
return data
.filter(e => test.filter(e, test.input))
.map(obj => {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return obj;
}
if (vFormat === 'v1') {
if (vFormat === BucketVersioningKeyFormat.v1) {
return {
key: `${DbPrefixes.Master}${obj.key}`,
value: obj.value,
@ -721,18 +970,22 @@ function getTestListing(test, data, vFormat) {
});
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`vFormat=${vFormat} Delimiter listing algorithm`, () => {
it('Should return good skipping value for DelimiterMaster', () => {
const delimiter = new DelimiterMaster({ delimiter: '/' });
for (let i = 0; i < 100; i++) {
delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
key: `${vFormat === BucketVersioningKeyFormat.v1 ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
value: '{}',
});
}
assert.strictEqual(delimiter.skipping(),
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
`${vFormat === BucketVersioningKeyFormat.v1 ? DbPrefixes.Master : ''}foo/`);
});
it('Should set Delimiter alphabeticalOrder field to the expected value', () => {
@ -759,7 +1012,7 @@ function getTestListing(test, data, vFormat) {
});
// Only v0 gets a listing of master and version keys together.
if (vFormat === 'v0') {
if (vFormat === BucketVersioningKeyFormat.v0) {
tests.forEach(test => {
it(`Should list master versions ${test.name}`, () => {
// Simulate skip scan done by LevelDB

View File

@ -13,7 +13,7 @@ const VSConst =
require('../../../../lib/versioning/constants').VersioningConstants;
const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
@ -35,16 +35,21 @@ const fakeLogger = {
};
function getListingKey(key, vFormat) {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return key;
}
if (vFormat === 'v1') {
if (vFormat === BucketVersioningKeyFormat.v1) {
return `${DbPrefixes.Master}${key}`;
}
return assert.fail(`bad vFormat ${vFormat}`);
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`Delimiter All masters listing algorithm vFormat=${vFormat}`, () => {
it('should return SKIP_NONE for DelimiterMaster when both NextMarker ' +
'and NextContinuationToken are undefined', () => {
@ -102,7 +107,7 @@ function getListingKey(key, vFormat) {
/* When a delimiter is set and the NextMarker ends with the
* delimiter it should return the next marker value. */
assert.strictEqual(delimiter.NextMarker, keyWithEndingDelimiter);
const skipKey = vFormat === 'v1' ?
const skipKey = vFormat === BucketVersioningKeyFormat.v1 ?
`${DbPrefixes.Master}${keyWithEndingDelimiter}` :
keyWithEndingDelimiter;
assert.strictEqual(delimiter.skipping(), skipKey);
@ -135,7 +140,8 @@ function getListingKey(key, vFormat) {
const listingKey = getListingKey(key, vFormat);
assert.strictEqual(delimiter.filter({ key: listingKey, value }), FILTER_ACCEPT);
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
assert.strictEqual(delimiter.prvKey, key);
}
assert.strictEqual(delimiter.NextMarker, key);
@ -206,7 +212,8 @@ function getListingKey(key, vFormat) {
});
});
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
it('should accept a PHD version as first input', () => {
const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
const keyPHD = 'keyPHD';

View File

@ -9,7 +9,7 @@ const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
class Test {
@ -32,75 +32,42 @@ const bar = '{"versionId":"bar"}';
const qux = '{"versionId":"qux"}';
const valuePHD = '{"isPHD":"true","versionId":"1234567890abcdefg"}';
const valueDeleteMarker = '{"hello":"world","isDeleteMarker":"true"}';
const dataVersioned = {
v0: [
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: 'notes/spring/2.txt', value: valuePHD },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: 'notes/summer/4.txt', value: valuePHD },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
{ key: 'notes/summer/44.txt', value: valuePHD },
{ key: 'notes/summer/444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/4444.txt', value: valuePHD },
{ key: 'notes/summer/44444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/444444.txt', value: valuePHD },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
],
v1: [ // we add M and V prefixes in getTestListing() due to the
// test cases needing the original key to filter
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
// Compared to v0, the two following keys are version keys
// that we give a version ID, because delete markers do not
// have a master key in v1.
{ key: `notes/summer/444.txt${VID_SEP}null`, value: valueDeleteMarker },
{ key: `notes/summer/44444.txt${VID_SEP}null`, value: valueDeleteMarker },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
],
};
const rawListingData = [
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: 'notes/spring/2.txt', value: valuePHD },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: 'notes/summer/4.txt', value: valuePHD },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
{ key: 'notes/summer/44.txt', value: valuePHD },
{ key: 'notes/summer/444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/4444.txt', value: valuePHD },
{ key: 'notes/summer/44444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/444444.txt', value: valuePHD },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
];
const receivedData = [
{ key: 'Pâtisserie=中文-español-English', value: bar, versionId: 'bar' },
{ key: 'Pâtisserie=中文-español-English', value: foo, versionId: 'foo' },
@ -130,9 +97,20 @@ const receivedData = [
];
const tests = [
new Test('all versions', {}, {
v0: {},
v1: [{ gte: DbPrefixes.Master, lt: inc(DbPrefixes.Master) },
{ gte: DbPrefixes.Version, lt: inc(DbPrefixes.Version) }],
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
gte: DbPrefixes.Version,
lt: inc(DbPrefixes.Version),
}],
}, {
Versions: receivedData,
CommonPrefixes: [],
@ -144,10 +122,17 @@ const tests = [
new Test('with valid key marker', {
keyMarker: receivedData[3].key,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `${receivedData[3].key}\u0001`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: [{
gt: `${receivedData[3].key}\u0001`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}${receivedData[3].key}${inc(VID_SEP)}`,
lt: inc(DbPrefixes.Master),
}, {
@ -166,10 +151,17 @@ const tests = [
keyMarker: 'zzzz',
delimiter: '/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `zzzz${inc(VID_SEP)}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: [{
gt: `zzzz${inc(VID_SEP)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}zzzz${inc(VID_SEP)}`,
lt: inc(DbPrefixes.Master),
}, {
@ -187,8 +179,14 @@ const tests = [
new Test('with maxKeys', {
maxKeys: 3,
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -206,8 +204,14 @@ const tests = [
new Test('with big maxKeys', {
maxKeys: 15000,
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -225,8 +229,14 @@ const tests = [
new Test('with delimiter', {
delimiter: '/',
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -247,8 +257,14 @@ const tests = [
new Test('with long delimiter', {
delimiter: 'notes/summer',
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -269,11 +285,15 @@ const tests = [
prefix: 'notes/summer/',
keyMarker: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer0${inc(VID_SEP)}`,
lt: `notes/summer${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer0${inc(VID_SEP)}`,
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/summer0${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
}, {
@ -292,11 +312,15 @@ const tests = [
delimiter: '/',
prefix: 'notes/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gte: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -323,11 +347,15 @@ const tests = [
prefix: 'notes/',
keyMarker: 'notes/year.txt',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -352,11 +380,15 @@ const tests = [
keyMarker: 'notes/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -378,11 +410,15 @@ const tests = [
keyMarker: 'notes/spring/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/spring/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/spring/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/spring/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -404,11 +440,15 @@ const tests = [
keyMarker: 'notes/summer/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/summer/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -432,11 +472,15 @@ const tests = [
keyMarker: 'notes/year.txt',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -460,11 +504,15 @@ const tests = [
keyMarker: 'notes/yore.rs',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/yore.rs${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/yore.rs${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/yore.rs${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -481,14 +529,16 @@ const tests = [
}, (e, input) => e.key > input.keyMarker),
];
function getTestListing(test, data, vFormat) {
return data
function getTestListing(test, vFormat) {
return rawListingData
.filter(e => vFormat !== BucketVersioningKeyFormat.v1 || e.value !== valuePHD)
.filter(e => test.filter(e, test.input))
.map(e => {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return e;
}
if (vFormat === 'v1') {
if (vFormat === BucketVersioningKeyFormat.v1) {
const keyPrefix = e.key.includes(VID_SEP) ?
DbPrefixes.Version : DbPrefixes.Master;
return {
@ -500,18 +550,22 @@ function getTestListing(test, data, vFormat) {
});
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
it('Should return good skipping value for DelimiterVersions', () => {
const delimiter = new DelimiterVersions({ delimiter: '/' });
for (let i = 0; i < 100; i++) {
delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
key: `${vFormat === BucketVersioningKeyFormat.v1 ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
value: '{}',
});
}
assert.strictEqual(delimiter.skipping(),
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
`${vFormat === BucketVersioningKeyFormat.v1 ? DbPrefixes.Master : ''}foo/`);
});
tests.forEach(test => {
@ -522,7 +576,7 @@ function getTestListing(test, data, vFormat) {
});
it(`Should list ${test.name}`, () => {
// Simulate skip scan done by LevelDB
const d = getTestListing(test, dataVersioned[vFormat], vFormat);
const d = getTestListing(test, vFormat);
const res = performListing(d, DelimiterVersions, test.input, logger, vFormat);
assert.deepStrictEqual(res, test.output);
});

View File

@ -2,10 +2,11 @@
const assert = require('assert');
const { checkLimit, inc, listingParamsMasterKeysV0ToV1 } =
const { checkLimit, inc, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig } =
require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
describe('checkLimit function', () => {
const tests = [
@ -102,3 +103,93 @@ describe('listingParamsMasterKeysV0ToV1', () => {
});
});
});
describe('listingParamsV0ToV0Mig', () => {
const testCases = [
{
v0params: {},
v0migparams: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gte: 'foo/bar',
lt: 'foo/bas',
},
v0migparams: {
gte: 'foo/bar',
lt: 'foo/bas',
},
}, {
v0params: {
gt: `foo/bar${inc(VID_SEP)}`,
},
v0migparams: [{
gt: `foo/bar${inc(VID_SEP)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gt: `foo/bar${VID_SEP}versionId`,
},
v0migparams: [{
gt: `foo/bar${VID_SEP}versionId`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gt: `foo/bar/baz${VID_SEP}versionId`,
lt: 'foo/bas',
},
v0migparams: {
gt: `foo/bar/baz${VID_SEP}versionId`,
lt: 'foo/bas',
},
}, {
v0params: {
gt: `éléphant rose${VID_SEP}versionId`,
},
v0migparams: {
gt: `éléphant rose${VID_SEP}versionId`,
},
}, {
v0params: {
gte: 'éléphant rose',
lt: 'éléphant rosf',
},
v0migparams: {
gte: 'éléphant rose',
lt: 'éléphant rosf',
},
}, {
v0params: {
gt: `${DbPrefixes.V1}foo`,
},
v0migparams: {
gt: inc(DbPrefixes.V1),
},
}, {
v0params: {
gte: `${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.V1}foo0`,
},
v0migparams: {
lt: '',
},
}];
testCases.forEach(({ v0params, v0migparams }) => {
it(`${JSON.stringify(v0params)} => ${JSON.stringify(v0migparams)}`, () => {
const converted = listingParamsV0ToV0Mig(v0params);
assert.deepStrictEqual(converted, v0migparams);
});
});
});