Compare commits

...

2 Commits

Author SHA1 Message Date
Jonathan Gramain 7f98c7b543 bugfix: S3C-2987 update listing algo for v0v1, v1mig migration steps
The v0v1 migration step (keys are both in v0 and v1 format) and v1mig
(keys in v0 are being deleted) both use the same logic than the plain
v1 vformat, because all v1 keys exist hence the listing can benefit
from the v1 enhancements in those migration steps.
2020-07-09 16:31:48 -07:00
Jonathan Gramain 87bdac0391 bugfix: S3C-2987 update listing algos for v0mig vformat migration state
Update the listing algos to work for buckets in v0mig state.

The main particularity of v0mig is to require a listing of two
disjoint key ranges: before and after the V1 prefix, to gather the
same listing than V0 without listing the V1 keys being migrated.

It will require a change in metadata RepdServer to allow for the
disjoint listing, triggered by the 'serial' attribute sent back in the
listing params.
2020-07-09 16:31:04 -07:00
11 changed files with 898 additions and 210 deletions

View File

@ -1,6 +1,6 @@
'use strict'; // eslint-disable-line strict
const { inc, checkLimit, listingParamsMasterKeysV0ToV1,
const { inc, checkLimit, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT } = require('./tools');
const DEFAULT_MAX_KEYS = 1000;
const VSConst = require('../../versioning/constants').VersioningConstants;
@ -44,6 +44,18 @@ class MultipartUploads {
genMDParams: this.genMDParamsV0,
getObjectKey: this.getObjectKeyV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
getObjectKey: this.getObjectKeyV0,
},
[BucketVersioningKeyFormat.v0v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
},
[BucketVersioningKeyFormat.v1mig]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
@ -73,6 +85,11 @@ class MultipartUploads {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
const v0params = this.genMDParamsV0();
return listingParamsMasterKeysV0ToV1(v0params);

View File

@ -1,7 +1,7 @@
'use strict'; // eslint-disable-line strict
const Extension = require('./Extension').default;
const { inc, listingParamsMasterKeysV0ToV1,
const { inc, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
const VSConst = require('../../versioning/constants').VersioningConstants;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
@ -99,6 +99,21 @@ class Delimiter extends Extension {
getObjectKey: this.getObjectKeyV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
getObjectKey: this.getObjectKeyV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1mig]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
@ -124,6 +139,11 @@ class Delimiter extends Extension {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
const params = this.genMDParamsV0();
return listingParamsMasterKeysV0ToV1(params);

View File

@ -38,6 +38,18 @@ class DelimiterMaster extends Delimiter {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0v1]: {
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1mig]: {
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1]: {
filter: this.filterV1,
skipping: this.skippingV1,

View File

@ -3,8 +3,8 @@
const Delimiter = require('./delimiter').Delimiter;
const Version = require('../../versioning/Version').Version;
const VSConst = require('../../versioning/constants').VersioningConstants;
const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
require('./tools');
const { inc, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } = require('./tools');
const VID_SEP = VSConst.VersionId.Separator;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
@ -40,6 +40,21 @@ class DelimiterVersions extends Delimiter {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0v1]: {
genMDParams: this.genMDParamsV1,
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1mig]: {
genMDParams: this.genMDParamsV1,
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
filter: this.filterV1,
@ -72,6 +87,11 @@ class DelimiterVersions extends Delimiter {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
// return an array of two listing params sets to ask for
// synchronized listing of M and V ranges

View File

@ -59,10 +59,59 @@ function listingParamsMasterKeysV0ToV1(v0params) {
return v1params;
}
function listingParamsV0ToV0Mig(v0params) {
if ((v0params.gt !== undefined && v0params.gt >= inc(DbPrefixes.V1))
|| (v0params.gte !== undefined && v0params.gte >= inc(DbPrefixes.V1))
|| (v0params.lt !== undefined && v0params.lt <= DbPrefixes.V1)) {
return v0params;
}
if ((v0params.gt !== undefined && v0params.gt >= DbPrefixes.V1)
|| (v0params.gte !== undefined && v0params.gte >= DbPrefixes.V1)) {
const v0migParams = Object.assign({}, v0params);
let greaterParam;
if (v0params.gt !== undefined) {
v0migParams.gt = inc(DbPrefixes.V1);
greaterParam = v0migParams.gt;
}
if (v0params.gte !== undefined) {
v0migParams.gte = inc(DbPrefixes.V1);
greaterParam = v0migParams.gte;
}
if (v0params.lt !== undefined && greaterParam !== undefined
&& v0params.lt <= greaterParam) {
// we annihilated the valid range during our v0mig
// transform to skip V1 prefix: return an empty range with
// a trick instead of an invalid combo
return { lt: '' };
}
return v0migParams;
}
const rangeParams1 = {
lt: DbPrefixes.V1,
};
if (v0params.gt !== undefined) {
rangeParams1.gt = v0params.gt;
}
if (v0params.gte !== undefined) {
rangeParams1.gte = v0params.gte;
}
const rangeParams2 = {
gte: inc(DbPrefixes.V1),
// tell RepdServer._listObject() that the second listing is to
// be done after the first, not in parallel
serial: true,
};
if (v0params.lt !== undefined) {
rangeParams2.lt = v0params.lt;
}
return [rangeParams1, rangeParams2];
}
module.exports = {
checkLimit,
inc,
listingParamsMasterKeysV0ToV1,
listingParamsV0ToV0Mig,
SKIP_NONE,
FILTER_END,
FILTER_SKIP,

View File

@ -3,6 +3,7 @@ module.exports.VersioningConstants = {
Separator: '\0',
},
DbPrefixes: {
V1: '\x7f',
Master: '\x7fM',
Version: '\x7fV',
},

View File

@ -1,14 +1,28 @@
'use strict'; // eslint-disable-line strict
const assert = require('assert');
const MultipartUploads =
require('../../../../lib/algos/list/MPU').MultipartUploads;
const MultipartUploads = require('../../../../lib/algos/list/MPU').MultipartUploads;
const { inc } = require('../../../../lib/algos/list/tools');
const werelogs = require('werelogs').Logger;
// eslint-disable-next-line new-cap
const logger = new werelogs('listMpuTest');
const performListing = require('../../../utils/performListing');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
function getListingKey(key, vFormat) {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return key;
}
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
return `${DbPrefixes.Master}${key}`;
}
assert.fail(`bad vFormat ${vFormat}`);
return undefined;
}
describe('Multipart Uploads listing algorithm', () => {
const splitter = '**';
@ -16,22 +30,14 @@ describe('Multipart Uploads listing algorithm', () => {
const storageClass = 'STANDARD';
const initiator1 = { ID: '1', DisplayName: 'initiator1' };
const initiator2 = { ID: '2', DisplayName: 'initiator2' };
const keys = {
v0: [`${overviewPrefix}test/1${splitter}uploadId1`,
`${overviewPrefix}test/2${splitter}uploadId2`,
`${overviewPrefix}test/3${splitter}uploadId3`,
`${overviewPrefix}testMore/4${splitter}uploadId4`,
`${overviewPrefix}testMore/5${splitter}uploadId5`,
`${overviewPrefix}prefixTest/5${splitter}uploadId5`,
],
v1: [`${DbPrefixes.Master}${overviewPrefix}test/1${splitter}uploadId1`,
`${DbPrefixes.Master}${overviewPrefix}test/2${splitter}uploadId2`,
`${DbPrefixes.Master}${overviewPrefix}test/3${splitter}uploadId3`,
`${DbPrefixes.Master}${overviewPrefix}testMore/4${splitter}uploadId4`,
`${DbPrefixes.Master}${overviewPrefix}testMore/5${splitter}uploadId5`,
`${DbPrefixes.Master}${overviewPrefix}prefixTest/5${splitter}uploadId5`,
],
};
const v0keys = [
`${overviewPrefix}test/1${splitter}uploadId1`,
`${overviewPrefix}test/2${splitter}uploadId2`,
`${overviewPrefix}test/3${splitter}uploadId3`,
`${overviewPrefix}testMore/4${splitter}uploadId4`,
`${overviewPrefix}testMore/5${splitter}uploadId5`,
`${overviewPrefix}prefixTest/5${splitter}uploadId5`,
];
const values = [
JSON.stringify({
'key': 'test/1',
@ -128,9 +134,15 @@ describe('Multipart Uploads listing algorithm', () => {
done();
});
['v0', 'v1'].forEach(vFormat => {
const dbListing = keys[vFormat].map((key, i) => ({
key,
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
const dbListing = v0keys.map((key, i) => ({
key: getListingKey(key, vFormat),
value: values[i],
}));
it(`should perform a vFormat=${vFormat} listing of all keys`, () => {
@ -171,4 +183,90 @@ describe('Multipart Uploads listing algorithm', () => {
assert.deepStrictEqual(listingResult, expectedResult);
});
});
describe('MultipartUploads.genMDParams()', () => {
[{
listingParams: {
splitter,
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
},
}, {
listingParams: {
splitter,
prefix: 'foo/bar',
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {
gte: 'foo/bar',
lt: 'foo/bas',
},
[BucketVersioningKeyFormat.v0mig]: {
gte: 'foo/bar',
lt: 'foo/bas',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}foo/bar`,
lt: `${DbPrefixes.Master}foo/bas`,
},
},
}, {
listingParams: {
splitter,
keyMarker: 'marker',
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {
gt: `${overviewPrefix}marker${inc(splitter)}`,
},
[BucketVersioningKeyFormat.v0mig]: [{
gt: `${overviewPrefix}marker${inc(splitter)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${overviewPrefix}marker${inc(splitter)}`,
lt: inc(DbPrefixes.Master),
},
},
}].forEach(testCase => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
it(`with vFormat=${vFormat}, listing params ${JSON.stringify(testCase.listingParams)}`, () => {
const delimiter = new MultipartUploads(
testCase.listingParams, logger, vFormat);
const mdParams = delimiter.genMDParams();
let paramsVFormat;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
// all above vformats are equivalent to v1 when it
// comes to generating md params
paramsVFormat = BucketVersioningKeyFormat.v1;
} else {
paramsVFormat = vFormat;
}
assert.deepStrictEqual(mdParams, testCase.mdParams[paramsVFormat]);
});
});
});
});
});

View File

@ -11,7 +11,7 @@ const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
class Test {
constructor(name, input, genMDParams, output, filter) {
@ -90,8 +90,14 @@ const receivedNonAlphaData = nonAlphabeticalData.map(
const tests = [
new Test('all elements', {}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -105,10 +111,17 @@ const tests = [
new Test('with valid marker', {
marker: receivedData[4].key,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -129,10 +142,17 @@ const tests = [
marker: 'zzzz',
delimiter: '/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -146,8 +166,14 @@ const tests = [
new Test('with makKeys', {
maxKeys: 3,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -161,8 +187,14 @@ const tests = [
new Test('with big makKeys', {
maxKeys: 15000,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -176,8 +208,14 @@ const tests = [
new Test('with delimiter', {
delimiter: '/',
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -193,8 +231,14 @@ const tests = [
new Test('with long delimiter', {
delimiter: 'notes/summer',
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -218,11 +262,15 @@ const tests = [
prefix: 'notes/summer/',
marker: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer${inc('/')}`,
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer${inc('/')}`,
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -237,11 +285,15 @@ const tests = [
delimiter: '/',
prefix: 'notes/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -264,11 +316,15 @@ const tests = [
prefix: 'notes/',
marker: 'notes/year.txt',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -289,11 +345,15 @@ const tests = [
marker: 'notes/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -311,11 +371,15 @@ const tests = [
marker: 'notes/spring/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/spring/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -333,11 +397,15 @@ const tests = [
marker: 'notes/summer/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -357,11 +425,15 @@ const tests = [
marker: 'notes/year.txt',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -381,11 +453,15 @@ const tests = [
marker: 'notes/yore.rs',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/yore.rs`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -400,8 +476,14 @@ const tests = [
new Test('all elements v2', {
v2: true,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -416,10 +498,17 @@ const tests = [
startAfter: receivedData[4].key,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -441,10 +530,17 @@ const tests = [
delimiter: '/',
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -459,10 +555,17 @@ const tests = [
continuationToken: receivedData[4].key,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -484,10 +587,17 @@ const tests = [
delimiter: '/',
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -503,11 +613,15 @@ const tests = [
prefix: 'notes/summer/',
startAfter: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -523,11 +637,15 @@ const tests = [
prefix: 'notes/summer/',
continuationToken: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -544,10 +662,17 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'notes/year.txt',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: inc(DbPrefixes.Master),
},
@ -568,11 +693,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -591,11 +720,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/spring/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -614,11 +747,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -639,11 +776,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -664,11 +805,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/yore.rs`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -680,6 +825,109 @@ const tests = [
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with startAfter after vformat V1 prefix', {
delimiter: '/',
startAfter: 'éléphant pâle',
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gt: 'éléphant pâle',
},
[BucketVersioningKeyFormat.v0mig]: {
gt: 'éléphant pâle',
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}éléphant pâle`,
lt: inc(DbPrefixes.Master),
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with startAfter inside vformat V1 prefix', {
delimiter: '/',
startAfter: `${DbPrefixes.V1}foo`,
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gt: `${DbPrefixes.V1}foo`,
},
// v0mig skips all V1-prefixed keys to start at the beginning
// of the second v0 range (to skip V1 keys being migrated)
[BucketVersioningKeyFormat.v0mig]: {
gt: inc(DbPrefixes.V1),
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${DbPrefixes.V1}foo`,
lt: inc(DbPrefixes.Master),
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with prefix after vformat V1 prefix', {
delimiter: '/',
prefix: 'éléphant pâle/',
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gte: 'éléphant pâle/',
lt: 'éléphant pâle0',
},
[BucketVersioningKeyFormat.v0mig]: {
gte: 'éléphant pâle/',
lt: 'éléphant pâle0',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}éléphant pâle/`,
lt: `${DbPrefixes.Master}éléphant pâle0`,
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with prefix inside vformat V1 prefix', {
delimiter: '/',
prefix: `${DbPrefixes.V1}foo/`,
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gte: `${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.V1}foo0`,
},
// v0mig skips the V1 prefix altogether to avoid returning V1
// keys being migrated. It uses a trick: passing "lt: ''"
// (empty string) forces to list an empty range
[BucketVersioningKeyFormat.v0mig]: {
lt: '',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.Master}${DbPrefixes.V1}foo0`,
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
];
const alphabeticalOrderTests = [
@ -708,10 +956,13 @@ function getTestListing(test, data, vFormat) {
return data
.filter(e => test.filter(e, test.input))
.map(obj => {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return obj;
}
if (vFormat === 'v1') {
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
return {
key: `${DbPrefixes.Master}${obj.key}`,
value: obj.value,
@ -721,18 +972,39 @@ function getTestListing(test, data, vFormat) {
});
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`vFormat=${vFormat} Delimiter listing algorithm`, () => {
it('Should return good skipping value for DelimiterMaster', () => {
const delimiter = new DelimiterMaster({ delimiter: '/' });
for (let i = 0; i < 100; i++) {
let key;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
key = `${DbPrefixes.Master}foo/${zpad(i)}`;
} else {
key = `foo/${zpad(i)}`;
}
delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
key,
value: '{}',
});
}
assert.strictEqual(delimiter.skipping(),
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
let skipping;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
skipping = `${DbPrefixes.Master}foo/`;
} else {
skipping = 'foo/';
}
assert.strictEqual(delimiter.skipping(), skipping);
});
it('Should set Delimiter alphabeticalOrder field to the expected value', () => {
@ -748,7 +1020,17 @@ function getTestListing(test, data, vFormat) {
it(`Should return metadata listing params to list ${test.name}`, () => {
const listing = new Delimiter(test.input, logger, vFormat);
const params = listing.genMDParams();
assert.deepStrictEqual(params, test.genMDParams[vFormat]);
let paramsVFormat;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
// all above vformats are equivalent to v1 when it
// comes to generating md params
paramsVFormat = BucketVersioningKeyFormat.v1;
} else {
paramsVFormat = vFormat;
}
assert.deepStrictEqual(params, test.genMDParams[paramsVFormat]);
});
it(`Should list ${test.name}`, () => {
// Simulate skip scan done by LevelDB
@ -759,7 +1041,7 @@ function getTestListing(test, data, vFormat) {
});
// Only v0 gets a listing of master and version keys together.
if (vFormat === 'v0') {
if (vFormat === BucketVersioningKeyFormat.v0) {
tests.forEach(test => {
it(`Should list master versions ${test.name}`, () => {
// Simulate skip scan done by LevelDB

View File

@ -13,7 +13,7 @@ const VSConst =
require('../../../../lib/versioning/constants').VersioningConstants;
const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
@ -35,16 +35,25 @@ const fakeLogger = {
};
function getListingKey(key, vFormat) {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return key;
}
if (vFormat === 'v1') {
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
return `${DbPrefixes.Master}${key}`;
}
return assert.fail(`bad vFormat ${vFormat}`);
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`Delimiter All masters listing algorithm vFormat=${vFormat}`, () => {
it('should return SKIP_NONE for DelimiterMaster when both NextMarker ' +
'and NextContinuationToken are undefined', () => {
@ -102,7 +111,9 @@ function getListingKey(key, vFormat) {
/* When a delimiter is set and the NextMarker ends with the
* delimiter it should return the next marker value. */
assert.strictEqual(delimiter.NextMarker, keyWithEndingDelimiter);
const skipKey = vFormat === 'v1' ?
const skipKey = [BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat) ?
`${DbPrefixes.Master}${keyWithEndingDelimiter}` :
keyWithEndingDelimiter;
assert.strictEqual(delimiter.skipping(), skipKey);
@ -135,7 +146,8 @@ function getListingKey(key, vFormat) {
const listingKey = getListingKey(key, vFormat);
assert.strictEqual(delimiter.filter({ key: listingKey, value }), FILTER_ACCEPT);
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
assert.strictEqual(delimiter.prvKey, key);
}
assert.strictEqual(delimiter.NextMarker, key);
@ -206,7 +218,8 @@ function getListingKey(key, vFormat) {
});
});
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
it('should accept a PHD version as first input', () => {
const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
const keyPHD = 'keyPHD';

View File

@ -9,7 +9,7 @@ const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
class Test {
@ -32,75 +32,42 @@ const bar = '{"versionId":"bar"}';
const qux = '{"versionId":"qux"}';
const valuePHD = '{"isPHD":"true","versionId":"1234567890abcdefg"}';
const valueDeleteMarker = '{"hello":"world","isDeleteMarker":"true"}';
const dataVersioned = {
v0: [
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: 'notes/spring/2.txt', value: valuePHD },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: 'notes/summer/4.txt', value: valuePHD },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
{ key: 'notes/summer/44.txt', value: valuePHD },
{ key: 'notes/summer/444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/4444.txt', value: valuePHD },
{ key: 'notes/summer/44444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/444444.txt', value: valuePHD },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
],
v1: [ // we add M and V prefixes in getTestListing() due to the
// test cases needing the original key to filter
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
// Compared to v0, the two following keys are version keys
// that we give a version ID, because delete markers do not
// have a master key in v1.
{ key: `notes/summer/444.txt${VID_SEP}null`, value: valueDeleteMarker },
{ key: `notes/summer/44444.txt${VID_SEP}null`, value: valueDeleteMarker },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
],
};
const rawListingData = [
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: 'notes/spring/2.txt', value: valuePHD },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: 'notes/summer/4.txt', value: valuePHD },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
{ key: 'notes/summer/44.txt', value: valuePHD },
{ key: 'notes/summer/444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/4444.txt', value: valuePHD },
{ key: 'notes/summer/44444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/444444.txt', value: valuePHD },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
];
const receivedData = [
{ key: 'Pâtisserie=中文-español-English', value: bar, versionId: 'bar' },
{ key: 'Pâtisserie=中文-español-English', value: foo, versionId: 'foo' },
@ -130,9 +97,20 @@ const receivedData = [
];
const tests = [
new Test('all versions', {}, {
v0: {},
v1: [{ gte: DbPrefixes.Master, lt: inc(DbPrefixes.Master) },
{ gte: DbPrefixes.Version, lt: inc(DbPrefixes.Version) }],
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
gte: DbPrefixes.Version,
lt: inc(DbPrefixes.Version),
}],
}, {
Versions: receivedData,
CommonPrefixes: [],
@ -144,10 +122,17 @@ const tests = [
new Test('with valid key marker', {
keyMarker: receivedData[3].key,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `${receivedData[3].key}\u0001`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: [{
gt: `${receivedData[3].key}\u0001`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}${receivedData[3].key}${inc(VID_SEP)}`,
lt: inc(DbPrefixes.Master),
}, {
@ -166,10 +151,17 @@ const tests = [
keyMarker: 'zzzz',
delimiter: '/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `zzzz${inc(VID_SEP)}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: [{
gt: `zzzz${inc(VID_SEP)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}zzzz${inc(VID_SEP)}`,
lt: inc(DbPrefixes.Master),
}, {
@ -187,8 +179,14 @@ const tests = [
new Test('with maxKeys', {
maxKeys: 3,
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -206,8 +204,14 @@ const tests = [
new Test('with big maxKeys', {
maxKeys: 15000,
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -225,8 +229,14 @@ const tests = [
new Test('with delimiter', {
delimiter: '/',
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -247,8 +257,14 @@ const tests = [
new Test('with long delimiter', {
delimiter: 'notes/summer',
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -269,11 +285,15 @@ const tests = [
prefix: 'notes/summer/',
keyMarker: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer0${inc(VID_SEP)}`,
lt: `notes/summer${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer0${inc(VID_SEP)}`,
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/summer0${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
}, {
@ -292,11 +312,15 @@ const tests = [
delimiter: '/',
prefix: 'notes/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gte: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -323,11 +347,15 @@ const tests = [
prefix: 'notes/',
keyMarker: 'notes/year.txt',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -352,11 +380,15 @@ const tests = [
keyMarker: 'notes/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -378,11 +410,15 @@ const tests = [
keyMarker: 'notes/spring/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/spring/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/spring/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/spring/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -404,11 +440,15 @@ const tests = [
keyMarker: 'notes/summer/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/summer/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -432,11 +472,15 @@ const tests = [
keyMarker: 'notes/year.txt',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -460,11 +504,15 @@ const tests = [
keyMarker: 'notes/yore.rs',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/yore.rs${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/yore.rs${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/yore.rs${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -481,14 +529,20 @@ const tests = [
}, (e, input) => e.key > input.keyMarker),
];
function getTestListing(test, data, vFormat) {
return data
function getTestListing(test, vFormat) {
return rawListingData
.filter(e => [BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)
|| e.value !== valuePHD)
.filter(e => test.filter(e, test.input))
.map(e => {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return e;
}
if (vFormat === 'v1') {
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
const keyPrefix = e.key.includes(VID_SEP) ?
DbPrefixes.Version : DbPrefixes.Master;
return {
@ -500,29 +554,60 @@ function getTestListing(test, data, vFormat) {
});
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
it('Should return good skipping value for DelimiterVersions', () => {
const delimiter = new DelimiterVersions({ delimiter: '/' });
for (let i = 0; i < 100; i++) {
let key;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
key = `${DbPrefixes.Master}foo/${zpad(i)}`;
} else {
key = `foo/${zpad(i)}`;
}
delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
key,
value: '{}',
});
}
assert.strictEqual(delimiter.skipping(),
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
let skipping;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
skipping = `${DbPrefixes.Master}foo/`;
} else {
skipping = 'foo/';
}
assert.strictEqual(delimiter.skipping(), skipping);
});
tests.forEach(test => {
it(`Should return metadata listing params to list ${test.name}`, () => {
const listing = new DelimiterVersions(test.input, logger, vFormat);
const params = listing.genMDParams();
assert.deepStrictEqual(params, test.genMDParams[vFormat]);
let paramsVFormat;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
// all above vformats are equivalent to v1 when it
// comes to generating md params
paramsVFormat = BucketVersioningKeyFormat.v1;
} else {
paramsVFormat = vFormat;
}
assert.deepStrictEqual(params, test.genMDParams[paramsVFormat]);
});
it(`Should list ${test.name}`, () => {
// Simulate skip scan done by LevelDB
const d = getTestListing(test, dataVersioned[vFormat], vFormat);
const d = getTestListing(test, vFormat);
const res = performListing(d, DelimiterVersions, test.input, logger, vFormat);
assert.deepStrictEqual(res, test.output);
});

View File

@ -2,10 +2,11 @@
const assert = require('assert');
const { checkLimit, inc, listingParamsMasterKeysV0ToV1 } =
const { checkLimit, inc, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig } =
require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
describe('checkLimit function', () => {
const tests = [
@ -102,3 +103,93 @@ describe('listingParamsMasterKeysV0ToV1', () => {
});
});
});
describe('listingParamsV0ToV0Mig', () => {
const testCases = [
{
v0params: {},
v0migparams: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gte: 'foo/bar',
lt: 'foo/bas',
},
v0migparams: {
gte: 'foo/bar',
lt: 'foo/bas',
},
}, {
v0params: {
gt: `foo/bar${inc(VID_SEP)}`,
},
v0migparams: [{
gt: `foo/bar${inc(VID_SEP)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gt: `foo/bar${VID_SEP}versionId`,
},
v0migparams: [{
gt: `foo/bar${VID_SEP}versionId`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gt: `foo/bar/baz${VID_SEP}versionId`,
lt: 'foo/bas',
},
v0migparams: {
gt: `foo/bar/baz${VID_SEP}versionId`,
lt: 'foo/bas',
},
}, {
v0params: {
gt: `éléphant rose${VID_SEP}versionId`,
},
v0migparams: {
gt: `éléphant rose${VID_SEP}versionId`,
},
}, {
v0params: {
gte: 'éléphant rose',
lt: 'éléphant rosf',
},
v0migparams: {
gte: 'éléphant rose',
lt: 'éléphant rosf',
},
}, {
v0params: {
gt: `${DbPrefixes.V1}foo`,
},
v0migparams: {
gt: inc(DbPrefixes.V1),
},
}, {
v0params: {
gte: `${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.V1}foo0`,
},
v0migparams: {
lt: '',
},
}];
testCases.forEach(({ v0params, v0migparams }) => {
it(`${JSON.stringify(v0params)} => ${JSON.stringify(v0migparams)}`, () => {
const converted = listingParamsV0ToV0Mig(v0params);
assert.deepStrictEqual(converted, v0migparams);
});
});
});