Compare commits

...

4 Commits

Author SHA1 Message Date
Jonathan Gramain d5844dfea4 Merge branch 'bugfix/S3C-2987-serialStreamHelper' into bugfix/S3C-2987-featureBranch 2020-07-09 16:32:17 -07:00
Jonathan Gramain 7f98c7b543 bugfix: S3C-2987 update listing algo for v0v1, v1mig migration steps
The v0v1 migration step (keys are both in v0 and v1 format) and v1mig
(keys in v0 are being deleted) both use the same logic than the plain
v1 vformat, because all v1 keys exist hence the listing can benefit
from the v1 enhancements in those migration steps.
2020-07-09 16:31:48 -07:00
Jonathan Gramain 87bdac0391 bugfix: S3C-2987 update listing algos for v0mig vformat migration state
Update the listing algos to work for buckets in v0mig state.

The main particularity of v0mig is to require a listing of two
disjoint key ranges: before and after the V1 prefix, to gather the
same listing than V0 without listing the V1 keys being migrated.

It will require a change in metadata RepdServer to allow for the
disjoint listing, triggered by the 'serial' attribute sent back in the
listing params.
2020-07-09 16:31:04 -07:00
Jonathan Gramain 94c10e4383 S3C-2987 helper class algo.stream.SerialStream
This helper concatenates two object streams into a single stream.

Will be used by the listing code in RepdServer to handle v0mig
migration stage where two listing on two different ranges need to be
done one after the other.
2020-07-09 16:30:14 -07:00
16 changed files with 1149 additions and 244 deletions

View File

@ -28,6 +28,7 @@ module.exports = {
LRUCache: require('./lib/algos/cache/LRUCache'),
},
stream: {
SerialStream: require('./lib/algos/stream/SerialStream'),
MergeStream: require('./lib/algos/stream/MergeStream'),
},
},

View File

@ -1,6 +1,6 @@
'use strict'; // eslint-disable-line strict
const { inc, checkLimit, listingParamsMasterKeysV0ToV1,
const { inc, checkLimit, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT } = require('./tools');
const DEFAULT_MAX_KEYS = 1000;
const VSConst = require('../../versioning/constants').VersioningConstants;
@ -44,6 +44,18 @@ class MultipartUploads {
genMDParams: this.genMDParamsV0,
getObjectKey: this.getObjectKeyV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
getObjectKey: this.getObjectKeyV0,
},
[BucketVersioningKeyFormat.v0v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
},
[BucketVersioningKeyFormat.v1mig]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
@ -73,6 +85,11 @@ class MultipartUploads {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
const v0params = this.genMDParamsV0();
return listingParamsMasterKeysV0ToV1(v0params);

View File

@ -1,7 +1,7 @@
'use strict'; // eslint-disable-line strict
const Extension = require('./Extension').default;
const { inc, listingParamsMasterKeysV0ToV1,
const { inc, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
const VSConst = require('../../versioning/constants').VersioningConstants;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
@ -99,6 +99,21 @@ class Delimiter extends Extension {
getObjectKey: this.getObjectKeyV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
getObjectKey: this.getObjectKeyV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1mig]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
getObjectKey: this.getObjectKeyV1,
@ -124,6 +139,11 @@ class Delimiter extends Extension {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
const params = this.genMDParamsV0();
return listingParamsMasterKeysV0ToV1(params);

View File

@ -38,6 +38,18 @@ class DelimiterMaster extends Delimiter {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0v1]: {
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1mig]: {
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1]: {
filter: this.filterV1,
skipping: this.skippingV1,

View File

@ -3,8 +3,8 @@
const Delimiter = require('./delimiter').Delimiter;
const Version = require('../../versioning/Version').Version;
const VSConst = require('../../versioning/constants').VersioningConstants;
const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
require('./tools');
const { inc, listingParamsV0ToV0Mig,
FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } = require('./tools');
const VID_SEP = VSConst.VersionId.Separator;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
@ -40,6 +40,21 @@ class DelimiterVersions extends Delimiter {
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0mig]: {
genMDParams: this.genMDParamsV0Mig,
filter: this.filterV0,
skipping: this.skippingV0,
},
[BucketVersioningKeyFormat.v0v1]: {
genMDParams: this.genMDParamsV1,
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1mig]: {
genMDParams: this.genMDParamsV1,
filter: this.filterV1,
skipping: this.skippingV1,
},
[BucketVersioningKeyFormat.v1]: {
genMDParams: this.genMDParamsV1,
filter: this.filterV1,
@ -72,6 +87,11 @@ class DelimiterVersions extends Delimiter {
return params;
}
genMDParamsV0Mig() {
const v0params = this.genMDParamsV0();
return listingParamsV0ToV0Mig(v0params);
}
genMDParamsV1() {
// return an array of two listing params sets to ask for
// synchronized listing of M and V ranges

View File

@ -59,10 +59,59 @@ function listingParamsMasterKeysV0ToV1(v0params) {
return v1params;
}
function listingParamsV0ToV0Mig(v0params) {
if ((v0params.gt !== undefined && v0params.gt >= inc(DbPrefixes.V1))
|| (v0params.gte !== undefined && v0params.gte >= inc(DbPrefixes.V1))
|| (v0params.lt !== undefined && v0params.lt <= DbPrefixes.V1)) {
return v0params;
}
if ((v0params.gt !== undefined && v0params.gt >= DbPrefixes.V1)
|| (v0params.gte !== undefined && v0params.gte >= DbPrefixes.V1)) {
const v0migParams = Object.assign({}, v0params);
let greaterParam;
if (v0params.gt !== undefined) {
v0migParams.gt = inc(DbPrefixes.V1);
greaterParam = v0migParams.gt;
}
if (v0params.gte !== undefined) {
v0migParams.gte = inc(DbPrefixes.V1);
greaterParam = v0migParams.gte;
}
if (v0params.lt !== undefined && greaterParam !== undefined
&& v0params.lt <= greaterParam) {
// we annihilated the valid range during our v0mig
// transform to skip V1 prefix: return an empty range with
// a trick instead of an invalid combo
return { lt: '' };
}
return v0migParams;
}
const rangeParams1 = {
lt: DbPrefixes.V1,
};
if (v0params.gt !== undefined) {
rangeParams1.gt = v0params.gt;
}
if (v0params.gte !== undefined) {
rangeParams1.gte = v0params.gte;
}
const rangeParams2 = {
gte: inc(DbPrefixes.V1),
// tell RepdServer._listObject() that the second listing is to
// be done after the first, not in parallel
serial: true,
};
if (v0params.lt !== undefined) {
rangeParams2.lt = v0params.lt;
}
return [rangeParams1, rangeParams2];
}
module.exports = {
checkLimit,
inc,
listingParamsMasterKeysV0ToV1,
listingParamsV0ToV0Mig,
SKIP_NONE,
FILTER_END,
FILTER_SKIP,

View File

@ -0,0 +1,56 @@
const stream = require('stream');
class SerialStream extends stream.Readable {
constructor(stream1, stream2) {
super({ objectMode: true });
this._streams = [stream1, stream2];
this._currentStream = stream1;
this._streamToResume = null;
stream1.on('data', item => this._onItem(stream1, item));
stream1.once('end', () => this._onEndStream1());
stream1.once('error', err => this._onError(stream1, err));
}
_read() {
if (this._streamToResume) {
this._streamToResume.resume();
this._streamToResume = null;
}
}
_destroy(err, callback) {
this._currentStream.destroy();
if (this._currentStream === this._streams[0]) {
this._streams[1].destroy();
}
callback();
}
_onItem(myStream, item) {
if (!this.push(item)) {
myStream.pause();
this._streamToResume = myStream;
}
}
_onEndStream1() {
// stream1 is done, now move on with data from stream2
const stream2 = this._streams[1];
stream2.on('data', item => this._onItem(stream2, item));
stream2.once('end', () => this._onEnd());
stream2.once('error', err => this._onError(stream2, err));
}
_onEnd() {
this.push(null);
}
_onError(myStream, err) {
this.emit('error', err);
this._destroy(err, () => {});
}
}
module.exports = SerialStream;

View File

@ -3,6 +3,7 @@ module.exports.VersioningConstants = {
Separator: '\0',
},
DbPrefixes: {
V1: '\x7f',
Master: '\x7fM',
Version: '\x7fV',
},

View File

@ -1,14 +1,28 @@
'use strict'; // eslint-disable-line strict
const assert = require('assert');
const MultipartUploads =
require('../../../../lib/algos/list/MPU').MultipartUploads;
const MultipartUploads = require('../../../../lib/algos/list/MPU').MultipartUploads;
const { inc } = require('../../../../lib/algos/list/tools');
const werelogs = require('werelogs').Logger;
// eslint-disable-next-line new-cap
const logger = new werelogs('listMpuTest');
const performListing = require('../../../utils/performListing');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
function getListingKey(key, vFormat) {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return key;
}
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
return `${DbPrefixes.Master}${key}`;
}
assert.fail(`bad vFormat ${vFormat}`);
return undefined;
}
describe('Multipart Uploads listing algorithm', () => {
const splitter = '**';
@ -16,22 +30,14 @@ describe('Multipart Uploads listing algorithm', () => {
const storageClass = 'STANDARD';
const initiator1 = { ID: '1', DisplayName: 'initiator1' };
const initiator2 = { ID: '2', DisplayName: 'initiator2' };
const keys = {
v0: [`${overviewPrefix}test/1${splitter}uploadId1`,
`${overviewPrefix}test/2${splitter}uploadId2`,
`${overviewPrefix}test/3${splitter}uploadId3`,
`${overviewPrefix}testMore/4${splitter}uploadId4`,
`${overviewPrefix}testMore/5${splitter}uploadId5`,
`${overviewPrefix}prefixTest/5${splitter}uploadId5`,
],
v1: [`${DbPrefixes.Master}${overviewPrefix}test/1${splitter}uploadId1`,
`${DbPrefixes.Master}${overviewPrefix}test/2${splitter}uploadId2`,
`${DbPrefixes.Master}${overviewPrefix}test/3${splitter}uploadId3`,
`${DbPrefixes.Master}${overviewPrefix}testMore/4${splitter}uploadId4`,
`${DbPrefixes.Master}${overviewPrefix}testMore/5${splitter}uploadId5`,
`${DbPrefixes.Master}${overviewPrefix}prefixTest/5${splitter}uploadId5`,
],
};
const v0keys = [
`${overviewPrefix}test/1${splitter}uploadId1`,
`${overviewPrefix}test/2${splitter}uploadId2`,
`${overviewPrefix}test/3${splitter}uploadId3`,
`${overviewPrefix}testMore/4${splitter}uploadId4`,
`${overviewPrefix}testMore/5${splitter}uploadId5`,
`${overviewPrefix}prefixTest/5${splitter}uploadId5`,
];
const values = [
JSON.stringify({
'key': 'test/1',
@ -128,9 +134,15 @@ describe('Multipart Uploads listing algorithm', () => {
done();
});
['v0', 'v1'].forEach(vFormat => {
const dbListing = keys[vFormat].map((key, i) => ({
key,
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
const dbListing = v0keys.map((key, i) => ({
key: getListingKey(key, vFormat),
value: values[i],
}));
it(`should perform a vFormat=${vFormat} listing of all keys`, () => {
@ -171,4 +183,90 @@ describe('Multipart Uploads listing algorithm', () => {
assert.deepStrictEqual(listingResult, expectedResult);
});
});
describe('MultipartUploads.genMDParams()', () => {
[{
listingParams: {
splitter,
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
},
}, {
listingParams: {
splitter,
prefix: 'foo/bar',
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {
gte: 'foo/bar',
lt: 'foo/bas',
},
[BucketVersioningKeyFormat.v0mig]: {
gte: 'foo/bar',
lt: 'foo/bas',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}foo/bar`,
lt: `${DbPrefixes.Master}foo/bas`,
},
},
}, {
listingParams: {
splitter,
keyMarker: 'marker',
},
mdParams: {
[BucketVersioningKeyFormat.v0]: {
gt: `${overviewPrefix}marker${inc(splitter)}`,
},
[BucketVersioningKeyFormat.v0mig]: [{
gt: `${overviewPrefix}marker${inc(splitter)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${overviewPrefix}marker${inc(splitter)}`,
lt: inc(DbPrefixes.Master),
},
},
}].forEach(testCase => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
it(`with vFormat=${vFormat}, listing params ${JSON.stringify(testCase.listingParams)}`, () => {
const delimiter = new MultipartUploads(
testCase.listingParams, logger, vFormat);
const mdParams = delimiter.genMDParams();
let paramsVFormat;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
// all above vformats are equivalent to v1 when it
// comes to generating md params
paramsVFormat = BucketVersioningKeyFormat.v1;
} else {
paramsVFormat = vFormat;
}
assert.deepStrictEqual(mdParams, testCase.mdParams[paramsVFormat]);
});
});
});
});
});

View File

@ -11,7 +11,7 @@ const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
class Test {
constructor(name, input, genMDParams, output, filter) {
@ -90,8 +90,14 @@ const receivedNonAlphaData = nonAlphabeticalData.map(
const tests = [
new Test('all elements', {}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -105,10 +111,17 @@ const tests = [
new Test('with valid marker', {
marker: receivedData[4].key,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -129,10 +142,17 @@ const tests = [
marker: 'zzzz',
delimiter: '/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -146,8 +166,14 @@ const tests = [
new Test('with makKeys', {
maxKeys: 3,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -161,8 +187,14 @@ const tests = [
new Test('with big makKeys', {
maxKeys: 15000,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -176,8 +208,14 @@ const tests = [
new Test('with delimiter', {
delimiter: '/',
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -193,8 +231,14 @@ const tests = [
new Test('with long delimiter', {
delimiter: 'notes/summer',
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -218,11 +262,15 @@ const tests = [
prefix: 'notes/summer/',
marker: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer${inc('/')}`,
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer${inc('/')}`,
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -237,11 +285,15 @@ const tests = [
delimiter: '/',
prefix: 'notes/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -264,11 +316,15 @@ const tests = [
prefix: 'notes/',
marker: 'notes/year.txt',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -289,11 +345,15 @@ const tests = [
marker: 'notes/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -311,11 +371,15 @@ const tests = [
marker: 'notes/spring/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/spring/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -333,11 +397,15 @@ const tests = [
marker: 'notes/summer/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -357,11 +425,15 @@ const tests = [
marker: 'notes/year.txt',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -381,11 +453,15 @@ const tests = [
marker: 'notes/yore.rs',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/yore.rs`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -400,8 +476,14 @@ const tests = [
new Test('all elements v2', {
v2: true,
}, {
v0: {},
v1: {
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
},
@ -416,10 +498,17 @@ const tests = [
startAfter: receivedData[4].key,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -441,10 +530,17 @@ const tests = [
delimiter: '/',
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -459,10 +555,17 @@ const tests = [
continuationToken: receivedData[4].key,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: receivedData[4].key,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: receivedData[4].key,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${receivedData[4].key}`,
lt: inc(DbPrefixes.Master),
},
@ -484,10 +587,17 @@ const tests = [
delimiter: '/',
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'zzzz',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'zzzz',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}zzzz`,
lt: inc(DbPrefixes.Master),
},
@ -503,11 +613,15 @@ const tests = [
prefix: 'notes/summer/',
startAfter: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -523,11 +637,15 @@ const tests = [
prefix: 'notes/summer/',
continuationToken: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/summer/',
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
},
@ -544,10 +662,17 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
},
v1: {
[BucketVersioningKeyFormat.v0mig]: [{
gt: 'notes/year.txt',
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: inc(DbPrefixes.Master),
},
@ -568,11 +693,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -591,11 +720,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/spring/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/spring/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -614,11 +747,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/summer/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/summer/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -639,11 +776,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/year.txt',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/year.txt`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -664,11 +805,15 @@ const tests = [
maxKeys: 1,
v2: true,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
v1: {
[BucketVersioningKeyFormat.v0mig]: {
gt: 'notes/yore.rs',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}notes/yore.rs`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
},
@ -680,6 +825,109 @@ const tests = [
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with startAfter after vformat V1 prefix', {
delimiter: '/',
startAfter: 'éléphant pâle',
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gt: 'éléphant pâle',
},
[BucketVersioningKeyFormat.v0mig]: {
gt: 'éléphant pâle',
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}éléphant pâle`,
lt: inc(DbPrefixes.Master),
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with startAfter inside vformat V1 prefix', {
delimiter: '/',
startAfter: `${DbPrefixes.V1}foo`,
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gt: `${DbPrefixes.V1}foo`,
},
// v0mig skips all V1-prefixed keys to start at the beginning
// of the second v0 range (to skip V1 keys being migrated)
[BucketVersioningKeyFormat.v0mig]: {
gt: inc(DbPrefixes.V1),
},
[BucketVersioningKeyFormat.v1]: {
gt: `${DbPrefixes.Master}${DbPrefixes.V1}foo`,
lt: inc(DbPrefixes.Master),
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with prefix after vformat V1 prefix', {
delimiter: '/',
prefix: 'éléphant pâle/',
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gte: 'éléphant pâle/',
lt: 'éléphant pâle0',
},
[BucketVersioningKeyFormat.v0mig]: {
gte: 'éléphant pâle/',
lt: 'éléphant pâle0',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}éléphant pâle/`,
lt: `${DbPrefixes.Master}éléphant pâle0`,
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
new Test('with prefix inside vformat V1 prefix', {
delimiter: '/',
prefix: `${DbPrefixes.V1}foo/`,
maxKeys: 3,
v2: true,
}, {
[BucketVersioningKeyFormat.v0]: {
gte: `${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.V1}foo0`,
},
// v0mig skips the V1 prefix altogether to avoid returning V1
// keys being migrated. It uses a trick: passing "lt: ''"
// (empty string) forces to list an empty range
[BucketVersioningKeyFormat.v0mig]: {
lt: '',
},
[BucketVersioningKeyFormat.v1]: {
gte: `${DbPrefixes.Master}${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.Master}${DbPrefixes.V1}foo0`,
},
}, {
Contents: [],
CommonPrefixes: [],
Delimiter: '/',
IsTruncated: false,
NextContinuationToken: undefined,
}, (e, input) => e.key > input.startAfter),
];
const alphabeticalOrderTests = [
@ -708,10 +956,13 @@ function getTestListing(test, data, vFormat) {
return data
.filter(e => test.filter(e, test.input))
.map(obj => {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return obj;
}
if (vFormat === 'v1') {
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
return {
key: `${DbPrefixes.Master}${obj.key}`,
value: obj.value,
@ -721,18 +972,39 @@ function getTestListing(test, data, vFormat) {
});
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`vFormat=${vFormat} Delimiter listing algorithm`, () => {
it('Should return good skipping value for DelimiterMaster', () => {
const delimiter = new DelimiterMaster({ delimiter: '/' });
for (let i = 0; i < 100; i++) {
let key;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
key = `${DbPrefixes.Master}foo/${zpad(i)}`;
} else {
key = `foo/${zpad(i)}`;
}
delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
key,
value: '{}',
});
}
assert.strictEqual(delimiter.skipping(),
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
let skipping;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
skipping = `${DbPrefixes.Master}foo/`;
} else {
skipping = 'foo/';
}
assert.strictEqual(delimiter.skipping(), skipping);
});
it('Should set Delimiter alphabeticalOrder field to the expected value', () => {
@ -748,7 +1020,17 @@ function getTestListing(test, data, vFormat) {
it(`Should return metadata listing params to list ${test.name}`, () => {
const listing = new Delimiter(test.input, logger, vFormat);
const params = listing.genMDParams();
assert.deepStrictEqual(params, test.genMDParams[vFormat]);
let paramsVFormat;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
// all above vformats are equivalent to v1 when it
// comes to generating md params
paramsVFormat = BucketVersioningKeyFormat.v1;
} else {
paramsVFormat = vFormat;
}
assert.deepStrictEqual(params, test.genMDParams[paramsVFormat]);
});
it(`Should list ${test.name}`, () => {
// Simulate skip scan done by LevelDB
@ -759,7 +1041,7 @@ function getTestListing(test, data, vFormat) {
});
// Only v0 gets a listing of master and version keys together.
if (vFormat === 'v0') {
if (vFormat === BucketVersioningKeyFormat.v0) {
tests.forEach(test => {
it(`Should list master versions ${test.name}`, () => {
// Simulate skip scan done by LevelDB

View File

@ -13,7 +13,7 @@ const VSConst =
require('../../../../lib/versioning/constants').VersioningConstants;
const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
@ -35,16 +35,25 @@ const fakeLogger = {
};
function getListingKey(key, vFormat) {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return key;
}
if (vFormat === 'v1') {
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
return `${DbPrefixes.Master}${key}`;
}
return assert.fail(`bad vFormat ${vFormat}`);
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`Delimiter All masters listing algorithm vFormat=${vFormat}`, () => {
it('should return SKIP_NONE for DelimiterMaster when both NextMarker ' +
'and NextContinuationToken are undefined', () => {
@ -102,7 +111,9 @@ function getListingKey(key, vFormat) {
/* When a delimiter is set and the NextMarker ends with the
* delimiter it should return the next marker value. */
assert.strictEqual(delimiter.NextMarker, keyWithEndingDelimiter);
const skipKey = vFormat === 'v1' ?
const skipKey = [BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat) ?
`${DbPrefixes.Master}${keyWithEndingDelimiter}` :
keyWithEndingDelimiter;
assert.strictEqual(delimiter.skipping(), skipKey);
@ -135,7 +146,8 @@ function getListingKey(key, vFormat) {
const listingKey = getListingKey(key, vFormat);
assert.strictEqual(delimiter.filter({ key: listingKey, value }), FILTER_ACCEPT);
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
assert.strictEqual(delimiter.prvKey, key);
}
assert.strictEqual(delimiter.NextMarker, key);
@ -206,7 +218,8 @@ function getListingKey(key, vFormat) {
});
});
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
it('should accept a PHD version as first input', () => {
const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
const keyPHD = 'keyPHD';

View File

@ -9,7 +9,7 @@ const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
class Test {
@ -32,75 +32,42 @@ const bar = '{"versionId":"bar"}';
const qux = '{"versionId":"qux"}';
const valuePHD = '{"isPHD":"true","versionId":"1234567890abcdefg"}';
const valueDeleteMarker = '{"hello":"world","isDeleteMarker":"true"}';
const dataVersioned = {
v0: [
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: 'notes/spring/2.txt', value: valuePHD },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: 'notes/summer/4.txt', value: valuePHD },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
{ key: 'notes/summer/44.txt', value: valuePHD },
{ key: 'notes/summer/444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/4444.txt', value: valuePHD },
{ key: 'notes/summer/44444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/444444.txt', value: valuePHD },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
],
v1: [ // we add M and V prefixes in getTestListing() due to the
// test cases needing the original key to filter
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
// Compared to v0, the two following keys are version keys
// that we give a version ID, because delete markers do not
// have a master key in v1.
{ key: `notes/summer/444.txt${VID_SEP}null`, value: valueDeleteMarker },
{ key: `notes/summer/44444.txt${VID_SEP}null`, value: valueDeleteMarker },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
],
};
const rawListingData = [
{ key: 'Pâtisserie=中文-español-English', value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}bar`, value: bar },
{ key: `Pâtisserie=中文-español-English${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/1.txt', value: bar },
{ key: `notes/spring/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/1.txt${VID_SEP}foo`, value: foo },
{ key: `notes/spring/1.txt${VID_SEP}qux`, value: qux },
{ key: 'notes/spring/2.txt', value: valuePHD },
{ key: `notes/spring/2.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/spring/2.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/spring/march/1.txt',
value: '{"versionId":"null","isNull":true}' },
{ key: `notes/spring/march/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/spring/march/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/1.txt', value: bar },
{ key: `notes/summer/1.txt${VID_SEP}bar`, value: bar },
{ key: `notes/summer/1.txt${VID_SEP}foo`, value: foo },
{ key: 'notes/summer/2.txt', value: bar },
{ key: `notes/summer/2.txt${VID_SEP}bar`, value: bar },
{ key: 'notes/summer/4.txt', value: valuePHD },
{ key: `notes/summer/4.txt${VID_SEP}bar`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}foo`, value: valueDeleteMarker },
{ key: `notes/summer/4.txt${VID_SEP}qux`, value: valueDeleteMarker },
{ key: 'notes/summer/44.txt', value: valuePHD },
{ key: 'notes/summer/444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/4444.txt', value: valuePHD },
{ key: 'notes/summer/44444.txt', value: valueDeleteMarker },
{ key: 'notes/summer/444444.txt', value: valuePHD },
{ key: 'notes/summer/august/1.txt', value },
{ key: 'notes/year.txt', value },
{ key: 'notes/yore.rs', value },
{ key: 'notes/zaphod/Beeblebrox.txt', value },
];
const receivedData = [
{ key: 'Pâtisserie=中文-español-English', value: bar, versionId: 'bar' },
{ key: 'Pâtisserie=中文-español-English', value: foo, versionId: 'foo' },
@ -130,9 +97,20 @@ const receivedData = [
];
const tests = [
new Test('all versions', {}, {
v0: {},
v1: [{ gte: DbPrefixes.Master, lt: inc(DbPrefixes.Master) },
{ gte: DbPrefixes.Version, lt: inc(DbPrefixes.Version) }],
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
gte: DbPrefixes.Version,
lt: inc(DbPrefixes.Version),
}],
}, {
Versions: receivedData,
CommonPrefixes: [],
@ -144,10 +122,17 @@ const tests = [
new Test('with valid key marker', {
keyMarker: receivedData[3].key,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `${receivedData[3].key}\u0001`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: [{
gt: `${receivedData[3].key}\u0001`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}${receivedData[3].key}${inc(VID_SEP)}`,
lt: inc(DbPrefixes.Master),
}, {
@ -166,10 +151,17 @@ const tests = [
keyMarker: 'zzzz',
delimiter: '/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `zzzz${inc(VID_SEP)}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: [{
gt: `zzzz${inc(VID_SEP)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}zzzz${inc(VID_SEP)}`,
lt: inc(DbPrefixes.Master),
}, {
@ -187,8 +179,14 @@ const tests = [
new Test('with maxKeys', {
maxKeys: 3,
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -206,8 +204,14 @@ const tests = [
new Test('with big maxKeys', {
maxKeys: 15000,
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -225,8 +229,14 @@ const tests = [
new Test('with delimiter', {
delimiter: '/',
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -247,8 +257,14 @@ const tests = [
new Test('with long delimiter', {
delimiter: 'notes/summer',
}, {
v0: {},
v1: [{
[BucketVersioningKeyFormat.v0]: {},
[BucketVersioningKeyFormat.v0mig]: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
[BucketVersioningKeyFormat.v1]: [{
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
}, {
@ -269,11 +285,15 @@ const tests = [
prefix: 'notes/summer/',
keyMarker: 'notes/summer0',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer0${inc(VID_SEP)}`,
lt: `notes/summer${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer0${inc(VID_SEP)}`,
lt: `notes/summer${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/summer0${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes/summer${inc('/')}`,
}, {
@ -292,11 +312,15 @@ const tests = [
delimiter: '/',
prefix: 'notes/',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gte: 'notes/',
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gte: `${DbPrefixes.Master}notes/`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -323,11 +347,15 @@ const tests = [
prefix: 'notes/',
keyMarker: 'notes/year.txt',
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -352,11 +380,15 @@ const tests = [
keyMarker: 'notes/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -378,11 +410,15 @@ const tests = [
keyMarker: 'notes/spring/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/spring/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/spring/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/spring/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -404,11 +440,15 @@ const tests = [
keyMarker: 'notes/summer/',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/summer/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/summer/${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/summer/${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -432,11 +472,15 @@ const tests = [
keyMarker: 'notes/year.txt',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/year.txt${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/year.txt${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -460,11 +504,15 @@ const tests = [
keyMarker: 'notes/yore.rs',
maxKeys: 1,
}, {
v0: {
[BucketVersioningKeyFormat.v0]: {
gt: `notes/yore.rs${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
v1: [{
[BucketVersioningKeyFormat.v0mig]: {
gt: `notes/yore.rs${inc(VID_SEP)}`,
lt: `notes${inc('/')}`,
},
[BucketVersioningKeyFormat.v1]: [{
gt: `${DbPrefixes.Master}notes/yore.rs${inc(VID_SEP)}`,
lt: `${DbPrefixes.Master}notes${inc('/')}`,
}, {
@ -481,14 +529,20 @@ const tests = [
}, (e, input) => e.key > input.keyMarker),
];
function getTestListing(test, data, vFormat) {
return data
function getTestListing(test, vFormat) {
return rawListingData
.filter(e => [BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)
|| e.value !== valuePHD)
.filter(e => test.filter(e, test.input))
.map(e => {
if (vFormat === 'v0') {
if ([BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig].includes(vFormat)) {
return e;
}
if (vFormat === 'v1') {
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
const keyPrefix = e.key.includes(VID_SEP) ?
DbPrefixes.Version : DbPrefixes.Master;
return {
@ -500,29 +554,60 @@ function getTestListing(test, data, vFormat) {
});
}
['v0', 'v1'].forEach(vFormat => {
[
BucketVersioningKeyFormat.v0,
BucketVersioningKeyFormat.v0mig,
BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1,
].forEach(vFormat => {
describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
it('Should return good skipping value for DelimiterVersions', () => {
const delimiter = new DelimiterVersions({ delimiter: '/' });
for (let i = 0; i < 100; i++) {
let key;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
key = `${DbPrefixes.Master}foo/${zpad(i)}`;
} else {
key = `foo/${zpad(i)}`;
}
delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
key,
value: '{}',
});
}
assert.strictEqual(delimiter.skipping(),
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
let skipping;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
skipping = `${DbPrefixes.Master}foo/`;
} else {
skipping = 'foo/';
}
assert.strictEqual(delimiter.skipping(), skipping);
});
tests.forEach(test => {
it(`Should return metadata listing params to list ${test.name}`, () => {
const listing = new DelimiterVersions(test.input, logger, vFormat);
const params = listing.genMDParams();
assert.deepStrictEqual(params, test.genMDParams[vFormat]);
let paramsVFormat;
if ([BucketVersioningKeyFormat.v0v1,
BucketVersioningKeyFormat.v1mig,
BucketVersioningKeyFormat.v1].includes(vFormat)) {
// all above vformats are equivalent to v1 when it
// comes to generating md params
paramsVFormat = BucketVersioningKeyFormat.v1;
} else {
paramsVFormat = vFormat;
}
assert.deepStrictEqual(params, test.genMDParams[paramsVFormat]);
});
it(`Should list ${test.name}`, () => {
// Simulate skip scan done by LevelDB
const d = getTestListing(test, dataVersioned[vFormat], vFormat);
const d = getTestListing(test, vFormat);
const res = performListing(d, DelimiterVersions, test.input, logger, vFormat);
assert.deepStrictEqual(res, test.output);
});

View File

@ -2,10 +2,11 @@
const assert = require('assert');
const { checkLimit, inc, listingParamsMasterKeysV0ToV1 } =
const { checkLimit, inc, listingParamsMasterKeysV0ToV1, listingParamsV0ToV0Mig } =
require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const { DbPrefixes } = VSConst;
const VID_SEP = VSConst.VersionId.Separator;
describe('checkLimit function', () => {
const tests = [
@ -102,3 +103,93 @@ describe('listingParamsMasterKeysV0ToV1', () => {
});
});
});
describe('listingParamsV0ToV0Mig', () => {
const testCases = [
{
v0params: {},
v0migparams: [{
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gte: 'foo/bar',
lt: 'foo/bas',
},
v0migparams: {
gte: 'foo/bar',
lt: 'foo/bas',
},
}, {
v0params: {
gt: `foo/bar${inc(VID_SEP)}`,
},
v0migparams: [{
gt: `foo/bar${inc(VID_SEP)}`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gt: `foo/bar${VID_SEP}versionId`,
},
v0migparams: [{
gt: `foo/bar${VID_SEP}versionId`,
lt: DbPrefixes.V1,
}, {
gte: inc(DbPrefixes.V1),
serial: true,
}],
}, {
v0params: {
gt: `foo/bar/baz${VID_SEP}versionId`,
lt: 'foo/bas',
},
v0migparams: {
gt: `foo/bar/baz${VID_SEP}versionId`,
lt: 'foo/bas',
},
}, {
v0params: {
gt: `éléphant rose${VID_SEP}versionId`,
},
v0migparams: {
gt: `éléphant rose${VID_SEP}versionId`,
},
}, {
v0params: {
gte: 'éléphant rose',
lt: 'éléphant rosf',
},
v0migparams: {
gte: 'éléphant rose',
lt: 'éléphant rosf',
},
}, {
v0params: {
gt: `${DbPrefixes.V1}foo`,
},
v0migparams: {
gt: inc(DbPrefixes.V1),
},
}, {
v0params: {
gte: `${DbPrefixes.V1}foo/`,
lt: `${DbPrefixes.V1}foo0`,
},
v0migparams: {
lt: '',
},
}];
testCases.forEach(({ v0params, v0migparams }) => {
it(`${JSON.stringify(v0params)} => ${JSON.stringify(v0migparams)}`, () => {
const converted = listingParamsV0ToV0Mig(v0params);
assert.deepStrictEqual(converted, v0migparams);
});
});
});

View File

@ -1,38 +1,6 @@
const assert = require('assert');
const stream = require('stream');
const MergeStream = require('../../../../lib/algos/stream/MergeStream');
class Streamify extends stream.Readable {
constructor(objectsToSend, errorAtEnd) {
super({ objectMode: true });
this._remaining = Array.from(objectsToSend);
this._remaining.reverse();
this._errorAtEnd = errorAtEnd || false;
this._ended = false;
this._destroyed = false;
}
_read() {
process.nextTick(() => {
while (this._remaining.length > 0) {
const item = this._remaining.pop();
if (!this.push(item)) {
return undefined;
}
}
if (this._errorAtEnd) {
return this.emit('error', new Error('OOPS'));
}
this._ended = true;
return this.push(null);
});
}
_destroy(err, callback) {
this._destroyed = true;
callback();
}
}
const Streamify = require('./Streamify');
function readAll(stream, usePauseResume, cb) {
const result = [];
@ -168,12 +136,16 @@ describe('MergeStream', () => {
`${testCasePretty(testCase, false)}` +
`${usePauseResume ? ' with pause/resume' : ''}` +
`${errorAtEnd ? ' with error' : ''}`;
const testDescRev =
`${testCasePretty(testCase, true)}` +
`${usePauseResume ? ' with pause/resume' : ''}` +
`${errorAtEnd ? ' with error' : ''}`;
it(`should cover ${testDesc}`, done => {
testMergeStreamWithIntegers(
testCase.stream1, testCase.stream2,
usePauseResume, errorAtEnd, done);
});
it(`should cover ${testDesc}`, done => {
it(`should cover ${testDescRev}`, done => {
testMergeStreamWithIntegers(
testCase.stream2, testCase.stream1,
usePauseResume, errorAtEnd, done);

View File

@ -0,0 +1,153 @@
const assert = require('assert');
const SerialStream = require('../../../../lib/algos/stream/SerialStream');
const Streamify = require('./Streamify');
function readAll(stream, usePauseResume, cb) {
const result = [];
stream.on('data', item => {
result.push(item);
if (usePauseResume) {
stream.pause();
setTimeout(() => stream.resume(), 1);
}
});
stream.once('end', () => cb(null, result));
stream.once('error', err => cb(err));
}
function testSerialStreamWithIntegers(contents1, contents2,
usePauseResume, errorAtEnd, cb) {
const expectedItems = contents1.concat(contents2);
const serialStream = new SerialStream(
new Streamify(contents1, errorAtEnd)
.on('error', () => {}),
new Streamify(contents2)
.on('error', () => {}));
readAll(serialStream, usePauseResume, (err, readItems) => {
if (errorAtEnd) {
assert(err);
} else {
assert.ifError(err);
assert.deepStrictEqual(readItems, expectedItems);
}
cb();
});
}
function testCasePretty(testCase, reversed) {
const desc1 = JSON.stringify(
reversed ? testCase.stream2 : testCase.stream1);
const desc2 = JSON.stringify(
reversed ? testCase.stream1 : testCase.stream2);
return `${desc1} concatenated with ${desc2}`;
}
describe('SerialStream', () => {
[
{
stream1: [],
stream2: [],
},
{
stream1: [0],
stream2: [],
},
{
stream1: [0, 1, 2, 3, 4],
stream2: [],
},
{
stream1: [0],
stream2: [1],
},
{
stream1: [1, 2, 3, 4, 5],
stream2: [6],
},
{
stream1: [1, 2, 3, 4, 5],
stream2: [6, 7, 8, 9, 10],
},
].forEach(testCase => {
[false, true].forEach(usePauseResume => {
[false, true].forEach(errorAtEnd => {
const testDesc =
`${testCasePretty(testCase, false)}` +
`${usePauseResume ? ' with pause/resume' : ''}` +
`${errorAtEnd ? ' with error' : ''}`;
const testDescRev =
`${testCasePretty(testCase, true)}` +
`${usePauseResume ? ' with pause/resume' : ''}` +
`${errorAtEnd ? ' with error' : ''}`;
it(`should cover ${testDesc}`, done => {
testSerialStreamWithIntegers(
testCase.stream1, testCase.stream2,
usePauseResume, errorAtEnd, done);
});
it(`should cover ${testDescRev}`, done => {
testSerialStreamWithIntegers(
testCase.stream2, testCase.stream1,
usePauseResume, errorAtEnd, done);
});
});
});
});
[100, 1000, 10000, 100000].forEach(nbEntries => {
[false, true].forEach(usePauseResume => {
[false, true].forEach(errorAtEnd => {
if ((!usePauseResume && !errorAtEnd) || nbEntries <= 1000) {
const fixtureDesc =
`${usePauseResume ? ' with pause/resume' : ''}` +
`${errorAtEnd ? ' with error' : ''}`;
it(`${nbEntries} entries${fixtureDesc}`,
function bigConcatSequential(done) {
this.timeout(10000);
const stream1 = [];
const stream2 = [];
for (let i = 0; i < nbEntries / 2; ++i) {
stream1.push(i);
}
for (let i = nbEntries / 2; i < nbEntries; ++i) {
stream2.push(i);
}
testSerialStreamWithIntegers(
stream1, stream2, usePauseResume, errorAtEnd, done);
});
}
});
});
});
// with 3 items per input stream, we reach the end of stream even
// though destroy() has been called (due to buffering), while with
// 100 items input streams are aborted before emitting the 'end'
// event, so it's useful to test both cases
[3, 100].forEach(nbItemsPerStream => {
it(`destroy() should destroy both inner streams with ${nbItemsPerStream} items per stream`,
done => {
const stream1 = new Streamify(new Array(nbItemsPerStream).fill()
.map((e, i) => i));
const stream2 = new Streamify(new Array(nbItemsPerStream).fill()
.map((e, i) => nbItemsPerStream + i));
const serialStream = new SerialStream(stream1, stream2);
serialStream.on('data', item => {
if (item === 5) {
serialStream.destroy();
const s1ended = stream1._ended;
const s2ended = stream2._ended;
setTimeout(() => {
if (!s1ended) {
assert(stream1._destroyed);
}
if (!s2ended) {
assert(stream2._destroyed);
}
done();
}, 10);
}
});
serialStream.once('error', err => {
assert.fail(`unexpected error: ${err.message}`);
});
});
});
});

View File

@ -0,0 +1,35 @@
const stream = require('stream');
class Streamify extends stream.Readable {
constructor(objectsToSend, errorAtEnd) {
super({ objectMode: true });
this._remaining = Array.from(objectsToSend);
this._remaining.reverse();
this._errorAtEnd = errorAtEnd || false;
this._ended = false;
this._destroyed = false;
}
_read() {
process.nextTick(() => {
while (this._remaining.length > 0) {
const item = this._remaining.pop();
if (!this.push(item)) {
return undefined;
}
}
if (this._errorAtEnd) {
return this.emit('error', new Error('OOPS'));
}
this._ended = true;
return this.push(null);
});
}
_destroy(err, callback) {
this._destroyed = true;
callback();
}
}
module.exports = Streamify;