Compare commits
No commits in common. "7293f93c229cb33995e9ff7a0b4adfdb177a7e19" and "4c6712741b77caf4be4f4ee7a9394cc3102f10e6" have entirely different histories.
7293f93c22
...
4c6712741b
|
|
@ -271,4 +271,4 @@ class Delimiter extends Extension {
|
|||
}
|
||||
}
|
||||
|
||||
module.exports = { Delimiter };
|
||||
module.exports = { Delimiter, getCommonPrefix };
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ class DelimiterMaster extends Delimiter {
|
|||
this.prvKey = undefined;
|
||||
this.prvPHDKey = undefined;
|
||||
this.inReplayPrefix = false;
|
||||
this.prefixKeySeen = false;
|
||||
this.prefixEndsWithDelim = this.prefix && this.prefix.endsWith(this.delimiter);
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
|
|
@ -62,6 +64,10 @@ class DelimiterMaster extends Delimiter {
|
|||
let key = obj.key;
|
||||
const value = obj.value;
|
||||
|
||||
if (key === this.prefix) {
|
||||
this.prefixKeySeen = true;
|
||||
}
|
||||
|
||||
if (key.startsWith(DbPrefixes.Replay)) {
|
||||
this.inReplayPrefix = true;
|
||||
return FILTER_SKIP;
|
||||
|
|
@ -95,9 +101,8 @@ class DelimiterMaster extends Delimiter {
|
|||
* NextMarker to the common prefix instead of the whole key
|
||||
* value. (TODO: remove this test once ZENKO-1048 is fixed)
|
||||
* */
|
||||
if (key === this.prvKey || key === this[this.nextContinueMarker] ||
|
||||
(this.delimiter &&
|
||||
key.startsWith(this[this.nextContinueMarker]))) {
|
||||
if (key === this.prvKey || key === this[this.nextContinueMarker]
|
||||
|| (this.delimiter && key.startsWith(this[this.nextContinueMarker]))) {
|
||||
/* master version already filtered */
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
|
|
@ -127,6 +132,14 @@ class DelimiterMaster extends Delimiter {
|
|||
return FILTER_ACCEPT;
|
||||
}
|
||||
this.prvKey = key;
|
||||
if (this.prefixEndsWithDelim) {
|
||||
/* When the prefix ends with a delimiter, update nextContinueMarker
|
||||
* to be able to skip ranges of the form prefix/subprefix/ as an optimization.
|
||||
* The marker may also end up being prefix/, in which case .skipping will determine
|
||||
* if a skip over the full range is allowed or a smaller skipping range of prefix/{VID_SEP}
|
||||
* must be used. */
|
||||
this[this.nextContinueMarker] = key.slice(0, key.lastIndexOf(this.delimiter) + 1);
|
||||
}
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
|
||||
|
|
@ -162,6 +175,27 @@ class DelimiterMaster extends Delimiter {
|
|||
return super.filter(obj);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if a nextContinueMarker ending with a delimiter
|
||||
* can be skipped.
|
||||
* @returns {bool}
|
||||
*/
|
||||
|
||||
allowDelimiterRangeSkip() {
|
||||
if (!this.prefixKeySeen) {
|
||||
// A prefix key is a master key equal to the prefix. If it has
|
||||
// not been encountered, can skip.
|
||||
return true;
|
||||
}
|
||||
const marker = this[this.nextContinueMarker];
|
||||
// prefix = prefix, key = prefix/. Can skip since key will be part of commonPrefixes.
|
||||
if (marker.length > this.prefix.length && marker.startsWith(this.prefix)) {
|
||||
return true;
|
||||
}
|
||||
const lastIdx = marker.lastIndexOf(this.delimiter); // prefix/foo is a masterKey following a prefix key.
|
||||
return marker.slice(0, lastIdx + 1) !== this.prefix; // cannot skip the full range prefix/ range.
|
||||
}
|
||||
|
||||
skippingBase() {
|
||||
if (this[this.nextContinueMarker]) {
|
||||
// next marker or next continuation token:
|
||||
|
|
@ -169,7 +203,7 @@ class DelimiterMaster extends Delimiter {
|
|||
// - foo : skipping foo.
|
||||
const index = this[this.nextContinueMarker].
|
||||
lastIndexOf(this.delimiter);
|
||||
if (index === this[this.nextContinueMarker].length - 1) {
|
||||
if (index === this[this.nextContinueMarker].length - 1 && this.allowDelimiterRangeSkip()) {
|
||||
return this[this.nextContinueMarker];
|
||||
}
|
||||
return this[this.nextContinueMarker] + VID_SEP;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"version": "7.10.36-1",
|
||||
"version": "7.10.36",
|
||||
"description": "Common utilities for the S3 project components",
|
||||
"main": "build/index.js",
|
||||
"repository": {
|
||||
|
|
@ -62,6 +62,7 @@
|
|||
"@types/jest": "^27.4.1",
|
||||
"@types/node": "^17.0.21",
|
||||
"@types/xml2js": "^0.4.11",
|
||||
"chance": "^1.1.8",
|
||||
"eslint": "^8.12.0",
|
||||
"eslint-config-airbnb": "6.2.0",
|
||||
"eslint-config-scality": "scality/Guidelines#7.10.2",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const chance = require('chance').Chance(); // eslint-disable-line
|
||||
const DelimiterMaster =
|
||||
require('../../../../lib/algos/list/delimiterMaster').DelimiterMaster;
|
||||
const {
|
||||
|
|
@ -16,8 +16,6 @@ const Version = require('../../../../lib/versioning/Version').Version;
|
|||
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
|
||||
const { DbPrefixes } = VSConst;
|
||||
const zpad = require('../../helpers').zpad;
|
||||
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const EmptyResult = {
|
||||
CommonPrefixes: [],
|
||||
|
|
@ -488,6 +486,336 @@ function getListingKey(key, vFormat) {
|
|||
// ...it should return to skipping by prefix as usual
|
||||
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
|
||||
});
|
||||
|
||||
it('should not skip over whole prefix when a key equals the prefix and ends with delimiter', () => {
|
||||
for (const prefix of ['prefix/', 'prefix/subprefix/']) {
|
||||
const delimiter = new DelimiterMaster({
|
||||
prefix,
|
||||
delimiter: '/',
|
||||
}, fakeLogger, vFormat);
|
||||
for (const testEntry of [
|
||||
{
|
||||
key: prefix,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}${VID_SEP}v1`,
|
||||
value: '{}',
|
||||
expectedRes: FILTER_SKIP, // versions get skipped after master
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}deleted`,
|
||||
isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP, // delete markers get skipped
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}deleted${VID_SEP}v1`,
|
||||
isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}deleted${VID_SEP}v2`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}notdeleted`,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}notdeleted${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}notdeleted${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}notdeleted${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}subprefix1/key-1`,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}subprefix1/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}subprefix1/key-1${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}subprefix1/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}subprefix1/key-2`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}subprefix1/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}subprefix1/key-2${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}subprefix1/`,
|
||||
},
|
||||
]) {
|
||||
const entry = {
|
||||
key: testEntry.key,
|
||||
};
|
||||
if (testEntry.isDeleteMarker) {
|
||||
entry.value = '{"isDeleteMarker":true}';
|
||||
} else {
|
||||
entry.value = '{}';
|
||||
}
|
||||
const res = delimiter.filter(entry);
|
||||
const skipping = delimiter.skipping();
|
||||
assert.strictEqual(res, testEntry.expectedRes);
|
||||
assert.strictEqual(skipping, testEntry.expectedSkipping);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should skip over whole prefix when a key equals the prefix and does not end with delimiter', () => {
|
||||
for (const prefix of ['prefix', 'prefix/subprefix']) {
|
||||
const delimiter = new DelimiterMaster({
|
||||
prefix,
|
||||
delimiter: '/',
|
||||
}, fakeLogger, vFormat);
|
||||
for (const testEntry of [
|
||||
{
|
||||
key: prefix,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}/`,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}/${VID_SEP}v1`,
|
||||
value: '{}',
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}/`, // common prefix already seen
|
||||
},
|
||||
{
|
||||
key: `${prefix}/deleted`,
|
||||
isDeleteMarker: true, // skipped delete marker
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}/`, // already added to common prefix
|
||||
},
|
||||
{
|
||||
key: `${prefix}/notdeleted`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}/`, // already added to common prefix
|
||||
},
|
||||
{
|
||||
key: `${prefix}ed`,
|
||||
isDeleteMarker: false,
|
||||
expectedRes: FILTER_ACCEPT, // new master key seen
|
||||
expectedSkipping: `${prefix}ed${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}ed/`,
|
||||
expectedRes: FILTER_ACCEPT, // new master key ending with prefix
|
||||
expectedSkipping: `${prefix}ed/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}ed/subprefix1/key-1`,
|
||||
expectedRes: FILTER_SKIP, // already have prefixed/ common prefix
|
||||
expectedSkipping: `${prefix}ed/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}ed/subprefix1/key-1${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}ed/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}ed/subprefix1/key-2`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}ed/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}ed/subprefix1/key-2${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}ed/`,
|
||||
},
|
||||
]) {
|
||||
const entry = {
|
||||
key: testEntry.key,
|
||||
};
|
||||
if (testEntry.isDeleteMarker) {
|
||||
entry.value = '{"isDeleteMarker":true}';
|
||||
} else {
|
||||
entry.value = '{}';
|
||||
}
|
||||
const res = delimiter.filter(entry);
|
||||
const skipping = delimiter.skipping();
|
||||
assert.strictEqual(res, testEntry.expectedRes);
|
||||
assert.strictEqual(skipping, testEntry.expectedSkipping);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should not skip over whole prefix when key equals the prefix and prefix key has delete marker ' +
|
||||
'and prefix ends with delimiter', () => {
|
||||
for (const prefix of ['prefix/', 'prefix/subprefix/']) {
|
||||
const delimiter = new DelimiterMaster({
|
||||
prefix,
|
||||
delimiter: '/',
|
||||
}, fakeLogger, vFormat);
|
||||
for (const testEntry of [
|
||||
{
|
||||
key: prefix,
|
||||
isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}subprefix-1`,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}subprefix-1${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}subprefix-1/foo`,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}subprefix-1/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}subprefix-1/bar`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}subprefix-1/`, // already added to common prefix
|
||||
},
|
||||
]) {
|
||||
const entry = {
|
||||
key: testEntry.key,
|
||||
};
|
||||
if (testEntry.isDeleteMarker) {
|
||||
entry.value = '{"isDeleteMarker":true}';
|
||||
} else {
|
||||
entry.value = '{}';
|
||||
}
|
||||
const res = delimiter.filter(entry);
|
||||
const skipping = delimiter.skipping();
|
||||
assert.strictEqual(res, testEntry.expectedRes);
|
||||
assert.strictEqual(skipping, testEntry.expectedSkipping);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should skip over whole prefix when key equals the prefix, prefix key has delete marker ' +
|
||||
'and prefix does not end with delimiter', () => {
|
||||
for (const prefix of ['prefix', 'prefix/subprefix']) {
|
||||
const delimiter = new DelimiterMaster({
|
||||
prefix,
|
||||
delimiter: '/',
|
||||
}, fakeLogger, vFormat);
|
||||
for (const testEntry of [
|
||||
{
|
||||
key: prefix,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}${VID_SEP}v1`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}/`,
|
||||
isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}/subprefix-1`,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}/subprefix-1/foo`,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}aa`,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}aa${VID_SEP}`, // already added to common prefix
|
||||
},
|
||||
]) {
|
||||
const entry = {
|
||||
key: testEntry.key,
|
||||
};
|
||||
if (testEntry.isDeleteMarker) {
|
||||
entry.value = '{"isDeleteMarker":true}';
|
||||
} else {
|
||||
entry.value = '{}';
|
||||
}
|
||||
const res = delimiter.filter(entry);
|
||||
const skipping = delimiter.skipping();
|
||||
assert.strictEqual(res, testEntry.expectedRes);
|
||||
assert.strictEqual(skipping, testEntry.expectedSkipping);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should be able to skip subprefixes within deleteMarker keys when a prefix key ' +
|
||||
'ending with delimiter is seen', () => {
|
||||
for (const prefix of ['prefix/', 'prefix/subprefix/']) {
|
||||
const delimiter = new DelimiterMaster({
|
||||
prefix,
|
||||
delimiter: '/',
|
||||
}, fakeLogger, vFormat);
|
||||
for (const testEntry of [
|
||||
{
|
||||
key: prefix,
|
||||
expectedRes: FILTER_ACCEPT,
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}${VID_SEP}v1`,
|
||||
// isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP, // versions get skipped after master
|
||||
expectedSkipping: `${prefix}${VID_SEP}`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}foo/`,
|
||||
isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}foo/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}foo/1`,
|
||||
isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}foo/`,
|
||||
},
|
||||
{
|
||||
key: `${prefix}foo/2`,
|
||||
isDeleteMarker: true,
|
||||
expectedRes: FILTER_SKIP,
|
||||
expectedSkipping: `${prefix}foo/`,
|
||||
},
|
||||
]) {
|
||||
const entry = {
|
||||
key: testEntry.key,
|
||||
};
|
||||
if (testEntry.isDeleteMarker) {
|
||||
entry.value = '{"isDeleteMarker":true}';
|
||||
} else {
|
||||
entry.value = '{}';
|
||||
}
|
||||
const res = delimiter.filter(entry);
|
||||
const skipping = delimiter.skipping();
|
||||
assert.strictEqual(res, testEntry.expectedRes);
|
||||
assert.strictEqual(skipping, testEntry.expectedSkipping);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2144,6 +2144,11 @@ chalk@^4.0.0:
|
|||
ansi-styles "^4.1.0"
|
||||
supports-color "^7.1.0"
|
||||
|
||||
chance@^1.1.8:
|
||||
version "1.1.8"
|
||||
resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.8.tgz#5d6c2b78c9170bf6eb9df7acdda04363085be909"
|
||||
integrity sha512-v7fi5Hj2VbR6dJEGRWLmJBA83LJMS47pkAbmROFxHWd9qmE1esHRZW8Clf1Fhzr3rjxnNZVCjOEv/ivFxeIMtg==
|
||||
|
||||
char-regex@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
|
||||
|
|
|
|||
Loading…
Reference in New Issue