Compare commits

..

No commits in common. "7293f93c229cb33995e9ff7a0b4adfdb177a7e19" and "4c6712741b77caf4be4f4ee7a9394cc3102f10e6" have entirely different histories.

5 changed files with 377 additions and 9 deletions

View File

@ -271,4 +271,4 @@ class Delimiter extends Extension {
}
}
module.exports = { Delimiter };
module.exports = { Delimiter, getCommonPrefix };

View File

@ -33,6 +33,8 @@ class DelimiterMaster extends Delimiter {
this.prvKey = undefined;
this.prvPHDKey = undefined;
this.inReplayPrefix = false;
this.prefixKeySeen = false;
this.prefixEndsWithDelim = this.prefix && this.prefix.endsWith(this.delimiter);
Object.assign(this, {
[BucketVersioningKeyFormat.v0]: {
@ -62,6 +64,10 @@ class DelimiterMaster extends Delimiter {
let key = obj.key;
const value = obj.value;
if (key === this.prefix) {
this.prefixKeySeen = true;
}
if (key.startsWith(DbPrefixes.Replay)) {
this.inReplayPrefix = true;
return FILTER_SKIP;
@ -95,9 +101,8 @@ class DelimiterMaster extends Delimiter {
* NextMarker to the common prefix instead of the whole key
* value. (TODO: remove this test once ZENKO-1048 is fixed)
* */
if (key === this.prvKey || key === this[this.nextContinueMarker] ||
(this.delimiter &&
key.startsWith(this[this.nextContinueMarker]))) {
if (key === this.prvKey || key === this[this.nextContinueMarker]
|| (this.delimiter && key.startsWith(this[this.nextContinueMarker]))) {
/* master version already filtered */
return FILTER_SKIP;
}
@ -127,6 +132,14 @@ class DelimiterMaster extends Delimiter {
return FILTER_ACCEPT;
}
this.prvKey = key;
if (this.prefixEndsWithDelim) {
/* When the prefix ends with a delimiter, update nextContinueMarker
* to be able to skip ranges of the form prefix/subprefix/ as an optimization.
* The marker may also end up being prefix/, in which case .skipping will determine
* if a skip over the full range is allowed or a smaller skipping range of prefix/{VID_SEP}
* must be used. */
this[this.nextContinueMarker] = key.slice(0, key.lastIndexOf(this.delimiter) + 1);
}
return FILTER_SKIP;
}
@ -162,6 +175,27 @@ class DelimiterMaster extends Delimiter {
return super.filter(obj);
}
/**
* Determine if a nextContinueMarker ending with a delimiter
* can be skipped.
* @returns {bool}
*/
allowDelimiterRangeSkip() {
if (!this.prefixKeySeen) {
// A prefix key is a master key equal to the prefix. If it has
// not been encountered, can skip.
return true;
}
const marker = this[this.nextContinueMarker];
// prefix = prefix, key = prefix/. Can skip since key will be part of commonPrefixes.
if (marker.length > this.prefix.length && marker.startsWith(this.prefix)) {
return true;
}
const lastIdx = marker.lastIndexOf(this.delimiter); // prefix/foo is a masterKey following a prefix key.
return marker.slice(0, lastIdx + 1) !== this.prefix; // cannot skip the full range prefix/ range.
}
skippingBase() {
if (this[this.nextContinueMarker]) {
// next marker or next continuation token:
@ -169,7 +203,7 @@ class DelimiterMaster extends Delimiter {
// - foo : skipping foo.
const index = this[this.nextContinueMarker].
lastIndexOf(this.delimiter);
if (index === this[this.nextContinueMarker].length - 1) {
if (index === this[this.nextContinueMarker].length - 1 && this.allowDelimiterRangeSkip()) {
return this[this.nextContinueMarker];
}
return this[this.nextContinueMarker] + VID_SEP;

View File

@ -3,7 +3,7 @@
"engines": {
"node": ">=16"
},
"version": "7.10.36-1",
"version": "7.10.36",
"description": "Common utilities for the S3 project components",
"main": "build/index.js",
"repository": {
@ -62,6 +62,7 @@
"@types/jest": "^27.4.1",
"@types/node": "^17.0.21",
"@types/xml2js": "^0.4.11",
"chance": "^1.1.8",
"eslint": "^8.12.0",
"eslint-config-airbnb": "6.2.0",
"eslint-config-scality": "scality/Guidelines#7.10.2",

View File

@ -1,7 +1,7 @@
'use strict'; // eslint-disable-line strict
const assert = require('assert');
const chance = require('chance').Chance(); // eslint-disable-line
const DelimiterMaster =
require('../../../../lib/algos/list/delimiterMaster').DelimiterMaster;
const {
@ -16,8 +16,6 @@ const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
const { DbPrefixes } = VSConst;
const zpad = require('../../helpers').zpad;
const VID_SEP = VSConst.VersionId.Separator;
const EmptyResult = {
CommonPrefixes: [],
@ -488,6 +486,336 @@ function getListingKey(key, vFormat) {
// ...it should return to skipping by prefix as usual
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
});
it('should not skip over whole prefix when a key equals the prefix and ends with delimiter', () => {
for (const prefix of ['prefix/', 'prefix/subprefix/']) {
const delimiter = new DelimiterMaster({
prefix,
delimiter: '/',
}, fakeLogger, vFormat);
for (const testEntry of [
{
key: prefix,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}${VID_SEP}v1`,
value: '{}',
expectedRes: FILTER_SKIP, // versions get skipped after master
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}deleted`,
isDeleteMarker: true,
expectedRes: FILTER_SKIP, // delete markers get skipped
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}deleted${VID_SEP}v1`,
isDeleteMarker: true,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}deleted${VID_SEP}v2`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}notdeleted`,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}notdeleted${VID_SEP}`,
},
{
key: `${prefix}notdeleted${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}notdeleted${VID_SEP}`,
},
{
key: `${prefix}subprefix1/key-1`,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}subprefix1/`,
},
{
key: `${prefix}subprefix1/key-1${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}subprefix1/`,
},
{
key: `${prefix}subprefix1/key-2`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}subprefix1/`,
},
{
key: `${prefix}subprefix1/key-2${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}subprefix1/`,
},
]) {
const entry = {
key: testEntry.key,
};
if (testEntry.isDeleteMarker) {
entry.value = '{"isDeleteMarker":true}';
} else {
entry.value = '{}';
}
const res = delimiter.filter(entry);
const skipping = delimiter.skipping();
assert.strictEqual(res, testEntry.expectedRes);
assert.strictEqual(skipping, testEntry.expectedSkipping);
}
}
});
it('should skip over whole prefix when a key equals the prefix and does not end with delimiter', () => {
for (const prefix of ['prefix', 'prefix/subprefix']) {
const delimiter = new DelimiterMaster({
prefix,
delimiter: '/',
}, fakeLogger, vFormat);
for (const testEntry of [
{
key: prefix,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}/`,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}/`,
},
{
key: `${prefix}/${VID_SEP}v1`,
value: '{}',
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}/`, // common prefix already seen
},
{
key: `${prefix}/deleted`,
isDeleteMarker: true, // skipped delete marker
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}/`, // already added to common prefix
},
{
key: `${prefix}/notdeleted`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}/`, // already added to common prefix
},
{
key: `${prefix}ed`,
isDeleteMarker: false,
expectedRes: FILTER_ACCEPT, // new master key seen
expectedSkipping: `${prefix}ed${VID_SEP}`,
},
{
key: `${prefix}ed/`,
expectedRes: FILTER_ACCEPT, // new master key ending with prefix
expectedSkipping: `${prefix}ed/`,
},
{
key: `${prefix}ed/subprefix1/key-1`,
expectedRes: FILTER_SKIP, // already have prefixed/ common prefix
expectedSkipping: `${prefix}ed/`,
},
{
key: `${prefix}ed/subprefix1/key-1${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}ed/`,
},
{
key: `${prefix}ed/subprefix1/key-2`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}ed/`,
},
{
key: `${prefix}ed/subprefix1/key-2${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}ed/`,
},
]) {
const entry = {
key: testEntry.key,
};
if (testEntry.isDeleteMarker) {
entry.value = '{"isDeleteMarker":true}';
} else {
entry.value = '{}';
}
const res = delimiter.filter(entry);
const skipping = delimiter.skipping();
assert.strictEqual(res, testEntry.expectedRes);
assert.strictEqual(skipping, testEntry.expectedSkipping);
}
}
});
it('should not skip over whole prefix when key equals the prefix and prefix key has delete marker ' +
'and prefix ends with delimiter', () => {
for (const prefix of ['prefix/', 'prefix/subprefix/']) {
const delimiter = new DelimiterMaster({
prefix,
delimiter: '/',
}, fakeLogger, vFormat);
for (const testEntry of [
{
key: prefix,
isDeleteMarker: true,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}subprefix-1`,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}subprefix-1${VID_SEP}`,
},
{
key: `${prefix}subprefix-1/foo`,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}subprefix-1/`,
},
{
key: `${prefix}subprefix-1/bar`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}subprefix-1/`, // already added to common prefix
},
]) {
const entry = {
key: testEntry.key,
};
if (testEntry.isDeleteMarker) {
entry.value = '{"isDeleteMarker":true}';
} else {
entry.value = '{}';
}
const res = delimiter.filter(entry);
const skipping = delimiter.skipping();
assert.strictEqual(res, testEntry.expectedRes);
assert.strictEqual(skipping, testEntry.expectedSkipping);
}
}
});
it('should skip over whole prefix when key equals the prefix, prefix key has delete marker ' +
'and prefix does not end with delimiter', () => {
for (const prefix of ['prefix', 'prefix/subprefix']) {
const delimiter = new DelimiterMaster({
prefix,
delimiter: '/',
}, fakeLogger, vFormat);
for (const testEntry of [
{
key: prefix,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}${VID_SEP}v1`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}/`,
isDeleteMarker: true,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}/subprefix-1`,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}/`,
},
{
key: `${prefix}/subprefix-1/foo`,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}/`,
},
{
key: `${prefix}aa`,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}aa${VID_SEP}`, // already added to common prefix
},
]) {
const entry = {
key: testEntry.key,
};
if (testEntry.isDeleteMarker) {
entry.value = '{"isDeleteMarker":true}';
} else {
entry.value = '{}';
}
const res = delimiter.filter(entry);
const skipping = delimiter.skipping();
assert.strictEqual(res, testEntry.expectedRes);
assert.strictEqual(skipping, testEntry.expectedSkipping);
}
}
});
it('should be able to skip subprefixes within deleteMarker keys when a prefix key ' +
'ending with delimiter is seen', () => {
for (const prefix of ['prefix/', 'prefix/subprefix/']) {
const delimiter = new DelimiterMaster({
prefix,
delimiter: '/',
}, fakeLogger, vFormat);
for (const testEntry of [
{
key: prefix,
expectedRes: FILTER_ACCEPT,
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}${VID_SEP}v1`,
// isDeleteMarker: true,
expectedRes: FILTER_SKIP, // versions get skipped after master
expectedSkipping: `${prefix}${VID_SEP}`,
},
{
key: `${prefix}foo/`,
isDeleteMarker: true,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}foo/`,
},
{
key: `${prefix}foo/1`,
isDeleteMarker: true,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}foo/`,
},
{
key: `${prefix}foo/2`,
isDeleteMarker: true,
expectedRes: FILTER_SKIP,
expectedSkipping: `${prefix}foo/`,
},
]) {
const entry = {
key: testEntry.key,
};
if (testEntry.isDeleteMarker) {
entry.value = '{"isDeleteMarker":true}';
} else {
entry.value = '{}';
}
const res = delimiter.filter(entry);
const skipping = delimiter.skipping();
assert.strictEqual(res, testEntry.expectedRes);
assert.strictEqual(skipping, testEntry.expectedSkipping);
}
}
});
}
});
});

View File

@ -2144,6 +2144,11 @@ chalk@^4.0.0:
ansi-styles "^4.1.0"
supports-color "^7.1.0"
chance@^1.1.8:
version "1.1.8"
resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.8.tgz#5d6c2b78c9170bf6eb9df7acdda04363085be909"
integrity sha512-v7fi5Hj2VbR6dJEGRWLmJBA83LJMS47pkAbmROFxHWd9qmE1esHRZW8Clf1Fhzr3rjxnNZVCjOEv/ivFxeIMtg==
char-regex@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"