Compare commits

...

4 Commits

Author SHA1 Message Date
Jonathan Gramain 5772c37b00 ARSN-42 bump version to 7.4.13
(cherry picked from commit 5ce057a498)
2021-12-08 15:54:46 -08:00
Jonathan Gramain f636af1854 improvement: ARSN-42 get/set ObjectMD.nullUploadId
Add getNullUploadId/setNullUploadId helpers to ObjectMD, to store the
null version uploadId, so that it can be passed to the metadata layer
as "replayId" when deleting the null version from another master key

(cherry picked from commit 8c3f88e233)
2021-12-08 15:54:00 -08:00
Jonathan Gramain 9dfe412a7c ARSN-38 bump arsenal version
(cherry picked from commit 04581abbf6)
2021-12-06 16:06:08 -08:00
Jonathan Gramain fece72a35e feature: ARSN-38 introduce replay prefix hidden in listings
- Add a new DB prefix for replay keys, similar to existing v1 vformat
  prefixes

- Hide this prefix for v0 listing algos DelimiterMaster and
  DelimiterVersions: skip keys beginning with this prefix, and update
  the "skipping" value to be able to skip the entire prefix after the
  streak length is reached (similar to how regular prefixes are
  skipped)

- fix an existing unit test in DelimiterVersions

(cherry picked from commit abfbe90a57)
2021-12-06 16:01:49 -08:00
8 changed files with 130 additions and 7 deletions

View File

@ -32,6 +32,7 @@ class DelimiterMaster extends Delimiter {
// non-PHD master version or a version whose master is a PHD version
this.prvKey = undefined;
this.prvPHDKey = undefined;
this.inReplayPrefix = false;
Object.assign(this, {
[BucketVersioningKeyFormat.v0]: {
@ -61,6 +62,12 @@ class DelimiterMaster extends Delimiter {
let key = obj.key;
const value = obj.value;
if (key.startsWith(DbPrefixes.Replay)) {
this.inReplayPrefix = true;
return FILTER_SKIP;
}
this.inReplayPrefix = false;
/* Skip keys not starting with the prefix or not alphabetically
* ordered. */
if ((this.prefix && !key.startsWith(this.prefix))
@ -155,7 +162,7 @@ class DelimiterMaster extends Delimiter {
return super.filter(obj);
}
skippingV0() {
skippingBase() {
if (this[this.nextContinueMarker]) {
// next marker or next continuation token:
// - foo/ : skipping foo/
@ -170,8 +177,15 @@ class DelimiterMaster extends Delimiter {
return SKIP_NONE;
}
skippingV0() {
if (this.inReplayPrefix) {
return DbPrefixes.Replay;
}
return this.skippingBase();
}
skippingV1() {
const skipTo = this.skippingV0();
const skipTo = this.skippingBase();
if (skipTo === SKIP_NONE) {
return SKIP_NONE;
}

View File

@ -33,6 +33,7 @@ class DelimiterVersions extends Delimiter {
// listing results
this.NextMarker = parameters.keyMarker;
this.NextVersionIdMarker = undefined;
this.inReplayPrefix = false;
Object.assign(this, {
[BucketVersioningKeyFormat.v0]: {
@ -163,6 +164,12 @@ class DelimiterVersions extends Delimiter {
* @return {number} - indicates if iteration should continue
*/
filterV0(obj) {
if (obj.key.startsWith(DbPrefixes.Replay)) {
this.inReplayPrefix = true;
return FILTER_SKIP;
}
this.inReplayPrefix = false;
if (Version.isPHD(obj.value)) {
// return accept to avoid skipping the next values in range
return FILTER_ACCEPT;
@ -224,6 +231,9 @@ class DelimiterVersions extends Delimiter {
}
skippingV0() {
if (this.inReplayPrefix) {
return DbPrefixes.Replay;
}
if (this.NextMarker) {
const index = this.NextMarker.lastIndexOf(this.delimiter);
if (index === this.NextMarker.length - 1) {

View File

@ -110,6 +110,7 @@ class ObjectMD {
// should be undefined when not set explicitly
'isNull': undefined,
'nullVersionId': undefined,
'nullUploadId': undefined,
'isDeleteMarker': undefined,
'versionId': undefined,
'uploadId': undefined,
@ -632,6 +633,27 @@ class ObjectMD {
return this._data.nullVersionId;
}
/**
* Set metadata nullUploadId value
*
* @param {string} nullUploadId - The upload ID used to complete
* the MPU of the null version
* @return {ObjectMD} itself
*/
setNullUploadId(nullUploadId) {
this._data.nullUploadId = nullUploadId;
return this;
}
/**
* Get metadata nullUploadId value
*
* @return {string|undefined} The object nullUploadId
*/
getNullUploadId() {
return this._data.nullUploadId;
}
/**
* Set metadata isDeleteMarker value
*

View File

@ -5,6 +5,7 @@ module.exports.VersioningConstants = {
DbPrefixes: {
Master: '\x7fM',
Version: '\x7fV',
Replay: '\x7fR',
},
BucketVersioningKeyFormat: {
current: 'v1',

View File

@ -3,7 +3,7 @@
"engines": {
"node": ">=6.9.5"
},
"version": "7.10.2",
"version": "7.10.4",
"description": "Common utilities for the S3 project components",
"main": "index.js",
"repository": {

View File

@ -8,12 +8,14 @@ const {
FILTER_ACCEPT,
FILTER_SKIP,
SKIP_NONE,
inc,
} = require('../../../../lib/algos/list/tools');
const VSConst =
require('../../../../lib/versioning/constants').VersioningConstants;
const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
const { DbPrefixes } = VSConst;
const zpad = require('../../helpers').zpad;
const VID_SEP = VSConst.VersionId.Separator;
@ -453,6 +455,39 @@ function getListingKey(key, vFormat) {
assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
});
it('should return good skipping value for DelimiterMaster on replay keys', () => {
const delimiter = new DelimiterMaster(
{ delimiter: '/', v2: true },
fakeLogger, vFormat);
for (let i = 0; i < 10; i++) {
delimiter.filter({
key: `foo/${zpad(i)}`,
value: '{}',
});
}
// simulate a listing that goes through a replay key, ...
assert.strictEqual(
delimiter.filter({
key: `${DbPrefixes.Replay}xyz`,
value: 'abcdef',
}),
FILTER_SKIP);
// ...it should skip the whole replay prefix
assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
// simulate a listing that reaches regular object keys
// beyond the replay prefix, ...
assert.strictEqual(
delimiter.filter({
key: `${inc(DbPrefixes.Replay)}foo/bar`,
value: '{}',
}),
FILTER_ACCEPT);
// ...it should return to skipping by prefix as usual
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
});
}
});
});

View File

@ -7,12 +7,12 @@ const {
FILTER_ACCEPT,
FILTER_SKIP,
SKIP_NONE,
inc,
} = require('../../../../lib/algos/list/tools');
const Werelogs = require('werelogs').Logger;
const logger = new Werelogs('listTest');
const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID');
@ -520,17 +520,55 @@ function getTestListing(test, data, vFormat) {
['v0', 'v1'].forEach(vFormat => {
describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
it('Should return good skipping value for DelimiterVersions', () => {
const delimiter = new DelimiterVersions({ delimiter: '/' });
const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
for (let i = 0; i < 100; i++) {
delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
value: '{}',
});
}
assert.strictEqual(delimiter.skipping(),
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
if (vFormat === 'v1') {
assert.deepStrictEqual(delimiter.skipping(), [
`${DbPrefixes.Master}foo/`,
`${DbPrefixes.Version}foo/`,
]);
} else {
assert.strictEqual(delimiter.skipping(), 'foo/');
}
});
if (vFormat === 'v0') {
it('Should return good skipping value for DelimiterVersions on replay keys', () => {
const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
for (let i = 0; i < 10; i++) {
delimiter.filter({
key: `foo/${zpad(i)}`,
value: '{}',
});
}
// simulate a listing that goes through a replay key, ...
assert.strictEqual(
delimiter.filter({
key: `${DbPrefixes.Replay}xyz`,
value: 'abcdef',
}),
FILTER_SKIP);
// ...it should skip the whole replay prefix
assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
// simulate a listing that reaches regular object keys
// beyond the replay prefix, ...
assert.strictEqual(
delimiter.filter({
key: `${inc(DbPrefixes.Replay)}foo/bar`,
value: '{}',
}),
FILTER_ACCEPT);
// ...it should return to skipping by prefix as usual
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
});
}
tests.forEach(test => {
it(`Should return metadata listing params to list ${test.name}`, () => {
const listing = new DelimiterVersions(test.input, logger, vFormat);

View File

@ -69,6 +69,8 @@ describe('ObjectMD class setters/getters', () => {
['IsNull', true],
['NullVersionId', null, undefined],
['NullVersionId', '111111'],
['NullUploadId', null, undefined],
['NullUploadId', 'abcdefghi'],
['IsDeleteMarker', null, false],
['IsDeleteMarker', true],
['VersionId', null, undefined],
@ -334,6 +336,7 @@ describe('getAttributes static method', () => {
'location': true,
'isNull': true,
'nullVersionId': true,
'nullUploadId': true,
'isDeleteMarker': true,
'versionId': true,
'tags': true,