Compare commits

...

7 Commits

Author SHA1 Message Date
Jonathan Gramain f4f867d9fb ARSN-42 bump version to 7.4.13
(cherry picked from commit 5ce057a498)
2021-12-08 15:52:39 -08:00
Jonathan Gramain 165da3f272 improvement: ARSN-42 get/set ObjectMD.nullUploadId
Add getNullUploadId/setNullUploadId helpers to ObjectMD, to store the
null version uploadId, so that it can be passed to the metadata layer
as "replayId" when deleting the null version from another master key

(cherry picked from commit 8c3f88e233)
2021-12-08 15:52:39 -08:00
Jonathan Gramain 1d8aa8deaf ARSN-38 bump arsenal version
(cherry picked from commit 04581abbf6)
2021-12-07 16:26:21 -08:00
Jonathan Gramain d5f252a3b3 feature: ARSN-38 introduce replay prefix hidden in listings
- Add a new DB prefix for replay keys, similar to existing v1 vformat
  prefixes

- Hide this prefix for v0 listing algos DelimiterMaster and
  DelimiterVersions: skip keys beginning with this prefix, and update
  the "skipping" value to be able to skip the entire prefix after the
  streak length is reached (similar to how regular prefixes are
  skipped)

- fix an existing unit test in DelimiterVersions

(cherry picked from commit abfbe90a57)
2021-12-07 16:26:10 -08:00
Thomas Carmet 02e3d71180 ARSN-17 align package.json with releases
(cherry picked from commit 36f6ca47e9)
2021-12-07 16:26:00 -08:00
Jonathan Gramain c18110d1e7 feature: ARSN-37 ObjectMD getUploadId/setUploadId
Add getter/setter for the "uploadId" field, used for MPUs in progress.

(cherry picked from commit b1c9474159)
2021-12-07 15:09:34 -08:00
Thomas Carmet 8202a6aa44 ARSN-20 migrate to github actions
Co-authored-by: Ronnie <halfpint1170@gmail.com>
(cherry picked from commit 4b08dd5263)
2021-09-23 15:43:12 -07:00
13 changed files with 201 additions and 120 deletions

47
.github/workflows/tests.yaml vendored Normal file
View File

@ -0,0 +1,47 @@
---
name: tests
on:
push:
branches-ignore:
- 'development/**'
jobs:
test:
runs-on: ubuntu-latest
services:
# Label used to access the service container
redis:
# Docker Hub image
image: redis
# Set health checks to wait until redis has started
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps port 6379 on service container to the host
- 6379:6379
steps:
- name: Checkout
uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: '10'
cache: 'yarn'
- name: install dependencies
run: yarn install --frozen-lockfile
- name: lint yaml
run: yarn --silent lint_yml
- name: lint javascript
run: yarn --silent lint -- --max-warnings 0
- name: lint markdown
run: yarn --silent lint_md
- name: run unit tests
run: yarn --silent test
- name: run functional tests
run: yarn ft_test
- name: run executables tests
run: yarn install && yarn test
working-directory: 'lib/executables/pensieveCreds/'

View File

@ -1,43 +0,0 @@
---
version: 0.2
branches:
default:
stage: pre-merge
stages:
pre-merge:
worker: &master-worker
type: docker
path: eve/workers/master
volumes:
- '/home/eve/workspace'
steps:
- Git:
name: fetch source
repourl: '%(prop:git_reference)s'
shallow: True
retryFetch: True
haltOnFailure: True
- ShellCommand:
name: install dependencies
command: yarn install --frozen-lockfile
- ShellCommand:
name: run lint yml
command: yarn run --silent lint_yml
- ShellCommand:
name: run lint
command: yarn run --silent lint -- --max-warnings 0
- ShellCommand:
name: run lint_md
command: yarn run --silent lint_md
- ShellCommand:
name: run test
command: yarn run --silent test
- ShellCommand:
name: run ft_test
command: yarn run ft_test
- ShellCommand:
name: run executables tests
command: yarn install && yarn test
workdir: '%(prop:builddir)s/build/lib/executables/pensieveCreds/'

View File

@ -1,57 +0,0 @@
FROM ubuntu:trusty
#
# Install apt packages needed by the buildchain
#
ENV LANG C.UTF-8
COPY buildbot_worker_packages.list arsenal_packages.list /tmp/
RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
&& apt-get install -qy software-properties-common python-software-properties \
&& curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
&& echo "deb https://deb.nodesource.com/node_10.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
&& curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
&& add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update -q \
&& cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
&& cat /tmp/arsenal_packages.list | xargs apt-get install -qy \
&& pip install pip==9.0.1 \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /tmp/*_packages.list
#
# Install usefull nodejs dependencies
#
RUN yarn global add mocha
#
# Add user eve
#
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
&& adduser eve sudo \
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
#
# Run buildbot-worker on startup
#
ARG BUILDBOT_VERSION=0.9.12
RUN pip install yamllint
RUN pip install buildbot-worker==$BUILDBOT_VERSION
USER eve
ENV HOME /home/eve
#
# Setup nodejs environmnent
#
ENV CXX=g++-4.9
ENV LANG C.UTF-8
WORKDIR /home/eve/workspace
CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
&& sudo service redis-server start \
&& buildbot-worker start --nodaemon

View File

@ -1,4 +0,0 @@
nodejs
redis-server
g++-4.9
yarn

View File

@ -1,9 +0,0 @@
ca-certificates
git
libffi-dev
libssl-dev
python2.7
python2.7-dev
python-pip
software-properties-common
sudo

View File

@ -32,6 +32,7 @@ class DelimiterMaster extends Delimiter {
// non-PHD master version or a version whose master is a PHD version // non-PHD master version or a version whose master is a PHD version
this.prvKey = undefined; this.prvKey = undefined;
this.prvPHDKey = undefined; this.prvPHDKey = undefined;
this.inReplayPrefix = false;
Object.assign(this, { Object.assign(this, {
[BucketVersioningKeyFormat.v0]: { [BucketVersioningKeyFormat.v0]: {
@ -61,6 +62,12 @@ class DelimiterMaster extends Delimiter {
let key = obj.key; let key = obj.key;
const value = obj.value; const value = obj.value;
if (key.startsWith(DbPrefixes.Replay)) {
this.inReplayPrefix = true;
return FILTER_SKIP;
}
this.inReplayPrefix = false;
/* Skip keys not starting with the prefix or not alphabetically /* Skip keys not starting with the prefix or not alphabetically
* ordered. */ * ordered. */
if ((this.prefix && !key.startsWith(this.prefix)) if ((this.prefix && !key.startsWith(this.prefix))
@ -155,7 +162,7 @@ class DelimiterMaster extends Delimiter {
return super.filter(obj); return super.filter(obj);
} }
skippingV0() { skippingBase() {
if (this[this.nextContinueMarker]) { if (this[this.nextContinueMarker]) {
// next marker or next continuation token: // next marker or next continuation token:
// - foo/ : skipping foo/ // - foo/ : skipping foo/
@ -170,8 +177,15 @@ class DelimiterMaster extends Delimiter {
return SKIP_NONE; return SKIP_NONE;
} }
skippingV0() {
if (this.inReplayPrefix) {
return DbPrefixes.Replay;
}
return this.skippingBase();
}
skippingV1() { skippingV1() {
const skipTo = this.skippingV0(); const skipTo = this.skippingBase();
if (skipTo === SKIP_NONE) { if (skipTo === SKIP_NONE) {
return SKIP_NONE; return SKIP_NONE;
} }

View File

@ -33,6 +33,7 @@ class DelimiterVersions extends Delimiter {
// listing results // listing results
this.NextMarker = parameters.keyMarker; this.NextMarker = parameters.keyMarker;
this.NextVersionIdMarker = undefined; this.NextVersionIdMarker = undefined;
this.inReplayPrefix = false;
Object.assign(this, { Object.assign(this, {
[BucketVersioningKeyFormat.v0]: { [BucketVersioningKeyFormat.v0]: {
@ -163,6 +164,12 @@ class DelimiterVersions extends Delimiter {
* @return {number} - indicates if iteration should continue * @return {number} - indicates if iteration should continue
*/ */
filterV0(obj) { filterV0(obj) {
if (obj.key.startsWith(DbPrefixes.Replay)) {
this.inReplayPrefix = true;
return FILTER_SKIP;
}
this.inReplayPrefix = false;
if (Version.isPHD(obj.value)) { if (Version.isPHD(obj.value)) {
// return accept to avoid skipping the next values in range // return accept to avoid skipping the next values in range
return FILTER_ACCEPT; return FILTER_ACCEPT;
@ -224,6 +231,9 @@ class DelimiterVersions extends Delimiter {
} }
skippingV0() { skippingV0() {
if (this.inReplayPrefix) {
return DbPrefixes.Replay;
}
if (this.NextMarker) { if (this.NextMarker) {
const index = this.NextMarker.lastIndexOf(this.delimiter); const index = this.NextMarker.lastIndexOf(this.delimiter);
if (index === this.NextMarker.length - 1) { if (index === this.NextMarker.length - 1) {

View File

@ -110,8 +110,10 @@ class ObjectMD {
// should be undefined when not set explicitly // should be undefined when not set explicitly
'isNull': undefined, 'isNull': undefined,
'nullVersionId': undefined, 'nullVersionId': undefined,
'nullUploadId': undefined,
'isDeleteMarker': undefined, 'isDeleteMarker': undefined,
'versionId': undefined, 'versionId': undefined,
'uploadId': undefined,
'tags': {}, 'tags': {},
'replicationInfo': { 'replicationInfo': {
status: '', status: '',
@ -630,6 +632,27 @@ class ObjectMD {
return this._data.nullVersionId; return this._data.nullVersionId;
} }
/**
* Set metadata nullUploadId value
*
* @param {string} nullUploadId - The upload ID used to complete
* the MPU of the null version
* @return {ObjectMD} itself
*/
setNullUploadId(nullUploadId) {
this._data.nullUploadId = nullUploadId;
return this;
}
/**
* Get metadata nullUploadId value
*
* @return {string|undefined} The object nullUploadId
*/
getNullUploadId() {
return this._data.nullUploadId;
}
/** /**
* Set metadata isDeleteMarker value * Set metadata isDeleteMarker value
* *
@ -680,6 +703,26 @@ class ObjectMD {
return VersionIDUtils.encode(this.getVersionId()); return VersionIDUtils.encode(this.getVersionId());
} }
/**
* Set metadata uploadId value
*
* @param {string} uploadId - The upload ID used to complete the MPU object
* @return {ObjectMD} itself
*/
setUploadId(uploadId) {
this._data.uploadId = uploadId;
return this;
}
/**
* Get metadata uploadId value
*
* @return {string|undefined} The object uploadId
*/
getUploadId() {
return this._data.uploadId;
}
/** /**
* Set tags * Set tags
* *

View File

@ -5,6 +5,7 @@ module.exports.VersioningConstants = {
DbPrefixes: { DbPrefixes: {
Master: '\x7fM', Master: '\x7fM',
Version: '\x7fV', Version: '\x7fV',
Replay: '\x7fR',
}, },
BucketVersioningKeyFormat: { BucketVersioningKeyFormat: {
current: 'v1', current: 'v1',

View File

@ -3,7 +3,7 @@
"engines": { "engines": {
"node": ">=6.9.5" "node": ">=6.9.5"
}, },
"version": "7.5.0", "version": "7.4.13",
"description": "Common utilities for the S3 project components", "description": "Common utilities for the S3 project components",
"main": "index.js", "main": "index.js",
"repository": { "repository": {

View File

@ -8,12 +8,14 @@ const {
FILTER_ACCEPT, FILTER_ACCEPT,
FILTER_SKIP, FILTER_SKIP,
SKIP_NONE, SKIP_NONE,
inc,
} = require('../../../../lib/algos/list/tools'); } = require('../../../../lib/algos/list/tools');
const VSConst = const VSConst =
require('../../../../lib/versioning/constants').VersioningConstants; require('../../../../lib/versioning/constants').VersioningConstants;
const Version = require('../../../../lib/versioning/Version').Version; const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID'); const { generateVersionId } = require('../../../../lib/versioning/VersionID');
const { DbPrefixes } = VSConst; const { DbPrefixes } = VSConst;
const zpad = require('../../helpers').zpad;
const VID_SEP = VSConst.VersionId.Separator; const VID_SEP = VSConst.VersionId.Separator;
@ -453,6 +455,39 @@ function getListingKey(key, vFormat) {
assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP); assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
}); });
it('should return good skipping value for DelimiterMaster on replay keys', () => {
const delimiter = new DelimiterMaster(
{ delimiter: '/', v2: true },
fakeLogger, vFormat);
for (let i = 0; i < 10; i++) {
delimiter.filter({
key: `foo/${zpad(i)}`,
value: '{}',
});
}
// simulate a listing that goes through a replay key, ...
assert.strictEqual(
delimiter.filter({
key: `${DbPrefixes.Replay}xyz`,
value: 'abcdef',
}),
FILTER_SKIP);
// ...it should skip the whole replay prefix
assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
// simulate a listing that reaches regular object keys
// beyond the replay prefix, ...
assert.strictEqual(
delimiter.filter({
key: `${inc(DbPrefixes.Replay)}foo/bar`,
value: '{}',
}),
FILTER_ACCEPT);
// ...it should return to skipping by prefix as usual
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
});
} }
}); });
}); });

View File

@ -7,12 +7,12 @@ const {
FILTER_ACCEPT, FILTER_ACCEPT,
FILTER_SKIP, FILTER_SKIP,
SKIP_NONE, SKIP_NONE,
inc,
} = require('../../../../lib/algos/list/tools'); } = require('../../../../lib/algos/list/tools');
const Werelogs = require('werelogs').Logger; const Werelogs = require('werelogs').Logger;
const logger = new Werelogs('listTest'); const logger = new Werelogs('listTest');
const performListing = require('../../../utils/performListing'); const performListing = require('../../../utils/performListing');
const zpad = require('../../helpers').zpad; const zpad = require('../../helpers').zpad;
const { inc } = require('../../../../lib/algos/list/tools');
const VSConst = require('../../../../lib/versioning/constants').VersioningConstants; const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
const Version = require('../../../../lib/versioning/Version').Version; const Version = require('../../../../lib/versioning/Version').Version;
const { generateVersionId } = require('../../../../lib/versioning/VersionID'); const { generateVersionId } = require('../../../../lib/versioning/VersionID');
@ -520,17 +520,55 @@ function getTestListing(test, data, vFormat) {
['v0', 'v1'].forEach(vFormat => { ['v0', 'v1'].forEach(vFormat => {
describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => { describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
it('Should return good skipping value for DelimiterVersions', () => { it('Should return good skipping value for DelimiterVersions', () => {
const delimiter = new DelimiterVersions({ delimiter: '/' }); const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
for (let i = 0; i < 100; i++) { for (let i = 0; i < 100; i++) {
delimiter.filter({ delimiter.filter({
key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`, key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
value: '{}', value: '{}',
}); });
} }
assert.strictEqual(delimiter.skipping(), if (vFormat === 'v1') {
`${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`); assert.deepStrictEqual(delimiter.skipping(), [
`${DbPrefixes.Master}foo/`,
`${DbPrefixes.Version}foo/`,
]);
} else {
assert.strictEqual(delimiter.skipping(), 'foo/');
}
}); });
if (vFormat === 'v0') {
it('Should return good skipping value for DelimiterVersions on replay keys', () => {
const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
for (let i = 0; i < 10; i++) {
delimiter.filter({
key: `foo/${zpad(i)}`,
value: '{}',
});
}
// simulate a listing that goes through a replay key, ...
assert.strictEqual(
delimiter.filter({
key: `${DbPrefixes.Replay}xyz`,
value: 'abcdef',
}),
FILTER_SKIP);
// ...it should skip the whole replay prefix
assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
// simulate a listing that reaches regular object keys
// beyond the replay prefix, ...
assert.strictEqual(
delimiter.filter({
key: `${inc(DbPrefixes.Replay)}foo/bar`,
value: '{}',
}),
FILTER_ACCEPT);
// ...it should return to skipping by prefix as usual
assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
});
}
tests.forEach(test => { tests.forEach(test => {
it(`Should return metadata listing params to list ${test.name}`, () => { it(`Should return metadata listing params to list ${test.name}`, () => {
const listing = new DelimiterVersions(test.input, logger, vFormat); const listing = new DelimiterVersions(test.input, logger, vFormat);

View File

@ -64,6 +64,8 @@ describe('ObjectMD class setters/getters', () => {
['IsNull', true], ['IsNull', true],
['NullVersionId', null, undefined], ['NullVersionId', null, undefined],
['NullVersionId', '111111'], ['NullVersionId', '111111'],
['NullUploadId', null, undefined],
['NullUploadId', 'abcdefghi'],
['IsDeleteMarker', null, false], ['IsDeleteMarker', null, false],
['IsDeleteMarker', true], ['IsDeleteMarker', true],
['VersionId', null, undefined], ['VersionId', null, undefined],
@ -73,6 +75,8 @@ describe('ObjectMD class setters/getters', () => {
key: 'value', key: 'value',
}], }],
['Tags', null, {}], ['Tags', null, {}],
['UploadId', null, undefined],
['UploadId', 'abcdefghi'],
['ReplicationInfo', null, { ['ReplicationInfo', null, {
status: '', status: '',
backends: [], backends: [],
@ -307,9 +311,11 @@ describe('getAttributes static method', () => {
'location': true, 'location': true,
'isNull': true, 'isNull': true,
'nullVersionId': true, 'nullVersionId': true,
'nullUploadId': true,
'isDeleteMarker': true, 'isDeleteMarker': true,
'versionId': true, 'versionId': true,
'tags': true, 'tags': true,
'uploadId': true,
'replicationInfo': true, 'replicationInfo': true,
'dataStoreName': true, 'dataStoreName': true,
'last-modified': true, 'last-modified': true,