ARSN-42 bump version to 7.4.13

(cherry picked from commit 5ce057a498)

.github/workflows/tests.yaml

@@ -0,0 +1,47 @@
+---
+name: tests
+
+on:
+  push:
+    branches-ignore:
+    - 'development/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      # Label used to access the service container
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: '10'
+        cache: 'yarn'
+    - name: install dependencies
+      run: yarn install --frozen-lockfile
+    - name: lint yaml
+      run: yarn --silent lint_yml
+    - name: lint javascript
+      run: yarn --silent lint -- --max-warnings 0
+    - name: lint markdown
+      run: yarn --silent lint_md
+    - name: run unit tests
+      run: yarn --silent test
+    - name: run functional tests
+      run: yarn ft_test
+    - name: run executables tests
+      run: yarn install && yarn test
+      working-directory: 'lib/executables/pensieveCreds/'

eve/main.yml

@@ -1,43 +0,0 @@
----
-version: 0.2
-
-branches:
-  default:
-    stage: pre-merge
-
-stages:
-  pre-merge:
-    worker: &master-worker
-      type: docker
-      path: eve/workers/master
-      volumes:
-        - '/home/eve/workspace'
-    steps:
-      - Git:
-          name: fetch source
-          repourl: '%(prop:git_reference)s'
-          shallow: True
-          retryFetch: True
-          haltOnFailure: True
-      - ShellCommand:
-          name: install dependencies
-          command: yarn install --frozen-lockfile
-      - ShellCommand:
-          name: run lint yml
-          command: yarn run --silent lint_yml
-      - ShellCommand:
-          name: run lint
-          command: yarn run --silent lint -- --max-warnings 0
-      - ShellCommand:
-          name: run lint_md
-          command: yarn run --silent lint_md
-      - ShellCommand:
-          name: run test
-          command: yarn run --silent test
-      - ShellCommand:
-          name: run ft_test
-          command: yarn run ft_test
-      - ShellCommand:
-          name: run executables tests
-          command: yarn install && yarn test
-          workdir: '%(prop:builddir)s/build/lib/executables/pensieveCreds/'

eve/workers/master/Dockerfile

@@ -1,57 +0,0 @@
-FROM ubuntu:trusty
-
-#
-# Install apt packages needed by the buildchain
-#
-ENV LANG C.UTF-8
-COPY buildbot_worker_packages.list arsenal_packages.list /tmp/
-RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
-    && apt-get install -qy software-properties-common python-software-properties \
-    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
-    && echo "deb https://deb.nodesource.com/node_10.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
-    && curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
-    && add-apt-repository ppa:ubuntu-toolchain-r/test \
-    && apt-get update -q \
-    && cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
-    && cat /tmp/arsenal_packages.list | xargs apt-get install -qy \
-    && pip install pip==9.0.1 \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm -f /tmp/*_packages.list
-
-#
-# Install usefull nodejs dependencies
-#
-
-RUN yarn global add mocha
-
-#
-# Add user eve
-#
-
-RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
-    && adduser eve sudo \
-    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
-
-#
-# Run buildbot-worker on startup
-#
-
-ARG BUILDBOT_VERSION=0.9.12
-RUN pip install yamllint
-RUN pip install buildbot-worker==$BUILDBOT_VERSION
-
-USER eve
-ENV HOME /home/eve
-#
-# Setup nodejs environmnent
-#
-
-ENV CXX=g++-4.9
-ENV LANG C.UTF-8
-
-
-WORKDIR /home/eve/workspace
-CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
-    && sudo service redis-server start \
-    && buildbot-worker start --nodaemon

eve/workers/master/arsenal_packages.list

@@ -1,4 +0,0 @@
-nodejs
-redis-server
-g++-4.9
-yarn

eve/workers/master/buildbot_worker_packages.list

@@ -1,9 +0,0 @@
-ca-certificates
-git
-libffi-dev
-libssl-dev
-python2.7
-python2.7-dev
-python-pip
-software-properties-common
-sudo

lib/algos/list/delimiterMaster.js

@@ -32,6 +32,7 @@ class DelimiterMaster extends Delimiter {
         // non-PHD master version or a version whose master is a PHD version
         this.prvKey = undefined;
         this.prvPHDKey = undefined;
+        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -61,6 +62,12 @@ class DelimiterMaster extends Delimiter {
         let key = obj.key;
         const value = obj.value;
 
+        if (key.startsWith(DbPrefixes.Replay)) {
+            this.inReplayPrefix = true;
+            return FILTER_SKIP;
+        }
+        this.inReplayPrefix = false;
+
         /* Skip keys not starting with the prefix or not alphabetically
          * ordered. */
         if ((this.prefix && !key.startsWith(this.prefix))
@@ -155,7 +162,7 @@ class DelimiterMaster extends Delimiter {
         return super.filter(obj);
     }
 
-    skippingV0() {
+    skippingBase() {
         if (this[this.nextContinueMarker]) {
             // next marker or next continuation token:
             // - foo/ : skipping foo/
@@ -170,8 +177,15 @@ class DelimiterMaster extends Delimiter {
         return SKIP_NONE;
     }
 
+    skippingV0() {
+        if (this.inReplayPrefix) {
+            return DbPrefixes.Replay;
+        }
+        return this.skippingBase();
+    }
+
     skippingV1() {
-        const skipTo = this.skippingV0();
+        const skipTo = this.skippingBase();
         if (skipTo === SKIP_NONE) {
             return SKIP_NONE;
         }

lib/algos/list/delimiterVersions.js

@@ -33,6 +33,7 @@ class DelimiterVersions extends Delimiter {
         // listing results
         this.NextMarker = parameters.keyMarker;
         this.NextVersionIdMarker = undefined;
+        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -163,6 +164,12 @@ class DelimiterVersions extends Delimiter {
      *  @return {number}          - indicates if iteration should continue
      */
     filterV0(obj) {
+        if (obj.key.startsWith(DbPrefixes.Replay)) {
+            this.inReplayPrefix = true;
+            return FILTER_SKIP;
+        }
+        this.inReplayPrefix = false;
+
         if (Version.isPHD(obj.value)) {
             // return accept to avoid skipping the next values in range
             return FILTER_ACCEPT;
@@ -224,6 +231,9 @@ class DelimiterVersions extends Delimiter {
     }
 
     skippingV0() {
+        if (this.inReplayPrefix) {
+            return DbPrefixes.Replay;
+        }
         if (this.NextMarker) {
             const index = this.NextMarker.lastIndexOf(this.delimiter);
             if (index === this.NextMarker.length - 1) {

lib/models/ObjectMD.js

@@ -110,8 +110,10 @@ class ObjectMD {
             // should be undefined when not set explicitly
             'isNull': undefined,
             'nullVersionId': undefined,
+            'nullUploadId': undefined,
             'isDeleteMarker': undefined,
             'versionId': undefined,
+            'uploadId': undefined,
             'tags': {},
             'replicationInfo': {
                 status: '',
@@ -630,6 +632,27 @@ class ObjectMD {
         return this._data.nullVersionId;
     }
 
+    /**
+     * Set metadata nullUploadId value
+     *
+     * @param {string} nullUploadId - The upload ID used to complete
+     * the MPU of the null version
+     * @return {ObjectMD} itself
+     */
+    setNullUploadId(nullUploadId) {
+        this._data.nullUploadId = nullUploadId;
+        return this;
+    }
+
+    /**
+     * Get metadata nullUploadId value
+     *
+     * @return {string|undefined} The object nullUploadId
+     */
+    getNullUploadId() {
+        return this._data.nullUploadId;
+    }
+
     /**
      * Set metadata isDeleteMarker value
      *
@@ -680,6 +703,26 @@ class ObjectMD {
         return VersionIDUtils.encode(this.getVersionId());
     }
 
+    /**
+     * Set metadata uploadId value
+     *
+     * @param {string} uploadId - The upload ID used to complete the MPU object
+     * @return {ObjectMD} itself
+     */
+    setUploadId(uploadId) {
+        this._data.uploadId = uploadId;
+        return this;
+    }
+
+    /**
+     * Get metadata uploadId value
+     *
+     * @return {string|undefined} The object uploadId
+     */
+    getUploadId() {
+        return this._data.uploadId;
+    }
+
     /**
      * Set tags
      *

lib/versioning/constants.js

@@ -5,6 +5,7 @@ module.exports.VersioningConstants = {
     DbPrefixes: {
         Master: '\x7fM',
         Version: '\x7fV',
+        Replay: '\x7fR',
     },
     BucketVersioningKeyFormat: {
         current: 'v1',

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=6.9.5"
   },
-  "version": "7.5.0",
+  "version": "7.4.13",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/algos/list/delimiterMaster.js

@@ -8,12 +8,14 @@ const {
     FILTER_ACCEPT,
     FILTER_SKIP,
     SKIP_NONE,
+    inc,
 } = require('../../../../lib/algos/list/tools');
 const VSConst =
     require('../../../../lib/versioning/constants').VersioningConstants;
 const Version = require('../../../../lib/versioning/Version').Version;
 const { generateVersionId } = require('../../../../lib/versioning/VersionID');
 const { DbPrefixes } = VSConst;
+const zpad = require('../../helpers').zpad;
 
 
 const VID_SEP = VSConst.VersionId.Separator;
@@ -453,6 +455,39 @@ function getListingKey(key, vFormat) {
 
                 assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
             });
+
+            it('should return good skipping value for DelimiterMaster on replay keys', () => {
+                const delimiter = new DelimiterMaster(
+                    { delimiter: '/', v2: true },
+                    fakeLogger, vFormat);
+
+                for (let i = 0; i < 10; i++) {
+                    delimiter.filter({
+                        key: `foo/${zpad(i)}`,
+                        value: '{}',
+                    });
+                }
+                // simulate a listing that goes through a replay key, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${DbPrefixes.Replay}xyz`,
+                        value: 'abcdef',
+                    }),
+                    FILTER_SKIP);
+                // ...it should skip the whole replay prefix
+                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
+
+                // simulate a listing that reaches regular object keys
+                // beyond the replay prefix, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
+                        value: '{}',
+                    }),
+                    FILTER_ACCEPT);
+                // ...it should return to skipping by prefix as usual
+                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
+            });
         }
     });
 });

tests/unit/algos/list/delimiterVersions.js

@@ -7,12 +7,12 @@ const {
     FILTER_ACCEPT,
     FILTER_SKIP,
     SKIP_NONE,
+    inc,
 } = require('../../../../lib/algos/list/tools');
 const Werelogs = require('werelogs').Logger;
 const logger = new Werelogs('listTest');
 const performListing = require('../../../utils/performListing');
 const zpad = require('../../helpers').zpad;
-const { inc } = require('../../../../lib/algos/list/tools');
 const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
 const Version = require('../../../../lib/versioning/Version').Version;
 const { generateVersionId } = require('../../../../lib/versioning/VersionID');
@@ -520,17 +520,55 @@ function getTestListing(test, data, vFormat) {
 ['v0', 'v1'].forEach(vFormat => {
     describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
         it('Should return good skipping value for DelimiterVersions', () => {
-            const delimiter = new DelimiterVersions({ delimiter: '/' });
+            const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
             for (let i = 0; i < 100; i++) {
                 delimiter.filter({
                     key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
                     value: '{}',
                 });
             }
-            assert.strictEqual(delimiter.skipping(),
-                               `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
+            if (vFormat === 'v1') {
+                assert.deepStrictEqual(delimiter.skipping(), [
+                    `${DbPrefixes.Master}foo/`,
+                    `${DbPrefixes.Version}foo/`,
+                ]);
+            } else {
+                assert.strictEqual(delimiter.skipping(), 'foo/');
+            }
         });
 
+        if (vFormat === 'v0') {
+            it('Should return good skipping value for DelimiterVersions on replay keys', () => {
+                const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
+                for (let i = 0; i < 10; i++) {
+                    delimiter.filter({
+                        key: `foo/${zpad(i)}`,
+                        value: '{}',
+                    });
+                }
+                // simulate a listing that goes through a replay key, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${DbPrefixes.Replay}xyz`,
+                        value: 'abcdef',
+                    }),
+                    FILTER_SKIP);
+                // ...it should skip the whole replay prefix
+                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
+
+                // simulate a listing that reaches regular object keys
+                // beyond the replay prefix, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
+                        value: '{}',
+                    }),
+                    FILTER_ACCEPT);
+                // ...it should return to skipping by prefix as usual
+                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
+            });
+        }
+
         tests.forEach(test => {
             it(`Should return metadata listing params to list ${test.name}`, () => {
                 const listing = new DelimiterVersions(test.input, logger, vFormat);

tests/unit/models/object.js

@@ -64,6 +64,8 @@ describe('ObjectMD class setters/getters', () => {
         ['IsNull', true],
         ['NullVersionId', null, undefined],
         ['NullVersionId', '111111'],
+        ['NullUploadId', null, undefined],
+        ['NullUploadId', 'abcdefghi'],
         ['IsDeleteMarker', null, false],
         ['IsDeleteMarker', true],
         ['VersionId', null, undefined],
@@ -73,6 +75,8 @@ describe('ObjectMD class setters/getters', () => {
             key: 'value',
         }],
         ['Tags', null, {}],
+        ['UploadId', null, undefined],
+        ['UploadId', 'abcdefghi'],
         ['ReplicationInfo', null, {
             status: '',
             backends: [],
@@ -307,9 +311,11 @@ describe('getAttributes static method', () => {
             'location': true,
             'isNull': true,
             'nullVersionId': true,
+            'nullUploadId': true,
             'isDeleteMarker': true,
             'versionId': true,
             'tags': true,
+            'uploadId': true,
             'replicationInfo': true,
             'dataStoreName': true,
             'last-modified': true,