.github/workflows/tests.yaml

@@ -1,47 +0,0 @@
----
-name: tests
-
-on:
-  push:
-    branches-ignore:
-    - 'development/**'
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps port 6379 on service container to the host
-          - 6379:6379
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - uses: actions/setup-node@v2
-      with:
-        node-version: '10'
-        cache: 'yarn'
-    - name: install dependencies
-      run: yarn install --frozen-lockfile
-    - name: lint yaml
-      run: yarn --silent lint_yml
-    - name: lint javascript
-      run: yarn --silent lint -- --max-warnings 0
-    - name: lint markdown
-      run: yarn --silent lint_md
-    - name: run unit tests
-      run: yarn --silent test
-    - name: run functional tests
-      run: yarn ft_test
-    - name: run executables tests
-      run: yarn install && yarn test
-      working-directory: 'lib/executables/pensieveCreds/'

eve/main.yml

@@ -0,0 +1,43 @@
+---
+version: 0.2
+
+branches:
+  default:
+    stage: pre-merge
+
+stages:
+  pre-merge:
+    worker: &master-worker
+      type: docker
+      path: eve/workers/master
+      volumes:
+        - '/home/eve/workspace'
+    steps:
+      - Git:
+          name: fetch source
+          repourl: '%(prop:git_reference)s'
+          shallow: True
+          retryFetch: True
+          haltOnFailure: True
+      - ShellCommand:
+          name: install dependencies
+          command: yarn install --frozen-lockfile
+      - ShellCommand:
+          name: run lint yml
+          command: yarn run --silent lint_yml
+      - ShellCommand:
+          name: run lint
+          command: yarn run --silent lint -- --max-warnings 0
+      - ShellCommand:
+          name: run lint_md
+          command: yarn run --silent lint_md
+      - ShellCommand:
+          name: run test
+          command: yarn run --silent test
+      - ShellCommand:
+          name: run ft_test
+          command: yarn run ft_test
+      - ShellCommand:
+          name: run executables tests
+          command: yarn install && yarn test
+          workdir: '%(prop:builddir)s/build/lib/executables/pensieveCreds/'

eve/workers/master/Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:trusty
+
+#
+# Install apt packages needed by the buildchain
+#
+ENV LANG C.UTF-8
+COPY buildbot_worker_packages.list arsenal_packages.list /tmp/
+RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
+    && apt-get install -qy software-properties-common python-software-properties \
+    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && echo "deb https://deb.nodesource.com/node_10.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
+    && curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+    && add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update -q \
+    && cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
+    && cat /tmp/arsenal_packages.list | xargs apt-get install -qy \
+    && pip install pip==9.0.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/*_packages.list
+
+#
+# Install usefull nodejs dependencies
+#
+
+RUN yarn global add mocha
+
+#
+# Add user eve
+#
+
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+
+#
+# Run buildbot-worker on startup
+#
+
+ARG BUILDBOT_VERSION=0.9.12
+RUN pip install yamllint
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+USER eve
+ENV HOME /home/eve
+#
+# Setup nodejs environmnent
+#
+
+ENV CXX=g++-4.9
+ENV LANG C.UTF-8
+
+
+WORKDIR /home/eve/workspace
+CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
+    && sudo service redis-server start \
+    && buildbot-worker start --nodaemon

eve/workers/master/arsenal_packages.list

@@ -0,0 +1,4 @@
+nodejs
+redis-server
+g++-4.9
+yarn

eve/workers/master/buildbot_worker_packages.list

@@ -0,0 +1,9 @@
+ca-certificates
+git
+libffi-dev
+libssl-dev
+python2.7
+python2.7-dev
+python-pip
+software-properties-common
+sudo

lib/algos/list/delimiterMaster.js

@@ -32,7 +32,6 @@ class DelimiterMaster extends Delimiter {
         // non-PHD master version or a version whose master is a PHD version
         this.prvKey = undefined;
         this.prvPHDKey = undefined;
-        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -62,12 +61,6 @@ class DelimiterMaster extends Delimiter {
         let key = obj.key;
         const value = obj.value;
 
-        if (key.startsWith(DbPrefixes.Replay)) {
-            this.inReplayPrefix = true;
-            return FILTER_SKIP;
-        }
-        this.inReplayPrefix = false;
-
         /* Skip keys not starting with the prefix or not alphabetically
          * ordered. */
         if ((this.prefix && !key.startsWith(this.prefix))
@@ -162,7 +155,7 @@ class DelimiterMaster extends Delimiter {
         return super.filter(obj);
     }
 
-    skippingBase() {
+    skippingV0() {
         if (this[this.nextContinueMarker]) {
             // next marker or next continuation token:
             // - foo/ : skipping foo/
@@ -177,15 +170,8 @@ class DelimiterMaster extends Delimiter {
         return SKIP_NONE;
     }
 
-    skippingV0() {
-        if (this.inReplayPrefix) {
-            return DbPrefixes.Replay;
-        }
-        return this.skippingBase();
-    }
-
     skippingV1() {
-        const skipTo = this.skippingBase();
+        const skipTo = this.skippingV0();
         if (skipTo === SKIP_NONE) {
             return SKIP_NONE;
         }

lib/algos/list/delimiterVersions.js

@@ -33,7 +33,6 @@ class DelimiterVersions extends Delimiter {
         // listing results
         this.NextMarker = parameters.keyMarker;
         this.NextVersionIdMarker = undefined;
-        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -164,12 +163,6 @@ class DelimiterVersions extends Delimiter {
      *  @return {number}          - indicates if iteration should continue
      */
     filterV0(obj) {
-        if (obj.key.startsWith(DbPrefixes.Replay)) {
-            this.inReplayPrefix = true;
-            return FILTER_SKIP;
-        }
-        this.inReplayPrefix = false;
-
         if (Version.isPHD(obj.value)) {
             // return accept to avoid skipping the next values in range
             return FILTER_ACCEPT;
@@ -231,9 +224,6 @@ class DelimiterVersions extends Delimiter {
     }
 
     skippingV0() {
-        if (this.inReplayPrefix) {
-            return DbPrefixes.Replay;
-        }
         if (this.NextMarker) {
             const index = this.NextMarker.lastIndexOf(this.delimiter);
             if (index === this.NextMarker.length - 1) {

lib/models/ObjectMD.js

@@ -110,10 +110,8 @@ class ObjectMD {
             // should be undefined when not set explicitly
             'isNull': undefined,
             'nullVersionId': undefined,
-            'nullUploadId': undefined,
             'isDeleteMarker': undefined,
             'versionId': undefined,
-            'uploadId': undefined,
             'tags': {},
             'replicationInfo': {
                 status: '',
@@ -632,27 +630,6 @@ class ObjectMD {
         return this._data.nullVersionId;
     }
 
-    /**
-     * Set metadata nullUploadId value
-     *
-     * @param {string} nullUploadId - The upload ID used to complete
-     * the MPU of the null version
-     * @return {ObjectMD} itself
-     */
-    setNullUploadId(nullUploadId) {
-        this._data.nullUploadId = nullUploadId;
-        return this;
-    }
-
-    /**
-     * Get metadata nullUploadId value
-     *
-     * @return {string|undefined} The object nullUploadId
-     */
-    getNullUploadId() {
-        return this._data.nullUploadId;
-    }
-
     /**
      * Set metadata isDeleteMarker value
      *
@@ -703,26 +680,6 @@ class ObjectMD {
         return VersionIDUtils.encode(this.getVersionId());
     }
 
-    /**
-     * Set metadata uploadId value
-     *
-     * @param {string} uploadId - The upload ID used to complete the MPU object
-     * @return {ObjectMD} itself
-     */
-    setUploadId(uploadId) {
-        this._data.uploadId = uploadId;
-        return this;
-    }
-
-    /**
-     * Get metadata uploadId value
-     *
-     * @return {string|undefined} The object uploadId
-     */
-    getUploadId() {
-        return this._data.uploadId;
-    }
-
     /**
      * Set tags
      *

lib/versioning/constants.js

@@ -5,7 +5,6 @@ module.exports.VersioningConstants = {
     DbPrefixes: {
         Master: '\x7fM',
         Version: '\x7fV',
-        Replay: '\x7fR',
     },
     BucketVersioningKeyFormat: {
         current: 'v1',

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=6.9.5"
   },
-  "version": "7.4.13",
+  "version": "7.5.0",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/algos/list/delimiterMaster.js

@@ -8,14 +8,12 @@ const {
     FILTER_ACCEPT,
     FILTER_SKIP,
     SKIP_NONE,
-    inc,
 } = require('../../../../lib/algos/list/tools');
 const VSConst =
     require('../../../../lib/versioning/constants').VersioningConstants;
 const Version = require('../../../../lib/versioning/Version').Version;
 const { generateVersionId } = require('../../../../lib/versioning/VersionID');
 const { DbPrefixes } = VSConst;
-const zpad = require('../../helpers').zpad;
 
 
 const VID_SEP = VSConst.VersionId.Separator;
@@ -455,39 +453,6 @@ function getListingKey(key, vFormat) {
 
                 assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
             });
-
-            it('should return good skipping value for DelimiterMaster on replay keys', () => {
-                const delimiter = new DelimiterMaster(
-                    { delimiter: '/', v2: true },
-                    fakeLogger, vFormat);
-
-                for (let i = 0; i < 10; i++) {
-                    delimiter.filter({
-                        key: `foo/${zpad(i)}`,
-                        value: '{}',
-                    });
-                }
-                // simulate a listing that goes through a replay key, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${DbPrefixes.Replay}xyz`,
-                        value: 'abcdef',
-                    }),
-                    FILTER_SKIP);
-                // ...it should skip the whole replay prefix
-                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
-
-                // simulate a listing that reaches regular object keys
-                // beyond the replay prefix, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
-                        value: '{}',
-                    }),
-                    FILTER_ACCEPT);
-                // ...it should return to skipping by prefix as usual
-                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
-            });
         }
     });
 });

tests/unit/algos/list/delimiterVersions.js

@@ -7,12 +7,12 @@ const {
     FILTER_ACCEPT,
     FILTER_SKIP,
     SKIP_NONE,
-    inc,
 } = require('../../../../lib/algos/list/tools');
 const Werelogs = require('werelogs').Logger;
 const logger = new Werelogs('listTest');
 const performListing = require('../../../utils/performListing');
 const zpad = require('../../helpers').zpad;
+const { inc } = require('../../../../lib/algos/list/tools');
 const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
 const Version = require('../../../../lib/versioning/Version').Version;
 const { generateVersionId } = require('../../../../lib/versioning/VersionID');
@@ -520,55 +520,17 @@ function getTestListing(test, data, vFormat) {
 ['v0', 'v1'].forEach(vFormat => {
     describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
         it('Should return good skipping value for DelimiterVersions', () => {
-            const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
+            const delimiter = new DelimiterVersions({ delimiter: '/' });
             for (let i = 0; i < 100; i++) {
                 delimiter.filter({
                     key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
                     value: '{}',
                 });
             }
-            if (vFormat === 'v1') {
+            assert.strictEqual(delimiter.skipping(),
-                assert.deepStrictEqual(delimiter.skipping(), [
+                               `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
-                    `${DbPrefixes.Master}foo/`,
-                    `${DbPrefixes.Version}foo/`,
-                ]);
-            } else {
-                assert.strictEqual(delimiter.skipping(), 'foo/');
-            }
         });
 
-        if (vFormat === 'v0') {
-            it('Should return good skipping value for DelimiterVersions on replay keys', () => {
-                const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
-                for (let i = 0; i < 10; i++) {
-                    delimiter.filter({
-                        key: `foo/${zpad(i)}`,
-                        value: '{}',
-                    });
-                }
-                // simulate a listing that goes through a replay key, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${DbPrefixes.Replay}xyz`,
-                        value: 'abcdef',
-                    }),
-                    FILTER_SKIP);
-                // ...it should skip the whole replay prefix
-                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
-
-                // simulate a listing that reaches regular object keys
-                // beyond the replay prefix, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
-                        value: '{}',
-                    }),
-                    FILTER_ACCEPT);
-                // ...it should return to skipping by prefix as usual
-                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
-            });
-        }
-
         tests.forEach(test => {
             it(`Should return metadata listing params to list ${test.name}`, () => {
                 const listing = new DelimiterVersions(test.input, logger, vFormat);

tests/unit/models/object.js

@@ -64,8 +64,6 @@ describe('ObjectMD class setters/getters', () => {
         ['IsNull', true],
         ['NullVersionId', null, undefined],
         ['NullVersionId', '111111'],
-        ['NullUploadId', null, undefined],
-        ['NullUploadId', 'abcdefghi'],
         ['IsDeleteMarker', null, false],
         ['IsDeleteMarker', true],
         ['VersionId', null, undefined],
@@ -75,8 +73,6 @@ describe('ObjectMD class setters/getters', () => {
             key: 'value',
         }],
         ['Tags', null, {}],
-        ['UploadId', null, undefined],
-        ['UploadId', 'abcdefghi'],
         ['ReplicationInfo', null, {
             status: '',
             backends: [],
@@ -311,11 +307,9 @@ describe('getAttributes static method', () => {
             'location': true,
             'isNull': true,
             'nullVersionId': true,
-            'nullUploadId': true,
             'isDeleteMarker': true,
             'versionId': true,
             'tags': true,
-            'uploadId': true,
             'replicationInfo': true,
             'dataStoreName': true,
             'last-modified': true,