.github/workflows/tests.yaml

@@ -1,47 +0,0 @@
----
-name: tests
-
-on:
-  push:
-    branches-ignore:
-    - 'development/**'
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps port 6379 on service container to the host
-          - 6379:6379
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - uses: actions/setup-node@v2
-      with:
-        node-version: '10'
-        cache: 'yarn'
-    - name: install dependencies
-      run: yarn install --frozen-lockfile
-    - name: lint yaml
-      run: yarn --silent lint_yml
-    - name: lint javascript
-      run: yarn --silent lint -- --max-warnings 0
-    - name: lint markdown
-      run: yarn --silent lint_md
-    - name: run unit tests
-      run: yarn --silent test
-    - name: run functional tests
-      run: yarn ft_test
-    - name: run executables tests
-      run: yarn install && yarn test
-      working-directory: 'lib/executables/pensieveCreds/'

eve/main.yml

@@ -0,0 +1,43 @@
+---
+version: 0.2
+
+branches:
+  default:
+    stage: pre-merge
+
+stages:
+  pre-merge:
+    worker: &master-worker
+      type: docker
+      path: eve/workers/master
+      volumes:
+        - '/home/eve/workspace'
+    steps:
+      - Git:
+          name: fetch source
+          repourl: '%(prop:git_reference)s'
+          shallow: True
+          retryFetch: True
+          haltOnFailure: True
+      - ShellCommand:
+          name: install dependencies
+          command: yarn install --frozen-lockfile
+      - ShellCommand:
+          name: run lint yml
+          command: yarn run --silent lint_yml
+      - ShellCommand:
+          name: run lint
+          command: yarn run --silent lint -- --max-warnings 0
+      - ShellCommand:
+          name: run lint_md
+          command: yarn run --silent lint_md
+      - ShellCommand:
+          name: run test
+          command: yarn run --silent test
+      - ShellCommand:
+          name: run ft_test
+          command: yarn run ft_test
+      - ShellCommand:
+          name: run executables tests
+          command: yarn install && yarn test
+          workdir: '%(prop:builddir)s/build/lib/executables/pensieveCreds/'

eve/workers/master/Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:trusty
+
+#
+# Install apt packages needed by the buildchain
+#
+ENV LANG C.UTF-8
+COPY buildbot_worker_packages.list arsenal_packages.list /tmp/
+RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
+    && apt-get install -qy software-properties-common python-software-properties \
+    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && echo "deb https://deb.nodesource.com/node_10.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
+    && curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+    && add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update -q \
+    && cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
+    && cat /tmp/arsenal_packages.list | xargs apt-get install -qy \
+    && pip install pip==9.0.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/*_packages.list
+
+#
+# Install usefull nodejs dependencies
+#
+
+RUN yarn global add mocha
+
+#
+# Add user eve
+#
+
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+
+#
+# Run buildbot-worker on startup
+#
+
+ARG BUILDBOT_VERSION=0.9.12
+RUN pip install yamllint
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+USER eve
+ENV HOME /home/eve
+#
+# Setup nodejs environmnent
+#
+
+ENV CXX=g++-4.9
+ENV LANG C.UTF-8
+
+
+WORKDIR /home/eve/workspace
+CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
+    && sudo service redis-server start \
+    && buildbot-worker start --nodaemon

eve/workers/master/arsenal_packages.list

@@ -0,0 +1,4 @@
+nodejs
+redis-server
+g++-4.9
+yarn

eve/workers/master/buildbot_worker_packages.list

@@ -0,0 +1,9 @@
+ca-certificates
+git
+libffi-dev
+libssl-dev
+python2.7
+python2.7-dev
+python-pip
+software-properties-common
+sudo

lib/algos/list/delimiterMaster.js

@@ -32,7 +32,6 @@ class DelimiterMaster extends Delimiter {
         // non-PHD master version or a version whose master is a PHD version
         this.prvKey = undefined;
         this.prvPHDKey = undefined;
-        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -62,12 +61,6 @@ class DelimiterMaster extends Delimiter {
         let key = obj.key;
         const value = obj.value;
 
-        if (key.startsWith(DbPrefixes.Replay)) {
-            this.inReplayPrefix = true;
-            return FILTER_SKIP;
-        }
-        this.inReplayPrefix = false;
-
         /* Skip keys not starting with the prefix or not alphabetically
          * ordered. */
         if ((this.prefix && !key.startsWith(this.prefix))
@@ -87,13 +80,13 @@ class DelimiterMaster extends Delimiter {
              *   when a listing page ends on an accepted obj and the next page
              *   starts with a version of this object.
              *   In that case prvKey is default set to undefined
-             *   in the constructor and comparing to NextMarker is the only
+             *   in the constructor) and comparing to NextMarker is the only
              *   way to know  we should not accept this version. This test is
              *   not redundant with the one at the beginning of this function,
              *   we are comparing here the key without the version suffix,
              * - key startsWith the previous NextMarker happens because we set
              *   NextMarker to the common prefix instead of the whole key
-             *   value. (TODO: remove this test once ZENKO-1048 is fixed)
+             *   value. (TODO: remove this test once ZENKO-1048 is fixed. ).
              *   */
             if (key === this.prvKey || key === this[this.nextContinueMarker] ||
                 (this.delimiter &&
@@ -162,7 +155,7 @@ class DelimiterMaster extends Delimiter {
         return super.filter(obj);
     }
 
-    skippingBase() {
+    skippingV0() {
         if (this[this.nextContinueMarker]) {
             // next marker or next continuation token:
             // - foo/ : skipping foo/
@@ -177,15 +170,8 @@ class DelimiterMaster extends Delimiter {
         return SKIP_NONE;
     }
 
-    skippingV0() {
-        if (this.inReplayPrefix) {
-            return DbPrefixes.Replay;
-        }
-        return this.skippingBase();
-    }
-
     skippingV1() {
-        const skipTo = this.skippingBase();
+        const skipTo = this.skippingV0();
         if (skipTo === SKIP_NONE) {
             return SKIP_NONE;
         }

lib/algos/list/delimiterVersions.js

@@ -33,7 +33,6 @@ class DelimiterVersions extends Delimiter {
         // listing results
         this.NextMarker = parameters.keyMarker;
         this.NextVersionIdMarker = undefined;
-        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -164,15 +163,8 @@ class DelimiterVersions extends Delimiter {
      *  @return {number}          - indicates if iteration should continue
      */
     filterV0(obj) {
-        if (obj.key.startsWith(DbPrefixes.Replay)) {
-            this.inReplayPrefix = true;
-            return FILTER_SKIP;
-        }
-        this.inReplayPrefix = false;
-
         if (Version.isPHD(obj.value)) {
-            // return accept to avoid skipping the next values in range
+            return FILTER_ACCEPT; // trick repd to not increase its streak
-            return FILTER_ACCEPT;
         }
         return this.filterCommon(obj.key, obj.value);
     }
@@ -213,9 +205,8 @@ class DelimiterVersions extends Delimiter {
         } else {
             nonversionedKey = key.slice(0, versionIdIndex);
             versionId = key.slice(versionIdIndex + 1);
-            // skip a version key if it is the master version
             if (this.masterKey === nonversionedKey && this.masterVersionId === versionId) {
-                return FILTER_SKIP;
+                return FILTER_ACCEPT; // trick repd to not increase its streak
             }
             this.masterKey = undefined;
             this.masterVersionId = undefined;
@@ -231,9 +222,6 @@ class DelimiterVersions extends Delimiter {
     }
 
     skippingV0() {
-        if (this.inReplayPrefix) {
-            return DbPrefixes.Replay;
-        }
         if (this.NextMarker) {
             const index = this.NextMarker.lastIndexOf(this.delimiter);
             if (index === this.NextMarker.length - 1) {

lib/models/ObjectMD.js

@@ -110,10 +110,8 @@ class ObjectMD {
             // should be undefined when not set explicitly
             'isNull': undefined,
             'nullVersionId': undefined,
-            'nullUploadId': undefined,
             'isDeleteMarker': undefined,
             'versionId': undefined,
-            'uploadId': undefined,
             'tags': {},
             'replicationInfo': {
                 status: '',
@@ -633,27 +631,6 @@ class ObjectMD {
         return this._data.nullVersionId;
     }
 
-    /**
-     * Set metadata nullUploadId value
-     *
-     * @param {string} nullUploadId - The upload ID used to complete
-     * the MPU of the null version
-     * @return {ObjectMD} itself
-     */
-    setNullUploadId(nullUploadId) {
-        this._data.nullUploadId = nullUploadId;
-        return this;
-    }
-
-    /**
-     * Get metadata nullUploadId value
-     *
-     * @return {string|undefined} The object nullUploadId
-     */
-    getNullUploadId() {
-        return this._data.nullUploadId;
-    }
-
     /**
      * Set metadata isDeleteMarker value
      *
@@ -704,26 +681,6 @@ class ObjectMD {
         return VersionIDUtils.encode(this.getVersionId());
     }
 
-    /**
-     * Set metadata uploadId value
-     *
-     * @param {string} uploadId - The upload ID used to complete the MPU object
-     * @return {ObjectMD} itself
-     */
-    setUploadId(uploadId) {
-        this._data.uploadId = uploadId;
-        return this;
-    }
-
-    /**
-     * Get metadata uploadId value
-     *
-     * @return {string|undefined} The object uploadId
-     */
-    getUploadId() {
-        return this._data.uploadId;
-    }
-
     /**
      * Set tags
      *

lib/policyEvaluator/utils/actionMaps.js

@@ -23,7 +23,6 @@ const sharedActionMap = {
     bucketPutReplication: 's3:PutReplicationConfiguration',
     bucketPutVersioning: 's3:PutBucketVersioning',
     bucketPutWebsite: 's3:PutBucketWebsite',
-    bypassGovernanceRetention: 's3:BypassGovernanceRetention',
     listMultipartUploads: 's3:ListBucketMultipartUploads',
     listParts: 's3:ListMultipartUploadParts',
     multipartDelete: 's3:AbortMultipartUpload',

lib/s3routes/routesUtils.js

@@ -85,18 +85,8 @@ const XMLResponseBackend = {
         });
     },
 
-    errorResponse: function errorXMLResponse(errCode, response, log, corsHeaders) {
+    errorResponse: function errorXMLResponse(errCode, response, log,
-        setCommonResponseHeaders(corsHeaders, response, log);
+                                             corsHeaders) {
-        // early return to avoid extra headers and XML data
-        if (errCode.code === 304) {
-            response.writeHead(errCode.code);
-            return response.end('', 'utf8', () => {
-                log.end().info('responded with empty body', {
-                    httpCode: response.statusCode,
-                });
-            });
-        }
-
         log.trace('sending error xml response', { errCode });
         /*
          <?xml version="1.0" encoding="UTF-8"?>
@@ -122,6 +112,7 @@ const XMLResponseBackend = {
         log.addDefaultFields({
             bytesSent,
         });
+        setCommonResponseHeaders(corsHeaders, response, log);
         response.writeHead(errCode.code,
             { 'Content-Type': 'application/xml',
                 'Content-Length': bytesSent });

lib/storage/metadata/bucketclient/LogConsumer.js

@@ -6,7 +6,6 @@ const jsonStream = require('JSONStream');
 const werelogs = require('werelogs');
 
 const errors = require('../../../errors');
-const jsutil = require('../../../jsutil');
 
 class ListRecordStream extends stream.Transform {
     constructor(logger) {
@@ -88,7 +87,6 @@ class LogConsumer {
     readRecords(params, cb) {
         const recordStream = new ListRecordStream(this.logger);
         const _params = params || {};
-        const cbOnce = jsutil.once(cb);
 
         this.bucketClient.getRaftLog(
             this.raftSession, _params.startSeq, _params.limit,
@@ -98,26 +96,26 @@ class LogConsumer {
                         // no such raft session, log and ignore
                         this.logger.warn('raft session does not exist yet',
                                          { raftId: this.raftSession });
-                        return cbOnce(null, { info: { start: null,
+                        return cb(null, { info: { start: null,
                             end: null } });
                     }
                     if (err.code === 416) {
                         // requested range not satisfiable
                         this.logger.debug('no new log record to process',
                                           { raftId: this.raftSession });
-                        return cbOnce(null, { info: { start: null,
+                        return cb(null, { info: { start: null,
                             end: null } });
                     }
                     this.logger.error(
                         'Error handling record log request', { error: err });
-                    return cbOnce(err);
+                    return cb(err);
                 }
                 // setup a temporary listener until the 'header' event
                 // is emitted
                 recordStream.on('error', err => {
                     this.logger.error('error receiving raft log',
                                       { error: err.message });
-                    return cbOnce(errors.InternalError);
+                    return cb(errors.InternalError);
                 });
                 const jsonResponse = stream.pipe(jsonStream.parse('log.*'));
                 jsonResponse.pipe(recordStream);
@@ -126,7 +124,7 @@ class LogConsumer {
                     .on('header', header => {
                         // remove temporary listener
                         recordStream.removeAllListeners('error');
-                        return cbOnce(null, { info: header.info,
+                        return cb(null, { info: header.info,
                                           log: recordStream });
                     })
                     .on('error', err => recordStream.emit('error', err));

lib/versioning/constants.js

@@ -5,7 +5,6 @@ module.exports.VersioningConstants = {
     DbPrefixes: {
         Master: '\x7fM',
         Version: '\x7fV',
-        Replay: '\x7fR',
     },
     BucketVersioningKeyFormat: {
         current: 'v1',

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=6.9.5"
   },
-  "version": "7.7.1",
+  "version": "7.7.0",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/algos/list/delimiterMaster.js

@@ -8,14 +8,12 @@ const {
     FILTER_ACCEPT,
     FILTER_SKIP,
     SKIP_NONE,
-    inc,
 } = require('../../../../lib/algos/list/tools');
 const VSConst =
     require('../../../../lib/versioning/constants').VersioningConstants;
 const Version = require('../../../../lib/versioning/Version').Version;
 const { generateVersionId } = require('../../../../lib/versioning/VersionID');
 const { DbPrefixes } = VSConst;
-const zpad = require('../../helpers').zpad;
 
 
 const VID_SEP = VSConst.VersionId.Separator;
@@ -66,6 +64,7 @@ function getListingKey(key, vFormat) {
                                                   fakeLogger, vFormat);
 
             /* Filter a master version to set NextMarker. */
+            // TODO: useless once S3C-1628 is fixed.
             const listingKey = getListingKey(key, vFormat);
             delimiter.filter({ key: listingKey, value: '' });
             assert.strictEqual(delimiter.NextMarker, key);
@@ -216,8 +215,8 @@ function getListingKey(key, vFormat) {
                     value: Version.generatePHDVersion(generateVersionId('', '')),
                 };
 
-                /* When filtered, it should return FILTER_ACCEPT and set the prvKey
+                /* When filtered, it should return FILTER_ACCEPT and set the prvKey.
-                 * to undefined. It should not be added to the result content or common
+                 * to undefined. It should not be added to result the content or common
                  * prefixes. */
                 assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
                 assert.strictEqual(delimiter.prvKey, undefined);
@@ -239,7 +238,7 @@ function getListingKey(key, vFormat) {
                  * and element in result content. */
                 delimiter.filter({ key, value });
 
-                /* When filtered, it should return FILTER_ACCEPT and set the prvKey
+                /* When filtered, it should return FILTER_ACCEPT and set the prvKey.
                  * to undefined. It should not be added to the result content or common
                  * prefixes. */
                 assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
@@ -284,7 +283,7 @@ function getListingKey(key, vFormat) {
                 });
             });
 
-            it('should skip a delete marker version', () => {
+            it('should accept a delete marker', () => {
                 const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
                 const version = new Version({ isDeleteMarker: true });
                 const key = 'key';
@@ -301,7 +300,7 @@ function getListingKey(key, vFormat) {
                 assert.deepStrictEqual(delimiter.result(), EmptyResult);
             });
 
-            it('should skip version after a delete marker master', () => {
+            it('should skip version after a delete marker', () => {
                 const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
                 const version = new Version({ isDeleteMarker: true });
                 const key = 'key';
@@ -317,7 +316,7 @@ function getListingKey(key, vFormat) {
                 assert.deepStrictEqual(delimiter.result(), EmptyResult);
             });
 
-            it('should accept a new master key after a delete marker master', () => {
+            it('should accept a new key after a delete marker', () => {
                 const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
                 const version = new Version({ isDeleteMarker: true });
                 const key1 = 'key1';
@@ -455,39 +454,6 @@ function getListingKey(key, vFormat) {
 
                 assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
             });
-
-            it('should return good skipping value for DelimiterMaster on replay keys', () => {
-                const delimiter = new DelimiterMaster(
-                    { delimiter: '/', v2: true },
-                    fakeLogger, vFormat);
-
-                for (let i = 0; i < 10; i++) {
-                    delimiter.filter({
-                        key: `foo/${zpad(i)}`,
-                        value: '{}',
-                    });
-                }
-                // simulate a listing that goes through a replay key, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${DbPrefixes.Replay}xyz`,
-                        value: 'abcdef',
-                    }),
-                    FILTER_SKIP);
-                // ...it should skip the whole replay prefix
-                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
-
-                // simulate a listing that reaches regular object keys
-                // beyond the replay prefix, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
-                        value: '{}',
-                    }),
-                    FILTER_ACCEPT);
-                // ...it should return to skipping by prefix as usual
-                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
-            });
         }
     });
 });

tests/unit/algos/list/delimiterVersions.js

@@ -3,29 +3,14 @@
 const assert = require('assert');
 const DelimiterVersions =
     require('../../../../lib/algos/list/delimiterVersions').DelimiterVersions;
-const {
-    FILTER_ACCEPT,
-    FILTER_SKIP,
-    SKIP_NONE,
-    inc,
-} = require('../../../../lib/algos/list/tools');
 const Werelogs = require('werelogs').Logger;
 const logger = new Werelogs('listTest');
 const performListing = require('../../../utils/performListing');
 const zpad = require('../../helpers').zpad;
+const { inc } = require('../../../../lib/algos/list/tools');
 const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
-const Version = require('../../../../lib/versioning/Version').Version;
-const { generateVersionId } = require('../../../../lib/versioning/VersionID');
 const { DbPrefixes } = VSConst;
 const VID_SEP = VSConst.VersionId.Separator;
-const EmptyResult = {
-    Versions: [],
-    CommonPrefixes: [],
-    IsTruncated: false,
-    NextKeyMarker: undefined,
-    NextVersionIdMarker: undefined,
-    Delimiter: undefined,
-};
 
 class Test {
     constructor(name, input, genMDParams, output, filter) {
@@ -279,7 +264,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }),
-    new Test('with bad key marker and good prefix', {
+    new Test('bad key marker and good prefix', {
         delimiter: '/',
         prefix: 'notes/summer/',
         keyMarker: 'notes/summer0',
@@ -303,7 +288,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
-    new Test('with delimiter and prefix (related to #147)', {
+    new Test('delimiter and prefix (related to #147)', {
         delimiter: '/',
         prefix: 'notes/',
     }, {
@@ -333,7 +318,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }),
-    new Test('with delimiter, prefix and marker (related to #147)', {
+    new Test('delimiter, prefix and marker (related to #147)', {
         delimiter: '/',
         prefix: 'notes/',
         keyMarker: 'notes/year.txt',
@@ -361,7 +346,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
-    new Test('with all parameters 1/3', {
+    new Test('all parameters 1/3', {
         delimiter: '/',
         prefix: 'notes/',
         keyMarker: 'notes/',
@@ -387,7 +372,7 @@ const tests = [
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('with all parameters 2/3', {
+    new Test('all parameters 2/3', {
         delimiter: '/',
         prefix: 'notes/', // prefix
         keyMarker: 'notes/spring/',
@@ -413,7 +398,7 @@ const tests = [
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('with all parameters 3/3', {
+    new Test('all parameters 3/3', {
         delimiter: '/',
         prefix: 'notes/', // prefix
         keyMarker: 'notes/summer/',
@@ -441,7 +426,7 @@ const tests = [
         NextVersionIdMarker: receivedData[19].versionId,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('with all parameters 4/3', {
+    new Test('all parameters 4/3', {
         delimiter: '/',
         prefix: 'notes/', // prefix
         keyMarker: 'notes/year.txt',
@@ -469,7 +454,7 @@ const tests = [
         NextVersionIdMarker: receivedData[20].versionId,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('with all parameters 5/3', {
+    new Test('all parameters 5/3', {
         delimiter: '/',
         prefix: 'notes/',
         keyMarker: 'notes/yore.rs',
@@ -496,79 +481,39 @@ const tests = [
     }, (e, input) => e.key > input.keyMarker),
 ];
 
-function getListingKey(key, vFormat) {
-    if (vFormat === 'v0') {
-        return key;
-    }
-    if (vFormat === 'v1') {
-        const keyPrefix = key.includes(VID_SEP) ?
-              DbPrefixes.Version : DbPrefixes.Master;
-        return `${keyPrefix}${key}`;
-    }
-    return assert.fail(`bad format ${vFormat}`);
-}
-
 function getTestListing(test, data, vFormat) {
     return data
         .filter(e => test.filter(e, test.input))
-        .map(e => ({
+        .map(e => {
-            key: getListingKey(e.key, vFormat),
+            if (vFormat === 'v0') {
-            value: e.value,
+                return e;
-        }));
+            }
+            if (vFormat === 'v1') {
+                const keyPrefix = e.key.includes(VID_SEP) ?
+                      DbPrefixes.Version : DbPrefixes.Master;
+                return {
+                    key: `${keyPrefix}${e.key}`,
+                    value: e.value,
+                };
+            }
+            return assert.fail(`bad format ${vFormat}`);
+        });
 }
 
 ['v0', 'v1'].forEach(vFormat => {
     describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
         it('Should return good skipping value for DelimiterVersions', () => {
-            const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
+            const delimiter = new DelimiterVersions({ delimiter: '/' });
             for (let i = 0; i < 100; i++) {
                 delimiter.filter({
                     key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
                     value: '{}',
                 });
             }
-            if (vFormat === 'v1') {
+            assert.strictEqual(delimiter.skipping(),
-                assert.deepStrictEqual(delimiter.skipping(), [
+                               `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
-                    `${DbPrefixes.Master}foo/`,
-                    `${DbPrefixes.Version}foo/`,
-                ]);
-            } else {
-                assert.strictEqual(delimiter.skipping(), 'foo/');
-            }
         });
 
-        if (vFormat === 'v0') {
-            it('Should return good skipping value for DelimiterVersions on replay keys', () => {
-                const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
-                for (let i = 0; i < 10; i++) {
-                    delimiter.filter({
-                        key: `foo/${zpad(i)}`,
-                        value: '{}',
-                    });
-                }
-                // simulate a listing that goes through a replay key, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${DbPrefixes.Replay}xyz`,
-                        value: 'abcdef',
-                    }),
-                    FILTER_SKIP);
-                // ...it should skip the whole replay prefix
-                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
-
-                // simulate a listing that reaches regular object keys
-                // beyond the replay prefix, ...
-                assert.strictEqual(
-                    delimiter.filter({
-                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
-                        value: '{}',
-                    }),
-                    FILTER_ACCEPT);
-                // ...it should return to skipping by prefix as usual
-                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
-            });
-        }
-
         tests.forEach(test => {
             it(`Should return metadata listing params to list ${test.name}`, () => {
                 const listing = new DelimiterVersions(test.input, logger, vFormat);
@@ -582,442 +527,5 @@ function getTestListing(test, data, vFormat) {
                 assert.deepStrictEqual(res, test.output);
             });
         });
-
-        it('skipping() should return SKIP_NONE when NextMarker is undefined', () => {
-            const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
-
-            assert.strictEqual(delimiter.NextMarker, undefined);
-            assert.strictEqual(delimiter.skipping(), SKIP_NONE);
-        });
-
-        it('skipping() should return SKIP_NONE when marker is set and ' +
-        'does not contain the delimiter', () => {
-            const key = 'foo';
-            const delimiter = new DelimiterVersions({ delimiter: '/', marker: key },
-                                                    logger, vFormat);
-
-            /* Filter a master version to set NextMarker. */
-            const listingKey = getListingKey(key, vFormat);
-            delimiter.filter({ key: listingKey, value: '' });
-            assert.strictEqual(delimiter.NextMarker, 'foo');
-            assert.strictEqual(delimiter.skipping(), SKIP_NONE);
-        });
-
-        it('skipping() should return prefix to skip when marker is set and ' +
-        'contains the delimiter', () => {
-            const key = 'foo/bar';
-            const delimiter = new DelimiterVersions({ delimiter: '/', marker: key },
-                                                    logger, vFormat);
-
-            /* Filter a master version to set NextMarker. */
-            const listingKey = getListingKey(key, vFormat);
-            delimiter.filter({ key: listingKey, value: '' });
-            assert.strictEqual(delimiter.NextMarker, 'foo/');
-
-            if (vFormat === 'v0') {
-                assert.strictEqual(delimiter.skipping(), 'foo/');
-            } else {
-                assert.deepStrictEqual(delimiter.skipping(), [
-                    `${DbPrefixes.Master}foo/`,
-                    `${DbPrefixes.Version}foo/`,
-                ]);
-            }
-        });
-
-        it('skipping() should return prefix when marker is set and ' +
-        'ends with the delimiter', () => {
-            const key = 'foo/';
-            const delimiter = new DelimiterVersions({ delimiter: '/', marker: key },
-                                                    logger, vFormat);
-
-            /* Filter a master version to set NextMarker. */
-            const listingKey = getListingKey(key, vFormat);
-            delimiter.filter({ key: listingKey, value: '' });
-            assert.strictEqual(delimiter.NextMarker, 'foo/');
-
-            if (vFormat === 'v0') {
-                assert.strictEqual(delimiter.skipping(), 'foo/');
-            } else {
-                assert.deepStrictEqual(delimiter.skipping(), [
-                    `${DbPrefixes.Master}foo/`,
-                    `${DbPrefixes.Version}foo/`,
-                ]);
-            }
-        });
-
-        it('should skip entries not starting with prefix', () => {
-            const delimiter = new DelimiterVersions({ prefix: 'prefix' }, logger, vFormat);
-
-            const listingKey = getListingKey('wrong', vFormat);
-            assert.strictEqual(delimiter.filter({ key: listingKey, value: '' }), FILTER_SKIP);
-            assert.strictEqual(delimiter.NextMarker, undefined);
-            assert.strictEqual(delimiter.prvKey, undefined);
-            assert.deepStrictEqual(delimiter.result(), EmptyResult);
-        });
-
-        it('should accept a master version', () => {
-            const delimiter = new DelimiterVersions({}, logger, vFormat);
-            const key = 'key';
-            const value = '';
-
-            const listingKey = getListingKey(key, vFormat);
-            assert.strictEqual(delimiter.filter({ key: listingKey, value }), FILTER_ACCEPT);
-            assert.strictEqual(delimiter.NextMarker, key);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: [],
-                Versions: [{
-                    key: 'key',
-                    value: '',
-                    versionId: 'null',
-                }],
-                Delimiter: undefined,
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-        });
-
-        it('should return good values for entries with different common prefixes', () => {
-            const delimiter = new DelimiterVersions({ delimiter: '/' },
-                                                    logger, vFormat);
-
-            /* Filter the first entry with a common prefix. It should be
-             * accepted and added to the result. */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey('commonPrefix1/key1', vFormat),
-                value: '',
-            }),
-                               FILTER_ACCEPT);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: ['commonPrefix1/'],
-                Versions: [],
-                Delimiter: '/',
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-
-            /* Filter the second entry with the same common prefix than the
-             * first entry. It should be skipped and not added to the result. */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey('commonPrefix1/key2', vFormat),
-                value: '',
-            }),
-                               FILTER_SKIP);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: ['commonPrefix1/'],
-                Versions: [],
-                Delimiter: '/',
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-
-            /* Filter an entry with a new common prefix. It should be accepted
-             * and not added to the result. */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey('commonPrefix2/key1', vFormat),
-                value: '',
-            }),
-                               FILTER_ACCEPT);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: ['commonPrefix1/', 'commonPrefix2/'],
-                Versions: [],
-                Delimiter: '/',
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-        });
-
-        it('should accept a delete marker version', () => {
-            const delimiter = new DelimiterVersions({}, logger, vFormat);
-            const version = new Version({ isDeleteMarker: true });
-            const key = 'key';
-            const obj = {
-                key: getListingKey(`${key}${VID_SEP}version`, vFormat),
-                value: version.toString(),
-            };
-
-            /* When filtered, it should return FILTER_ACCEPT and
-             * should be added to the result content. */
-            assert.strictEqual(delimiter.filter(obj), FILTER_ACCEPT);
-            assert.strictEqual(delimiter.NextMarker, key);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: [],
-                Versions: [{
-                    key: 'key',
-                    value: version.toString(),
-                    versionId: 'version',
-                }],
-                Delimiter: undefined,
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-        });
-
-        it('should accept a version after a delete marker master', () => {
-            const delimiter = new DelimiterVersions({}, logger, vFormat);
-            const version = new Version({ isDeleteMarker: true });
-            const key = 'key';
-            const versionKey = `${key}${VID_SEP}version`;
-
-            delimiter.filter({ key: getListingKey(key, vFormat), value: version.toString() });
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(versionKey, vFormat),
-                value: 'value',
-            }), FILTER_ACCEPT);
-            assert.strictEqual(delimiter.NextMarker, key);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: [],
-                Versions: [{
-                    key: 'key',
-                    value: version.toString(),
-                    versionId: 'null',
-                }, {
-                    key: 'key',
-                    value: 'value',
-                    versionId: 'version',
-                }],
-                Delimiter: undefined,
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-        });
-
-        it('should accept a new master key w/ version after a delete marker master', () => {
-            const delimiter = new DelimiterVersions({}, logger, vFormat);
-            const version = new Version({ isDeleteMarker: true });
-            const key1 = 'key1';
-            const key2 = 'key2';
-            const value2 = '{"versionId":"version"}';
-
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(key1, vFormat),
-                value: version.toString(),
-            }), FILTER_ACCEPT);
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(key2, vFormat),
-                value: value2,
-            }), FILTER_ACCEPT);
-            assert.strictEqual(delimiter.NextMarker, key2);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: [],
-                Delimiter: undefined,
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-                Versions: [{
-                    key: 'key1',
-                    value: '{"isDeleteMarker":true}',
-                    versionId: 'null',
-                }, {
-                    key: 'key2',
-                    value: '{"versionId":"version"}',
-                    versionId: 'version',
-                }],
-            });
-        });
-
-        it('should accept a version after skipping an object because of its commonPrefix', () => {
-            const commonPrefix1 = 'commonPrefix1/';
-            const commonPrefix2 = 'commonPrefix2/';
-            const prefix1Key1 = 'commonPrefix1/key1';
-            const prefix1Key2 = 'commonPrefix1/key2';
-            const prefix2VersionKey1 = `commonPrefix2/key1${VID_SEP}version`;
-            const value = '{"versionId":"version"}';
-
-            const delimiter = new DelimiterVersions({ delimiter: '/' },
-                                                    logger, vFormat);
-
-            /* Filter the two first entries with the same common prefix to add
-             * it to the result and reach the state where an entry is skipped
-             * because of an already present common prefix in the result. */
-            delimiter.filter({ key: getListingKey(prefix1Key1, vFormat), value });
-            delimiter.filter({ key: getListingKey(prefix1Key2, vFormat), value });
-
-            /* Filter an object with a key containing a version part and a new
-             * common prefix. It should be accepted and the new common prefix
-             * added to the result. */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(prefix2VersionKey1, vFormat),
-                value,
-            }), FILTER_ACCEPT);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: [commonPrefix1, commonPrefix2],
-                Versions: [],
-                Delimiter: '/',
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-        });
-
-        it('should skip first version key if equal to master', () => {
-            const delimiter = new DelimiterVersions({}, logger, vFormat);
-            const masterKey = 'key';
-            const versionKey1 = `${masterKey}${VID_SEP}version1`;
-            const versionKey2 = `${masterKey}${VID_SEP}version2`;
-            const value2 = 'value2';
-
-            /* Filter the master version for version1 */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(masterKey, vFormat),
-                value: '{"versionId":"version1"}',
-            }), FILTER_ACCEPT);
-
-            /* Filter the version key for version1 */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(versionKey1, vFormat),
-                value: '{"versionId":"version1"}',
-            }), FILTER_SKIP);
-
-            /* Filter the version key for version2 */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(versionKey2, vFormat),
-                value: value2,
-            }), FILTER_ACCEPT);
-
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: [],
-                Versions: [{
-                    key: 'key',
-                    value: '{"versionId":"version1"}',
-                    versionId: 'version1',
-                }, {
-                    key: 'key',
-                    value: 'value2',
-                    versionId: 'version2',
-                }],
-                Delimiter: undefined,
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-        });
-
-        it('should skip master and version key if under a known prefix', () => {
-            const commonPrefix1 = 'commonPrefix/';
-            const prefixKey1 = 'commonPrefix/key1';
-            const prefixKey2 = 'commonPrefix/key2';
-            const prefixVersionKey1 = `commonPrefix/key2${VID_SEP}version`;
-            const value = '{"versionId":"version"}';
-
-            const delimiter = new DelimiterVersions({ delimiter: '/' },
-                                                    logger, vFormat);
-
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(prefixKey1, vFormat),
-                value,
-            }), FILTER_ACCEPT);
-            /* The second master key of the same common prefix should be skipped */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(prefixKey2, vFormat),
-                value,
-            }), FILTER_SKIP);
-
-            /* The version key of the same common prefix should also be skipped */
-            assert.strictEqual(delimiter.filter({
-                key: getListingKey(prefixVersionKey1, vFormat),
-                value,
-            }), FILTER_SKIP);
-            assert.deepStrictEqual(delimiter.result(), {
-                CommonPrefixes: [commonPrefix1],
-                Versions: [],
-                Delimiter: '/',
-                IsTruncated: false,
-                NextKeyMarker: undefined,
-                NextVersionIdMarker: undefined,
-            });
-        });
-
-        if (vFormat === 'v0') {
-            it('should accept a PHD version as first input', () => {
-                const delimiter = new DelimiterVersions({}, logger, vFormat);
-                const keyPHD = 'keyPHD';
-                const objPHD = {
-                    key: keyPHD,
-                    value: Version.generatePHDVersion(generateVersionId('', '')),
-                };
-
-                /* When filtered, it should return FILTER_ACCEPT and set the prvKey
-                 * to undefined. It should not be added to the result content or common
-                 * prefixes. */
-                assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
-                assert.strictEqual(delimiter.prvKey, undefined);
-                assert.strictEqual(delimiter.NextMarker, undefined);
-                assert.deepStrictEqual(delimiter.result(), EmptyResult);
-            });
-
-            it('should accept a PHD version', () => {
-                const delimiter = new DelimiterVersions({}, logger, vFormat);
-                const key = 'keyA';
-                const value = '';
-                const keyPHD = 'keyBPHD';
-                const objPHD = {
-                    key: keyPHD,
-                    value: Version.generatePHDVersion(generateVersionId('', '')),
-                };
-
-                /* Filter a master version to set the NextMarker and add
-                 * and element in result content. */
-                delimiter.filter({ key, value });
-
-                /* When filtered, it should return FILTER_ACCEPT. It
-                 * should not be added to the result content or common
-                 * prefixes. */
-                assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
-                assert.strictEqual(delimiter.prvKey, undefined);
-                assert.strictEqual(delimiter.NextMarker, key);
-                assert.deepStrictEqual(delimiter.result(), {
-                    CommonPrefixes: [],
-                    Versions: [{
-                        key: 'keyA',
-                        value: '',
-                        versionId: 'null',
-                    }],
-                    Delimiter: undefined,
-                    IsTruncated: false,
-                    NextKeyMarker: undefined,
-                    NextVersionIdMarker: undefined,
-                });
-            });
-
-            it('should accept a version after a PHD', () => {
-                const delimiter = new DelimiterVersions({}, logger, vFormat);
-                const masterKey = 'key';
-                const keyVersion = `${masterKey}${VID_SEP}version`;
-                const value = '';
-                const objPHD = {
-                    key: masterKey,
-                    value: Version.generatePHDVersion(generateVersionId('', '')),
-                };
-
-                /* Filter the PHD object. */
-                delimiter.filter(objPHD);
-
-                /* The filtering of the PHD object has no impact, the version is
-                 * accepted and added to the result. */
-                assert.strictEqual(delimiter.filter({
-                    key: keyVersion,
-                    value,
-                }), FILTER_ACCEPT);
-                assert.strictEqual(delimiter.NextMarker, masterKey);
-                assert.deepStrictEqual(delimiter.result(), {
-                    CommonPrefixes: [],
-                    Versions: [{
-                        key: 'key',
-                        value: '',
-                        versionId: 'version',
-                    }],
-                    Delimiter: undefined,
-                    IsTruncated: false,
-                    NextKeyMarker: undefined,
-                    NextVersionIdMarker: undefined,
-                });
-            });
-        }
     });
 });

tests/unit/models/object.js

@@ -69,8 +69,6 @@ describe('ObjectMD class setters/getters', () => {
         ['IsNull', true],
         ['NullVersionId', null, undefined],
         ['NullVersionId', '111111'],
-        ['NullUploadId', null, undefined],
-        ['NullUploadId', 'abcdefghi'],
         ['IsDeleteMarker', null, false],
         ['IsDeleteMarker', true],
         ['VersionId', null, undefined],
@@ -80,8 +78,6 @@ describe('ObjectMD class setters/getters', () => {
             key: 'value',
         }],
         ['Tags', null, {}],
-        ['UploadId', null, undefined],
-        ['UploadId', 'abcdefghi'],
         ['ReplicationInfo', null, {
             status: '',
             backends: [],
@@ -336,11 +332,9 @@ describe('getAttributes static method', () => {
             'location': true,
             'isNull': true,
             'nullVersionId': true,
-            'nullUploadId': true,
             'isDeleteMarker': true,
             'versionId': true,
             'tags': true,
-            'uploadId': true,
             'replicationInfo': true,
             'dataStoreName': true,
             'last-modified': true,