ARSN-20 migrate to github actions

Co-authored-by: Ronnie <halfpint1170@gmail.com>
(cherry picked from commit 4b08dd5263)

.github/workflows/tests.yaml

@@ -0,0 +1,47 @@
+---
+name: tests
+
+on:
+  push:
+    branches-ignore:
+    - 'development/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      # Label used to access the service container
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: '10'
+        cache: 'yarn'
+    - name: install dependencies
+      run: yarn install --frozen-lockfile
+    - name: lint yaml
+      run: yarn --silent lint_yml
+    - name: lint javascript
+      run: yarn --silent lint -- --max-warnings 0
+    - name: lint markdown
+      run: yarn --silent lint_md
+    - name: run unit tests
+      run: yarn --silent test
+    - name: run functional tests
+      run: yarn ft_test
+    - name: run executables tests
+      run: yarn install && yarn test
+      working-directory: 'lib/executables/pensieveCreds/'

eve/main.yml

@@ -1,43 +0,0 @@
----
-version: 0.2
-
-branches:
-  default:
-    stage: pre-merge
-
-stages:
-  pre-merge:
-    worker: &master-worker
-      type: docker
-      path: eve/workers/master
-      volumes:
-        - '/home/eve/workspace'
-    steps:
-      - Git:
-          name: fetch source
-          repourl: '%(prop:git_reference)s'
-          shallow: True
-          retryFetch: True
-          haltOnFailure: True
-      - ShellCommand:
-          name: install dependencies
-          command: yarn install --frozen-lockfile
-      - ShellCommand:
-          name: run lint yml
-          command: yarn run --silent lint_yml
-      - ShellCommand:
-          name: run lint
-          command: yarn run --silent lint -- --max-warnings 0
-      - ShellCommand:
-          name: run lint_md
-          command: yarn run --silent lint_md
-      - ShellCommand:
-          name: run test
-          command: yarn run --silent test
-      - ShellCommand:
-          name: run ft_test
-          command: yarn run ft_test
-      - ShellCommand:
-          name: run executables tests
-          command: yarn install && yarn test
-          workdir: '%(prop:builddir)s/build/lib/executables/pensieveCreds/'

eve/workers/master/Dockerfile

@@ -1,57 +0,0 @@
-FROM ubuntu:trusty
-
-#
-# Install apt packages needed by the buildchain
-#
-ENV LANG C.UTF-8
-COPY buildbot_worker_packages.list arsenal_packages.list /tmp/
-RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
-    && apt-get install -qy software-properties-common python-software-properties \
-    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
-    && echo "deb https://deb.nodesource.com/node_10.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
-    && curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
-    && add-apt-repository ppa:ubuntu-toolchain-r/test \
-    && apt-get update -q \
-    && cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
-    && cat /tmp/arsenal_packages.list | xargs apt-get install -qy \
-    && pip install pip==9.0.1 \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm -f /tmp/*_packages.list
-
-#
-# Install usefull nodejs dependencies
-#
-
-RUN yarn global add mocha
-
-#
-# Add user eve
-#
-
-RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
-    && adduser eve sudo \
-    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
-
-#
-# Run buildbot-worker on startup
-#
-
-ARG BUILDBOT_VERSION=0.9.12
-RUN pip install yamllint
-RUN pip install buildbot-worker==$BUILDBOT_VERSION
-
-USER eve
-ENV HOME /home/eve
-#
-# Setup nodejs environmnent
-#
-
-ENV CXX=g++-4.9
-ENV LANG C.UTF-8
-
-
-WORKDIR /home/eve/workspace
-CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
-    && sudo service redis-server start \
-    && buildbot-worker start --nodaemon

eve/workers/master/arsenal_packages.list

@@ -1,4 +0,0 @@
-nodejs
-redis-server
-g++-4.9
-yarn

eve/workers/master/buildbot_worker_packages.list

@@ -1,9 +0,0 @@
-ca-certificates
-git
-libffi-dev
-libssl-dev
-python2.7
-python2.7-dev
-python-pip
-software-properties-common
-sudo

lib/algos/list/delimiterMaster.js

@@ -32,6 +32,7 @@ class DelimiterMaster extends Delimiter {
         // non-PHD master version or a version whose master is a PHD version
         this.prvKey = undefined;
         this.prvPHDKey = undefined;
+        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -61,6 +62,12 @@ class DelimiterMaster extends Delimiter {
         let key = obj.key;
         const value = obj.value;
 
+        if (key.startsWith(DbPrefixes.Replay)) {
+            this.inReplayPrefix = true;
+            return FILTER_SKIP;
+        }
+        this.inReplayPrefix = false;
+
         /* Skip keys not starting with the prefix or not alphabetically
          * ordered. */
         if ((this.prefix && !key.startsWith(this.prefix))
@@ -80,13 +87,13 @@ class DelimiterMaster extends Delimiter {
              *   when a listing page ends on an accepted obj and the next page
              *   starts with a version of this object.
              *   In that case prvKey is default set to undefined
-             *   in the constructor) and comparing to NextMarker is the only
+             *   in the constructor and comparing to NextMarker is the only
              *   way to know  we should not accept this version. This test is
              *   not redundant with the one at the beginning of this function,
              *   we are comparing here the key without the version suffix,
              * - key startsWith the previous NextMarker happens because we set
              *   NextMarker to the common prefix instead of the whole key
-             *   value. (TODO: remove this test once ZENKO-1048 is fixed. ).
+             *   value. (TODO: remove this test once ZENKO-1048 is fixed)
              *   */
             if (key === this.prvKey || key === this[this.nextContinueMarker] ||
                 (this.delimiter &&
@@ -155,7 +162,7 @@ class DelimiterMaster extends Delimiter {
         return super.filter(obj);
     }
 
-    skippingV0() {
+    skippingBase() {
         if (this[this.nextContinueMarker]) {
             // next marker or next continuation token:
             // - foo/ : skipping foo/
@@ -170,8 +177,15 @@ class DelimiterMaster extends Delimiter {
         return SKIP_NONE;
     }
 
+    skippingV0() {
+        if (this.inReplayPrefix) {
+            return DbPrefixes.Replay;
+        }
+        return this.skippingBase();
+    }
+
     skippingV1() {
-        const skipTo = this.skippingV0();
+        const skipTo = this.skippingBase();
         if (skipTo === SKIP_NONE) {
             return SKIP_NONE;
         }

lib/algos/list/delimiterVersions.js

@@ -33,6 +33,7 @@ class DelimiterVersions extends Delimiter {
         // listing results
         this.NextMarker = parameters.keyMarker;
         this.NextVersionIdMarker = undefined;
+        this.inReplayPrefix = false;
 
         Object.assign(this, {
             [BucketVersioningKeyFormat.v0]: {
@@ -163,8 +164,15 @@ class DelimiterVersions extends Delimiter {
      *  @return {number}          - indicates if iteration should continue
      */
     filterV0(obj) {
+        if (obj.key.startsWith(DbPrefixes.Replay)) {
+            this.inReplayPrefix = true;
+            return FILTER_SKIP;
+        }
+        this.inReplayPrefix = false;
+
         if (Version.isPHD(obj.value)) {
-            return FILTER_ACCEPT; // trick repd to not increase its streak
+            // return accept to avoid skipping the next values in range
+            return FILTER_ACCEPT;
         }
         return this.filterCommon(obj.key, obj.value);
     }
@@ -205,8 +213,9 @@ class DelimiterVersions extends Delimiter {
         } else {
             nonversionedKey = key.slice(0, versionIdIndex);
             versionId = key.slice(versionIdIndex + 1);
+            // skip a version key if it is the master version
             if (this.masterKey === nonversionedKey && this.masterVersionId === versionId) {
-                return FILTER_ACCEPT; // trick repd to not increase its streak
+                return FILTER_SKIP;
             }
             this.masterKey = undefined;
             this.masterVersionId = undefined;
@@ -222,6 +231,9 @@ class DelimiterVersions extends Delimiter {
     }
 
     skippingV0() {
+        if (this.inReplayPrefix) {
+            return DbPrefixes.Replay;
+        }
         if (this.NextMarker) {
             const index = this.NextMarker.lastIndexOf(this.delimiter);
             if (index === this.NextMarker.length - 1) {

lib/models/ObjectMD.js

@@ -110,8 +110,10 @@ class ObjectMD {
             // should be undefined when not set explicitly
             'isNull': undefined,
             'nullVersionId': undefined,
+            'nullUploadId': undefined,
             'isDeleteMarker': undefined,
             'versionId': undefined,
+            'uploadId': undefined,
             'tags': {},
             'replicationInfo': {
                 status: '',
@@ -631,6 +633,27 @@ class ObjectMD {
         return this._data.nullVersionId;
     }
 
+    /**
+     * Set metadata nullUploadId value
+     *
+     * @param {string} nullUploadId - The upload ID used to complete
+     * the MPU of the null version
+     * @return {ObjectMD} itself
+     */
+    setNullUploadId(nullUploadId) {
+        this._data.nullUploadId = nullUploadId;
+        return this;
+    }
+
+    /**
+     * Get metadata nullUploadId value
+     *
+     * @return {string|undefined} The object nullUploadId
+     */
+    getNullUploadId() {
+        return this._data.nullUploadId;
+    }
+
     /**
      * Set metadata isDeleteMarker value
      *
@@ -681,6 +704,26 @@ class ObjectMD {
         return VersionIDUtils.encode(this.getVersionId());
     }
 
+    /**
+     * Set metadata uploadId value
+     *
+     * @param {string} uploadId - The upload ID used to complete the MPU object
+     * @return {ObjectMD} itself
+     */
+    setUploadId(uploadId) {
+        this._data.uploadId = uploadId;
+        return this;
+    }
+
+    /**
+     * Get metadata uploadId value
+     *
+     * @return {string|undefined} The object uploadId
+     */
+    getUploadId() {
+        return this._data.uploadId;
+    }
+
     /**
      * Set tags
      *

lib/policyEvaluator/utils/actionMaps.js

@@ -23,6 +23,7 @@ const sharedActionMap = {
     bucketPutReplication: 's3:PutReplicationConfiguration',
     bucketPutVersioning: 's3:PutBucketVersioning',
     bucketPutWebsite: 's3:PutBucketWebsite',
+    bypassGovernanceRetention: 's3:BypassGovernanceRetention',
     listMultipartUploads: 's3:ListBucketMultipartUploads',
     listParts: 's3:ListMultipartUploadParts',
     multipartDelete: 's3:AbortMultipartUpload',

lib/s3routes/routesUtils.js

@@ -85,8 +85,18 @@ const XMLResponseBackend = {
         });
     },
 
-    errorResponse: function errorXMLResponse(errCode, response, log,
+    errorResponse: function errorXMLResponse(errCode, response, log, corsHeaders) {
-                                             corsHeaders) {
+        setCommonResponseHeaders(corsHeaders, response, log);
+        // early return to avoid extra headers and XML data
+        if (errCode.code === 304) {
+            response.writeHead(errCode.code);
+            return response.end('', 'utf8', () => {
+                log.end().info('responded with empty body', {
+                    httpCode: response.statusCode,
+                });
+            });
+        }
+
         log.trace('sending error xml response', { errCode });
         /*
          <?xml version="1.0" encoding="UTF-8"?>
@@ -112,7 +122,6 @@ const XMLResponseBackend = {
         log.addDefaultFields({
             bytesSent,
         });
-        setCommonResponseHeaders(corsHeaders, response, log);
         response.writeHead(errCode.code,
             { 'Content-Type': 'application/xml',
                 'Content-Length': bytesSent });

lib/storage/metadata/bucketclient/LogConsumer.js

@@ -6,6 +6,7 @@ const jsonStream = require('JSONStream');
 const werelogs = require('werelogs');
 
 const errors = require('../../../errors');
+const jsutil = require('../../../jsutil');
 
 class ListRecordStream extends stream.Transform {
     constructor(logger) {
@@ -87,6 +88,7 @@ class LogConsumer {
     readRecords(params, cb) {
         const recordStream = new ListRecordStream(this.logger);
         const _params = params || {};
+        const cbOnce = jsutil.once(cb);
 
         this.bucketClient.getRaftLog(
             this.raftSession, _params.startSeq, _params.limit,
@@ -96,26 +98,26 @@ class LogConsumer {
                         // no such raft session, log and ignore
                         this.logger.warn('raft session does not exist yet',
                                          { raftId: this.raftSession });
-                        return cb(null, { info: { start: null,
+                        return cbOnce(null, { info: { start: null,
                             end: null } });
                     }
                     if (err.code === 416) {
                         // requested range not satisfiable
                         this.logger.debug('no new log record to process',
                                           { raftId: this.raftSession });
-                        return cb(null, { info: { start: null,
+                        return cbOnce(null, { info: { start: null,
                             end: null } });
                     }
                     this.logger.error(
                         'Error handling record log request', { error: err });
-                    return cb(err);
+                    return cbOnce(err);
                 }
                 // setup a temporary listener until the 'header' event
                 // is emitted
                 recordStream.on('error', err => {
                     this.logger.error('error receiving raft log',
                                       { error: err.message });
-                    return cb(errors.InternalError);
+                    return cbOnce(errors.InternalError);
                 });
                 const jsonResponse = stream.pipe(jsonStream.parse('log.*'));
                 jsonResponse.pipe(recordStream);
@@ -124,7 +126,7 @@ class LogConsumer {
                     .on('header', header => {
                         // remove temporary listener
                         recordStream.removeAllListeners('error');
-                        return cb(null, { info: header.info,
+                        return cbOnce(null, { info: header.info,
                                           log: recordStream });
                     })
                     .on('error', err => recordStream.emit('error', err));

lib/versioning/constants.js

@@ -5,6 +5,7 @@ module.exports.VersioningConstants = {
     DbPrefixes: {
         Master: '\x7fM',
         Version: '\x7fV',
+        Replay: '\x7fR',
     },
     BucketVersioningKeyFormat: {
         current: 'v1',

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=6.9.5"
   },
-  "version": "7.7.0",
+  "version": "7.7.1",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/algos/list/delimiterMaster.js

@@ -8,12 +8,14 @@ const {
     FILTER_ACCEPT,
     FILTER_SKIP,
     SKIP_NONE,
+    inc,
 } = require('../../../../lib/algos/list/tools');
 const VSConst =
     require('../../../../lib/versioning/constants').VersioningConstants;
 const Version = require('../../../../lib/versioning/Version').Version;
 const { generateVersionId } = require('../../../../lib/versioning/VersionID');
 const { DbPrefixes } = VSConst;
+const zpad = require('../../helpers').zpad;
 
 
 const VID_SEP = VSConst.VersionId.Separator;
@@ -64,7 +66,6 @@ function getListingKey(key, vFormat) {
                                                   fakeLogger, vFormat);
 
             /* Filter a master version to set NextMarker. */
-            // TODO: useless once S3C-1628 is fixed.
             const listingKey = getListingKey(key, vFormat);
             delimiter.filter({ key: listingKey, value: '' });
             assert.strictEqual(delimiter.NextMarker, key);
@@ -215,8 +216,8 @@ function getListingKey(key, vFormat) {
                     value: Version.generatePHDVersion(generateVersionId('', '')),
                 };
 
-                /* When filtered, it should return FILTER_ACCEPT and set the prvKey.
+                /* When filtered, it should return FILTER_ACCEPT and set the prvKey
-                 * to undefined. It should not be added to result the content or common
+                 * to undefined. It should not be added to the result content or common
                  * prefixes. */
                 assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
                 assert.strictEqual(delimiter.prvKey, undefined);
@@ -238,7 +239,7 @@ function getListingKey(key, vFormat) {
                  * and element in result content. */
                 delimiter.filter({ key, value });
 
-                /* When filtered, it should return FILTER_ACCEPT and set the prvKey.
+                /* When filtered, it should return FILTER_ACCEPT and set the prvKey
                  * to undefined. It should not be added to the result content or common
                  * prefixes. */
                 assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
@@ -283,7 +284,7 @@ function getListingKey(key, vFormat) {
                 });
             });
 
-            it('should accept a delete marker', () => {
+            it('should skip a delete marker version', () => {
                 const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
                 const version = new Version({ isDeleteMarker: true });
                 const key = 'key';
@@ -300,7 +301,7 @@ function getListingKey(key, vFormat) {
                 assert.deepStrictEqual(delimiter.result(), EmptyResult);
             });
 
-            it('should skip version after a delete marker', () => {
+            it('should skip version after a delete marker master', () => {
                 const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
                 const version = new Version({ isDeleteMarker: true });
                 const key = 'key';
@@ -316,7 +317,7 @@ function getListingKey(key, vFormat) {
                 assert.deepStrictEqual(delimiter.result(), EmptyResult);
             });
 
-            it('should accept a new key after a delete marker', () => {
+            it('should accept a new master key after a delete marker master', () => {
                 const delimiter = new DelimiterMaster({}, fakeLogger, vFormat);
                 const version = new Version({ isDeleteMarker: true });
                 const key1 = 'key1';
@@ -454,6 +455,39 @@ function getListingKey(key, vFormat) {
 
                 assert.strictEqual(delimiter.filter({ key, value }), FILTER_SKIP);
             });
+
+            it('should return good skipping value for DelimiterMaster on replay keys', () => {
+                const delimiter = new DelimiterMaster(
+                    { delimiter: '/', v2: true },
+                    fakeLogger, vFormat);
+
+                for (let i = 0; i < 10; i++) {
+                    delimiter.filter({
+                        key: `foo/${zpad(i)}`,
+                        value: '{}',
+                    });
+                }
+                // simulate a listing that goes through a replay key, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${DbPrefixes.Replay}xyz`,
+                        value: 'abcdef',
+                    }),
+                    FILTER_SKIP);
+                // ...it should skip the whole replay prefix
+                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
+
+                // simulate a listing that reaches regular object keys
+                // beyond the replay prefix, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
+                        value: '{}',
+                    }),
+                    FILTER_ACCEPT);
+                // ...it should return to skipping by prefix as usual
+                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
+            });
         }
     });
 });

tests/unit/algos/list/delimiterVersions.js

@@ -3,14 +3,29 @@
 const assert = require('assert');
 const DelimiterVersions =
     require('../../../../lib/algos/list/delimiterVersions').DelimiterVersions;
+const {
+    FILTER_ACCEPT,
+    FILTER_SKIP,
+    SKIP_NONE,
+    inc,
+} = require('../../../../lib/algos/list/tools');
 const Werelogs = require('werelogs').Logger;
 const logger = new Werelogs('listTest');
 const performListing = require('../../../utils/performListing');
 const zpad = require('../../helpers').zpad;
-const { inc } = require('../../../../lib/algos/list/tools');
 const VSConst = require('../../../../lib/versioning/constants').VersioningConstants;
+const Version = require('../../../../lib/versioning/Version').Version;
+const { generateVersionId } = require('../../../../lib/versioning/VersionID');
 const { DbPrefixes } = VSConst;
 const VID_SEP = VSConst.VersionId.Separator;
+const EmptyResult = {
+    Versions: [],
+    CommonPrefixes: [],
+    IsTruncated: false,
+    NextKeyMarker: undefined,
+    NextVersionIdMarker: undefined,
+    Delimiter: undefined,
+};
 
 class Test {
     constructor(name, input, genMDParams, output, filter) {
@@ -264,7 +279,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }),
-    new Test('bad key marker and good prefix', {
+    new Test('with bad key marker and good prefix', {
         delimiter: '/',
         prefix: 'notes/summer/',
         keyMarker: 'notes/summer0',
@@ -288,7 +303,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
-    new Test('delimiter and prefix (related to #147)', {
+    new Test('with delimiter and prefix (related to #147)', {
         delimiter: '/',
         prefix: 'notes/',
     }, {
@@ -318,7 +333,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }),
-    new Test('delimiter, prefix and marker (related to #147)', {
+    new Test('with delimiter, prefix and marker (related to #147)', {
         delimiter: '/',
         prefix: 'notes/',
         keyMarker: 'notes/year.txt',
@@ -346,7 +361,7 @@ const tests = [
         NextKeyMarker: undefined,
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
-    new Test('all parameters 1/3', {
+    new Test('with all parameters 1/3', {
         delimiter: '/',
         prefix: 'notes/',
         keyMarker: 'notes/',
@@ -372,7 +387,7 @@ const tests = [
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('all parameters 2/3', {
+    new Test('with all parameters 2/3', {
         delimiter: '/',
         prefix: 'notes/', // prefix
         keyMarker: 'notes/spring/',
@@ -398,7 +413,7 @@ const tests = [
         NextVersionIdMarker: undefined,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('all parameters 3/3', {
+    new Test('with all parameters 3/3', {
         delimiter: '/',
         prefix: 'notes/', // prefix
         keyMarker: 'notes/summer/',
@@ -426,7 +441,7 @@ const tests = [
         NextVersionIdMarker: receivedData[19].versionId,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('all parameters 4/3', {
+    new Test('with all parameters 4/3', {
         delimiter: '/',
         prefix: 'notes/', // prefix
         keyMarker: 'notes/year.txt',
@@ -454,7 +469,7 @@ const tests = [
         NextVersionIdMarker: receivedData[20].versionId,
     }, (e, input) => e.key > input.keyMarker),
 
-    new Test('all parameters 5/3', {
+    new Test('with all parameters 5/3', {
         delimiter: '/',
         prefix: 'notes/',
         keyMarker: 'notes/yore.rs',
@@ -481,39 +496,79 @@ const tests = [
     }, (e, input) => e.key > input.keyMarker),
 ];
 
+function getListingKey(key, vFormat) {
+    if (vFormat === 'v0') {
+        return key;
+    }
+    if (vFormat === 'v1') {
+        const keyPrefix = key.includes(VID_SEP) ?
+              DbPrefixes.Version : DbPrefixes.Master;
+        return `${keyPrefix}${key}`;
+    }
+    return assert.fail(`bad format ${vFormat}`);
+}
+
 function getTestListing(test, data, vFormat) {
     return data
         .filter(e => test.filter(e, test.input))
-        .map(e => {
+        .map(e => ({
-            if (vFormat === 'v0') {
+            key: getListingKey(e.key, vFormat),
-                return e;
+            value: e.value,
-            }
+        }));
-            if (vFormat === 'v1') {
-                const keyPrefix = e.key.includes(VID_SEP) ?
-                      DbPrefixes.Version : DbPrefixes.Master;
-                return {
-                    key: `${keyPrefix}${e.key}`,
-                    value: e.value,
-                };
-            }
-            return assert.fail(`bad format ${vFormat}`);
-        });
 }
 
 ['v0', 'v1'].forEach(vFormat => {
     describe(`Delimiter All Versions listing algorithm vFormat=${vFormat}`, () => {
         it('Should return good skipping value for DelimiterVersions', () => {
-            const delimiter = new DelimiterVersions({ delimiter: '/' });
+            const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
             for (let i = 0; i < 100; i++) {
                 delimiter.filter({
                     key: `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/${zpad(i)}`,
                     value: '{}',
                 });
             }
-            assert.strictEqual(delimiter.skipping(),
+            if (vFormat === 'v1') {
-                               `${vFormat === 'v1' ? DbPrefixes.Master : ''}foo/`);
+                assert.deepStrictEqual(delimiter.skipping(), [
+                    `${DbPrefixes.Master}foo/`,
+                    `${DbPrefixes.Version}foo/`,
+                ]);
+            } else {
+                assert.strictEqual(delimiter.skipping(), 'foo/');
+            }
         });
 
+        if (vFormat === 'v0') {
+            it('Should return good skipping value for DelimiterVersions on replay keys', () => {
+                const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
+                for (let i = 0; i < 10; i++) {
+                    delimiter.filter({
+                        key: `foo/${zpad(i)}`,
+                        value: '{}',
+                    });
+                }
+                // simulate a listing that goes through a replay key, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${DbPrefixes.Replay}xyz`,
+                        value: 'abcdef',
+                    }),
+                    FILTER_SKIP);
+                // ...it should skip the whole replay prefix
+                assert.strictEqual(delimiter.skipping(), DbPrefixes.Replay);
+
+                // simulate a listing that reaches regular object keys
+                // beyond the replay prefix, ...
+                assert.strictEqual(
+                    delimiter.filter({
+                        key: `${inc(DbPrefixes.Replay)}foo/bar`,
+                        value: '{}',
+                    }),
+                    FILTER_ACCEPT);
+                // ...it should return to skipping by prefix as usual
+                assert.strictEqual(delimiter.skipping(), `${inc(DbPrefixes.Replay)}foo/`);
+            });
+        }
+
         tests.forEach(test => {
             it(`Should return metadata listing params to list ${test.name}`, () => {
                 const listing = new DelimiterVersions(test.input, logger, vFormat);
@@ -527,5 +582,442 @@ function getTestListing(test, data, vFormat) {
                 assert.deepStrictEqual(res, test.output);
             });
         });
+
+        it('skipping() should return SKIP_NONE when NextMarker is undefined', () => {
+            const delimiter = new DelimiterVersions({ delimiter: '/' }, logger, vFormat);
+
+            assert.strictEqual(delimiter.NextMarker, undefined);
+            assert.strictEqual(delimiter.skipping(), SKIP_NONE);
+        });
+
+        it('skipping() should return SKIP_NONE when marker is set and ' +
+        'does not contain the delimiter', () => {
+            const key = 'foo';
+            const delimiter = new DelimiterVersions({ delimiter: '/', marker: key },
+                                                    logger, vFormat);
+
+            /* Filter a master version to set NextMarker. */
+            const listingKey = getListingKey(key, vFormat);
+            delimiter.filter({ key: listingKey, value: '' });
+            assert.strictEqual(delimiter.NextMarker, 'foo');
+            assert.strictEqual(delimiter.skipping(), SKIP_NONE);
+        });
+
+        it('skipping() should return prefix to skip when marker is set and ' +
+        'contains the delimiter', () => {
+            const key = 'foo/bar';
+            const delimiter = new DelimiterVersions({ delimiter: '/', marker: key },
+                                                    logger, vFormat);
+
+            /* Filter a master version to set NextMarker. */
+            const listingKey = getListingKey(key, vFormat);
+            delimiter.filter({ key: listingKey, value: '' });
+            assert.strictEqual(delimiter.NextMarker, 'foo/');
+
+            if (vFormat === 'v0') {
+                assert.strictEqual(delimiter.skipping(), 'foo/');
+            } else {
+                assert.deepStrictEqual(delimiter.skipping(), [
+                    `${DbPrefixes.Master}foo/`,
+                    `${DbPrefixes.Version}foo/`,
+                ]);
+            }
+        });
+
+        it('skipping() should return prefix when marker is set and ' +
+        'ends with the delimiter', () => {
+            const key = 'foo/';
+            const delimiter = new DelimiterVersions({ delimiter: '/', marker: key },
+                                                    logger, vFormat);
+
+            /* Filter a master version to set NextMarker. */
+            const listingKey = getListingKey(key, vFormat);
+            delimiter.filter({ key: listingKey, value: '' });
+            assert.strictEqual(delimiter.NextMarker, 'foo/');
+
+            if (vFormat === 'v0') {
+                assert.strictEqual(delimiter.skipping(), 'foo/');
+            } else {
+                assert.deepStrictEqual(delimiter.skipping(), [
+                    `${DbPrefixes.Master}foo/`,
+                    `${DbPrefixes.Version}foo/`,
+                ]);
+            }
+        });
+
+        it('should skip entries not starting with prefix', () => {
+            const delimiter = new DelimiterVersions({ prefix: 'prefix' }, logger, vFormat);
+
+            const listingKey = getListingKey('wrong', vFormat);
+            assert.strictEqual(delimiter.filter({ key: listingKey, value: '' }), FILTER_SKIP);
+            assert.strictEqual(delimiter.NextMarker, undefined);
+            assert.strictEqual(delimiter.prvKey, undefined);
+            assert.deepStrictEqual(delimiter.result(), EmptyResult);
+        });
+
+        it('should accept a master version', () => {
+            const delimiter = new DelimiterVersions({}, logger, vFormat);
+            const key = 'key';
+            const value = '';
+
+            const listingKey = getListingKey(key, vFormat);
+            assert.strictEqual(delimiter.filter({ key: listingKey, value }), FILTER_ACCEPT);
+            assert.strictEqual(delimiter.NextMarker, key);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: [],
+                Versions: [{
+                    key: 'key',
+                    value: '',
+                    versionId: 'null',
+                }],
+                Delimiter: undefined,
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+        });
+
+        it('should return good values for entries with different common prefixes', () => {
+            const delimiter = new DelimiterVersions({ delimiter: '/' },
+                                                    logger, vFormat);
+
+            /* Filter the first entry with a common prefix. It should be
+             * accepted and added to the result. */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey('commonPrefix1/key1', vFormat),
+                value: '',
+            }),
+                               FILTER_ACCEPT);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: ['commonPrefix1/'],
+                Versions: [],
+                Delimiter: '/',
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+
+            /* Filter the second entry with the same common prefix than the
+             * first entry. It should be skipped and not added to the result. */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey('commonPrefix1/key2', vFormat),
+                value: '',
+            }),
+                               FILTER_SKIP);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: ['commonPrefix1/'],
+                Versions: [],
+                Delimiter: '/',
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+
+            /* Filter an entry with a new common prefix. It should be accepted
+             * and not added to the result. */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey('commonPrefix2/key1', vFormat),
+                value: '',
+            }),
+                               FILTER_ACCEPT);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: ['commonPrefix1/', 'commonPrefix2/'],
+                Versions: [],
+                Delimiter: '/',
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+        });
+
+        it('should accept a delete marker version', () => {
+            const delimiter = new DelimiterVersions({}, logger, vFormat);
+            const version = new Version({ isDeleteMarker: true });
+            const key = 'key';
+            const obj = {
+                key: getListingKey(`${key}${VID_SEP}version`, vFormat),
+                value: version.toString(),
+            };
+
+            /* When filtered, it should return FILTER_ACCEPT and
+             * should be added to the result content. */
+            assert.strictEqual(delimiter.filter(obj), FILTER_ACCEPT);
+            assert.strictEqual(delimiter.NextMarker, key);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: [],
+                Versions: [{
+                    key: 'key',
+                    value: version.toString(),
+                    versionId: 'version',
+                }],
+                Delimiter: undefined,
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+        });
+
+        it('should accept a version after a delete marker master', () => {
+            const delimiter = new DelimiterVersions({}, logger, vFormat);
+            const version = new Version({ isDeleteMarker: true });
+            const key = 'key';
+            const versionKey = `${key}${VID_SEP}version`;
+
+            delimiter.filter({ key: getListingKey(key, vFormat), value: version.toString() });
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(versionKey, vFormat),
+                value: 'value',
+            }), FILTER_ACCEPT);
+            assert.strictEqual(delimiter.NextMarker, key);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: [],
+                Versions: [{
+                    key: 'key',
+                    value: version.toString(),
+                    versionId: 'null',
+                }, {
+                    key: 'key',
+                    value: 'value',
+                    versionId: 'version',
+                }],
+                Delimiter: undefined,
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+        });
+
+        it('should accept a new master key w/ version after a delete marker master', () => {
+            const delimiter = new DelimiterVersions({}, logger, vFormat);
+            const version = new Version({ isDeleteMarker: true });
+            const key1 = 'key1';
+            const key2 = 'key2';
+            const value2 = '{"versionId":"version"}';
+
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(key1, vFormat),
+                value: version.toString(),
+            }), FILTER_ACCEPT);
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(key2, vFormat),
+                value: value2,
+            }), FILTER_ACCEPT);
+            assert.strictEqual(delimiter.NextMarker, key2);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: [],
+                Delimiter: undefined,
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+                Versions: [{
+                    key: 'key1',
+                    value: '{"isDeleteMarker":true}',
+                    versionId: 'null',
+                }, {
+                    key: 'key2',
+                    value: '{"versionId":"version"}',
+                    versionId: 'version',
+                }],
+            });
+        });
+
+        it('should accept a version after skipping an object because of its commonPrefix', () => {
+            const commonPrefix1 = 'commonPrefix1/';
+            const commonPrefix2 = 'commonPrefix2/';
+            const prefix1Key1 = 'commonPrefix1/key1';
+            const prefix1Key2 = 'commonPrefix1/key2';
+            const prefix2VersionKey1 = `commonPrefix2/key1${VID_SEP}version`;
+            const value = '{"versionId":"version"}';
+
+            const delimiter = new DelimiterVersions({ delimiter: '/' },
+                                                    logger, vFormat);
+
+            /* Filter the two first entries with the same common prefix to add
+             * it to the result and reach the state where an entry is skipped
+             * because of an already present common prefix in the result. */
+            delimiter.filter({ key: getListingKey(prefix1Key1, vFormat), value });
+            delimiter.filter({ key: getListingKey(prefix1Key2, vFormat), value });
+
+            /* Filter an object with a key containing a version part and a new
+             * common prefix. It should be accepted and the new common prefix
+             * added to the result. */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(prefix2VersionKey1, vFormat),
+                value,
+            }), FILTER_ACCEPT);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: [commonPrefix1, commonPrefix2],
+                Versions: [],
+                Delimiter: '/',
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+        });
+
+        it('should skip first version key if equal to master', () => {
+            const delimiter = new DelimiterVersions({}, logger, vFormat);
+            const masterKey = 'key';
+            const versionKey1 = `${masterKey}${VID_SEP}version1`;
+            const versionKey2 = `${masterKey}${VID_SEP}version2`;
+            const value2 = 'value2';
+
+            /* Filter the master version for version1 */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(masterKey, vFormat),
+                value: '{"versionId":"version1"}',
+            }), FILTER_ACCEPT);
+
+            /* Filter the version key for version1 */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(versionKey1, vFormat),
+                value: '{"versionId":"version1"}',
+            }), FILTER_SKIP);
+
+            /* Filter the version key for version2 */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(versionKey2, vFormat),
+                value: value2,
+            }), FILTER_ACCEPT);
+
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: [],
+                Versions: [{
+                    key: 'key',
+                    value: '{"versionId":"version1"}',
+                    versionId: 'version1',
+                }, {
+                    key: 'key',
+                    value: 'value2',
+                    versionId: 'version2',
+                }],
+                Delimiter: undefined,
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+        });
+
+        it('should skip master and version key if under a known prefix', () => {
+            const commonPrefix1 = 'commonPrefix/';
+            const prefixKey1 = 'commonPrefix/key1';
+            const prefixKey2 = 'commonPrefix/key2';
+            const prefixVersionKey1 = `commonPrefix/key2${VID_SEP}version`;
+            const value = '{"versionId":"version"}';
+
+            const delimiter = new DelimiterVersions({ delimiter: '/' },
+                                                    logger, vFormat);
+
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(prefixKey1, vFormat),
+                value,
+            }), FILTER_ACCEPT);
+            /* The second master key of the same common prefix should be skipped */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(prefixKey2, vFormat),
+                value,
+            }), FILTER_SKIP);
+
+            /* The version key of the same common prefix should also be skipped */
+            assert.strictEqual(delimiter.filter({
+                key: getListingKey(prefixVersionKey1, vFormat),
+                value,
+            }), FILTER_SKIP);
+            assert.deepStrictEqual(delimiter.result(), {
+                CommonPrefixes: [commonPrefix1],
+                Versions: [],
+                Delimiter: '/',
+                IsTruncated: false,
+                NextKeyMarker: undefined,
+                NextVersionIdMarker: undefined,
+            });
+        });
+
+        if (vFormat === 'v0') {
+            it('should accept a PHD version as first input', () => {
+                const delimiter = new DelimiterVersions({}, logger, vFormat);
+                const keyPHD = 'keyPHD';
+                const objPHD = {
+                    key: keyPHD,
+                    value: Version.generatePHDVersion(generateVersionId('', '')),
+                };
+
+                /* When filtered, it should return FILTER_ACCEPT and set the prvKey
+                 * to undefined. It should not be added to the result content or common
+                 * prefixes. */
+                assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
+                assert.strictEqual(delimiter.prvKey, undefined);
+                assert.strictEqual(delimiter.NextMarker, undefined);
+                assert.deepStrictEqual(delimiter.result(), EmptyResult);
+            });
+
+            it('should accept a PHD version', () => {
+                const delimiter = new DelimiterVersions({}, logger, vFormat);
+                const key = 'keyA';
+                const value = '';
+                const keyPHD = 'keyBPHD';
+                const objPHD = {
+                    key: keyPHD,
+                    value: Version.generatePHDVersion(generateVersionId('', '')),
+                };
+
+                /* Filter a master version to set the NextMarker and add
+                 * and element in result content. */
+                delimiter.filter({ key, value });
+
+                /* When filtered, it should return FILTER_ACCEPT. It
+                 * should not be added to the result content or common
+                 * prefixes. */
+                assert.strictEqual(delimiter.filter(objPHD), FILTER_ACCEPT);
+                assert.strictEqual(delimiter.prvKey, undefined);
+                assert.strictEqual(delimiter.NextMarker, key);
+                assert.deepStrictEqual(delimiter.result(), {
+                    CommonPrefixes: [],
+                    Versions: [{
+                        key: 'keyA',
+                        value: '',
+                        versionId: 'null',
+                    }],
+                    Delimiter: undefined,
+                    IsTruncated: false,
+                    NextKeyMarker: undefined,
+                    NextVersionIdMarker: undefined,
+                });
+            });
+
+            it('should accept a version after a PHD', () => {
+                const delimiter = new DelimiterVersions({}, logger, vFormat);
+                const masterKey = 'key';
+                const keyVersion = `${masterKey}${VID_SEP}version`;
+                const value = '';
+                const objPHD = {
+                    key: masterKey,
+                    value: Version.generatePHDVersion(generateVersionId('', '')),
+                };
+
+                /* Filter the PHD object. */
+                delimiter.filter(objPHD);
+
+                /* The filtering of the PHD object has no impact, the version is
+                 * accepted and added to the result. */
+                assert.strictEqual(delimiter.filter({
+                    key: keyVersion,
+                    value,
+                }), FILTER_ACCEPT);
+                assert.strictEqual(delimiter.NextMarker, masterKey);
+                assert.deepStrictEqual(delimiter.result(), {
+                    CommonPrefixes: [],
+                    Versions: [{
+                        key: 'key',
+                        value: '',
+                        versionId: 'version',
+                    }],
+                    Delimiter: undefined,
+                    IsTruncated: false,
+                    NextKeyMarker: undefined,
+                    NextVersionIdMarker: undefined,
+                });
+            });
+        }
     });
 });

tests/unit/models/object.js

@@ -69,6 +69,8 @@ describe('ObjectMD class setters/getters', () => {
         ['IsNull', true],
         ['NullVersionId', null, undefined],
         ['NullVersionId', '111111'],
+        ['NullUploadId', null, undefined],
+        ['NullUploadId', 'abcdefghi'],
         ['IsDeleteMarker', null, false],
         ['IsDeleteMarker', true],
         ['VersionId', null, undefined],
@@ -78,6 +80,8 @@ describe('ObjectMD class setters/getters', () => {
             key: 'value',
         }],
         ['Tags', null, {}],
+        ['UploadId', null, undefined],
+        ['UploadId', 'abcdefghi'],
         ['ReplicationInfo', null, {
             status: '',
             backends: [],
@@ -332,9 +336,11 @@ describe('getAttributes static method', () => {
             'location': true,
             'isNull': true,
             'nullVersionId': true,
+            'nullUploadId': true,
             'isDeleteMarker': true,
             'versionId': true,
             'tags': true,
+            'uploadId': true,
             'replicationInfo': true,
             'dataStoreName': true,
             'last-modified': true,