CLDSRV-544: bump version

(cherry picked from commit ca0904f584)

.github/actions/setup-ci/action.yaml

@@ -16,7 +16,7 @@ runs:
       run: |-
         set -exu;
         mkdir -p /tmp/artifacts/${{ github.job }}/;
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v4
       with:
         node-version: '16'
         cache: 'yarn'

.github/docker/docker-compose.yaml

@@ -62,6 +62,6 @@ services:
   pykmip:
     network_mode: "host"
     profiles: ['pykmip']
-    image: registry.scality.com/cloudserver-dev/pykmip
+    image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
     volumes:
       - /tmp/artifacts/${JOB_NAME}:/artifacts

.github/workflows/codeql.yaml

@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: javascript, python, ruby
 
       - name: Build and analyze
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/dependency-review.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v3
+        uses: actions/dependency-review-action@v4

.github/workflows/release.yaml

@@ -11,36 +11,59 @@ on:
 
 jobs:
   build-federation-image:
-    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
+    runs-on: ubuntu-20.04
-    secrets: inherit
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push image for federation
+        uses: docker/build-push-action@v5
         with:
           push: true
-      registry: registry.scality.com
-      namespace: ${{ github.event.repository.name }}
-      name: ${{ github.event.repository.name }}
           context: .
           file: images/svc-base/Dockerfile
-      tag: ${{ github.event.inputs.tag }}-svc-base
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
+          cache-from: type=gha,scope=federation
+          cache-to: type=gha,mode=max,scope=federation
 
   build-image:
-    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
+    runs-on: ubuntu-20.04
-    secrets: inherit
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push image
+        uses: docker/build-push-action@v5
         with:
           push: true
-      registry: registry.scality.com
-      namespace: ${{ github.event.repository.name }}
-      name: ${{ github.event.repository.name }}
           context: .
-      file: Dockerfile
+          tags: |
-      tag: ${{ github.event.inputs.tag }}
+            ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
   github-release:
     runs-on: ubuntu-latest
     steps:
       - name: Create Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ github.token }}
         with:
           name: Release ${{ github.event.inputs.tag }}
           tag_name: ${{ github.event.inputs.tag }}

.github/workflows/tests.yaml

@@ -65,23 +65,24 @@ env:
   ENABLE_LOCAL_CACHE: "true"
   REPORT_TOKEN: "report-token-1"
   REMOTE_MANAGEMENT_DISABLE: "1"
-
+  # https://github.com/git-lfs/git-lfs/issues/5749
+  GIT_CLONE_PROTECTION_ACTIVE: 'false'
 jobs:
   linting-coverage:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v4
       with:
         node-version: '16'
         cache: yarn
     - name: install dependencies
       run: yarn install --frozen-lockfile --network-concurrency 1
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.9'
-    - uses: actions/cache@v2
+    - uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip
@@ -114,7 +115,7 @@ jobs:
         find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
       if: always()
     - name: Upload files to artifacts
-      uses: scality/action-artifacts@v2
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -127,58 +128,72 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1.6.0
+        uses: docker/setup-buildx-action@v3
       - name: Login to GitHub Registry
-        uses: docker/login-action@v1.10.0
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          password: ${{ github.token }}
-      - name: Login to Registry
-        uses: docker/login-action@v1
-        with:
-          registry: registry.scality.com
-          username: ${{ secrets.REGISTRY_LOGIN }}
-          password: ${{ secrets.REGISTRY_PASSWORD }}
       - name: Build and push cloudserver image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           push: true
           context: .
           provenance: false
           tags: |
-            ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+            ghcr.io/${{ github.repository }}:${{ github.sha }}
-            registry.scality.com/cloudserver-dev/cloudserver:${{ github.sha }}
           cache-from: type=gha,scope=cloudserver
           cache-to: type=gha,mode=max,scope=cloudserver
-
+      - name: Build and push pykmip image
-  build-federation-image:
+        uses: docker/build-push-action@v5
-    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
+        with:
-    secrets: inherit
+          push: true
+          context: .github/pykmip
+          tags: |
+            ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+          cache-from: type=gha,scope=pykmip
+          cache-to: type=gha,mode=max,scope=pykmip
+
+  build-federation-image:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push image for federation
+        uses: docker/build-push-action@v5
         with:
           push: true
-      registry: registry.scality.com
-      namespace: cloudserver-dev
-      name: cloudserver
           context: .
           file: images/svc-base/Dockerfile
-      tag: ${{ github.sha }}-svc-base
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.sha }}-svc-base
+          cache-from: type=gha,scope=federation
+          cache-to: type=gha,mode=max,scope=federation
 
   multiple-backend:
     runs-on: ubuntu-latest
     needs: build
     env:
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
       S3BACKEND: mem
       S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
       S3DATA: multiple
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Setup CI environment
@@ -194,7 +209,7 @@ jobs:
       env:
         S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -217,14 +232,14 @@ jobs:
     env:
       S3BACKEND: file
       S3VAULT: mem
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
       MPU_TESTING: "yes"
       ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
       JOB_NAME: ${{ matrix.job-name }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Setup CI environment
@@ -247,7 +262,7 @@ jobs:
         bash wait_for_local_port.bash 8000 40
         yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -263,12 +278,12 @@ jobs:
       ENABLE_UTAPI_V2: t
       S3BACKEND: mem
       BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Setup CI environment
@@ -282,7 +297,7 @@ jobs:
         bash wait_for_local_port.bash 8000 40
         yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -298,12 +313,13 @@ jobs:
       S3BACKEND: file
       S3VAULT: mem
       MPU_TESTING: true
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Setup CI environment
@@ -321,7 +337,7 @@ jobs:
         bash wait_for_local_port.bash 5696 40
         yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net

images/svc-base/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.scality.com/federation/nodesvc-base:7.10.6.0
+FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
 
 ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
 ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json

index.js

@@ -1,10 +1,10 @@
 'use strict'; // eslint-disable-line strict
 
-/**
+require('werelogs').stderrUtils.catchAndTimestampStderr(
- * Catch uncaught exceptions and add timestamp to aid debugging
+    undefined,
- */
+    // Do not exit as workers have their own listener that will exit
-process.on('uncaughtException', err => {
+    // But primary don't have another listener
-    process.stderr.write(`${new Date().toISOString()}: Uncaught exception: \n${err.stack}`);
+    require('cluster').isPrimary ? 1 : null,
-});
+);
 
 require('./lib/server.js')();

lib/Config.js

@@ -88,6 +88,47 @@ function parseSproxydConfig(configSproxyd) {
     return joi.attempt(configSproxyd, joiSchema, 'bad config');
 }
 
+function parseRedisConfig(redisConfig) {
+    const joiSchema = joi.object({
+        password: joi.string().allow(''),
+        host: joi.string(),
+        port: joi.number(),
+        retry: joi.object({
+            connectBackoff: joi.object({
+                min: joi.number().required(),
+                max: joi.number().required(),
+                jitter: joi.number().required(),
+                factor: joi.number().required(),
+                deadline: joi.number().required(),
+            }),
+        }),
+        // sentinel config
+        sentinels: joi.alternatives().try(
+            joi.string()
+                .pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
+                .custom(hosts => hosts.split(',').map(item => {
+                    const [host, port] = item.split(':');
+                    return { host, port: Number.parseInt(port, 10) };
+                })),
+            joi.array().items(
+                joi.object({
+                    host: joi.string().required(),
+                    port: joi.number().required(),
+                })
+            ).min(1),
+        ),
+        name: joi.string(),
+        sentinelPassword: joi.string().allow(''),
+    })
+          .and('host', 'port')
+          .and('sentinels', 'name')
+          .xor('host', 'sentinels')
+          .without('sentinels', ['host', 'port'])
+          .without('host', ['sentinels', 'sentinelPassword']);
+
+    return joi.attempt(redisConfig, joiSchema, 'bad config');
+}
+
 function restEndpointsAssert(restEndpoints, locationConstraints) {
     assert(typeof restEndpoints === 'object',
         'bad config: restEndpoints must be an object of endpoints');
@@ -294,18 +335,17 @@ function parseUtapiReindex(config) {
     const {
         enabled,
         schedule,
-        sentinel,
+        redis,
         bucketd,
         onlyCountLatestWhenObjectLocked,
     } = config;
     assert(typeof enabled === 'boolean',
            'bad config: utapi.reindex.enabled must be a boolean');
-    assert(typeof sentinel === 'object',
+
-        'bad config: utapi.reindex.sentinel must be an object');
+    const parsedRedis = parseRedisConfig(redis);
-    assert(typeof sentinel.port === 'number',
+    assert(Array.isArray(parsedRedis.sentinels),
-        'bad config: utapi.reindex.sentinel.port must be a number');
+           'bad config: utapi reindex redis config requires a list of sentinels');
-    assert(typeof sentinel.name === 'string',
+
-        'bad config: utapi.reindex.sentinel.name must be a string');
     assert(typeof bucketd === 'object',
         'bad config: utapi.reindex.bucketd must be an object');
     assert(typeof bucketd.port === 'number',
@@ -323,6 +363,13 @@ function parseUtapiReindex(config) {
             'bad config: utapi.reindex.schedule must be a valid ' +
             `cron schedule. ${e.message}.`);
     }
+    return {
+        enabled,
+        schedule,
+        redis: parsedRedis,
+        bucketd,
+        onlyCountLatestWhenObjectLocked,
+    };
 }
 
 function requestsConfigAssert(requestsConfig) {
@@ -761,8 +808,7 @@ class Config extends EventEmitter {
             assert(typeof config.localCache.port === 'number',
                 'config: invalid port for localCache. port must be a number');
             if (config.localCache.password !== undefined) {
-                assert(
+                assert(typeof config.localCache.password === 'string',
-                    this._verifyRedisPassword(config.localCache.password),
                     'config: invalid password for localCache. password must' +
                     ' be a string');
             }
@@ -774,55 +820,7 @@ class Config extends EventEmitter {
         }
 
         if (config.redis) {
-            if (config.redis.sentinels) {
+            this.redis = parseRedisConfig(config.redis);
-                this.redis = { sentinels: [], name: null };
-
-                assert(typeof config.redis.name === 'string',
-                    'bad config: redis sentinel name must be a string');
-                this.redis.name = config.redis.name;
-                assert(Array.isArray(config.redis.sentinels) ||
-                    typeof config.redis.sentinels === 'string',
-                    'bad config: redis sentinels must be an array or string');
-
-                if (typeof config.redis.sentinels === 'string') {
-                    config.redis.sentinels.split(',').forEach(item => {
-                        const [host, port] = item.split(':');
-                        this.redis.sentinels.push({ host,
-                            port: Number.parseInt(port, 10) });
-                    });
-                } else if (Array.isArray(config.redis.sentinels)) {
-                    config.redis.sentinels.forEach(item => {
-                        const { host, port } = item;
-                        assert(typeof host === 'string',
-                            'bad config: redis sentinel host must be a string');
-                        assert(typeof port === 'number',
-                            'bad config: redis sentinel port must be a number');
-                        this.redis.sentinels.push({ host, port });
-                    });
-                }
-
-                if (config.redis.sentinelPassword !== undefined) {
-                    assert(
-                    this._verifyRedisPassword(config.redis.sentinelPassword));
-                    this.redis.sentinelPassword = config.redis.sentinelPassword;
-                }
-            } else {
-                // check for standalone configuration
-                this.redis = {};
-                assert(typeof config.redis.host === 'string',
-                    'bad config: redis.host must be a string');
-                assert(typeof config.redis.port === 'number',
-                    'bad config: redis.port must be a number');
-                this.redis.host = config.redis.host;
-                this.redis.port = config.redis.port;
-            }
-            if (config.redis.password !== undefined) {
-                assert(
-                    this._verifyRedisPassword(config.redis.password),
-                    'bad config: invalid password for redis. password must ' +
-                    'be a string');
-                this.redis.password = config.redis.password;
-            }
         }
         if (config.utapi) {
             this.utapi = { component: 's3' };
@@ -851,65 +849,8 @@ class Config extends EventEmitter {
                 this.utapi.localCache = config.localCache;
                 assert(config.utapi.redis, 'missing required property of utapi ' +
                     'configuration: redis');
-                if (config.utapi.redis.sentinels) {
+                this.utapi.redis = parseRedisConfig(config.utapi.redis);
-                    this.utapi.redis = { sentinels: [], name: null };
+                if (this.utapi.redis.retry === undefined) {
-
-                    assert(typeof config.utapi.redis.name === 'string',
-                        'bad config: redis sentinel name must be a string');
-                    this.utapi.redis.name = config.utapi.redis.name;
-
-                    assert(Array.isArray(config.utapi.redis.sentinels),
-                        'bad config: redis sentinels must be an array');
-                    config.utapi.redis.sentinels.forEach(item => {
-                        const { host, port } = item;
-                        assert(typeof host === 'string',
-                            'bad config: redis sentinel host must be a string');
-                        assert(typeof port === 'number',
-                            'bad config: redis sentinel port must be a number');
-                        this.utapi.redis.sentinels.push({ host, port });
-                    });
-                } else {
-                    // check for standalone configuration
-                    this.utapi.redis = {};
-                    assert(typeof config.utapi.redis.host === 'string',
-                        'bad config: redis.host must be a string');
-                    assert(typeof config.utapi.redis.port === 'number',
-                        'bad config: redis.port must be a number');
-                    this.utapi.redis.host = config.utapi.redis.host;
-                    this.utapi.redis.port = config.utapi.redis.port;
-                }
-                if (config.utapi.redis.password !== undefined) {
-                    assert(
-                        this._verifyRedisPassword(config.utapi.redis.password),
-                        'config: invalid password for utapi redis. password' +
-                        ' must be a string');
-                    this.utapi.redis.password = config.utapi.redis.password;
-                }
-                if (config.utapi.redis.sentinelPassword !== undefined) {
-                    assert(
-                    this._verifyRedisPassword(config.utapi.redis.sentinelPassword),
-                        'config: invalid password for utapi redis. password' +
-                        ' must be a string');
-                    this.utapi.redis.sentinelPassword =
-                        config.utapi.redis.sentinelPassword;
-                }
-                if (config.utapi.redis.retry !== undefined) {
-                    if (config.utapi.redis.retry.connectBackoff !== undefined) {
-                        const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
-                        assert.strictEqual(typeof min, 'number',
-                        'utapi.redis.retry.connectBackoff: min must be a number');
-                        assert.strictEqual(typeof max, 'number',
-                        'utapi.redis.retry.connectBackoff: max must be a number');
-                        assert.strictEqual(typeof jitter, 'number',
-                        'utapi.redis.retry.connectBackoff: jitter must be a number');
-                        assert.strictEqual(typeof factor, 'number',
-                        'utapi.redis.retry.connectBackoff: factor must be a number');
-                        assert.strictEqual(typeof deadline, 'number',
-                        'utapi.redis.retry.connectBackoff: deadline must be a number');
-                    }
-
-                    this.utapi.redis.retry = config.utapi.redis.retry;
-                } else {
                     this.utapi.redis.retry = {
                         connectBackoff: {
                             min: 10,
@@ -920,6 +861,7 @@ class Config extends EventEmitter {
                         },
                     };
                 }
+
                 if (config.utapi.metrics) {
                     this.utapi.metrics = config.utapi.metrics;
                 }
@@ -988,8 +930,7 @@ class Config extends EventEmitter {
                 }
 
                 if (config.utapi && config.utapi.reindex) {
-                    parseUtapiReindex(config.utapi.reindex);
+                    this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
-                    this.utapi.reindex = config.utapi.reindex;
                 }
             }
 
@@ -1430,10 +1371,6 @@ class Config extends EventEmitter {
         };
     }
 
-    _verifyRedisPassword(password) {
-        return typeof password === 'string';
-    }
-
     setAuthDataAccounts(accounts) {
         this.authData.accounts = accounts;
         this.emit('authdata-update');
@@ -1547,6 +1484,7 @@ class Config extends EventEmitter {
 
 module.exports = {
     parseSproxydConfig,
+    parseRedisConfig,
     locationConstraintAssert,
     ConfigObject: Config,
     config: new Config(),

lib/api/apiUtils/object/versioning.js

@@ -193,7 +193,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
             // null keys are used, which is used as an optimization to
             // avoid having to check the versioned key since there can
             // be no more versioned key to clean up
-            if (mst.isNull && !mst.isNull2) {
+            if (mst.isNull && mst.versionId && !mst.isNull2) {
                 const delOptions = { versionId: mst.versionId };
                 return { options, delOptions };
             }
@@ -224,7 +224,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
     if (masterIsNull) {
         // if master is a null version or a non-versioned key,
         // copy it to a new null key
-        const nullVersionId = mst.isNull ? mst.versionId : nonVersionedObjId;
+        const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
         if (nullVersionCompatMode) {
             options.extraMD = {
                 nullVersionId,

lib/routes/routeBackbeat.js

@@ -49,6 +49,7 @@ const NAMESPACE = 'default';
 const CIPHER = null; // replication/lifecycle does not work on encrypted objects
 
 let { locationConstraints } = config;
+const { nullVersionCompatMode } = config;
 const { implName } = dataWrapper;
 let dataClient = dataWrapper.client;
 config.on('location-constraints-update', () => {
@@ -492,9 +493,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
         let isNull = false;
 
         if (versionId === 'null') {
-            if (!config.nullVersionCompatMode) {
             isNull = true;
-            }
             // Retrieve the null version id from the object metadata.
             versionId = objMd && objMd.versionId;
             if (!versionId) {
@@ -503,6 +502,16 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
                 // the flag is needed to allow cloudserver to know that the version
                 // is a null version and allow access to it using the "null" versionId.
                 omVal.isNull = true;
+                // If the new null keys logic (S3C-7352) is supported (not compatibility mode),
+                // create a null key with the isNull2 flag.
+                if (!nullVersionCompatMode) {
+                    omVal.isNull2 = true;
+                }
+                // Delete the version id from the version metadata payload to prevent issues
+                // with creating a non-version object (versioning set to false) that includes a version id.
+                // For example, this version ID might come from a null version of a suspended bucket being
+                // replicated to this bucket.
+                delete omVal.versionId;
                 if (versioning) {
                     // If the null version does not have a version id, it is a current null version.
                     // To update the metadata of a current version, versioning is set to false.
@@ -537,7 +546,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
         }
 
         const options = {
-            isNull,
             overheadField: constants.overheadField,
         };
 
@@ -558,6 +566,11 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
             options.versionId = versionId;
         }
 
+        // If the new null keys logic (S3C-7352) is not supported (compatibility mode), 'isNull' remains undefined.
+        if (!nullVersionCompatMode) {
+            options.isNull = isNull;
+        }
+
         log.trace('putting object version', {
             objectKey: request.objectKey, omVal, options });
         return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,

lib/utapi/utapi.js

@@ -1,3 +1,4 @@
+require('werelogs').stderrUtils.catchAndTimestampStderr();
 const _config = require('../Config').config;
 const { utapiVersion, UtapiServer: utapiServer } = require('utapi');
 

lib/utapi/utapiReindex.js

@@ -1,3 +1,4 @@
+require('werelogs').stderrUtils.catchAndTimestampStderr();
 const UtapiReindex = require('utapi').UtapiReindex;
 const { config } = require('../Config');
 

lib/utapi/utapiReplay.js

@@ -1,3 +1,4 @@
+require('werelogs').stderrUtils.catchAndTimestampStderr();
 const UtapiReplay = require('utapi').UtapiReplay;
 const _config = require('../Config').config;
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3",
-  "version": "7.70.45",
+  "version": "7.70.45-4",
   "description": "S3 connector",
   "main": "index.js",
   "engines": {
@@ -20,7 +20,7 @@
   "homepage": "https://github.com/scality/S3#readme",
   "dependencies": {
     "@hapi/joi": "^17.1.0",
-    "arsenal": "git+https://github.com/scality/arsenal#7.70.25",
+    "arsenal": "git+https://github.com/scality/arsenal#7.70.25-1",
     "async": "~2.5.0",
     "aws-sdk": "2.905.0",
     "azure-storage": "^2.1.0",
@@ -35,11 +35,11 @@
     "moment": "^2.26.0",
     "npm-run-all": "~4.1.5",
     "prom-client": "14.2.0",
-    "utapi": "git+https://github.com/scality/utapi#7.70.3",
+    "utapi": "git+https://github.com/scality/utapi#7.70.5",
     "utf8": "~2.1.1",
     "uuid": "^3.0.1",
     "vaultclient": "scality/vaultclient#7.10.13",
-    "werelogs": "scality/werelogs#8.1.0",
+    "werelogs": "scality/werelogs#8.1.0-1",
     "xml2js": "~0.4.16"
   },
   "devDependencies": {

tests/functional/raw-node/test/routes/routeBackbeat.js

@@ -1,15 +1,15 @@
 const assert = require('assert');
 const async = require('async');
 const crypto = require('crypto');
-const { models, versioning } = require('arsenal');
+const { versioning } = require('arsenal');
 const versionIdUtils = versioning.VersionID;
-const { ObjectMD } = models;
 
 const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest');
 const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
 
 const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1';
 const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe;
+const isNullVersionCompatMode = process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true';
 
 const backbeatAuthCredentials = {
     accessKey: 'accessKey1',
@@ -87,17 +87,15 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
 }
 
 function updateStorageClass(data, storageClass) {
-    let parsedBody;
+    let result;
     try {
-        parsedBody = JSON.parse(data.body);
+        const parsedBody = JSON.parse(JSON.parse(data.body).Body);
+        parsedBody['x-amz-storage-class'] = storageClass;
+        result = JSON.stringify(parsedBody);
     } catch (err) {
         return { error: err };
     }
-    const { result, error } = ObjectMD.createFromBlob(parsedBody.Body);
+
-    if (error) {
-        return { error };
-    }
-    result.setAmzStorageClass(storageClass);
     return { result };
 }
 
@@ -202,11 +200,11 @@ describeSkipIfAWS('backbeat routes', () => {
 
         it('should update metadata of a current null version', done => {
             let objMD;
-            return async.series([
+            return async.series({
-                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
+                putObject: next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
-                next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
+                enableVersioningSource: next => s3.putBucketVersioning(
-                    next),
+                    { Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
-                next => makeBackbeatRequest({
+                getMetadata: next => makeBackbeatRequest({
                     method: 'GET',
                     resourceType: 'metadata',
                     bucket,
@@ -226,7 +224,7 @@ describeSkipIfAWS('backbeat routes', () => {
                     objMD = result;
                     return next();
                 }),
-                next => makeBackbeatRequest({
+                putMetadata: next => makeBackbeatRequest({
                     method: 'PUT',
                     resourceType: 'metadata',
                     bucket,
@@ -235,19 +233,40 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
-                next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
+                headObject: next => s3.headObject(
-                next => s3.listObjectVersions({ Bucket: bucket }, next),
+                    { Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
-            ], (err, data) => {
+                getMetadataAfter: next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, next),
+                listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
+            }, (err, results) => {
                 if (err) {
                     return done(err);
                 }
-                const headObjectRes = data[4];
+
+                const headObjectRes = results.headObject;
                 assert.strictEqual(headObjectRes.VersionId, 'null');
                 assert.strictEqual(headObjectRes.StorageClass, storageClass);
 
-                const listObjectVersionsRes = data[5];
+                const getMetadataAfterRes = results.getMetadataAfter;
+                const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
+                const expectedMd = JSON.parse(objMD);
+                expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed
+                if (!isNullVersionCompatMode) {
+                    expectedMd.isNull2 = true; // TODO remove the line once CLDSRV-509 is fixed
+                }
+                assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd);
+
+                const listObjectVersionsRes = results.listObjectVersions;
                 const { Versions } = listObjectVersionsRes;
 
                 assert.strictEqual(Versions.length, 1);
@@ -261,18 +280,20 @@ describeSkipIfAWS('backbeat routes', () => {
         it('should update metadata of a non-current null version', done => {
             let objMD;
             let expectedVersionId;
-            return async.series([
+            return async.series({
-                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
+                putObjectInitial: next => s3.putObject(
-                next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
+                    { Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
-                    next),
+                enableVersioning: next => s3.putBucketVersioning(
-                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
+                    { Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
+                putObjectAgain: next => s3.putObject(
+                { Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
                     if (err) {
                         return next(err);
                     }
                     expectedVersionId = data.VersionId;
                     return next();
                 }),
-                next => makeBackbeatRequest({
+                getMetadata: next => makeBackbeatRequest({
                     method: 'GET',
                     resourceType: 'metadata',
                     bucket,
@@ -292,7 +313,7 @@ describeSkipIfAWS('backbeat routes', () => {
                     objMD = result;
                     return next();
                 }),
-                next => makeBackbeatRequest({
+                putMetadata: next => makeBackbeatRequest({
                     method: 'PUT',
                     resourceType: 'metadata',
                     bucket,
@@ -301,23 +322,36 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
-                next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
+                headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
-                next => s3.listObjectVersions({ Bucket: bucket }, next),
+                getMetadataAfter: next => makeBackbeatRequest({
-            ], (err, data) => {
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, next),
+                listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
+            }, (err, results) => {
                 if (err) {
                     return done(err);
                 }
-                const headObjectRes = data[5];
+                const headObjectRes = results.headObject;
                 assert.strictEqual(headObjectRes.VersionId, 'null');
                 assert.strictEqual(headObjectRes.StorageClass, storageClass);
 
-                const listObjectVersionsRes = data[6];
+                const getMetadataAfterRes = results.getMetadataAfter;
+                const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
+                assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
+
+                const listObjectVersionsRes = results.listObjectVersions;
                 const { Versions } = listObjectVersionsRes;
 
                 assert.strictEqual(Versions.length, 2);
-
                 const currentVersion = Versions.find(v => v.IsLatest);
                 assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId);
 
@@ -327,6 +361,160 @@ describeSkipIfAWS('backbeat routes', () => {
             });
         });
 
+        it('should update metadata of a suspended null version', done => {
+            let objMD;
+            return async.series({
+                suspendVersioning: next => s3.putBucketVersioning(
+                    { Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
+                putObject: next => s3.putObject(
+                    { Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
+                enableVersioning: next => s3.putBucketVersioning(
+                    { Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
+                getMetadata: next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    const { error, result } = updateStorageClass(data, storageClass);
+                    if (error) {
+                        return next(error);
+                    }
+                    objMD = result;
+                    return next();
+                }),
+                putUpdatedMetadata: next => makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: objMD,
+                }, next),
+                headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
+                getMetadataAfter: next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, next),
+                listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
+            }, (err, results) => {
+                if (err) {
+                    return done(err);
+                }
+                const headObjectRes = results.headObject;
+                assert.strictEqual(headObjectRes.VersionId, 'null');
+                assert.strictEqual(headObjectRes.StorageClass, storageClass);
+
+                const getMetadataAfterRes = results.getMetadataAfter;
+                const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
+                assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
+
+                const listObjectVersionsRes = results.listObjectVersions;
+                const { Versions } = listObjectVersionsRes;
+
+                assert.strictEqual(Versions.length, 1);
+
+                const [currentVersion] = Versions;
+                assertVersionIsNullAndUpdated(currentVersion);
+                return done();
+            });
+        });
+
+        it('should update metadata of a suspended null version with internal version id', done => {
+            let objMD;
+            return async.series({
+                suspendVersioning: next => s3.putBucketVersioning(
+                    { Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
+                putObject: next => s3.putObject(
+                    { Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
+                enableVersioning: next => s3.putBucketVersioning(
+                    { Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
+                putObjectTagging: next => s3.putObjectTagging({
+                    Bucket: bucket, Key: keyName, VersionId: 'null',
+                    Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] },
+                }, next),
+                getMetadata: next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    const { error, result } = updateStorageClass(data, storageClass);
+                    if (error) {
+                        return next(error);
+                    }
+                    objMD = result;
+                    return next();
+                }),
+                putUpdatedMetadata: next => makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: objMD,
+                }, next),
+                headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
+                getMetadataAfter: next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, next),
+                listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
+            }, (err, results) => {
+                if (err) {
+                    return done(err);
+                }
+                const headObjectRes = results.headObject;
+                assert.strictEqual(headObjectRes.VersionId, 'null');
+                assert.strictEqual(headObjectRes.StorageClass, storageClass);
+
+                const getMetadataAfterRes = results.getMetadataAfter;
+                const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
+                assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
+
+                const listObjectVersionsRes = results.listObjectVersions;
+                const { Versions } = listObjectVersionsRes;
+
+                assert.strictEqual(Versions.length, 1);
+
+                const [currentVersion] = Versions;
+                assertVersionIsNullAndUpdated(currentVersion);
+                return done();
+            });
+        });
+
         // Skipping is necessary because non-versioned buckets are not supported by S3C backbeat routes.
         it.skip('should update metadata of a non-version object', done => {
             let objMD;
@@ -361,7 +549,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -423,7 +611,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -486,7 +674,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -557,7 +745,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -622,7 +810,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -682,7 +870,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -743,7 +931,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
@@ -806,7 +994,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
                     next),
@@ -886,7 +1074,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -961,7 +1149,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
                 next => s3.listObjectVersions({ Bucket: bucket }, next),
@@ -1033,7 +1221,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
                 next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
@@ -1107,7 +1295,7 @@ describeSkipIfAWS('backbeat routes', () => {
                         versionId: 'null',
                     },
                     authCredentials: backbeatAuthCredentials,
-                    requestBody: objMD.getSerialized(),
+                    requestBody: objMD,
                 }, next),
                 next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
                     next),

tests/functional/raw-node/test/routes/routeBackbeatForReplication.js

@@ -100,6 +100,67 @@ describeSkipIfAWS('backbeat routes for replication', () => {
         });
     });
 
+    it('should successfully replicate a suspended null version', done => {
+        let objMD;
+
+        async.series({
+            suspendVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Suspended' } }, next),
+            putObject: next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, next),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObject: next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectRes = results.headObject;
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 1);
+
+            const [currentVersion] = Versions;
+            assert.strictEqual(currentVersion.IsLatest, true);
+            assert.strictEqual(currentVersion.VersionId, 'null');
+
+            return done();
+        });
+    });
+
     it('should successfully replicate a null version and update it', done => {
         let objMD;
 
@@ -178,4 +239,930 @@ describeSkipIfAWS('backbeat routes for replication', () => {
             return done();
         });
     });
+
+    it('should successfully put object after replicating a null version', done => {
+        let objMD;
+        let expectedVersionId;
+
+        async.series({
+            putObjectSource: next => s3.putObject(
+                { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, next),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            putObjectDestination: next => s3.putObject(
+            { Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                expectedVersionId = data.VersionId;
+                return next();
+            }),
+            headObject: next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectRes = results.headObject;
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+
+            const [currentVersion, nonCurrentVersion] = Versions;
+            assert.strictEqual(currentVersion.VersionId, expectedVersionId);
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+
+            return done();
+        });
+    });
+
+    it('should replicate/put metadata to a destination that has a version', done => {
+        let objMD;
+        let firstVersionId;
+        let secondVersionId;
+
+        async.series({
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            putObjectDestination: next => s3.putObject(
+            { Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                firstVersionId = data.VersionId;
+                return next();
+            }),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            putObjectSource: next => s3.putObject(
+            { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                secondVersionId = data.VersionId;
+                return next();
+            }),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: secondVersionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: secondVersionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObjectFirstVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: firstVersionId }, next),
+            headObjectSecondVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: secondVersionId }, next),
+            listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const firstHeadObjectRes = results.headObjectFirstVersion;
+            assert.strictEqual(firstHeadObjectRes.VersionId, firstVersionId);
+
+            const secondHeadObjectRes = results.headObjectSecondVersion;
+            assert.strictEqual(secondHeadObjectRes.VersionId, secondVersionId);
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, secondVersionId);
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, firstVersionId);
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+
+    it('should replicate/put metadata to a destination that has a null version', done => {
+        let objMD;
+        let versionId;
+
+        async.series({
+            putObjectDestinationInitial: next => s3.putObject(
+                { Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            putObjectSource: next => s3.putObject(
+            { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                versionId = data.VersionId;
+                return next();
+            }),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObjectNullVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            listObjectVersions: next => s3.listObjectVersions(
+                { Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectRes = results.headObjectNullVersion;
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, versionId);
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+
+    it('should replicate/put metadata to a destination that has a suspended null version', done => {
+        let objMD;
+        let versionId;
+
+        async.series({
+            suspendVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Suspended' } }, next),
+            putObjectDestinationInitial: next => s3.putObject(
+                { Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            putObjectSource: next => s3.putObject(
+            { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                versionId = data.VersionId;
+                return next();
+            }),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObjectNullVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectRes = results.headObjectNullVersion;
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, versionId);
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+
+    it('should replicate/put metadata to a destination that has a previously updated null version', done => {
+        let objMD;
+        let objMDNull;
+        let versionId;
+
+        async.series({
+            putObjectDestinationInitial: next => s3.putObject(
+                { Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            getMetadataNullVersion: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMDNull = JSON.parse(data.body).Body;
+                return next();
+            }),
+            updateMetadataNullVersion: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDNull,
+            }, next),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            putObjectSource: next => s3.putObject(
+            { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                versionId = data.VersionId;
+                return next();
+            }),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObjectNullVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectRes = results.headObjectNullVersion;
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, versionId);
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+
+    it('should replicate/put metadata to a destination that has a suspended null version with internal version',
+    done => {
+        const tagSet = [
+            {
+                Key: 'key1',
+                Value: 'value1',
+            },
+        ];
+        let objMD;
+        let versionId;
+
+        async.series({
+            suspendVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Suspended' } }, next),
+            putObjectDestinationInitial: next => s3.putObject(
+                { Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
+            putObjectTagging: next => s3.putObjectTagging(
+                { Bucket: bucketDestination, Key: keyName, Tagging: { TagSet: tagSet } }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            putObjectSource: next => s3.putObject(
+            { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                versionId = data.VersionId;
+                return next();
+            }),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObjectNullVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            getObjectTaggingNullVersion: next => s3.getObjectTagging(
+                { Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectRes = results.headObjectNullVersion;
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const getObjectTaggingRes = results.getObjectTaggingNullVersion;
+            assert.deepStrictEqual(getObjectTaggingRes.TagSet, tagSet);
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, versionId);
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+
+    it('should mimic null version replication by crrExistingObjects, then replicate version', done => {
+        let objMDNull;
+        let objMDNullReplicated;
+        let objMDVersion;
+        let versionId;
+
+        async.series({
+            createNullSoloMasterKey: next => s3.putObject(
+                { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, next),
+            enableVersioningSource: next => s3.putBucketVersioning(
+                { Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            simulateCrrExistingObjectsGetMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMDNull = JSON.parse(data.body).Body;
+                assert.strictEqual(JSON.parse(objMDNull).versionId, undefined);
+                return next();
+            }),
+            simulateCrrExistingObjectsPutMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDNull,
+            }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning(
+                { Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
+            replicateNullVersion: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMDNullReplicated = JSON.parse(data.body).Body;
+                return next();
+            }),
+            putReplicatedNullVersion: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDNullReplicated,
+            }, next),
+            putNewVersionSource: next => s3.putObject(
+            { Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                versionId = data.VersionId;
+                return next();
+            }),
+            simulateMetadataReplicationVersion: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMDVersion = JSON.parse(data.body).Body;
+                return next();
+            }),
+            listObjectVersionsBeforeReplicate: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+            putReplicatedVersion: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDVersion,
+            }, next),
+            checkReplicatedNullVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            checkReplicatedVersion: next => s3.headObject(
+                { Bucket: bucketDestination, Key: keyName, VersionId: versionId }, next),
+            listObjectVersionsAfterReplicate: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectNullVersionRes = results.checkReplicatedNullVersion;
+            assert.strictEqual(headObjectNullVersionRes.VersionId, 'null');
+
+            const headObjectVersionRes = results.checkReplicatedVersion;
+            assert.strictEqual(headObjectVersionRes.VersionId, versionId);
+
+            const listObjectVersionsRes = results.listObjectVersionsAfterReplicate;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, versionId);
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+
+    it('should replicate/put NULL metadata to a destination that has a version', done => {
+        let objMD;
+        let versionId;
+
+        async.series({
+            enableVersioningDestination: next => s3.putBucketVersioning({
+                Bucket: bucketDestination,
+                VersioningConfiguration: { Status: 'Enabled' },
+            }, next),
+            putObjectDestination: next => s3.putObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                Body: Buffer.from(testData),
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                versionId = data.VersionId;
+                return next();
+            }),
+            putObjectSource: next => s3.putObject({
+                Bucket: bucketSource,
+                Key: keyName,
+                Body: Buffer.from(testData),
+            }, next),
+            enableVersioningSource: next => s3.putBucketVersioning({
+                Bucket: bucketSource,
+                VersioningConfiguration: { Status: 'Enabled' },
+            }, next),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObjectByVersionId: next => s3.headObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                VersionId: versionId,
+            }, next),
+            headObjectByNullVersionId: next => s3.headObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                VersionId: 'null',
+            }, next),
+            listObjectVersions: next => s3.listObjectVersions({
+                Bucket: bucketDestination,
+            }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const firstHeadObjectRes = results.headObjectByVersionId;
+            assert.strictEqual(firstHeadObjectRes.VersionId, versionId);
+
+            const secondHeadObjectRes = results.headObjectByNullVersionId;
+            assert.strictEqual(secondHeadObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, 'null');
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, versionId);
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+
+    it('should replicate/put NULL metadata to a destination that has a null version', done => {
+        let objMD;
+
+        async.series({
+            putObjectDestinationInitial: next => s3.putObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                Body: Buffer.from(testData),
+            }, next),
+            enableVersioningDestination: next => s3.putBucketVersioning({
+                Bucket: bucketDestination,
+                VersioningConfiguration: { Status: 'Enabled' },
+            }, next),
+            putObjectSource: next => s3.putObject({
+                Bucket: bucketSource,
+                Key: keyName,
+                Body: Buffer.from(testData),
+            }, next),
+            enableVersioningSource: next => s3.putBucketVersioning({
+                Bucket: bucketSource,
+                VersioningConfiguration: { Status: 'Enabled' },
+            }, next),
+            putObjectTaggingSource: next => s3.putObjectTagging({
+                Bucket: bucketSource,
+                Key: keyName,
+                VersionId: 'null',
+                Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] },
+            }, next),
+            getMetadata: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            replicateMetadata: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            headObjectNullVersion: next => s3.headObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                VersionId: 'null',
+            }, next),
+            getObjectTaggingNullVersion: next => s3.getObjectTagging({
+                Bucket: bucketDestination,
+                Key: keyName,
+                VersionId: 'null',
+            }, next),
+            listObjectVersions: next => s3.listObjectVersions({
+                Bucket: bucketDestination,
+            }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const headObjectRes = results.headObjectNullVersion;
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const getObjectTaggingRes = results.getObjectTaggingNullVersion;
+            assert.deepStrictEqual(getObjectTaggingRes.TagSet, [{ Key: 'key1', Value: 'value1' }]);
+
+            const listObjectVersionsRes = results.listObjectVersions;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 1);
+            const [currentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, 'null');
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            return done();
+        });
+    });
+
+    it('should replicate/put a lifecycled NULL metadata to a destination that has a version', done => {
+        let objMDUpdated;
+        let objMDReplicated;
+        let versionId;
+
+        async.series({
+            enableVersioningDestination: next => s3.putBucketVersioning({
+                Bucket: bucketDestination,
+                VersioningConfiguration: { Status: 'Enabled' },
+            }, next),
+            putObjectDestination: next => s3.putObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                Body: Buffer.from(testData),
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                versionId = data.VersionId;
+                return next();
+            }),
+            putObjectSource: next => s3.putObject({
+                Bucket: bucketSource,
+                Key: keyName,
+                Body: Buffer.from(testData),
+            }, next),
+            enableVersioningSource: next => s3.putBucketVersioning({
+                Bucket: bucketSource,
+                VersioningConfiguration: { Status: 'Enabled' },
+            }, next),
+            simulateLifecycleNullVersion: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMDUpdated = JSON.parse(data.body).Body;
+                return next();
+            }),
+            updateMetadataSource: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDUpdated,
+            }, next),
+            getReplicatedNullVersion: next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMDReplicated = JSON.parse(data.body).Body;
+                return next();
+            }),
+            putReplicatedNullVersion: next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: { versionId: 'null' },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDReplicated,
+            }, next),
+            headObjectByVersionId: next => s3.headObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                VersionId: versionId,
+            }, next),
+            headObjectByNullVersion: next => s3.headObject({
+                Bucket: bucketDestination,
+                Key: keyName,
+                VersionId: 'null',
+            }, next),
+            listObjectVersionsDestination: next => s3.listObjectVersions({
+                Bucket: bucketDestination,
+            }, next),
+        }, (err, results) => {
+            if (err) {
+                return done(err);
+            }
+
+            const firstHeadObjectRes = results.headObjectByVersionId;
+            assert.strictEqual(firstHeadObjectRes.VersionId, versionId);
+
+            const secondHeadObjectRes = results.headObjectByNullVersion;
+            assert.strictEqual(secondHeadObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = results.listObjectVersionsDestination;
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, 'null');
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, versionId);
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
 });

tests/unit/testConfigs/parseRedisConfig.spec.js

@@ -0,0 +1,254 @@
+const assert = require('assert');
+const { parseRedisConfig } = require('../../../lib/Config');
+
+describe('parseRedisConfig', () => {
+    [
+        {
+            desc: 'with host and port',
+            input: {
+                host: 'localhost',
+                port: 6479,
+            },
+        },
+        {
+            desc: 'with host, port and password',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                password: 'mypass',
+            },
+        },
+        {
+            desc: 'with host, port and an empty password',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                password: '',
+            },
+        },
+        {
+            desc: 'with host, port and an empty retry config',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                retry: {
+                },
+            },
+        },
+        {
+            desc: 'with host, port and a custom retry config',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                retry: {
+                    connectBackoff: {
+                        min: 10,
+                        max: 1000,
+                        jitter: 0.1,
+                        factor: 1.5,
+                        deadline: 10000,
+                    },
+                },
+            },
+        },
+        {
+            desc: 'with a single sentinel and no sentinel password',
+            input: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: 'localhost',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with two sentinels and a sentinel password',
+            input: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                    {
+                        host: '10.20.30.41',
+                        port: 16479,
+                    },
+                ],
+                sentinelPassword: 'mypass',
+            },
+        },
+        {
+            desc: 'with a sentinel and an empty sentinel password',
+            input: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                ],
+                sentinelPassword: '',
+            },
+        },
+        {
+            desc: 'with a basic production-like config with sentinels',
+            input: {
+                name: 'scality-s3',
+                password: '',
+                sentinelPassword: '',
+                sentinels: [
+                    {
+                        host: 'storage-1',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-2',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-3',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-4',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-5',
+                        port: 16379,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with a single sentinel passed as a string',
+            input: {
+                name: 'myname',
+                sentinels: '10.20.30.40:16479',
+            },
+            output: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with a list of sentinels passed as a string',
+            input: {
+                name: 'myname',
+                sentinels: '10.20.30.40:16479,another-host:16480,10.20.30.42:16481',
+                sentinelPassword: 'mypass',
+            },
+            output: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                    {
+                        host: 'another-host',
+                        port: 16480,
+                    },
+                    {
+                        host: '10.20.30.42',
+                        port: 16481,
+                    },
+                ],
+                sentinelPassword: 'mypass',
+            },
+        },
+    ].forEach(testCase => {
+        it(`should parse a valid config ${testCase.desc}`, () => {
+            const redisConfig = parseRedisConfig(testCase.input);
+            assert.deepStrictEqual(redisConfig, testCase.output || testCase.input);
+        });
+    });
+
+    [
+        {
+            desc: 'that is empty',
+            input: {},
+        },
+        {
+            desc: 'with only a host',
+            input: {
+                host: 'localhost',
+            },
+        },
+        {
+            desc: 'with only a port',
+            input: {
+                port: 6479,
+            },
+        },
+        {
+            desc: 'with a custom retry config with missing values',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                retry: {
+                    connectBackoff: {
+                    },
+                },
+            },
+        },
+        {
+            desc: 'with a sentinel but no name',
+            input: {
+                sentinels: [
+                    {
+                        host: 'localhost',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with a sentinel but an empty name',
+            input: {
+                name: '',
+                sentinels: [
+                    {
+                        host: 'localhost',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with an empty list of sentinels',
+            input: {
+                name: 'myname',
+                sentinels: [],
+            },
+        },
+        {
+            desc: 'with an empty list of sentinels passed as a string',
+            input: {
+                name: 'myname',
+                sentinels: '',
+            },
+        },
+        {
+            desc: 'with an invalid list of sentinels passed as a string (missing port)',
+            input: {
+                name: 'myname',
+                sentinels: '10.20.30.40:16479,10.20.30.50',
+            },
+        },
+    ].forEach(testCase => {
+        it(`should fail to parse an invalid config ${testCase.desc}`, () => {
+            assert.throws(() => {
+                parseRedisConfig(testCase.input);
+            });
+        });
+    });
+});

yarn.lock

@@ -499,9 +499,9 @@ arraybuffer.slice@~0.0.7:
   optionalDependencies:
     ioctl "^2.0.2"
 
-"arsenal@git+https://github.com/scality/arsenal#7.70.25":
+"arsenal@git+https://github.com/scality/arsenal#7.70.25-1":
-  version "7.70.25"
+  version "7.70.25-1"
-  resolved "git+https://github.com/scality/arsenal#7423fac674529efb69fe81bab4404bfa9737382e"
+  resolved "git+https://github.com/scality/arsenal#9ba0900ff857d50c14bf3c5adf3ae3bb643d7fe6"
   dependencies:
     "@js-sdsl/ordered-set" "^4.4.2"
     "@types/async" "^3.2.12"
@@ -5476,9 +5476,9 @@ user-home@^2.0.0:
   dependencies:
     os-homedir "^1.0.0"
 
-"utapi@git+https://github.com/scality/utapi#7.70.3":
+"utapi@git+https://github.com/scality/utapi#7.70.5":
-  version "7.70.3"
+  version "7.70.5"
-  resolved "git+https://github.com/scality/utapi#e8882a28cc888b2a96479f0301a16f45ce2b0603"
+  resolved "git+https://github.com/scality/utapi#20667ff74176879da22d745fc6db0896270dd610"
   dependencies:
     "@hapi/joi" "^17.1.1"
     "@senx/warp10" "^1.0.14"
@@ -5606,6 +5606,12 @@ werelogs@scality/werelogs#8.1.0:
   dependencies:
     safe-json-stringify "1.0.3"
 
+werelogs@scality/werelogs#8.1.0-1:
+  version "8.1.0-1"
+  resolved "https://codeload.github.com/scality/werelogs/tar.gz/1a3a7b12aa15e9b72f7fa3231b7c3adbd74f75b3"
+  dependencies:
+    safe-json-stringify "1.0.3"
+
 werelogs@scality/werelogs#GA7.2.0.5:
   version "7.2.0"
   resolved "https://codeload.github.com/scality/werelogs/tar.gz/bc034589ebf7810d6e6d61932f94327976de6eef"