CLDSRV-499 Resolving S3C metadata replication issues for non-versioned objects

Context: In S3C, Replication Queue Populator is only triggering/replicating version keys.

Issue: The 'CrrExistingObjects' script does not replicate objects that were both created before versioning was enabled and have not been updated since versioning was enabled. We refer to these keys as 'solo null markers.'
CrrExistingObjects script should replicate metadata of versions/objects created before enabling replication and possibly before versioning. Objects set before enabling versioning should also be replicated. The issue is that these objects are represented only by a Master key without a version key if there are "solo null markers".

Solution:

Previously, the solution involved an additional putMetadata call to create a version key for the Master key, thereby initiating replication of the version representing the master key.
However, this method relied on a bug in PutMetadata of null versions rather than being a formal implementation. With the bug now fixed with https://scality.atlassian.net/browse/CLDSRV-497 , this approach is no longer viable.

To address this, we're introducing a new putMetdata header, "x-scal-migrate-solo-null-marker," specifically designed for this situation. It will add a version key to a specified null solo master key.

Alternative solution:
An alternative approach involves allowing the Replication Queue Populator to accept a null solo master version (like Artesca does). If we choose this solution, during replication, the put metadata operation will generate a new null solo master at the destination.
Today, putting version metadata, whether null or not null, works. However, putting a version on top of a null solo master using putMetadata, leads to the deletion of the null version, which creates a problem. To address this, it is needed to add the versioning.versioningPreprocessing() logic into the putMetadata process.
Note: The issue of "deletion of the null version" does not occur in Artesca. This is because Artesca uses the cloud providers' SDKs for master/version replication, whereas S3C depends on internal mechanisms such as BackbeatClient.PutMetadata and BackbeatClient.putData.

lib/api/apiUtils/object/versioning.js

@@ -170,7 +170,7 @@ function processVersioningState(mst, vstat) {
         options.versioning = true;
         if (mst.exists) {
             // store master version in a new key
-            const versionId = mst.isNull ? mst.versionId : nonVersionedObjId;
+            const versionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
             storeOptions.versionId = versionId;
             storeOptions.isNull = true;
             options.nullVersionId = versionId;

lib/routes/routeBackbeat.js

@@ -442,6 +442,31 @@ function putData(request, response, bucketInfo, objMd, log, callback) {
     });
 }
 
+/**
+ * Checks if the 'x-scal-migrate-null-solo-master' header is set to 'true'.
+ *
+ * This header is used by the crrExistingObjects.js script within S3Utils for migrating a null
+ * solo master key into a null master key along with a corresponding null version.
+ * The generation of this new internal null version initiates the replication process of the master key.
+ * @param {Object} headers - request headers.
+ * @return {boolean} Returns true if the 'x-scal-migrate-null-solo-master' header
+ *                    is equal to 'true', otherwise returns false.
+ */
+function _isNullSoloMaster(headers) {
+    return headers['x-scal-migrate-null-solo-master'] === 'true';
+}
+
+/**
+ * Check that the migration request is targeting a null solo master key within a bucket where versioning is enabled.
+ * @param {string} versionId - The version ID from the request
+ * @param {boolean} versioning - A flag indicating if versioning is enabled or not.
+ * @param {Object} objMd - The metadata object of the null solo master key, where `versionId` should be undefined
+ * @returns {boolean} - Returns `true` if the conditions for a valid null solo master are met, otherwise `false`.
+ */
+function _isValidNullSoloMaster(versionId, versioning, objMd) {
+    return versionId === 'null' && versioning && objMd && objMd.versionId === undefined;
+}
+
 function putMetadata(request, response, bucketInfo, objMd, log, callback) {
     return _getRequestPayload(request, (err, payload) => {
         if (err) {
@@ -478,7 +503,17 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
         let versioning = bucketInfo.isVersioningEnabled();
         let isNull = false;
 
-        if (versionId === 'null') {
+        // NOTE: condition only needed in S3C to replicate non-versioned objects (CLDSRV-499)
+        if (_isNullSoloMaster(headers)) {
+            if (!_isValidNullSoloMaster(versionId, versioning, objMd)) {
+                return callback(errors.InvalidArgument);
+            }
+
+            // The following creates an internal null version and updates the versionId of the master null key:
+            versionId = '';
+            isNull = true;
+            omVal.isNull = true;
+        } else if (versionId === 'null') {
             isNull = true;
             // Retrieve the null version id from the object metadata.
             versionId = objMd && objMd.versionId;
@@ -522,7 +557,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
         }
 
         const options = {
-            versionId,
             isNull,
         };
 
@@ -535,6 +569,14 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
             options.versioning = true;
         }
 
+        // NOTE: When options fields are sent to Metadata through the query string,
+        // they are converted to strings. As a result, Metadata interprets the value undefined
+        // in the versionId field as an empty string ('').
+        // To prevent this, the versionId field is only included in options when it is defined.
+        if (versionId !== undefined) {
+            options.versionId = versionId;
+        }
+
         log.trace('putting object version', {
             objectKey: request.objectKey, omVal, options });
         return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,

tests/functional/raw-node/test/routes/routeBackbeat.js

@@ -5,7 +5,7 @@ const { models, versioning } = require('arsenal');
 const versionIdUtils = versioning.VersionID;
 const { ObjectMD } = models;
 
-const { makeRequest } = require('../../utils/makeRequest');
+const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest');
 const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
 
 const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1';
@@ -86,39 +86,6 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
     });
 }
 
-/** makeBackbeatRequest - utility function to generate a request going
- * through backbeat route
- * @param {object} params - params for making request
- * @param {string} params.method - request method
- * @param {string} params.bucket - bucket name
- * @param {string} params.objectKey - object key
- * @param {string} params.subCommand - subcommand to backbeat
- * @param {object} [params.headers] - headers and their string values
- * @param {object} [params.authCredentials] - authentication credentials
- * @param {object} params.authCredentials.accessKey - access key
- * @param {object} params.authCredentials.secretKey - secret key
- * @param {string} [params.requestBody] - request body contents
- * @param {object} [params.queryObj] - query params
- * @param {function} callback - with error and response parameters
- * @return {undefined} - and call callback
- */
-function makeBackbeatRequest(params, callback) {
-    const { method, headers, bucket, objectKey, resourceType,
-            authCredentials, requestBody, queryObj } = params;
-    const options = {
-        authCredentials,
-        hostname: ipAddress,
-        port: 8000,
-        method,
-        headers,
-        path: `/_/backbeat/${resourceType}/${bucket}/${objectKey}`,
-        requestBody,
-        jsonResponse: true,
-        queryObj,
-    };
-    makeRequest(options, callback);
-}
-
 function updateStorageClass(data, storageClass) {
     let parsedBody;
     try {
@@ -360,6 +327,253 @@ describeSkipIfAWS('backbeat routes', () => {
             });
         });
 
+        it('should migrate null version', done => {
+            let objMD;
+            return async.series([
+                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
+                next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
+                    next),
+                next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    const { error, result } = updateStorageClass(data, storageClass);
+                    if (error) {
+                        return next(error);
+                    }
+                    objMD = result;
+                    return next();
+                }),
+                next => makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    headers: {
+                        'x-scal-migrate-null-solo-master': 'true',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: objMD.getSerialized(),
+                }, next),
+                next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
+                next => s3.listObjectVersions({ Bucket: bucket }, next),
+            ], (err, data) => {
+                if (err) {
+                    return done(err);
+                }
+                const headObjectRes = data[4];
+                assert.strictEqual(headObjectRes.VersionId, 'null');
+                assert.strictEqual(headObjectRes.StorageClass, storageClass);
+
+                const listObjectVersionsRes = data[5];
+                const { Versions } = listObjectVersionsRes;
+
+                assert.strictEqual(Versions.length, 1);
+
+                const [currentVersion] = Versions;
+                assertVersionIsNullAndUpdated(currentVersion);
+                return done();
+            });
+        });
+
+        it('should not migrate null version if not solo (has version)', done => {
+            let objMD;
+            return async.series([
+                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
+                next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
+                    next),
+                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
+                next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    objMD = JSON.parse(data.body).Body;
+                    return next();
+                }),
+                next => makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    headers: {
+                        'x-scal-migrate-null-solo-master': 'true',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: objMD,
+                }, next),
+            ], err => {
+                assert.notEqual(err, null, 'Expected failure but got success');
+                assert.strictEqual(err.code, 'InvalidArgument');
+                return done();
+            });
+        });
+
+        it('should not migrate a version that is not null', done => {
+            let objMD;
+            let versionId;
+            return async.series([
+                next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
+                    next),
+                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    versionId = data.VersionId;
+                    return next();
+                }),
+                next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId,
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    objMD = JSON.parse(data.body).Body;
+                    return next();
+                }),
+                next => makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId,
+                    },
+                    headers: {
+                        'x-scal-migrate-null-solo-master': 'true',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: objMD,
+                }, next),
+            ], err => {
+                assert.notEqual(err, null, 'Expected failure but got success');
+                assert.strictEqual(err.code, 'InvalidArgument');
+                return done();
+            });
+        });
+
+        it('should not migrate a null version that does not exist', done => {
+            let objMD;
+            let versionId;
+            return async.series([
+                next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
+                    next),
+                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    versionId = data.VersionId;
+                    return next();
+                }),
+                next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId,
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    objMD = JSON.parse(data.body).Body;
+                    return next();
+                }),
+                next => makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    headers: {
+                        'x-scal-migrate-null-solo-master': 'true',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: objMD,
+                }, next),
+            ], err => {
+                assert.notEqual(err, null, 'Expected failure but got success');
+                assert.strictEqual(err.code, 'InvalidArgument');
+                return done();
+            });
+        });
+
+        // Skipping is necessary because non-versioned buckets are not supported by S3C backbeat routes.
+        it.skip('should not migrate an object from a non-versioned bucket', done => {
+            let objMD;
+            return async.series([
+                next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
+                next => makeBackbeatRequest({
+                    method: 'GET',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                }, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    objMD = JSON.parse(data.body).Body;
+                    return next();
+                }),
+                next => makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    headers: {
+                        'x-scal-migrate-null-solo-master': 'true',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: objMD,
+                }, next),
+            ], err => {
+                assert.notEqual(err, null, 'Expected failure but got success');
+                assert.strictEqual(err.code, 'InvalidArgument');
+                return done();
+            });
+        });
+
         // Skipping is necessary because non-versioned buckets are not supported by S3C backbeat routes.
         it.skip('should update metadata of a non-version object', done => {
             let objMD;

tests/functional/raw-node/test/routes/routeBackbeatForReplication.js

@@ -0,0 +1,389 @@
+const assert = require('assert');
+const async = require('async');
+const { models } = require('arsenal');
+const { ObjectMD } = models;
+
+const { makeBackbeatRequest } = require('../../utils/makeRequest');
+const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
+
+const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe;
+
+const backbeatAuthCredentials = {
+    accessKey: 'accessKey1',
+    secretKey: 'verySecretKey1',
+};
+
+const testData = 'testkey data';
+
+describeSkipIfAWS('backbeat routes for replication', () => {
+    const bucketUtil = new BucketUtility(
+        'default', { signatureVersion: 'v4' });
+    const s3 = bucketUtil.s3;
+
+    const bucketSource = 'backbeatbucket-replication-source';
+    const bucketDestination = 'backbeatbucket-replication-destination';
+    const keyName = 'key0';
+    const storageClass = 'foo';
+
+    beforeEach(done =>
+        bucketUtil.emptyIfExists(bucketSource)
+            .then(() => s3.createBucket({ Bucket: bucketSource }).promise())
+            .then(() => bucketUtil.emptyIfExists(bucketDestination))
+            .then(() => s3.createBucket({ Bucket: bucketDestination }).promise())
+            .then(() => done(), err => done(err))
+    );
+
+    afterEach(done =>
+        bucketUtil.empty(bucketSource)
+            .then(() => s3.deleteBucket({ Bucket: bucketSource }).promise())
+            .then(() => bucketUtil.empty(bucketDestination))
+            .then(() => s3.deleteBucket({ Bucket: bucketDestination }).promise())
+            .then(() => done(), err => done(err))
+    );
+
+    it('should successfully replicate a null version', done => {
+        let objMD;
+        return async.series([
+            next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, next),
+            next => s3.putBucketVersioning({ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } },
+                next),
+            next => s3.putBucketVersioning({ Bucket: bucketDestination, VersioningConfiguration:
+                { Status: 'Enabled' } }, next),
+            next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        ], (err, data) => {
+            if (err) {
+                return done(err);
+            }
+            const headObjectRes = data[5];
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = data[6];
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 1);
+
+            const [currentVersion] = Versions;
+            assert.strictEqual(currentVersion.IsLatest, true);
+            assert.strictEqual(currentVersion.VersionId, 'null');
+            return done();
+        });
+    });
+
+    it('should successfully replicate a null version and update it', done => {
+        let objMD;
+        return async.series([
+            next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, next),
+            next => s3.putBucketVersioning({ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } },
+                next),
+            next => s3.putBucketVersioning({ Bucket: bucketDestination, VersioningConfiguration:
+                { Status: 'Enabled' } }, next),
+            next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            next => {
+                const { result, error } = ObjectMD.createFromBlob(objMD);
+                if (error) {
+                    return next(error);
+                }
+                result.setAmzStorageClass(storageClass);
+                return makeBackbeatRequest({
+                    method: 'PUT',
+                    resourceType: 'metadata',
+                    bucket: bucketDestination,
+                    objectKey: keyName,
+                    queryObj: {
+                        versionId: 'null',
+                    },
+                    authCredentials: backbeatAuthCredentials,
+                    requestBody: result.getSerialized(),
+                }, next);
+            },
+            next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        ], (err, data) => {
+            if (err) {
+                return done(err);
+            }
+            const headObjectRes = data[6];
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+            assert.strictEqual(headObjectRes.StorageClass, storageClass);
+
+            const listObjectVersionsRes = data[7];
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 1);
+
+            const [currentVersion] = Versions;
+            assert.strictEqual(currentVersion.IsLatest, true);
+            assert.strictEqual(currentVersion.VersionId, 'null');
+            assert.strictEqual(currentVersion.StorageClass, storageClass);
+            return done();
+        });
+    });
+
+    it('should successfully put object after replicating a null version', done => {
+        let objMD;
+        let expectedVersionId;
+        return async.series([
+            next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, next),
+            next => s3.putBucketVersioning({ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } },
+                next),
+            next => s3.putBucketVersioning({ Bucket: bucketDestination, VersioningConfiguration:
+                { Status: 'Enabled' } }, next),
+            next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                objMD = JSON.parse(data.body).Body;
+                return next();
+            }),
+            next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMD,
+            }, next),
+            next => s3.putObject({ Bucket: bucketDestination, Key: keyName, Body: new Buffer(testData) },
+            (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+                expectedVersionId = data.VersionId;
+                return next();
+            }),
+            next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        ], (err, data) => {
+            if (err) {
+                return done(err);
+            }
+            const headObjectRes = data[6];
+            assert.strictEqual(headObjectRes.VersionId, 'null');
+
+            const listObjectVersionsRes = data[7];
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+
+            const [currentVersion, nonCurrentVersion] = Versions;
+            assert.strictEqual(currentVersion.VersionId, expectedVersionId);
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+            return done();
+        });
+    });
+
+    it('should replicate a null solo master version an then replicate another version', done => {
+        let objMDNull;
+        let objMDNullAfterMigration;
+        let objMDVersion;
+        let versionId;
+        // Simulate a flow where a key, created before versioning, is replicated using the CRRexistingObjects script.
+        // Then, replicate another version of the same key.
+        return async.series([
+            // 1. Create null solo master key
+            next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, next),
+            next => s3.putBucketVersioning({ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } },
+                next),
+            // 2. Check that the version is a null solo master key
+            next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+
+                objMDNull = JSON.parse(data.body).Body;
+                assert.strictEqual(JSON.parse(objMDNull).versionId, undefined);
+                return next();
+            }),
+            // 3. Simulate the putMetadata with x-scal-migrate-null-solo-master to generate an internal null version
+            next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                headers: {
+                    'x-scal-migrate-null-solo-master': 'true',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDNull,
+            }, next),
+            // 4. Simulate the put metadata logic in Replication Queue Processor.
+            next => s3.putBucketVersioning({ Bucket: bucketDestination, VersioningConfiguration:
+                { Status: 'Enabled' } }, next),
+            next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+
+                objMDNullAfterMigration = JSON.parse(data.body).Body;
+                // 5. Check that the migration worked and that a versionId representing
+                // the new internal version attached is set.
+                assert.notEqual(JSON.parse(objMDNullAfterMigration).versionId, undefined);
+                return next();
+            }),
+            next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId: 'null',
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDNullAfterMigration,
+            }, next),
+            // 6. Put a new version in the source bucket to be replicated.
+            next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+
+                versionId = data.VersionId;
+                return next();
+            }),
+            // 7. Simulate the metadata replication of the version.
+            next => makeBackbeatRequest({
+                method: 'GET',
+                resourceType: 'metadata',
+                bucket: bucketSource,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                if (err) {
+                    return next(err);
+                }
+
+                objMDVersion = JSON.parse(data.body).Body;
+                return next();
+            }),
+            next => makeBackbeatRequest({
+                method: 'PUT',
+                resourceType: 'metadata',
+                bucket: bucketDestination,
+                objectKey: keyName,
+                queryObj: {
+                    versionId,
+                },
+                authCredentials: backbeatAuthCredentials,
+                requestBody: objMDVersion,
+            }, next),
+            // 8. Check that the null version does not get overwritten.
+            next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
+            next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: versionId }, next),
+            next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
+        ], (err, data) => {
+            if (err) {
+                return done(err);
+            }
+            const headObjectNullRes = data[10];
+            assert.strictEqual(headObjectNullRes.VersionId, 'null');
+
+            const headObjectVersionRes = data[11];
+            assert.strictEqual(headObjectVersionRes.VersionId, versionId);
+
+            const listObjectVersionsRes = data[12];
+            const { Versions } = listObjectVersionsRes;
+
+            assert.strictEqual(Versions.length, 2);
+
+            const [currentVersion, nonCurrentVersion] = Versions;
+
+            assert.strictEqual(currentVersion.VersionId, versionId);
+            assert.strictEqual(currentVersion.IsLatest, true);
+
+            assert.strictEqual(nonCurrentVersion.VersionId, 'null');
+            assert.strictEqual(nonCurrentVersion.IsLatest, false);
+
+            return done();
+        });
+    });
+});

tests/functional/raw-node/utils/makeRequest.js

@@ -190,8 +190,42 @@ function makeGcpRequest(params, callback) {
     makeRequest(options, callback);
 }
 
+/** makeBackbeatRequest - utility function to generate a request going
+ * through backbeat route
+ * @param {object} params - params for making request
+ * @param {string} params.method - request method
+ * @param {string} params.bucket - bucket name
+ * @param {string} params.objectKey - object key
+ * @param {string} params.subCommand - subcommand to backbeat
+ * @param {object} [params.headers] - headers and their string values
+ * @param {object} [params.authCredentials] - authentication credentials
+ * @param {object} params.authCredentials.accessKey - access key
+ * @param {object} params.authCredentials.secretKey - secret key
+ * @param {string} [params.requestBody] - request body contents
+ * @param {object} [params.queryObj] - query params
+ * @param {function} callback - with error and response parameters
+ * @return {undefined} - and call callback
+ */
+function makeBackbeatRequest(params, callback) {
+    const { method, headers, bucket, objectKey, resourceType,
+            authCredentials, requestBody, queryObj } = params;
+    const options = {
+        authCredentials,
+        hostname: ipAddress,
+        port: 8000,
+        method,
+        headers,
+        path: `/_/backbeat/${resourceType}/${bucket}/${objectKey}`,
+        requestBody,
+        jsonResponse: true,
+        queryObj,
+    };
+    makeRequest(options, callback);
+}
+
 module.exports = {
     makeRequest,
     makeS3Request,
     makeGcpRequest,
+    makeBackbeatRequest,
 };