CLDSRV-461 Backbeat API supports non versioned bucket

(cherry picked from commit d47b71d612)

lib/api/bucketPutVersioning.js

@@ -19,6 +19,10 @@ const invalidBucketStateMessage = 'A replication configuration is present on ' +
 const objectLockErrorMessage = 'An Object Lock configuration is present on ' +
   'this bucket, so the versioning state cannot be changed.';
 
+const replicationVersioningErrorMessage = 'A replication configuration is ' +
+'present on this bucket, so you cannot change the versioning state. To ' +
+'change the versioning state, first delete the replication configuration.';
+
 /**
  * Format of xml request:
 
@@ -72,6 +76,18 @@ function _checkBackendVersioningImplemented(bucket) {
     return true;
 }
 
+function _isValidReplicationVersioning(result, bucket) {
+    if (result.VersioningConfiguration &&
+        result.VersioningConfiguration.Status) {
+        // Is there a replication configuration set on the bucket and is the
+        // user attempting to suspend versioning?
+        if (bucket.getReplicationConfiguration()) {
+            return result.VersioningConfiguration.Status[0] !== 'Suspended';
+        }
+    }
+    return true;
+}
+
 /**
  * Bucket Put Versioning - Create or update bucket Versioning
  * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
@@ -110,6 +126,15 @@ function bucketPutVersioning(authInfo, request, log, callback) {
                     externalVersioningErrorMessage);
                 return next(error, bucket);
             }
+            if (!_isValidReplicationVersioning(result, bucket)) {
+                log.debug(replicationVersioningErrorMessage, {
+                    method: 'bucketPutVersioning',
+                    error: errors.InvalidBucketState,
+                });
+                const error = errors.InvalidBucketState
+                    .customizeDescription(replicationVersioningErrorMessage);
+                return next(error, bucket);
+            }
             const versioningConfiguration = {};
             if (result.VersioningConfiguration.Status) {
                 versioningConfiguration.Status =

lib/routes/routeBackbeat.js

@@ -1280,16 +1280,6 @@ function routeBackbeat(clientIP, request, response, log) {
                 return backbeatRoutes[request.method][request.resourceType]
                     [request.query.operation](request, response, log, next);
             }
-            const versioningConfig = bucketInfo.getVersioningConfiguration();
-            if (!versioningConfig || versioningConfig.Status !== 'Enabled') {
-                log.debug('bucket versioning is not enabled', {
-                    method: request.method,
-                    bucketName: request.bucketName,
-                    objectKey: request.objectKey,
-                    resourceType: request.resourceType,
-                });
-                return next(errors.InvalidBucketState);
-            }
             return backbeatRoutes[request.method][request.resourceType](
                 request, response, bucketInfo, objMd, log, next);
         }],

tests/functional/raw-node/test/routes/routeBackbeat.js

@@ -19,7 +19,7 @@ const backbeatAuthCredentials = {
 const TEST_BUCKET = 'backbeatbucket';
 const TEST_ENCRYPTED_BUCKET = 'backbeatbucket-encrypted';
 const TEST_KEY = 'fookey';
-const NONVERSIONED_BUCKET = 'backbeatbucket-non-versioned';
+const NONVERSIONED_BUCKET = 'backbeatbucket-non-versioned3';
 const BUCKET_FOR_NULL_VERSION = 'backbeatbucket-null-version';
 
 const testArn = 'aws::iam:123456789012:user/bart';
@@ -63,9 +63,49 @@ const testMd = {
     },
 };
 
-function checkObjectData(s3, objectKey, dataValue, done) {
+const nonVersionedTestMd = {
+    'owner-display-name': 'Bart',
+    'owner-id': ('79a59df900b949e55d96a1e698fbaced' +
+                 'fd6e09d98eacf8f8d5218e7cd47ef2be'),
+    'content-length': testData.length,
+    'content-md5': testDataMd5,
+    'x-amz-version-id': 'null',
+    'x-amz-server-version-id': '',
+    'x-amz-storage-class': 'awsbackend',
+    'x-amz-server-side-encryption': '',
+    'x-amz-server-side-encryption-aws-kms-key-id': '',
+    'x-amz-server-side-encryption-customer-algorithm': '',
+    'acl': {
+        Canned: 'private',
+        FULL_CONTROL: [],
+        WRITE_ACP: [],
+        READ: [],
+        READ_ACP: [],
+    },
+    'location': null,
+    'isNull': '',
+    'nullVersionId': '',
+    'isDeleteMarker': false,
+    'tags': {},
+    'replicationInfo': {
+        status: '',
+        backends: [],
+        content: [],
+        destination: '',
+        storageClass: '',
+        role: '',
+        storageType: '',
+        dataStoreVersionId: '',
+        isNFS: null,
+    },
+    'dataStoreName': 'us-east-1',
+    'last-modified': '2018-12-18T01:22:15.986Z',
+    'md-model-version': 3,
+};
+
+function checkObjectData(s3, bucket, objectKey, dataValue, done) {
     s3.getObject({
-        Bucket: TEST_BUCKET,
+        Bucket: bucket,
         Key: objectKey,
     }, (err, data) => {
         assert.ifError(err);
@@ -197,6 +237,7 @@ describeSkipIfAWS('backbeat routes', () => {
             .then(() => s3.deleteBucket({ Bucket: TEST_BUCKET }).promise())
             .then(() => bucketUtil.empty(TEST_ENCRYPTED_BUCKET))
             .then(() => s3.deleteBucket({ Bucket: TEST_ENCRYPTED_BUCKET }).promise())
+            .then(() => bucketUtil.empty(NONVERSIONED_BUCKET))
             .then(() =>
                 s3.deleteBucket({ Bucket: NONVERSIONED_BUCKET }).promise())
             .then(() => done(), err => done(err))
@@ -1284,6 +1325,60 @@ describeSkipIfAWS('backbeat routes', () => {
             });
         });
 
+        it('should PUT data and metadata for a non-versioned bucket', done => {
+            const bucket = NONVERSIONED_BUCKET;
+            const objectKey = 'non-versioned-key';
+            async.waterfall([
+                next =>
+                    makeBackbeatRequest({
+                        method: 'PUT',
+                        bucket,
+                        objectKey,
+                        resourceType: 'data',
+                        queryObj: { v2: '' },
+                        headers: {
+                            'content-length': testData.length,
+                            'content-md5': testDataMd5,
+                            'x-scal-canonical-id': testArn,
+                        },
+                        authCredentials: backbeatAuthCredentials,
+                        requestBody: testData,
+                    }, (err, response) => {
+                        console.log('error!!!!', err);
+                        assert.ifError(err);
+                        const metadata = Object.assign({}, nonVersionedTestMd, {
+                            location: JSON.parse(response.body),
+                        });
+                        return next(null, metadata);
+                    }),
+                (metadata, next) =>
+                    makeBackbeatRequest({
+                        method: 'PUT',
+                        bucket,
+                        objectKey,
+                        resourceType: 'metadata',
+                        authCredentials: backbeatAuthCredentials,
+                        requestBody: JSON.stringify(metadata),
+                    }, (err, response) => {
+                        console.log('error2!!!!', err);
+                        assert.ifError(err);
+                        assert.strictEqual(response.statusCode, 200);
+                        next();
+                    }),
+                next =>
+                    s3.headObject({
+                        Bucket: bucket,
+                        Key: objectKey,
+                    }, (err, data) => {
+                        console.log('error3!!!!', err);
+                        assert.ifError(err);
+                        assert.strictEqual(data.StorageClass, 'awsbackend');
+                        next();
+                    }),
+                next => checkObjectData(s3, bucket, objectKey, testData, next),
+            ], done);
+        });
+
         it('PUT metadata with "x-scal-replication-content: METADATA"' +
         'header should replicate metadata only', done => {
             async.waterfall([next => {
@@ -1347,39 +1442,6 @@ describeSkipIfAWS('backbeat routes', () => {
             });
         });
 
-        it('should refuse PUT data if bucket is not versioned',
-        done => makeBackbeatRequest({
-            method: 'PUT', bucket: NONVERSIONED_BUCKET,
-            objectKey: testKey, resourceType: 'data',
-            queryObj: { v2: '' },
-            headers: {
-                'content-length': testData.length,
-                'content-md5': testDataMd5,
-                'x-scal-canonical-id': testArn,
-            },
-            authCredentials: backbeatAuthCredentials,
-            requestBody: testData,
-        },
-        err => {
-            assert.strictEqual(err.code, 'InvalidBucketState');
-            done();
-        }));
-
-        it('should refuse PUT metadata if bucket is not versioned',
-        done => makeBackbeatRequest({
-            method: 'PUT', bucket: NONVERSIONED_BUCKET,
-            objectKey: testKey, resourceType: 'metadata',
-            queryObj: {
-                versionId: versionIdUtils.encode(testMd.versionId),
-            },
-            authCredentials: backbeatAuthCredentials,
-            requestBody: JSON.stringify(testMd),
-        },
-        err => {
-            assert.strictEqual(err.code, 'InvalidBucketState');
-            done();
-        }));
-
         it('should refuse PUT data if no x-scal-canonical-id header ' +
            'is provided', done => makeBackbeatRequest({
                method: 'PUT', bucket: TEST_BUCKET,
@@ -1516,7 +1578,7 @@ describeSkipIfAWS('backbeat routes', () => {
             }, (response, next) => {
                 assert.strictEqual(response.statusCode, 200);
                 // give some time for the async deletes to complete
-                setTimeout(() => checkObjectData(s3, testKey, testData, next),
+                setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, testData, next),
                            1000);
             }, next => {
                 // check that the object copy referencing the old data
@@ -1604,7 +1666,7 @@ describeSkipIfAWS('backbeat routes', () => {
             }, (response, next) => {
                 assert.strictEqual(response.statusCode, 200);
                 // give some time for the async deletes to complete
-                setTimeout(() => checkObjectData(s3, testKey, '', next),
+                setTimeout(() => checkObjectData(s3, TEST_BUCKET, testKey, '', next),
                            1000);
             }, next => {
                 // check that the object copy referencing the old data
@@ -1904,6 +1966,28 @@ describeSkipIfAWS('backbeat routes', () => {
             });
         });
     });
+
+    describe('GET Metadata route for non-versioned bucket', () => {
+        beforeEach(done => s3.putObject({
+            Bucket: NONVERSIONED_BUCKET,
+            Key: TEST_KEY,
+            Body: new Buffer('hello'),
+        }, done));
+
+        it('should return metadata blob for a versionId', done => {
+            makeBackbeatRequest({
+                method: 'GET', bucket: NONVERSIONED_BUCKET,
+                objectKey: TEST_KEY, resourceType: 'metadata',
+                authCredentials: backbeatAuthCredentials,
+            }, (err, data) => {
+                const parsedBody = JSON.parse(JSON.parse(data.body).Body);
+                assert.strictEqual(data.statusCode, 200);
+                assert.deepStrictEqual(parsedBody['content-length'], 5);
+                done();
+            });
+        });
+    });
+
     describe('Batch Delete Route', () => {
         it('should batch delete a location', done => {
             let versionId;

tests/unit/api/bucketPutVersioning.js

@@ -0,0 +1,181 @@
+const assert = require('assert');
+const async = require('async');
+
+const { errors } = require('arsenal');
+const { bucketPut } = require('../../../lib/api/bucketPut');
+const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning');
+const bucketPutReplication = require('../../../lib/api/bucketPutReplication');
+
+const { cleanup,
+    DummyRequestLogger,
+    makeAuthInfo } = require('../helpers');
+const metadata = require('../../../lib/metadata/wrapper');
+
+const xmlEnableVersioning =
+'<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
+'<Status>Enabled</Status>' +
+'</VersioningConfiguration>';
+
+const xmlSuspendVersioning =
+'<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
+'<Status>Suspended</Status>' +
+'</VersioningConfiguration>';
+
+const locConstraintVersioned =
+'<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
+'<LocationConstraint>awsbackend</LocationConstraint>' +
+'</CreateBucketConfiguration>';
+
+const locConstraintNonVersioned =
+'<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
+'<LocationConstraint>azurebackend</LocationConstraint>' +
+'</CreateBucketConfiguration>';
+
+const xmlReplicationConfiguration =
+'<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
+    '<Role>arn:aws:iam::account-id:role/src-resource,arn:aws:iam::account-id:role/dest-resource</Role>' +
+    '<Rule>' +
+        '<Prefix></Prefix>' +
+        '<Status>Enabled</Status>' +
+        '<Destination>' +
+            '<Bucket>arn:aws:s3:::destination-bucket</Bucket>' +
+            '<StorageClass>STANDARD</StorageClass>' +
+        '</Destination>' +
+    '</Rule>' +
+'</ReplicationConfiguration>';
+
+const externalVersioningErrorMessage = 'We do not currently support putting ' +
+'a versioned object to a location-constraint of type Azure or GCP.';
+
+const log = new DummyRequestLogger();
+const bucketName = 'bucketname';
+const authInfo = makeAuthInfo('accessKey1');
+
+function _getPutBucketRequest(xml) {
+    const request = {
+        bucketName,
+        headers: { host: `${bucketName}.s3.amazonaws.com` },
+        url: '/',
+    };
+    request.post = xml;
+    return request;
+}
+
+function _putReplicationRequest(xml) {
+    const request = {
+        bucketName,
+        headers: { host: `${bucketName}.s3.amazonaws.com` },
+        url: '/?replication',
+    };
+    request.post = xml;
+    return request;
+}
+
+function _putVersioningRequest(xml) {
+    const request = {
+        bucketName,
+        headers: { host: `${bucketName}.s3.amazonaws.com` },
+        url: '/?versioning',
+        query: { versioning: '' },
+    };
+    request.post = xml;
+    return request;
+}
+
+describe('bucketPutVersioning API', () => {
+    before(() => cleanup());
+    afterEach(() => cleanup());
+
+    describe('with version enabled location constraint', () => {
+        beforeEach(done => {
+            const request = _getPutBucketRequest(locConstraintVersioned);
+            bucketPut(authInfo, request, log, done);
+        });
+
+        const tests = [
+            {
+                msg: 'should successfully enable versioning on location ' +
+                'constraint with supportsVersioning set to true',
+                input: xmlEnableVersioning,
+                output: { Status: 'Enabled' },
+            },
+            {
+                msg: 'should successfully suspend versioning on location ' +
+                'constraint with supportsVersioning set to true',
+                input: xmlSuspendVersioning,
+                output: { Status: 'Suspended' },
+            },
+        ];
+        tests.forEach(test => it(test.msg, done => {
+            const request = _putVersioningRequest(test.input);
+            bucketPutVersioning(authInfo, request, log, err => {
+                assert.ifError(err,
+                    `Expected success, but got err: ${err}`);
+                metadata.getBucket(bucketName, log, (err, bucket) => {
+                    assert.ifError(err,
+                        `Expected success, but got err: ${err}`);
+                    assert.deepStrictEqual(bucket._versioningConfiguration,
+                        test.output);
+                    done();
+                });
+            });
+        }));
+
+        it('should not suspend versioning on bucket with replication', done => {
+            async.series([
+                // Enable versioning to allow putting a replication config.
+                next => {
+                    const request = _putVersioningRequest(xmlEnableVersioning);
+                    bucketPutVersioning(authInfo, request, log, next);
+                },
+                // Put the replication config on the bucket.
+                next => {
+                    const request =
+                        _putReplicationRequest(xmlReplicationConfiguration);
+                    bucketPutReplication(authInfo, request, log, next);
+                },
+                // Attempt to suspend versioning.
+                next => {
+                    const request = _putVersioningRequest(xmlSuspendVersioning);
+                    bucketPutVersioning(authInfo, request, log, err => {
+                        assert(err.InvalidBucketState);
+                        next();
+                    });
+                },
+            ], done);
+        });
+    });
+
+    describe('with version disabled location constraint', () => {
+        beforeEach(done => {
+            const request = _getPutBucketRequest(locConstraintNonVersioned);
+            bucketPut(authInfo, request, log, done);
+        });
+
+        const tests = [
+            {
+                msg: 'should return error if enabling versioning on location ' +
+                'constraint with supportsVersioning set to false',
+                input: xmlEnableVersioning,
+                output: { error: errors.NotImplemented.customizeDescription(
+                    externalVersioningErrorMessage) },
+            },
+            {
+                msg: 'should return error if suspending versioning on ' +
+                ' location constraint with supportsVersioning set to false',
+                input: xmlSuspendVersioning,
+                output: { error: errors.NotImplemented.customizeDescription(
+                    externalVersioningErrorMessage) },
+            },
+        ];
+        tests.forEach(test => it(test.msg, done => {
+            const putBucketVersioningRequest =
+                _putVersioningRequest(test.input);
+            bucketPutVersioning(authInfo, putBucketVersioningRequest, log,
+            err => {
+                assert.deepStrictEqual(err, test.output.error);
+                done();
+            });
+        }));
+    });
+});