feature: S3C-1182 add versioning support for GCP backend

Supports versioning with GCP backend

constants.js

@@ -127,7 +127,7 @@ const constants = {
     // for external backends, don't call unless at least 1 minute
     // (60,000 milliseconds) since last call
     externalBackendHealthCheckInterval: 60000,
-    versioningNotImplBackends: { azure: true, gcp: true },
+    versioningNotImplBackends: { azure: true },
     mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
     skipBatchDeleteBackends: { azure: true, gcp: true },
     s3HandledBackends: { azure: true, gcp: true },

lib/data/external/GcpClient.js

@@ -6,7 +6,8 @@ const { GCP, GcpUtils } = require('./GCP');
 const { createMpuKey } = GcpUtils;
 const AwsClient = require('./AwsClient');
 const { prepareStream } = require('../../api/apiUtils/object/prepareStream');
-const { logHelper, removeQuotes } = require('./utils');
+const createLogger = require('../multipleBackendLogger');
+const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils');
 const { config } = require('../../Config');
 
 const missingVerIdInternalError = errors.InternalError.customizeDescription(
@@ -80,9 +81,34 @@ class GcpClient extends AwsClient {
                 return cb(null, bucketResp);
             });
         };
+
+        const checkBucketVersioning = (headResp, cb) => {
+            if (headResp.err) {
+                return cb(null, headResp);
+            }
+            let bucketResp;
+            return this._client.getBucketVersioning(
+            { Bucket: headResp.gcpBucket }, (err, data) => {
+                bucketResp = { gcpBucket: headResp.gcpBucket };
+                if (err) {
+                    bucketResp.error = err;
+                } else if (!data.Status || data.Status === 'Suspended') {
+                    bucketResp.error = 'Versioning must be enabled';
+                    bucketResp.versioningStatus = data.Status;
+                } else {
+                    bucketResp.versioningStatus = data.Status;
+                    bucketResp.message = headResp.message;
+                }
+                return cb(null, bucketResp);
+            });
+        };
+
         const gcpResp = {};
         async.parallel({
-            main: done => checkBucketHealth(this._gcpBucketName, done),
+            main: done => async.waterfall([
+                next => checkBucketHealth(this._gcpBucketName, next),
+                (headResp, next) => checkBucketVersioning(headResp, next),
+            ], (err, result) => done(null, result)),
             mpu: done => checkBucketHealth(this._mpuBucketName, done),
         }, (err, result) => {
             if (err) {
@@ -97,6 +123,70 @@ class GcpClient extends AwsClient {
         });
     }
 
+    put(stream, size, keyContext, reqUids, callback) {
+        const gcpKey = this._createGcpKey(keyContext.bucketName,
+           keyContext.objectKey, this._bucketMatch);
+        const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
+        const log = createLogger(reqUids);
+
+        const putCb = (err, data) => {
+            if (err) {
+                logHelper(log, 'error', 'err from data backend',
+                  err, this._dataStoreName, this.clientType);
+                return callback(errors.ServiceUnavailable
+                  .customizeDescription('Error returned from ' +
+                  `${this.type}: ${err.message}`)
+                );
+            }
+            if (keyContext.isDeleteMarker) {
+                log.info('GCP delete marker: returning "0" as versiond id');
+                return callback(null, gcpKey, '0');
+            }
+            if (!data.VersionId) {
+                logHelper(log, 'error', 'missing version id for data ' +
+                    'backend object', missingVerIdInternalError,
+                    this._dataStoreName, this.clientType);
+                return callback(missingVerIdInternalError);
+            }
+            const dataStoreVersionId = data.VersionId;
+            return callback(null, gcpKey, dataStoreVersionId);
+        };
+
+        const params = {
+            Bucket: this._gcpBucketName,
+            Key: gcpKey,
+        };
+        // we call data.put to create a delete marker, but it's actually a
+        // delete request in call to AWS
+        if (keyContext.isDeleteMarker) {
+            return this._client.deleteObject(params, putCb);
+        }
+        const uploadParams = params;
+        uploadParams.Metadata = metaHeaders;
+        uploadParams.ContentLength = size;
+        if (keyContext.tagging) {
+            uploadParams.Tagging = keyContext.tagging;
+        }
+        if (keyContext.contentType !== undefined) {
+            uploadParams.ContentType = keyContext.contentType;
+        }
+        if (keyContext.cacheControl !== undefined) {
+            uploadParams.CacheControl = keyContext.cacheControl;
+        }
+        if (keyContext.contentDisposition !== undefined) {
+            uploadParams.ContentDisposition = keyContext.contentDisposition;
+        }
+        if (keyContext.contentEncoding !== undefined) {
+            uploadParams.ContentEncoding = keyContext.contentEncoding;
+        }
+        if (!stream) {
+            return this._client.putObject(uploadParams, putCb);
+        }
+
+        uploadParams.Body = stream;
+        return this._client.upload(uploadParams, putCb);
+    }
+
     createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
         cacheControl, contentDisposition, contentEncoding, log, callback) {
         const metaHeadersTrimmed = {};

tests/locationConfig/locationConfigTests.json

@@ -158,7 +158,7 @@
         "legacyAwsBehavior": true,
         "details": {
             "gcpEndpoint": "storage.googleapis.com",
-            "bucketName": "zenko-gcp-bucket",
+            "bucketName": "zenko-gcp-bucket-ver",
             "mpuBucketName": "zenko-gcp-mpu",
             "bucketMatch": true,
             "credentialsProfile": "google"
@@ -170,7 +170,7 @@
         "legacyAwsBehavior": true,
         "details": {
             "gcpEndpoint": "storage.googleapis.com",
-            "bucketName": "zenko-gcp-bucket-2",
+            "bucketName": "zenko-gcp-bucket-ver-2",
             "mpuBucketName": "zenko-gcp-mpu-2",
             "bucketMatch": true,
             "credentialsProfile": "google_2"
@@ -182,7 +182,7 @@
         "legacyAwsBehavior": true,
         "details": {
             "gcpEndpoint": "storage.googleapis.com",
-            "bucketName": "zenko-gcp-bucket",
+            "bucketName": "zenko-gcp-bucket-ver",
             "mpuBucketName": "zenko-gcp-mpu",
             "bucketMatch": false,
             "credentialsProfile": "google"

tests/unit/DummyService.js

@@ -56,6 +56,9 @@ class DummyService {
         }
         return callback(null, retObj);
     }
+    deleteObject(params, callback) {
+        return callback();
+    }
     // To-Do: add tests for other methods
 }
 

tests/unit/multipleBackend/ExternalBackendClient.js

@@ -15,6 +15,7 @@ const backendClients = [
             bucketName: 'awsTestBucketName',
             dataStoreName: 'awsDataStore',
             serverSideEncryption: false,
+            bucketMatch: true,
             type: 'aws',
         },
     },
@@ -26,6 +27,7 @@ const backendClients = [
             bucketName: 'gcpTestBucketName',
             mpuBucket: 'gcpTestMpuBucketName',
             dataStoreName: 'gcpDataStore',
+            bucketMatch: true,
             type: 'gcp',
         },
     },
@@ -104,3 +106,30 @@ describe('external backend clients', () => {
         // To-Do: test the other external client methods
     });
 });
+
+describe('Test GCP versioning delete marker', () => {
+    let testClient;
+
+    before(() => {
+        const backend = backendClients[1];
+        testClient = new backend.Class(backend.config);
+        testClient._client = new DummyService(backend.config);
+    });
+
+    it('should return "0" as delete marker versionId', done => {
+        const stream = 'testValue';
+        const size = stream.length;
+        const keyContext = {
+            objectKey: 'testKeyValue',
+            isDeleteMarker: true,
+        };
+        const reqUids = '1234';
+
+        testClient.put(stream, size, keyContext, reqUids,
+        (err, key, versionId) => {
+            assert.strictEqual(key, keyContext.objectKey);
+            assert.strictEqual(versionId, '0');
+            return done();
+        });
+    });
+});