Compare commits

...

1 Commits

Author SHA1 Message Date
Alexander Chan fc7d77be2d feature: S3C-1182 add versioning support for GCP backend
Supports versioning with GCP backend
2019-02-05 22:09:47 -08:00
5 changed files with 128 additions and 6 deletions

View File

@ -127,7 +127,7 @@ const constants = {
// for external backends, don't call unless at least 1 minute // for external backends, don't call unless at least 1 minute
// (60,000 milliseconds) since last call // (60,000 milliseconds) since last call
externalBackendHealthCheckInterval: 60000, externalBackendHealthCheckInterval: 60000,
versioningNotImplBackends: { azure: true, gcp: true }, versioningNotImplBackends: { azure: true },
mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true }, mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
skipBatchDeleteBackends: { azure: true, gcp: true }, skipBatchDeleteBackends: { azure: true, gcp: true },
s3HandledBackends: { azure: true, gcp: true }, s3HandledBackends: { azure: true, gcp: true },

View File

@ -6,7 +6,8 @@ const { GCP, GcpUtils } = require('./GCP');
const { createMpuKey } = GcpUtils; const { createMpuKey } = GcpUtils;
const AwsClient = require('./AwsClient'); const AwsClient = require('./AwsClient');
const { prepareStream } = require('../../api/apiUtils/object/prepareStream'); const { prepareStream } = require('../../api/apiUtils/object/prepareStream');
const { logHelper, removeQuotes } = require('./utils'); const createLogger = require('../multipleBackendLogger');
const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils');
const { config } = require('../../Config'); const { config } = require('../../Config');
const missingVerIdInternalError = errors.InternalError.customizeDescription( const missingVerIdInternalError = errors.InternalError.customizeDescription(
@ -80,9 +81,34 @@ class GcpClient extends AwsClient {
return cb(null, bucketResp); return cb(null, bucketResp);
}); });
}; };
const checkBucketVersioning = (headResp, cb) => {
if (headResp.err) {
return cb(null, headResp);
}
let bucketResp;
return this._client.getBucketVersioning(
{ Bucket: headResp.gcpBucket }, (err, data) => {
bucketResp = { gcpBucket: headResp.gcpBucket };
if (err) {
bucketResp.error = err;
} else if (!data.Status || data.Status === 'Suspended') {
bucketResp.error = 'Versioning must be enabled';
bucketResp.versioningStatus = data.Status;
} else {
bucketResp.versioningStatus = data.Status;
bucketResp.message = headResp.message;
}
return cb(null, bucketResp);
});
};
const gcpResp = {}; const gcpResp = {};
async.parallel({ async.parallel({
main: done => checkBucketHealth(this._gcpBucketName, done), main: done => async.waterfall([
next => checkBucketHealth(this._gcpBucketName, next),
(headResp, next) => checkBucketVersioning(headResp, next),
], (err, result) => done(null, result)),
mpu: done => checkBucketHealth(this._mpuBucketName, done), mpu: done => checkBucketHealth(this._mpuBucketName, done),
}, (err, result) => { }, (err, result) => {
if (err) { if (err) {
@ -97,6 +123,70 @@ class GcpClient extends AwsClient {
}); });
} }
put(stream, size, keyContext, reqUids, callback) {
const gcpKey = this._createGcpKey(keyContext.bucketName,
keyContext.objectKey, this._bucketMatch);
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
const log = createLogger(reqUids);
const putCb = (err, data) => {
if (err) {
logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
if (keyContext.isDeleteMarker) {
log.info('GCP delete marker: returning "0" as versiond id');
return callback(null, gcpKey, '0');
}
if (!data.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
const dataStoreVersionId = data.VersionId;
return callback(null, gcpKey, dataStoreVersionId);
};
const params = {
Bucket: this._gcpBucketName,
Key: gcpKey,
};
// we call data.put to create a delete marker, but it's actually a
// delete request in call to AWS
if (keyContext.isDeleteMarker) {
return this._client.deleteObject(params, putCb);
}
const uploadParams = params;
uploadParams.Metadata = metaHeaders;
uploadParams.ContentLength = size;
if (keyContext.tagging) {
uploadParams.Tagging = keyContext.tagging;
}
if (keyContext.contentType !== undefined) {
uploadParams.ContentType = keyContext.contentType;
}
if (keyContext.cacheControl !== undefined) {
uploadParams.CacheControl = keyContext.cacheControl;
}
if (keyContext.contentDisposition !== undefined) {
uploadParams.ContentDisposition = keyContext.contentDisposition;
}
if (keyContext.contentEncoding !== undefined) {
uploadParams.ContentEncoding = keyContext.contentEncoding;
}
if (!stream) {
return this._client.putObject(uploadParams, putCb);
}
uploadParams.Body = stream;
return this._client.upload(uploadParams, putCb);
}
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType, createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
cacheControl, contentDisposition, contentEncoding, log, callback) { cacheControl, contentDisposition, contentEncoding, log, callback) {
const metaHeadersTrimmed = {}; const metaHeadersTrimmed = {};

View File

@ -158,7 +158,7 @@
"legacyAwsBehavior": true, "legacyAwsBehavior": true,
"details": { "details": {
"gcpEndpoint": "storage.googleapis.com", "gcpEndpoint": "storage.googleapis.com",
"bucketName": "zenko-gcp-bucket", "bucketName": "zenko-gcp-bucket-ver",
"mpuBucketName": "zenko-gcp-mpu", "mpuBucketName": "zenko-gcp-mpu",
"bucketMatch": true, "bucketMatch": true,
"credentialsProfile": "google" "credentialsProfile": "google"
@ -170,7 +170,7 @@
"legacyAwsBehavior": true, "legacyAwsBehavior": true,
"details": { "details": {
"gcpEndpoint": "storage.googleapis.com", "gcpEndpoint": "storage.googleapis.com",
"bucketName": "zenko-gcp-bucket-2", "bucketName": "zenko-gcp-bucket-ver-2",
"mpuBucketName": "zenko-gcp-mpu-2", "mpuBucketName": "zenko-gcp-mpu-2",
"bucketMatch": true, "bucketMatch": true,
"credentialsProfile": "google_2" "credentialsProfile": "google_2"
@ -182,7 +182,7 @@
"legacyAwsBehavior": true, "legacyAwsBehavior": true,
"details": { "details": {
"gcpEndpoint": "storage.googleapis.com", "gcpEndpoint": "storage.googleapis.com",
"bucketName": "zenko-gcp-bucket", "bucketName": "zenko-gcp-bucket-ver",
"mpuBucketName": "zenko-gcp-mpu", "mpuBucketName": "zenko-gcp-mpu",
"bucketMatch": false, "bucketMatch": false,
"credentialsProfile": "google" "credentialsProfile": "google"

View File

@ -56,6 +56,9 @@ class DummyService {
} }
return callback(null, retObj); return callback(null, retObj);
} }
deleteObject(params, callback) {
return callback();
}
// To-Do: add tests for other methods // To-Do: add tests for other methods
} }

View File

@ -15,6 +15,7 @@ const backendClients = [
bucketName: 'awsTestBucketName', bucketName: 'awsTestBucketName',
dataStoreName: 'awsDataStore', dataStoreName: 'awsDataStore',
serverSideEncryption: false, serverSideEncryption: false,
bucketMatch: true,
type: 'aws', type: 'aws',
}, },
}, },
@ -26,6 +27,7 @@ const backendClients = [
bucketName: 'gcpTestBucketName', bucketName: 'gcpTestBucketName',
mpuBucket: 'gcpTestMpuBucketName', mpuBucket: 'gcpTestMpuBucketName',
dataStoreName: 'gcpDataStore', dataStoreName: 'gcpDataStore',
bucketMatch: true,
type: 'gcp', type: 'gcp',
}, },
}, },
@ -104,3 +106,30 @@ describe('external backend clients', () => {
// To-Do: test the other external client methods // To-Do: test the other external client methods
}); });
}); });
describe('Test GCP versioning delete marker', () => {
let testClient;
before(() => {
const backend = backendClients[1];
testClient = new backend.Class(backend.config);
testClient._client = new DummyService(backend.config);
});
it('should return "0" as delete marker versionId', done => {
const stream = 'testValue';
const size = stream.length;
const keyContext = {
objectKey: 'testKeyValue',
isDeleteMarker: true,
};
const reqUids = '1234';
testClient.put(stream, size, keyContext, reqUids,
(err, key, versionId) => {
assert.strictEqual(key, keyContext.objectKey);
assert.strictEqual(versionId, '0');
return done();
});
});
});