Compare commits
1 Commits
developmen
...
feature/S3
Author | SHA1 | Date |
---|---|---|
Alexander Chan | fc7d77be2d |
|
@ -127,7 +127,7 @@ const constants = {
|
||||||
// for external backends, don't call unless at least 1 minute
|
// for external backends, don't call unless at least 1 minute
|
||||||
// (60,000 milliseconds) since last call
|
// (60,000 milliseconds) since last call
|
||||||
externalBackendHealthCheckInterval: 60000,
|
externalBackendHealthCheckInterval: 60000,
|
||||||
versioningNotImplBackends: { azure: true, gcp: true },
|
versioningNotImplBackends: { azure: true },
|
||||||
mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
|
mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
|
||||||
skipBatchDeleteBackends: { azure: true, gcp: true },
|
skipBatchDeleteBackends: { azure: true, gcp: true },
|
||||||
s3HandledBackends: { azure: true, gcp: true },
|
s3HandledBackends: { azure: true, gcp: true },
|
||||||
|
|
|
@ -6,7 +6,8 @@ const { GCP, GcpUtils } = require('./GCP');
|
||||||
const { createMpuKey } = GcpUtils;
|
const { createMpuKey } = GcpUtils;
|
||||||
const AwsClient = require('./AwsClient');
|
const AwsClient = require('./AwsClient');
|
||||||
const { prepareStream } = require('../../api/apiUtils/object/prepareStream');
|
const { prepareStream } = require('../../api/apiUtils/object/prepareStream');
|
||||||
const { logHelper, removeQuotes } = require('./utils');
|
const createLogger = require('../multipleBackendLogger');
|
||||||
|
const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils');
|
||||||
const { config } = require('../../Config');
|
const { config } = require('../../Config');
|
||||||
|
|
||||||
const missingVerIdInternalError = errors.InternalError.customizeDescription(
|
const missingVerIdInternalError = errors.InternalError.customizeDescription(
|
||||||
|
@ -80,9 +81,34 @@ class GcpClient extends AwsClient {
|
||||||
return cb(null, bucketResp);
|
return cb(null, bucketResp);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const checkBucketVersioning = (headResp, cb) => {
|
||||||
|
if (headResp.err) {
|
||||||
|
return cb(null, headResp);
|
||||||
|
}
|
||||||
|
let bucketResp;
|
||||||
|
return this._client.getBucketVersioning(
|
||||||
|
{ Bucket: headResp.gcpBucket }, (err, data) => {
|
||||||
|
bucketResp = { gcpBucket: headResp.gcpBucket };
|
||||||
|
if (err) {
|
||||||
|
bucketResp.error = err;
|
||||||
|
} else if (!data.Status || data.Status === 'Suspended') {
|
||||||
|
bucketResp.error = 'Versioning must be enabled';
|
||||||
|
bucketResp.versioningStatus = data.Status;
|
||||||
|
} else {
|
||||||
|
bucketResp.versioningStatus = data.Status;
|
||||||
|
bucketResp.message = headResp.message;
|
||||||
|
}
|
||||||
|
return cb(null, bucketResp);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const gcpResp = {};
|
const gcpResp = {};
|
||||||
async.parallel({
|
async.parallel({
|
||||||
main: done => checkBucketHealth(this._gcpBucketName, done),
|
main: done => async.waterfall([
|
||||||
|
next => checkBucketHealth(this._gcpBucketName, next),
|
||||||
|
(headResp, next) => checkBucketVersioning(headResp, next),
|
||||||
|
], (err, result) => done(null, result)),
|
||||||
mpu: done => checkBucketHealth(this._mpuBucketName, done),
|
mpu: done => checkBucketHealth(this._mpuBucketName, done),
|
||||||
}, (err, result) => {
|
}, (err, result) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
|
@ -97,6 +123,70 @@ class GcpClient extends AwsClient {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
put(stream, size, keyContext, reqUids, callback) {
|
||||||
|
const gcpKey = this._createGcpKey(keyContext.bucketName,
|
||||||
|
keyContext.objectKey, this._bucketMatch);
|
||||||
|
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
|
||||||
|
const log = createLogger(reqUids);
|
||||||
|
|
||||||
|
const putCb = (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
logHelper(log, 'error', 'err from data backend',
|
||||||
|
err, this._dataStoreName, this.clientType);
|
||||||
|
return callback(errors.ServiceUnavailable
|
||||||
|
.customizeDescription('Error returned from ' +
|
||||||
|
`${this.type}: ${err.message}`)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (keyContext.isDeleteMarker) {
|
||||||
|
log.info('GCP delete marker: returning "0" as versiond id');
|
||||||
|
return callback(null, gcpKey, '0');
|
||||||
|
}
|
||||||
|
if (!data.VersionId) {
|
||||||
|
logHelper(log, 'error', 'missing version id for data ' +
|
||||||
|
'backend object', missingVerIdInternalError,
|
||||||
|
this._dataStoreName, this.clientType);
|
||||||
|
return callback(missingVerIdInternalError);
|
||||||
|
}
|
||||||
|
const dataStoreVersionId = data.VersionId;
|
||||||
|
return callback(null, gcpKey, dataStoreVersionId);
|
||||||
|
};
|
||||||
|
|
||||||
|
const params = {
|
||||||
|
Bucket: this._gcpBucketName,
|
||||||
|
Key: gcpKey,
|
||||||
|
};
|
||||||
|
// we call data.put to create a delete marker, but it's actually a
|
||||||
|
// delete request in call to AWS
|
||||||
|
if (keyContext.isDeleteMarker) {
|
||||||
|
return this._client.deleteObject(params, putCb);
|
||||||
|
}
|
||||||
|
const uploadParams = params;
|
||||||
|
uploadParams.Metadata = metaHeaders;
|
||||||
|
uploadParams.ContentLength = size;
|
||||||
|
if (keyContext.tagging) {
|
||||||
|
uploadParams.Tagging = keyContext.tagging;
|
||||||
|
}
|
||||||
|
if (keyContext.contentType !== undefined) {
|
||||||
|
uploadParams.ContentType = keyContext.contentType;
|
||||||
|
}
|
||||||
|
if (keyContext.cacheControl !== undefined) {
|
||||||
|
uploadParams.CacheControl = keyContext.cacheControl;
|
||||||
|
}
|
||||||
|
if (keyContext.contentDisposition !== undefined) {
|
||||||
|
uploadParams.ContentDisposition = keyContext.contentDisposition;
|
||||||
|
}
|
||||||
|
if (keyContext.contentEncoding !== undefined) {
|
||||||
|
uploadParams.ContentEncoding = keyContext.contentEncoding;
|
||||||
|
}
|
||||||
|
if (!stream) {
|
||||||
|
return this._client.putObject(uploadParams, putCb);
|
||||||
|
}
|
||||||
|
|
||||||
|
uploadParams.Body = stream;
|
||||||
|
return this._client.upload(uploadParams, putCb);
|
||||||
|
}
|
||||||
|
|
||||||
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
|
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
|
||||||
cacheControl, contentDisposition, contentEncoding, log, callback) {
|
cacheControl, contentDisposition, contentEncoding, log, callback) {
|
||||||
const metaHeadersTrimmed = {};
|
const metaHeadersTrimmed = {};
|
||||||
|
|
|
@ -158,7 +158,7 @@
|
||||||
"legacyAwsBehavior": true,
|
"legacyAwsBehavior": true,
|
||||||
"details": {
|
"details": {
|
||||||
"gcpEndpoint": "storage.googleapis.com",
|
"gcpEndpoint": "storage.googleapis.com",
|
||||||
"bucketName": "zenko-gcp-bucket",
|
"bucketName": "zenko-gcp-bucket-ver",
|
||||||
"mpuBucketName": "zenko-gcp-mpu",
|
"mpuBucketName": "zenko-gcp-mpu",
|
||||||
"bucketMatch": true,
|
"bucketMatch": true,
|
||||||
"credentialsProfile": "google"
|
"credentialsProfile": "google"
|
||||||
|
@ -170,7 +170,7 @@
|
||||||
"legacyAwsBehavior": true,
|
"legacyAwsBehavior": true,
|
||||||
"details": {
|
"details": {
|
||||||
"gcpEndpoint": "storage.googleapis.com",
|
"gcpEndpoint": "storage.googleapis.com",
|
||||||
"bucketName": "zenko-gcp-bucket-2",
|
"bucketName": "zenko-gcp-bucket-ver-2",
|
||||||
"mpuBucketName": "zenko-gcp-mpu-2",
|
"mpuBucketName": "zenko-gcp-mpu-2",
|
||||||
"bucketMatch": true,
|
"bucketMatch": true,
|
||||||
"credentialsProfile": "google_2"
|
"credentialsProfile": "google_2"
|
||||||
|
@ -182,7 +182,7 @@
|
||||||
"legacyAwsBehavior": true,
|
"legacyAwsBehavior": true,
|
||||||
"details": {
|
"details": {
|
||||||
"gcpEndpoint": "storage.googleapis.com",
|
"gcpEndpoint": "storage.googleapis.com",
|
||||||
"bucketName": "zenko-gcp-bucket",
|
"bucketName": "zenko-gcp-bucket-ver",
|
||||||
"mpuBucketName": "zenko-gcp-mpu",
|
"mpuBucketName": "zenko-gcp-mpu",
|
||||||
"bucketMatch": false,
|
"bucketMatch": false,
|
||||||
"credentialsProfile": "google"
|
"credentialsProfile": "google"
|
||||||
|
|
|
@ -56,6 +56,9 @@ class DummyService {
|
||||||
}
|
}
|
||||||
return callback(null, retObj);
|
return callback(null, retObj);
|
||||||
}
|
}
|
||||||
|
deleteObject(params, callback) {
|
||||||
|
return callback();
|
||||||
|
}
|
||||||
// To-Do: add tests for other methods
|
// To-Do: add tests for other methods
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,6 +15,7 @@ const backendClients = [
|
||||||
bucketName: 'awsTestBucketName',
|
bucketName: 'awsTestBucketName',
|
||||||
dataStoreName: 'awsDataStore',
|
dataStoreName: 'awsDataStore',
|
||||||
serverSideEncryption: false,
|
serverSideEncryption: false,
|
||||||
|
bucketMatch: true,
|
||||||
type: 'aws',
|
type: 'aws',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -26,6 +27,7 @@ const backendClients = [
|
||||||
bucketName: 'gcpTestBucketName',
|
bucketName: 'gcpTestBucketName',
|
||||||
mpuBucket: 'gcpTestMpuBucketName',
|
mpuBucket: 'gcpTestMpuBucketName',
|
||||||
dataStoreName: 'gcpDataStore',
|
dataStoreName: 'gcpDataStore',
|
||||||
|
bucketMatch: true,
|
||||||
type: 'gcp',
|
type: 'gcp',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -104,3 +106,30 @@ describe('external backend clients', () => {
|
||||||
// To-Do: test the other external client methods
|
// To-Do: test the other external client methods
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Test GCP versioning delete marker', () => {
|
||||||
|
let testClient;
|
||||||
|
|
||||||
|
before(() => {
|
||||||
|
const backend = backendClients[1];
|
||||||
|
testClient = new backend.Class(backend.config);
|
||||||
|
testClient._client = new DummyService(backend.config);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return "0" as delete marker versionId', done => {
|
||||||
|
const stream = 'testValue';
|
||||||
|
const size = stream.length;
|
||||||
|
const keyContext = {
|
||||||
|
objectKey: 'testKeyValue',
|
||||||
|
isDeleteMarker: true,
|
||||||
|
};
|
||||||
|
const reqUids = '1234';
|
||||||
|
|
||||||
|
testClient.put(stream, size, keyContext, reqUids,
|
||||||
|
(err, key, versionId) => {
|
||||||
|
assert.strictEqual(key, keyContext.objectKey);
|
||||||
|
assert.strictEqual(versionId, '0');
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
Loading…
Reference in New Issue