Compare commits

...

3 Commits

Author SHA1 Message Date
Bennett Buchanan 8e9cfb514c FT: Add replication information to object MD 2017-06-12 13:46:38 -07:00
Bennett Buchanan 332c0e141a FT: Add deleteBucketReplication API 2017-06-12 13:44:58 -07:00
Bennett Buchanan aedced5ff5 FT: Add getBucketReplication API 2017-06-12 13:43:35 -07:00
22 changed files with 844 additions and 5 deletions

View File

@ -16,6 +16,8 @@ const bucketPutCors = require('./bucketPutCors');
const bucketPutVersioning = require('./bucketPutVersioning'); const bucketPutVersioning = require('./bucketPutVersioning');
const bucketPutWebsite = require('./bucketPutWebsite'); const bucketPutWebsite = require('./bucketPutWebsite');
const bucketPutReplication = require('./bucketPutReplication'); const bucketPutReplication = require('./bucketPutReplication');
const bucketGetReplication = require('./bucketGetReplication');
const bucketDeleteReplication = require('./bucketDeleteReplication');
const corsPreflight = require('./corsPreflight'); const corsPreflight = require('./corsPreflight');
const completeMultipartUpload = require('./completeMultipartUpload'); const completeMultipartUpload = require('./completeMultipartUpload');
const initiateMultipartUpload = require('./initiateMultipartUpload'); const initiateMultipartUpload = require('./initiateMultipartUpload');
@ -160,6 +162,8 @@ const api = {
bucketPutVersioning, bucketPutVersioning,
bucketPutWebsite, bucketPutWebsite,
bucketPutReplication, bucketPutReplication,
bucketGetReplication,
bucketDeleteReplication,
corsPreflight, corsPreflight,
completeMultipartUpload, completeMultipartUpload,
initiateMultipartUpload, initiateMultipartUpload,

View File

@ -13,4 +13,12 @@ function getReplicationConfiguration(xml, log, cb) {
}); });
} }
module.exports = getReplicationConfiguration; // Get the XML representation of the bucket replication configuration.
function getReplicationConfigurationXML(config) {
return ReplicationConfiguration.getConfigXML(config);
}
module.exports = {
getReplicationConfiguration,
getReplicationConfigurationXML,
};

View File

@ -314,6 +314,37 @@ class ReplicationConfiguration {
return undefined; return undefined;
} }
/**
* Get the XML representation of the configuration object
* @param {object} config - The bucket replication configuration
* @return {string} - The XML representation of the configuration
*/
static getConfigXML(config) {
const { role, destination, rules } = config;
const Role = `<Role>${role}</Role>`;
const Bucket = `<Bucket>${destination}</Bucket>`;
const rulesXML = rules.map(rule => {
const { prefix, enabled, storageClass, id } = rule;
const Prefix = prefix === '' ? '<Prefix/>' :
`<Prefix>${prefix}</Prefix>`;
const Status =
`<Status>${enabled ? 'Enabled' : 'Disabled'}</Status>`;
const StorageClass = storageClass ?
`<StorageClass>${storageClass}</StorageClass>` : '';
const Destination =
`<Destination>${Bucket}${StorageClass}</Destination>`;
// If the ID property was omitted in the configuration object, we
// create an ID for the rule. Hence it is always defined.
const ID = `<ID>${id}</ID>`;
return `<Rule>${ID}${Prefix}${Status}${Destination}</Rule>`;
}).join('');
return '<?xml version="1.0" encoding="UTF-8"?>' +
'<ReplicationConfiguration ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
`${rulesXML}${Role}` +
'</ReplicationConfiguration>';
}
/** /**
* Validate the bucket metadata replication configuration structure and * Validate the bucket metadata replication configuration structure and
* value types * value types

View File

@ -11,6 +11,7 @@ const locationConstraintCheck = require('./locationConstraintCheck');
const { versioningPreprocessing } = require('./versioning'); const { versioningPreprocessing } = require('./versioning');
const removeAWSChunked = require('./removeAWSChunked'); const removeAWSChunked = require('./removeAWSChunked');
const { decodeVersionId } = require('./versioning'); const { decodeVersionId } = require('./versioning');
const { buildReplicationInfoForObject } = require('./replication');
const { config } = require('../../../Config'); const { config } = require('../../../Config');
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
@ -108,6 +109,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
metadataStoreParams.tagging = request.headers['x-amz-tagging']; metadataStoreParams.tagging = request.headers['x-amz-tagging'];
} }
buildReplicationInfoForObject(metadataStoreParams, objectKey, bucketMD,
isDeleteMarker);
const decodedVidResult = decodeVersionId(request.query); const decodedVidResult = decodeVersionId(request.query);
if (decodedVidResult instanceof Error) { if (decodedVidResult instanceof Error) {
log.trace('invalid versionId query', { log.trace('invalid versionId query', {

View File

@ -0,0 +1,57 @@
/**
* Check that a bucket replication rule applies for the given object key. If it
* does, assign replication information to the given object.
* @param {object} obj - The object to set replicationInfo for
* @param {string} objKey - The key of the object
* @param {object} bucketMD - The bucket metadata
* @param {array} content - The content type that should be replicated
* @return {undefined}
*/
function buildReplicationInfo(obj, objKey, bucketMD, content) {
const config = bucketMD.getReplicationConfiguration();
// If bucket does not have a replication configuration, do not replicate.
if (config) {
const rule = config.rules.find(rule => objKey.startsWith(rule.prefix));
if (rule) {
// eslint-disable-next-line no-param-reassign
obj.replicationInfo = {
status: 'PENDING',
content,
destination: config.destination,
storageClass: rule.storageClass || '',
};
}
}
return undefined;
}
/**
* Set the object replicationInfo to replicate data and metadata, or only
* metadata if the object is a delete marker
* @param {object} obj - The object to set replicationInfo for
* @param {string} objKey - The key of the object
* @param {object} bucketMD - The bucket metadata
* @param {boolean} isDeleteMarker - Whether the object is a deleteMarker
* @return {undefined}
*/
function buildReplicationInfoForObject(obj, objKey, bucketMD, isDeleteMarker) {
// Delete markers have no data, so we only replicate metadata.
const content = isDeleteMarker ? ['METADATA'] : ['DATA', 'METADATA'];
return buildReplicationInfo(obj, objKey, bucketMD, content);
}
/**
* Set the object replicationInfo to replicate only metadata
* @param {object} obj - The object to set replicationInfo for
* @param {string} objKey - The key of the object
* @param {object} bucketMD - The bucket metadata
* @return {undefined}
*/
function buildReplicationInfoForObjectMD(obj, objKey, bucketMD) {
return buildReplicationInfo(obj, objKey, bucketMD, ['METADATA']);
}
module.exports = {
buildReplicationInfoForObject,
buildReplicationInfoForObjectMD,
};

View File

@ -0,0 +1,56 @@
const metadata = require('../metadata/wrapper');
const { metadataValidateBucket } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
/**
* bucketDeleteReplication - Delete the bucket replication configuration
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
* @param {object} request - http request object
* @param {object} log - Werelogs logger
* @param {function} callback - callback to server
* @return {undefined}
*/
function bucketDeleteReplication(authInfo, request, log, callback) {
log.debug('processing request', { method: 'bucketDeleteReplication' });
const { bucketName, headers, method } = request;
const metadataValParams = {
authInfo,
bucketName,
requestType: 'bucketOwnerAction',
};
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
if (err) {
log.debug('error processing request', {
error: err,
method: 'bucketDeleteReplication',
});
return callback(err, corsHeaders);
}
if (!bucket.getReplicationConfiguration()) {
log.trace('no existing replication configuration', {
method: 'bucketDeleteReplication',
});
pushMetric('deleteBucketReplication', log, {
authInfo,
bucket: bucketName,
});
return callback(null, corsHeaders);
}
log.trace('deleting replication configuration in metadata');
bucket.setReplicationConfiguration(null);
return metadata.updateBucket(bucketName, bucket, log, err => {
if (err) {
return callback(err, corsHeaders);
}
pushMetric('deleteBucketReplication', log, {
authInfo,
bucket: bucketName,
});
return callback(null, corsHeaders);
});
});
}
module.exports = bucketDeleteReplication;

View File

@ -0,0 +1,52 @@
const { errors } = require('arsenal');
const { metadataValidateBucket } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
const { getReplicationConfigurationXML } =
require('./apiUtils/bucket/getReplicationConfiguration');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
/**
* bucketGetReplication - Get the bucket replication configuration
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
* @param {object} request - http request object
* @param {object} log - Werelogs logger
* @param {function} callback - callback to server
* @return {undefined}
*/
function bucketGetReplication(authInfo, request, log, callback) {
log.debug('processing request', { method: 'bucketGetReplication' });
const { bucketName, headers, method } = request;
const metadataValParams = {
authInfo,
bucketName,
requestType: 'bucketOwnerAction',
};
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
if (err) {
log.debug('error processing request', {
error: err,
method: 'bucketGetReplication',
});
return callback(err, null, corsHeaders);
}
const replicationConfig = bucket.getReplicationConfiguration();
if (!replicationConfig) {
log.debug('error processing request', {
error: errors.ReplicationConfigurationNotFoundError,
method: 'bucketGetReplication',
});
return callback(errors.ReplicationConfigurationNotFoundError, null,
corsHeaders);
}
const xml = getReplicationConfigurationXML(replicationConfig);
pushMetric('getBucketReplication', log, {
authInfo,
bucket: bucketName,
});
return callback(null, xml, corsHeaders);
});
}
module.exports = bucketGetReplication;

View File

@ -4,7 +4,7 @@ const { errors } = require('arsenal');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const { metadataValidateBucket } = require('../metadata/metadataUtils'); const { metadataValidateBucket } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const getReplicationConfiguration = const { getReplicationConfiguration } =
require('./apiUtils/bucket/getReplicationConfiguration'); require('./apiUtils/bucket/getReplicationConfiguration');
const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../utilities/collectCorsHeaders');

View File

@ -8,6 +8,8 @@ const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const { buildReplicationInfoForObjectMD } =
require('./apiUtils/object/replication');
/** /**
* Object Delete Tagging - Delete tag set from an object * Object Delete Tagging - Delete tag set from an object
@ -68,6 +70,7 @@ function objectDeleteTagging(authInfo, request, log, callback) {
objectMD.tags = {}; objectMD.tags = {};
const params = objectMD.versionId ? { versionId: const params = objectMD.versionId ? { versionId:
objectMD.versionId } : {}; objectMD.versionId } : {};
buildReplicationInfoForObjectMD(objectMD, objectKey, bucket);
metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params,
log, err => log, err =>
next(err, bucket, objectMD)); next(err, bucket, objectMD));

View File

@ -6,6 +6,8 @@ const { decodeVersionId, getVersionIdResHeader } =
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const { buildReplicationInfoForObjectMD } =
require('./apiUtils/object/replication');
const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const { parseTagXml } = require('./apiUtils/object/tagging'); const { parseTagXml } = require('./apiUtils/object/tagging');
@ -74,6 +76,7 @@ function objectPutTagging(authInfo, request, log, callback) {
objectMD.tags = tags; objectMD.tags = tags;
const params = objectMD.versionId ? { versionId: const params = objectMD.versionId ? { versionId:
objectMD.versionId } : {}; objectMD.versionId } : {};
buildReplicationInfoForObjectMD(objectMD, objectKey, bucket);
metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params,
log, err => log, err =>
next(err, bucket, objectMD)); next(err, bucket, objectMD));

View File

@ -300,6 +300,14 @@ class BucketInfo {
this._replicationConfiguration = replicationConfiguration; this._replicationConfiguration = replicationConfiguration;
return this; return this;
} }
/**
* Get replication configuration information
* @return {object|null} replication configuration information or `null` if
* the bucket does not have a replication configuration
*/
getReplicationConfiguration() {
return this._replicationConfiguration;
}
/** /**
* Get cors resource * Get cors resource
* @return {object[]} cors * @return {object[]} cors

View File

@ -1,5 +1,7 @@
const { errors } = require('arsenal'); const { errors } = require('arsenal');
const { buildReplicationInfoForObjectMD } =
require('../api/apiUtils/object/replication');
const aclUtils = require('../utilities/aclUtils'); const aclUtils = require('../utilities/aclUtils');
const constants = require('../../constants'); const constants = require('../../constants');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
@ -16,6 +18,7 @@ const acl = {
log.trace('updating object acl in metadata'); log.trace('updating object acl in metadata');
// eslint-disable-next-line no-param-reassign // eslint-disable-next-line no-param-reassign
objectMD.acl = addACLParams; objectMD.acl = addACLParams;
buildReplicationInfoForObjectMD(objectMD, objectKey, bucket);
metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log,
cb); cb);
}, },

View File

@ -53,6 +53,12 @@ module.exports = class ObjectMD {
'isDeleteMarker': '', 'isDeleteMarker': '',
'versionId': undefined, // If no versionId, it should be undefined 'versionId': undefined, // If no versionId, it should be undefined
'tags': {}, 'tags': {},
'replicationInfo': {
status: '',
content: [],
destination: '',
storageClass: '',
},
}; };
} }
@ -570,6 +576,32 @@ module.exports = class ObjectMD {
return this._data.tags; return this._data.tags;
} }
/**
* Set replication information
*
* @param {object} replicationInfo - replication information object
* @return {ObjectMD} itself
*/
setReplicationInfo(replicationInfo) {
const { status, content, destination, storageClass } = replicationInfo;
this._data.replicationInfo = {
status,
content,
destination,
storageClass: storageClass || '',
};
return this;
}
/**
* Get replication information
*
* @return {object} replication object
*/
getReplicationInfo() {
return this._data.replicationInfo;
}
/** /**
* Set custom meta headers * Set custom meta headers
* *

View File

@ -34,6 +34,13 @@ function routeDELETE(request, response, log, statsClient) {
return routesUtils.responseNoBody(err, corsHeaders, return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log); response, 204, log);
}); });
} else if (request.query.replication !== undefined) {
return api.callApiMethod('bucketDeleteReplication', request,
response, log, (err, corsHeaders) => {
statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
} }
api.callApiMethod('bucketDelete', request, response, log, api.callApiMethod('bucketDelete', request, response, log,
(err, corsHeaders) => { (err, corsHeaders) => {

View File

@ -24,6 +24,13 @@ function routerGET(request, response, log, statsClient) {
return routesUtils.responseXMLBody(err, xml, response, log, return routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders); corsHeaders);
}); });
} else if (request.query.replication !== undefined) {
api.callApiMethod('bucketGetReplication', request, response, log,
(err, xml, corsHeaders) => {
statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders);
});
} else if (request.query.cors !== undefined) { } else if (request.query.cors !== undefined) {
api.callApiMethod('bucketGetCors', request, response, log, api.callApiMethod('bucketGetCors', request, response, log,
(err, xml, corsHeaders) => { (err, xml, corsHeaders) => {

View File

@ -92,8 +92,8 @@ const services = {
const { objectKey, authInfo, size, contentMD5, metaHeaders, const { objectKey, authInfo, size, contentMD5, metaHeaders,
contentType, cacheControl, contentDisposition, contentEncoding, contentType, cacheControl, contentDisposition, contentEncoding,
expires, multipart, headers, overrideMetadata, log, expires, multipart, headers, overrideMetadata, log,
lastModifiedDate, versioning, versionId, tagging, taggingCopy } lastModifiedDate, versioning, versionId, tagging, taggingCopy,
= params; replicationInfo } = params;
log.trace('storing object in metadata'); log.trace('storing object in metadata');
assert.strictEqual(typeof bucketName, 'string'); assert.strictEqual(typeof bucketName, 'string');
const md = new ObjectMD(); const md = new ObjectMD();
@ -125,6 +125,9 @@ const services = {
if (headers && headers['x-amz-website-redirect-location']) { if (headers && headers['x-amz-website-redirect-location']) {
md.setRedirectLocation(headers['x-amz-website-redirect-location']); md.setRedirectLocation(headers['x-amz-website-redirect-location']);
} }
if (replicationInfo) {
md.setReplicationInfo(replicationInfo);
}
// options to send to metadata to create or overwrite versions // options to send to metadata to create or overwrite versions
// when putting the object MD // when putting the object MD
const options = {}; const options = {};

View File

@ -0,0 +1,102 @@
const assert = require('assert');
const { S3 } = require('aws-sdk');
const { series } = require('async');
const { errors } = require('arsenal');
const getConfig = require('../support/config');
const BucketUtility = require('../../lib/utility/bucket-util');
const bucket = 'source-bucket';
const replicationConfig = {
Role: 'arn:partition:service::account-id:resourcetype/resource',
Rules: [
{
Destination: { Bucket: 'arn:aws:s3:::destination-bucket' },
Prefix: 'test-prefix',
Status: 'Enabled',
ID: 'test-id',
},
],
};
describe.only('aws-node-sdk test deleteBucketReplication', () => {
let s3;
let otherAccountS3;
const config = getConfig('default', { signatureVersion: 'v4' });
function putVersioningOnBucket(bucket, cb) {
return s3.putBucketVersioning({
Bucket: bucket,
VersioningConfiguration: { Status: 'Enabled' },
}, cb);
}
function putReplicationOnBucket(bucket, cb) {
return s3.putBucketReplication({
Bucket: bucket,
ReplicationConfiguration: replicationConfig,
}, cb);
}
function deleteReplicationAndCheckResponse(bucket, cb) {
return s3.deleteBucketReplication({ Bucket: bucket }, (err, data) => {
assert.strictEqual(err, null);
assert.deepStrictEqual(data, {});
return cb();
});
}
beforeEach(done => {
s3 = new S3(config);
otherAccountS3 = new BucketUtility('lisa', {}).s3;
return s3.createBucket({ Bucket: bucket }, done);
});
afterEach(done => s3.deleteBucket({ Bucket: bucket }, done));
it('should return empty object if bucket is not versioned enabled', done =>
deleteReplicationAndCheckResponse(bucket, done));
it('should return empty object if bucket is version enabled but has no ' +
'replication config', done => series([
next => putVersioningOnBucket(bucket, next),
next => deleteReplicationAndCheckResponse(bucket, next),
], done));
it('should delete a bucket replication config when it has one', done =>
series([
next => putVersioningOnBucket(bucket, next),
next => putReplicationOnBucket(bucket, next),
next => deleteReplicationAndCheckResponse(bucket, next),
], done));
// TODO: Include test when getBucketReplication API is merged.
it.skip('should return ReplicationConfigurationNotFoundError if getting ' +
'replication config after it has been deleted', done =>
series([
next => putVersioningOnBucket(bucket, next),
next => putReplicationOnBucket(bucket, next),
next => s3.getBucketReplication({ Bucket: bucket }, (err, data) => {
if (err) {
return next(err);
}
assert.deepStrictEqual(data, {
ReplicationConfiguration: replicationConfig,
});
return next();
}),
next => deleteReplicationAndCheckResponse(bucket, next),
next => s3.getBucketReplication({ Bucket: bucket }, err => {
assert(errors.ReplicationConfigurationNotFoundError[err.code]);
return next();
}),
], done));
it('should return AccessDenied if user is not bucket owner', done =>
otherAccountS3.deleteBucketReplication({ Bucket: bucket }, err => {
assert(err);
assert.strictEqual(err.code, 'AccessDenied');
assert.strictEqual(err.statusCode, 403);
return done();
}));
});

View File

@ -0,0 +1,78 @@
const assert = require('assert');
const { S3 } = require('aws-sdk');
const { series } = require('async');
const { errors } = require('arsenal');
const getConfig = require('../support/config');
const BucketUtility = require('../../lib/utility/bucket-util');
const bucket = 'source-bucket';
const replicationConfig = {
Role: 'arn:partition:service::account-id:resourcetype/resource',
Rules: [
{
Destination: { Bucket: 'arn:aws:s3:::destination-bucket' },
Prefix: 'test-prefix',
Status: 'Enabled',
ID: 'test-id',
},
],
};
describe('aws-node-sdk test getBucketReplication', () => {
let s3;
let otherAccountS3;
beforeEach(done => {
const config = getConfig('default', { signatureVersion: 'v4' });
s3 = new S3(config);
otherAccountS3 = new BucketUtility('lisa', {}).s3;
return series([
next => s3.createBucket({ Bucket: bucket }, next),
next => s3.putBucketVersioning({
Bucket: bucket,
VersioningConfiguration: {
Status: 'Enabled',
},
}, next),
], done);
});
afterEach(done => s3.deleteBucket({ Bucket: bucket }, done));
it("should return 'ReplicationConfigurationNotFoundError' if bucket does " +
'not have a replication configuration', done =>
s3.getBucketReplication({ Bucket: bucket }, err => {
assert(errors.ReplicationConfigurationNotFoundError[err.code]);
return done();
}));
it('should get the replication configuration that was put on a bucket',
done => s3.putBucketReplication({
Bucket: bucket,
ReplicationConfiguration: replicationConfig,
}, err => {
if (err) {
return done(err);
}
return s3.getBucketReplication({ Bucket: bucket }, (err, data) => {
if (err) {
return done(err);
}
const expectedObj = {
ReplicationConfiguration: replicationConfig,
};
assert.deepStrictEqual(data, expectedObj);
return done();
});
}));
it('should return AccessDenied if user is not bucket owner', done =>
otherAccountS3.getBucketReplication({ Bucket: bucket }, err => {
assert(err);
assert.strictEqual(err.code, 'AccessDenied');
assert.strictEqual(err.statusCode, 403);
return done();
}));
});

View File

@ -0,0 +1,92 @@
const assert = require('assert');
const { parseString } = require('xml2js');
const { DummyRequestLogger } = require('../helpers');
const { getReplicationConfigurationXML } =
require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration');
// Compare the values from the parsedXML with the original configuration values.
function checkXML(parsedXML, config) {
const { Rule, Role } = parsedXML.ReplicationConfiguration;
const { role, destination } = config;
assert.strictEqual(Role[0], role);
assert.strictEqual(Array.isArray(Rule), true);
for (let i = 0; i < Rule.length; i++) {
const { ID, Prefix, Status, Destination } = Rule[i];
const { Bucket, StorageClass } = Destination[0];
const { id, prefix, enabled, storageClass } = config.rules[i];
assert.strictEqual(ID[0], id);
assert.strictEqual(Prefix[0], prefix);
assert.strictEqual(Status[0], enabled ? 'Enabled' : 'Disabled');
if (storageClass !== undefined) {
assert.strictEqual(StorageClass[0], storageClass);
}
assert.strictEqual(Bucket[0], destination);
}
}
// Get the replication XML, parse it, and check that values are as expected.
function getAndCheckXML(bucketReplicationConfig, cb) {
const log = new DummyRequestLogger();
const xml = getReplicationConfigurationXML(bucketReplicationConfig, log);
return parseString(xml, (err, res) => {
if (err) {
return cb(err);
}
checkXML(res, bucketReplicationConfig);
return cb(null, res);
});
}
// Get an example bucket replication configuration.
function getReplicationConfig() {
return {
role: 'arn:partition:service::account-id:resourcetype/resource',
destination: 'destination-bucket',
rules: [
{
id: 'test-id-1',
prefix: 'test-prefix-1',
enabled: true,
storageClass: 'STANDARD',
},
],
};
}
describe("'getReplicationConfigurationXML' function", () => {
it('should return XML from the bucket replication configuration', done =>
getAndCheckXML(getReplicationConfig(), done));
it('should not return XML with StorageClass tag if `storageClass` ' +
'property is omitted', done => {
const config = getReplicationConfig();
delete config.rules[0].storageClass;
return getAndCheckXML(config, done);
});
it("should return XML with StorageClass tag set to 'Disabled' if " +
'`enabled` property is false', done => {
const config = getReplicationConfig();
config.rules[0].enabled = false;
return getAndCheckXML(config, done);
});
it('should return XML with a self-closing Prefix tag if `prefix` ' +
"property is ''", done => {
const config = getReplicationConfig();
config.rules[0].prefix = '';
return getAndCheckXML(config, done);
});
it('should return XML from the bucket replication configuration with ' +
'multiple rules', done => {
const config = getReplicationConfig();
config.rules.push({
id: 'test-id-2',
prefix: 'test-prefix-2',
enabled: true,
});
return getAndCheckXML(config, done);
});
});

View File

@ -1,7 +1,7 @@
const assert = require('assert'); const assert = require('assert');
const { DummyRequestLogger } = require('../helpers'); const { DummyRequestLogger } = require('../helpers');
const getReplicationConfiguration = const { getReplicationConfiguration } =
require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration');
const replicationUtils = const replicationUtils =
require('../../functional/aws-node-sdk/lib/utility/replication'); require('../../functional/aws-node-sdk/lib/utility/replication');

View File

@ -0,0 +1,276 @@
const assert = require('assert');
const async = require('async');
const BucketInfo = require('../../../lib/metadata/BucketInfo');
const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } =
require('../helpers');
const { metadata } = require('../../../lib/metadata/in_memory/metadata');
const DummyRequest = require('../DummyRequest');
const objectDelete = require('../../../lib/api/objectDelete');
const objectPut = require('../../../lib/api/objectPut');
const objectPutACL = require('../../../lib/api/objectPutACL');
const objectPutTagging = require('../../../lib/api/objectPutTagging');
const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging');
const log = new DummyRequestLogger();
const canonicalID = 'accessKey1';
const authInfo = makeAuthInfo(canonicalID);
const ownerID = authInfo.getCanonicalID();
const namespace = 'default';
const bucketName = 'source-bucket';
const bucketARN = `arn:aws:s3:::${bucketName}`;
const storageClassType = 'STANDARD';
const keyA = 'key-A';
const keyB = 'key-B';
const deleteReq = new DummyRequest({
bucketName,
namespace,
objectKey: keyA,
headers: {},
url: `/${bucketName}/${keyA}`,
});
const objectACLReq = {
bucketName,
namespace,
objectKey: keyA,
headers: {
'x-amz-grant-read': `id=${ownerID}`,
'x-amz-grant-read-acp': `id=${ownerID}`,
},
url: `/${bucketName}/${keyA}?acl`,
query: { acl: '' },
};
// Get an object request with the given key.
function getObjectPutReq(key) {
return new DummyRequest({
bucketName,
namespace,
objectKey: key,
headers: {},
url: `/${bucketName}/${key}`,
}, Buffer.from('body content', 'utf8'));
}
const taggingPutReq = new TaggingConfigTester()
.createObjectTaggingRequest('PUT', bucketName, keyA);
const taggingDeleteReq = new TaggingConfigTester()
.createObjectTaggingRequest('DELETE', bucketName, keyA);
const emptyReplicationMD = {
status: '',
content: [],
destination: '',
storageClass: '',
};
// Check that the object key has the expected replication information.
function checkObjectReplicationInfo(key, expected) {
const objectMD = metadata.keyMaps.get(bucketName).get(key);
assert.deepStrictEqual(objectMD.replicationInfo, expected);
}
// Put the object key and check the replication information.
function putObjectAndCheckMD(key, expected, cb) {
return objectPut(authInfo, getObjectPutReq(key), undefined, log, err => {
if (err) {
return cb(err);
}
checkObjectReplicationInfo(key, expected);
return cb();
});
}
// Create the bucket in metadata.
function createBucket() {
metadata
.buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', ''));
metadata.keyMaps.set(bucketName, new Map);
}
// Create the bucket in metadata with versioning and a replication config.
function createBucketWithReplication(hasStorageClass) {
createBucket();
const config = {
role: 'arn:partition:service::account-id:resourcetype/resource',
destination: 'arn:aws:s3:::source-bucket',
rules: [{
prefix: keyA,
enabled: true,
id: 'test-id',
}],
};
if (hasStorageClass) {
config.rules[0].storageClass = storageClassType;
}
Object.assign(metadata.buckets.get(bucketName), {
_versioningConfiguration: { status: 'Enabled' },
_replicationConfiguration: config,
});
}
describe('Replication object MD without bucket replication config', () => {
beforeEach(() => {
cleanup();
createBucket();
});
afterEach(() => cleanup());
it('should not update object metadata', done =>
putObjectAndCheckMD(keyA, emptyReplicationMD, done));
it('should not update object metadata if putting object ACL', done =>
async.series([
next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
next => objectPutACL(authInfo, objectACLReq, log, next),
], err => {
if (err) {
return done(err);
}
checkObjectReplicationInfo(keyA, emptyReplicationMD);
return done();
}));
describe('Object tagging', () => {
beforeEach(done => async.series([
next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
next => objectPutTagging(authInfo, taggingPutReq, log, next),
], err => done(err)));
it('should not update object metadata if putting tag', done => {
checkObjectReplicationInfo(keyA, emptyReplicationMD);
return done();
});
it('should not update object metadata if deleting tag', done =>
async.series([
// Put a new version to update replication MD content array.
next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
next => objectDeleteTagging(authInfo, taggingDeleteReq, log,
next),
], err => {
if (err) {
return done(err);
}
checkObjectReplicationInfo(keyA, emptyReplicationMD);
return done();
}));
});
});
[true, false].forEach(hasStorageClass => {
describe('Replication object MD with bucket replication config ' +
`${hasStorageClass ? 'with' : 'without'} storage class`, () => {
const replicationMD = {
status: 'PENDING',
content: ['DATA', 'METADATA'],
destination: bucketARN,
storageClass: '',
};
const newReplicationMD = hasStorageClass ? Object.assign(replicationMD,
{ storageClass: storageClassType }) : replicationMD;
const replicateMetadataOnly = Object.assign({}, newReplicationMD,
{ content: ['METADATA'] });
beforeEach(() => {
cleanup();
createBucketWithReplication(hasStorageClass);
});
afterEach(() => cleanup());
it('should update metadata when replication config prefix matches ' +
'an object key', done =>
putObjectAndCheckMD(keyA, newReplicationMD, done));
it('should update metadata when replication config prefix matches ' +
'the start of an object key', done =>
putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done));
it('should not update metadata when replication config prefix does ' +
'not match the start of an object key', done =>
putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done));
it('should not update metadata when replication config prefix does ' +
'not apply', done =>
putObjectAndCheckMD(keyB, emptyReplicationMD, done));
it("should update status to 'PENDING' if putting a new version", done =>
putObjectAndCheckMD(keyA, newReplicationMD, err => {
if (err) {
return done(err);
}
const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
// Update metadata to a status after replication has occurred.
objectMD.replicationInfo.status = 'COMPLETED';
return putObjectAndCheckMD(keyA, newReplicationMD, done);
}));
it("should update status to 'PENDING' and content to '['METADATA']' " +
'if putting object ACL', done =>
async.series([
next => putObjectAndCheckMD(keyA, newReplicationMD, next),
next => objectPutACL(authInfo, objectACLReq, log, next),
], err => {
if (err) {
return done(err);
}
checkObjectReplicationInfo(keyA, replicateMetadataOnly);
return done();
}));
it('should update metadata if putting a delete marker', done =>
async.series([
next => putObjectAndCheckMD(keyA, newReplicationMD, err => {
if (err) {
return next(err);
}
const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
// Set metadata to a status after replication has occurred.
objectMD.replicationInfo.status = 'COMPLETED';
return next();
}),
next => objectDelete(authInfo, deleteReq, log, next),
], err => {
if (err) {
return done(err);
}
const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
assert.strictEqual(objectMD.isDeleteMarker, true);
checkObjectReplicationInfo(keyA, replicateMetadataOnly);
return done();
}));
describe('Object tagging', () => {
beforeEach(done => async.series([
next => putObjectAndCheckMD(keyA, newReplicationMD, next),
next => objectPutTagging(authInfo, taggingPutReq, log, next),
], err => done(err)));
it("should update status to 'PENDING' and content to " +
"'['METADATA']'if putting tag", done => {
checkObjectReplicationInfo(keyA, replicateMetadataOnly);
return done();
});
it("should update status to 'PENDING' and content to " +
"'['METADATA']' if deleting tag", done =>
async.series([
// Put a new version to update replication MD content array.
next => putObjectAndCheckMD(keyA, newReplicationMD, next),
next => objectDeleteTagging(authInfo, taggingDeleteReq, log,
next),
], err => {
if (err) {
return done(err);
}
checkObjectReplicationInfo(keyA, replicateMetadataOnly);
return done();
}));
});
});
});

View File

@ -72,6 +72,19 @@ describe('ObjectMD class setters/getters', () => {
['Tags', { ['Tags', {
key: 'value', key: 'value',
}], }],
['Tags', null, {}],
['ReplicationInfo', null, {
status: '',
content: [],
destination: '',
storageClass: '',
}],
['ReplicationInfo', {
status: 'PENDING',
content: ['DATA', 'METADATA'],
destination: 'destination-bucket',
storageClass: 'STANDARD',
}],
].forEach(test => { ].forEach(test => {
const property = test[0]; const property = test[0];
const testValue = test[1]; const testValue = test[1];