Compare commits

...

1 Commits

Author SHA1 Message Date
Ilke c3ba866caa ci test 2020-06-14 18:54:51 -07:00
7 changed files with 512 additions and 9 deletions

View File

@ -148,6 +148,12 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
removeAWSChunked(request.headers['content-encoding']);
metadataStoreParams.expires = request.headers.expires;
metadataStoreParams.tagging = request.headers['x-amz-tagging'];
const defaultObjectLockConfiguration
= bucketMD.getObjectLockConfiguration();
if (defaultObjectLockConfiguration) {
metadataStoreParams.defaultRetention
= defaultObjectLockConfiguration;
}
}
// if creating new delete marker and there is an existing object, copy

View File

@ -0,0 +1,97 @@
const { errors } = require('arsenal');
const moment = require('moment');
/**
* Calculates retain until date for the locked object version
* @param {object} retention - includes days or years retention period
* @return {object} the date until the object version remains locked
*/
function calculateRetainUntilDate(retention) {
const { days, years } = retention;
const date = moment();
// Calculate the number of days to retain the lock on the object
const retainUntilDays = days || years * 365;
const retainUntilDate
= date.add(retainUntilDays, 'Days');
return retainUntilDate.toISOString();
}
/**
* Validates object lock headers
* @param {object} bucket - bucket metadata
* @param {object} headers - request headers
* @param {object} log - the log request
* @return {object} - object with error if validation fails
*/
function validateHeaders(bucket, headers, log) {
const bucketObjectLockEnabled = bucket.isObjectLockEnabled();
const objectLegalHold = headers['x-amz-object-lock-legal-hold'];
const objectLockDate = headers['x-amz-object-lock-retain-until-date'];
const objectLockMode = headers['x-amz-object-lock-mode'];
// If retention headers or legal hold header present but
// object lock is not enabled on the bucket return error
if ((objectLockDate || objectLockMode || objectLegalHold)
&& !bucketObjectLockEnabled) {
log.trace('bucket is missing ObjectLockConfiguration');
return errors.InvalidRequest.customizeDescription(
'Bucket is missing ObjectLockConfiguration');
}
if ((objectLockMode || objectLockDate) &&
!(objectLockMode && objectLockDate)) {
return errors.InvalidArgument.customizeDescription(
'x-amz-object-lock-retain-until-date and ' +
'x-amz-object-lock-mode must both be supplied'
);
}
const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']);
if (objectLockMode && !validModes.has(objectLockMode)) {
return errors.InvalidArgument.customizeDescription(
'Unknown wormMode directive');
}
const validLegalHolds = new Set(['ON', 'OFF']);
if (objectLegalHold && !validLegalHolds.has(objectLegalHold)) {
return errors.InvalidArgument.customizeDescription(
'Legal hold status must be one of "ON", "OFF"');
}
const currentDate = new Date().toISOString();
if (objectLockMode && objectLockDate <= currentDate) {
return errors.InvalidArgument.customizeDescription(
'The retain until date must be in the future!');
}
return null;
}
/**
* Sets object retention ond/or legal hold information on object's metadata
* @param {object} headers - request headers
* @param {object} md - object metadata
* @param {(object|null)} defaultRetention - bucket retention configuration if
* bucket has any configuration set
* @return {undefined}
*/
function setObjectLockInformation(headers, md, defaultRetention) {
// Stores retention information if object either has its own retention
// configuration or default retention configuration from its bucket
const headerMode = headers['x-amz-object-lock-mode'];
const headerDate = headers['x-amz-object-lock-retain-until-date'];
const objectRetention = headers && headerMode && headerDate;
if (objectRetention || defaultRetention) {
const mode = headerMode || defaultRetention.rule.mode;
const date = headerDate
|| calculateRetainUntilDate(defaultRetention.rule);
const retention = {
mode,
retainUntilDate: date,
};
md.setRetentionInfo(retention);
}
const headerLegalHold = headers['x-amz-object-lock-legal-hold'];
if (headers && headerLegalHold) {
const legalHold = headerLegalHold === 'ON';
md.setLegalHold(legalHold);
}
}
module.exports = {
calculateRetainUntilDate,
setObjectLockInformation,
validateHeaders,
};

View File

@ -8,6 +8,7 @@ const createAndStoreObject = require('./apiUtils/object/createAndStoreObject');
const { checkQueryVersionId } = require('./apiUtils/object/versioning');
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
const { validateHeaders } = require('./apiUtils/object/objectLockHelpers');
const kms = require('../kms/wrapper');
const checkObjectEncryption = require('./apiUtils/object/checkEncryption');
@ -34,18 +35,24 @@ const versionIdUtils = versioning.VersionID;
*/
function objectPut(authInfo, request, streamingV4Params, log, callback) {
log.debug('processing request', { method: 'objectPut' });
if (!aclUtils.checkGrantHeaderValidity(request.headers)) {
const {
bucketName,
headers,
method,
objectKey,
parsedContentLength,
query,
} = request;
if (!aclUtils.checkGrantHeaderValidity(headers)) {
log.trace('invalid acl header');
return callback(errors.InvalidArgument);
}
const queryContainsVersionId = checkQueryVersionId(request.query);
const queryContainsVersionId = checkQueryVersionId(query);
if (queryContainsVersionId instanceof Error) {
return callback(queryContainsVersionId);
}
const invalidSSEError = errors.InvalidArgument.customizeDescription(
'The encryption method specified is not supported');
const bucketName = request.bucketName;
const objectKey = request.objectKey;
const requestType = 'objectPut';
const valParams = { authInfo, bucketName, objectKey, requestType };
const canonicalID = authInfo.getCanonicalID();
@ -53,8 +60,8 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
return metadataValidateBucketAndObj(valParams, log,
(err, bucket, objMD) => {
const responseHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket);
const responseHeaders = collectCorsHeaders(headers.origin,
method, bucket);
if (err) {
log.trace('error processing request', {
error: err,
@ -84,6 +91,14 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
return kms.createCipherBundle(
serverSideEncryption, log, next);
}
return next(null);
},
function validateObjectLockConditions(next) {
const objectLockValidationError
= validateHeaders(bucket, headers, log);
if (objectLockValidationError) {
return next(objectLockValidationError);
}
return next(null, null);
},
function objectCreateAndStore(cipherBundle, next) {
@ -95,7 +110,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
if (err) {
return callback(err, responseHeaders);
}
const newByteLength = request.parsedContentLength;
const newByteLength = parsedContentLength;
// Utapi expects null or a number for oldByteLength:
// * null - new object

View File

@ -10,6 +10,8 @@ const constants = require('../constants');
const data = require('./data/wrapper');
const metadata = require('./metadata/wrapper');
const logger = require('./utilities/logger');
const { setObjectLockInformation }
= require('./api/apiUtils/object/objectLockHelpers');
const removeAWSChunked = require('./api/apiUtils/object/removeAWSChunked');
const { parseTagFromQuery } = s3middleware.tagging;
const { config } = require('./Config');
@ -95,7 +97,7 @@ const services = {
contentType, cacheControl, contentDisposition, contentEncoding,
expires, multipart, headers, overrideMetadata, log,
lastModifiedDate, versioning, versionId, tagging, taggingCopy,
replicationInfo, dataStoreName } = params;
replicationInfo, defaultRetention, dataStoreName } = params;
log.trace('storing object in metadata');
assert.strictEqual(typeof bucketName, 'string');
const md = new ObjectMD();
@ -128,6 +130,18 @@ const services = {
if (headers && headers['x-amz-website-redirect-location']) {
md.setRedirectLocation(headers['x-amz-website-redirect-location']);
}
if (headers) {
// Stores retention information if object either has its own retention
// configuration or default retention configuration from its bucket
const headerMode = headers['x-amz-object-lock-mode'];
const headerDate = headers['x-amz-object-lock-retain-until-date'];
const headerLegalHold = headers['x-amz-object-lock-legal-hold'];
const objectRetention = headers && headerMode && headerDate;
const objectLegalHold = headers && headerLegalHold;
if (objectRetention || defaultRetention || objectLegalHold) {
setObjectLockInformation(headers, md, defaultRetention);
}
}
if (replicationInfo) {
md.setReplicationInfo(replicationInfo);
}

View File

@ -1,8 +1,10 @@
const assert = require('assert');
const async = require('async');
const moment = require('moment');
const { errors, s3middleware } = require('arsenal');
const { bucketPut } = require('../../../lib/api/bucketPut');
const bucketPutObjectLock = require('../../../lib/api/bucketPutObjectLock');
const bucketPutACL = require('../../../lib/api/bucketPutACL');
const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning');
const { parseTagFromQuery } = s3middleware.tagging;
@ -11,6 +13,7 @@ const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils }
const { ds } = require('../../../lib/data/in_memory/backend');
const metadata = require('../metadataswitch');
const objectPut = require('../../../lib/api/objectPut');
const { objectLockTestUtils } = require('../helpers');
const DummyRequest = require('../DummyRequest');
const log = new DummyRequestLogger();
@ -20,13 +23,22 @@ const namespace = 'default';
const bucketName = 'bucketname';
const postBody = Buffer.from('I am a body', 'utf8');
const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
const mockDate = new Date(2050, 10, 12);
const testPutBucketRequest = new DummyRequest({
bucketName,
namespace,
headers: { host: `${bucketName}.s3.amazonaws.com` },
url: '/',
});
const testPutBucketRequestLock = new DummyRequest({
bucketName,
namespace,
headers: {
'host': `${bucketName}.s3.amazonaws.com`,
'x-amz-bucket-object-lock-enabled': true,
},
url: '/',
});
const objectName = 'objectName';
let testPutObjectRequest;
@ -162,6 +174,154 @@ describe('objectPut API', () => {
});
});
const mockModes = ['GOVERNANCE', 'COMPLIANCE'];
mockModes.forEach(mockMode => {
it(`should put an object with valid date & ${mockMode} mode`, done => {
const testPutObjectRequest = new DummyRequest({
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-object-lock-retain-until-date': mockDate,
'x-amz-object-lock-mode': mockMode,
},
url: `/${bucketName}/${objectName}`,
calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==',
}, postBody);
bucketPut(authInfo, testPutBucketRequestLock, log, () => {
objectPut(authInfo, testPutObjectRequest, undefined, log,
(err, headers) => {
assert.ifError(err);
assert.strictEqual(headers.ETag, `"${correctMD5}"`);
metadata.getObjectMD(bucketName, objectName, {}, log,
(err, md) => {
const { mode, retainUntilDate }
= md.retentionInfo;
assert.ifError(err);
assert(md);
assert.strictEqual(mode, mockMode);
assert.strictEqual(retainUntilDate, mockDate);
done();
});
});
});
});
});
const formatTime = time => time.slice(0, 21);
const testObjectLockConfigs = [
{
testMode: 'COMPLIANCE',
val: 30,
type: 'Days',
},
{
testMode: 'GOVERNANCE',
val: 5,
type: 'Years',
},
];
testObjectLockConfigs.forEach(config => {
const { testMode, type, val } = config;
it('should put an object with default retention if object does not ' +
'have retention configuration but bucket has', done => {
const testPutObjectRequest = new DummyRequest({
bucketName,
namespace,
objectKey: objectName,
headers: {},
url: `/${bucketName}/${objectName}`,
calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==',
}, postBody);
const testObjLockRequest = {
bucketName,
headers: { host: `${bucketName}.s3.amazonaws.com` },
post: objectLockTestUtils.generateXml(testMode, val, type),
};
bucketPut(authInfo, testPutBucketRequestLock, log, () => {
bucketPutObjectLock(authInfo, testObjLockRequest, log, () => {
objectPut(authInfo, testPutObjectRequest, undefined, log,
(err, headers) => {
assert.ifError(err);
assert.strictEqual(headers.ETag, `"${correctMD5}"`);
metadata.getObjectMD(bucketName, objectName, {},
log, (err, md) => {
const { mode, retainUntilDate: retainDate }
= md.retentionInfo;
const date = moment();
const days
= type === 'Days' ? val : val * 365;
const expectedDate
= date.add(days, 'Days');
assert.ifError(err);
assert.strictEqual(mode, testMode);
assert.strictEqual(formatTime(retainDate),
formatTime(expectedDate.toISOString()));
done();
});
});
});
});
});
});
it('should successfully put an object with legal hold ON', done => {
const request = new DummyRequest({
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-object-lock-legal-hold': 'ON',
},
url: `/${bucketName}/${objectName}`,
calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==',
}, postBody);
bucketPut(authInfo, testPutBucketRequestLock, log, () => {
objectPut(authInfo, request, undefined, log, (err, headers) => {
assert.ifError(err);
assert.strictEqual(headers.ETag, `"${correctMD5}"`);
metadata.getObjectMD(bucketName, objectName, {}, log,
(err, md) => {
assert.ifError(err);
assert.strictEqual(md.legalHold, true);
done();
});
});
});
});
it('should successfully put an object with legal hold OFF', done => {
const request = new DummyRequest({
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-object-lock-legal-hold': 'OFF',
},
url: `/${bucketName}/${objectName}`,
calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==',
}, postBody);
bucketPut(authInfo, testPutBucketRequestLock, log, () => {
objectPut(authInfo, request, undefined, log, (err, headers) => {
assert.ifError(err);
assert.strictEqual(headers.ETag, `"${correctMD5}"`);
metadata.getObjectMD(bucketName, objectName, {}, log,
(err, md) => {
assert.ifError(err);
assert(md);
assert.strictEqual(md.legalHold, false);
done();
});
});
});
});
it('should successfully put an object with user metadata', done => {
const testPutObjectRequest = new DummyRequest({
bucketName,
@ -271,6 +431,30 @@ describe('objectPut API', () => {
});
});
it('should not put object with retention configuration if object lock ' +
'is not enabled on the bucket', done => {
const testPutObjectRequest = new DummyRequest({
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-object-lock-retain-until-date': mockDate,
'x-amz-object-lock-mode': 'GOVERNANCE',
},
url: `/${bucketName}/${objectName}`,
calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==',
}, postBody);
bucketPut(authInfo, testPutBucketRequest, log, () => {
objectPut(authInfo, testPutObjectRequest, undefined, log, err => {
assert.deepStrictEqual(err, errors.InvalidRequest
.customizeDescription(
'Bucket is missing ObjectLockConfiguration'));
done();
});
});
});
describe('objectPut API with versioning', () => {
beforeEach(() => {
cleanup();

View File

@ -342,6 +342,18 @@ class CorsConfigTester {
}
}
const objectLockTestUtils = {
generateXml: (mode, num, daysOrYears) =>
'<ObjectLockConfiguration ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
'<ObjectLockEnabled>Enabled</ObjectLockEnabled>' +
'<Rule><DefaultRetention>' +
`<Mode>${mode}</Mode>` +
`<${daysOrYears}>${num}</${daysOrYears}>` +
'</DefaultRetention></Rule>' +
'</ObjectLockConfiguration>',
};
const versioningTestUtils = {
createPutObjectRequest: (bucketName, keyName, body) => {
const params = {
@ -485,6 +497,7 @@ module.exports = {
makeAuthInfo,
WebsiteConfig,
CorsConfigTester,
objectLockTestUtils,
versioningTestUtils,
TaggingConfigTester,
AccessControlPolicy,

View File

@ -0,0 +1,174 @@
const assert = require('assert');
const moment = require('moment');
const { errors } = require('arsenal');
const BucketInfo = require('arsenal').models.BucketInfo;
const { DummyRequestLogger } = require('../helpers');
const {
calculateRetainUntilDate,
validateHeaders,
} = require('../../../lib/api/apiUtils/object/objectLockHelpers');
const mockName = 'testbucket';
const mockOwner = 'someCanonicalId';
const mockOwnerDisplayName = 'accountDisplayName';
const mockCreationDate = new Date().toJSON();
const bucketInfo = new BucketInfo(
mockName, mockOwner, mockOwnerDisplayName, mockCreationDate,
null, null, null, null, null, null, null, null, null, null,
null, null, true);
const objLockDisabledBucketInfo = new BucketInfo(
mockName, mockOwner, mockOwnerDisplayName, mockCreationDate,
null, null, null, null, null, null, null, null, null, null,
null, null, false);
const log = new DummyRequestLogger();
describe('objectLockHelpers: validateHeaders', () => {
it('should fail if object lock is not enabled on the bucket', () => {
const headers = {
'x-amz-object-lock-retain-until-date': '2050-10-12',
'x-amz-object-lock-mode': 'COMPLIANCE',
};
const objectLockValidationError
= validateHeaders(objLockDisabledBucketInfo, headers, log);
const expectedError = errors.InvalidRequest.customizeDescription(
'Bucket is missing ObjectLockConfiguration');
assert.strictEqual(objectLockValidationError.InvalidRequest, true);
assert.strictEqual(objectLockValidationError.description,
expectedError.description);
});
it('should pass with valid retention headers', () => {
const headers = {
'x-amz-object-lock-retain-until-date': '2050-10-12',
'x-amz-object-lock-mode': 'COMPLIANCE',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
assert.strictEqual(objectLockValidationError, null);
});
it('should pass with valid legal hold header', () => {
const headers = {
'x-amz-object-lock-legal-hold': 'ON',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
assert.strictEqual(objectLockValidationError, null);
});
it('should pass with valid legal hold header', () => {
const headers = {
'x-amz-object-lock-legal-hold': 'OFF',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
assert.strictEqual(objectLockValidationError, null);
});
it('should pass with both legal hold and retention headers', () => {
const headers = {
'x-amz-object-lock-retain-until-date': '2050-10-12',
'x-amz-object-lock-mode': 'GOVERNANCE',
'x-amz-object-lock-legal-hold': 'ON',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
assert.strictEqual(objectLockValidationError, null);
});
it('should fail with missing object-lock-mode header', () => {
const headers = {
'x-amz-object-lock-retain-until-date': '2005-10-12',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
const expectedError = errors.InvalidArgument.customizeDescription(
'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' +
'must both be supplied');
assert.strictEqual(objectLockValidationError.InvalidArgument, true);
assert.strictEqual(objectLockValidationError.description,
expectedError.description);
});
it('should fail with missing object-lock-retain-until-date header', () => {
const headers = {
'x-amz-object-lock-mode': 'GOVERNANCE',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
const expectedError = errors.InvalidArgument.customizeDescription(
'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' +
'must both be supplied');
assert.strictEqual(objectLockValidationError.InvalidArgument, true);
assert.strictEqual(objectLockValidationError.description,
expectedError.description);
});
it('should fail with past retention date header', () => {
const headers = {
'x-amz-object-lock-retain-until-date': '2005-10-12',
'x-amz-object-lock-mode': 'COMPLIANCE',
};
const expectedError = errors.InvalidArgument.customizeDescription(
'The retain until date must be in the future!');
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
assert.strictEqual(objectLockValidationError.InvalidArgument, true);
assert.strictEqual(objectLockValidationError.description,
expectedError.description);
});
it('should fail if object lock legal hold is not either ON or OFF', () => {
const headers = {
'x-amz-object-lock-legal-hold': 'on',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
assert.strictEqual(objectLockValidationError.InvalidArgument, true);
});
it('should fail with invalid retention period header', () => {
const headers = {
'x-amz-object-lock-retain-until-date': '2050-10-12',
'x-amz-object-lock-mode': 'Governance',
};
const objectLockValidationError
= validateHeaders(bucketInfo, headers, log);
const expectedError = errors.InvalidArgument.customizeDescription(
'Unknown wormMode directive');
assert.strictEqual(objectLockValidationError.InvalidArgument, true);
assert.strictEqual(objectLockValidationError.description,
expectedError.description);
});
});
describe('objectLockHelpers: calculateRetainUntilDate', () => {
it('should calculate retainUntilDate for config with days', () => {
const mockConfigWithDays = {
mode: 'GOVERNANCE',
days: 90,
};
const date = moment();
const expectedRetainUntilDate
= date.add(mockConfigWithDays.days, 'Days');
const retainUntilDate = calculateRetainUntilDate(mockConfigWithDays);
assert.strictEqual(retainUntilDate.slice(0, 21),
expectedRetainUntilDate.toISOString().slice(0, 21));
});
it('should calculate retainUntilDate for config with years', () => {
const mockConfigWithYears = {
mode: 'GOVERNANCE',
years: 3,
};
const date = moment();
const expectedRetainUntilDate
= date.add(mockConfigWithYears.years * 365, 'Days');
const retainUntilDate = calculateRetainUntilDate(mockConfigWithYears);
assert.strictEqual(retainUntilDate.slice(0, 21),
expectedRetainUntilDate.toISOString().slice(0, 21));
});
});