Compare commits

...

9 Commits

Author SHA1 Message Date
Dora Korpar 6ff95c7ebb ft: S3C-2787 put object retention 2020-05-18 17:00:26 -07:00
Dora Korpar d400da4725 unit tests 2020-05-18 11:12:26 -07:00
Dora Korpar e4058581b8 get retention info 2020-05-18 11:12:26 -07:00
Dora Korpar 722bfd998a ret info start 2020-05-18 11:12:26 -07:00
Dora Korpar 5c92c491d0 getretentioninfo 2020-05-18 11:12:25 -07:00
Dora Korpar 2c01244f05 ft: S3C-2787-objectlock-object-retention 2020-05-18 11:12:25 -07:00
Dora Korpar 69b7139677 update ModelVersion readme 2020-05-18 08:21:14 -07:00
Dora Korpar 6ff5f14fbb update Arsenal 2020-05-15 15:37:15 -07:00
Dora Korpar 62654f375b ft: S3C 2789 put obj lock config api 2020-05-15 15:33:26 -07:00
15 changed files with 395 additions and 21 deletions

View File

@ -93,7 +93,6 @@ const constants = {
'logging',
'metrics',
'notification',
'object-lock',
'policyStatus',
'publicAccessBlock',
'requestPayment',

View File

@ -22,6 +22,7 @@ const bucketPutWebsite = require('./bucketPutWebsite');
const bucketPutReplication = require('./bucketPutReplication');
const bucketPutLifecycle = require('./bucketPutLifecycle');
const bucketPutPolicy = require('./bucketPutPolicy');
const bucketPutObjectLock = require('./bucketPutObjectLock');
const bucketGetReplication = require('./bucketGetReplication');
const bucketDeleteReplication = require('./bucketDeleteReplication');
const corsPreflight = require('./corsPreflight');
@ -43,6 +44,7 @@ const objectPutACL = require('./objectPutACL');
const objectPutTagging = require('./objectPutTagging');
const objectPutPart = require('./objectPutPart');
const objectPutCopyPart = require('./objectPutCopyPart');
const objectPutRetention = require('./objectPutRetention');
const prepareRequestContexts
= require('./apiUtils/authorization/prepareRequestContexts');
const serviceGet = require('./serviceGet');
@ -187,6 +189,7 @@ const api = {
bucketPutPolicy,
bucketGetPolicy,
bucketDeletePolicy,
bucketPutObjectLock,
corsPreflight,
completeMultipartUpload,
initiateMultipartUpload,
@ -206,6 +209,7 @@ const api = {
objectPutTagging,
objectPutPart,
objectPutCopyPart,
objectPutRetention,
serviceGet,
websiteGet,
websiteHead,

View File

@ -1,12 +1,18 @@
const { policies } = require('arsenal');
const { config } = require('../../../Config');
const RequestContext = policies.RequestContext;
const requestUtils = policies.requestUtils;
const { RequestContext, requestUtils } = policies;
let apiMethodAfterVersionCheck;
const apiMethodWithVersion = { objectGetACL: true, objectPutACL: true,
objectGet: true, objectDelete: true, objectPutTagging: true,
objectGetTagging: true, objectDeleteTagging: true };
const apiMethodWithVersion = {
objectGetACL: true,
objectPutACL: true,
objectGet: true,
objectDelete: true,
objectPutTagging: true,
objectGetTagging: true,
objectDeleteTagging: true,
objectPutRetention: true,
};
function isHeaderAcl(headers) {
return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] ||

View File

@ -11,6 +11,7 @@ const locationConstraintCheck = require('./locationConstraintCheck');
const { versioningPreprocessing } = require('./versioning');
const removeAWSChunked = require('./removeAWSChunked');
const getReplicationInfo = require('./getReplicationInfo');
const getRetentionInfo = require('./getRetentionInfo');
const { config } = require('../../../Config');
const validateWebsiteHeader = require('./websiteServing')
.validateWebsiteHeader;
@ -137,6 +138,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
headers,
isDeleteMarker,
replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size),
retentionInfo: getRetentionInfo(bucketMD),
log,
};
if (!isDeleteMarker) {

View File

@ -0,0 +1,20 @@
function getRetentionInfo(bucketMD) {
const objLockConfig = bucketMD.getObjectLockConfiguration();
const objRetention = {};
if (objLockConfig) {
objRetention.mode = objLockConfig.rule.mode;
const date = new Date();
const day = objLockConfig.rule.day;
if (day) {
date.setDate(date.getDate() + day);
} else {
date.setFullYear(date.getFullYear() + objLockConfig.rule.year);
}
objRetention.retainUntilDate = date;
return objRetention;
}
return undefined;
}
module.exports = getRetentionInfo;

View File

@ -0,0 +1,82 @@
const { waterfall } = require('async');
const arsenal = require('arsenal');
const errors = arsenal.errors;
const ObjectLockConfiguration = arsenal.models.ObjectLockConfiguration;
const parseXML = require('../utilities/parseXML');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const metadata = require('../metadata/wrapper');
const { metadataValidateBucket } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
/**
* Bucket Put Object Lock - Create or update bucket object lock configuration
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
* @param {object} request - http request object
* @param {object} log - Werelogs logger
* @param {function} callback - callback to server
* @return {undefined}
*/
function bucketPutObjectLock(authInfo, request, log, callback) {
log.debug('processing request', { method: 'bucketPutObjectLock' });
const bucketName = request.bucketName;
const metadataValParams = {
authInfo,
bucketName,
requestType: 'bucketPutObjectLock',
};
return waterfall([
next => parseXML(request.post, log, next),
(parsedXml, next) => {
const lockConfigClass = new ObjectLockConfiguration(parsedXml);
// if there was an error getting object lock configuration,
// returned configObj will contain 'error' key
process.nextTick(() => {
const configObj = lockConfigClass.
getValidatedObjectLockConfiguration();
return next(configObj.error || null, configObj);
});
},
(objectLockConfig, next) => metadataValidateBucket(metadataValParams,
log, (err, bucket) => {
if (err) {
return next(err, bucket);
}
return next(null, bucket, objectLockConfig);
}),
(bucket, objectLockConfig, next) => {
const isObjectLockEnabled = bucket.isObjectLockEnabled();
process.nextTick(() => {
if (!isObjectLockEnabled) {
return next(errors.InvalidBucketState.customizeDescription(
'Object Lock configuration cannot be enabled on ' +
'existing buckets'), bucket);
}
return next(null, bucket, objectLockConfig);
});
},
(bucket, objectLockConfig, next) => {
bucket.setObjectLockConfiguration(objectLockConfig);
metadata.updateBucket(bucket.getName(), bucket, log, err =>
next(err, bucket));
},
], (err, bucket) => {
const corsHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket);
if (err) {
log.trace('error processing request', { error: err,
method: 'bucketPutObjectLock' });
return callback(err, corsHeaders);
}
pushMetric('putBucketObjectLock', log, {
authInfo,
bucket: bucketName,
});
return callback(null, corsHeaders);
});
}
module.exports = bucketPutObjectLock;

View File

@ -5,6 +5,7 @@ const { errors, versioning, s3middleware } = require('arsenal');
const convertToXml = s3middleware.convertToXml;
const { pushMetric } = require('../utapi/utilities');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
const getRetentionInfo = require('./apiUtils/object/getRetentionInfo');
const { validateAndFilterMpuParts, generateMpuPartStorageInfo } =
require('./apiUtils/object/processMpuParts');
const { config } = require('../Config');
@ -304,6 +305,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
multipart: true,
replicationInfo: getReplicationInfo(objectKey, destBucket,
false, calculatedSize, REPLICATION_ACTION),
retentionInfo: getRetentionInfo(destBucket),
log,
};
if (storedMetadata['x-amz-tagging']) {

View File

@ -11,6 +11,7 @@ const locationConstraintCheck
const { checkQueryVersionId, versioningPreprocessing }
= require('./apiUtils/object/versioning');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
const getRetentionInfo = require('./apiUtils/object/getRetentionInfo');
const data = require('../data/wrapper');
const logger = require('../utilities/logger');
const services = require('../services');
@ -162,6 +163,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination,
replicationInfo: getReplicationInfo(objectKey, destBucketMD, false,
sourceObjMD['content-length']),
locationMatch,
retentionInfo: getRetentionInfo(destBucketMD),
};
// In case whichMetadata === 'REPLACE' but contentType is undefined in copy

View File

@ -0,0 +1,118 @@
const async = require('async');
const { errors, s3middleware } = require('arsenal');
const { decodeVersionId, getVersionIdResHeader } =
require('./apiUtils/object/versioning');
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const metadata = require('../metadata/wrapper');
const { config } = require('../Config');
const multipleBackendGateway = require('../data/multipleBackendGateway');
const { parseRetentionXml } = s3middleware.objectRetention;
const REPLICATION_ACTION = 'PUT_RETENTION';
/**
* Object Put Retention - Adds retention information to object
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
* @param {object} request - http request object
* @param {object} log - Werelogs logger
* @param {function} callback - callback to server
* @return {undefined}
*/
function objectPutRetention(authInfo, request, log, callback) {
log.debug('processing request', { method: 'objectPutRetention' });
const { bucketName, objectKey } = request;
const decodedVidResult = decodeVersionId(request.query);
if (decodedVidResult instanceof Error) {
log.trace('invalid versionId query', {
versionId: request.query.versionId,
error: decodedVidResult,
});
return process.nextTick(() => callback(decodedVidResult));
}
const reqVersionId = decodedVidResult;
const metadataValParams = {
authInfo,
bucketName,
objectKey,
requestType: 'objectPutRetention',
versionId: reqVersionId,
};
return async.waterfall([
next => metadataValidateBucketAndObj(metadataValParams, log,
(err, bucket, objectMD) => {
if (err) {
log.trace('request authorization failed',
{ method: 'objectPutRetention', error: err });
return next(err);
}
if (!objectMD) {
const err = reqVersionId ? errors.NoSuchVersion :
errors.NoSuchKey;
log.trace('error no object metadata found',
{ method: 'objectPutRetention', error: err });
return next(err, bucket);
}
if (objectMD.isDeleteMarker) {
log.trace('version is a delete marker',
{ method: 'objectPutRetention' });
return next(errors.MethodNotAllowed, bucket);
}
return next(null, bucket, objectMD);
}),
(bucket, objectMD, next) => {
log.trace('parsing retention information');
parseRetentionXml(request.post, log,
(err, retentionInfo) => next(err, bucket, retentionInfo, objectMD));
},
(bucket, retentionInfo, objectMD, next) => {
// eslint-disable-next-line no-param-reassign
objectMD.retentionInfo = retentionInfo;
const params = objectMD.versionId ?
{ versionId: objectMD.versionId } : {};
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
0, REPLICATION_ACTION, objectMD);
if (replicationInfo) {
// eslint-disable-next-line no-param-reassign
objectMD.replicationInfo = Object.assign({},
objectMD.replicationInfo, replicationInfo);
}
metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params,
log, err => next(err, bucket, objectMD));
},
(bucket, objectMD, next) => {
if (config.backends.data === 'multiple') {
return multipleBackendGateway.objectRetention('Put', objectKey,
bucket, objectMD, log, err => next(err, bucket, objectMD));
}
return next(null, bucket, objectMD);
},
], (err, bucket, objectMD) => {
const additionalResHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket);
if (err) {
log.trace('error processing request',
{ error: err, method: 'objectPutRetention' });
} else {
pushMetric('putObjectRetention', log, {
authInfo,
bucket: bucketName,
keys: [objectKey],
});
const verCfg = bucket.getVersioningConfiguration();
additionalResHeaders['x-amz-version-id'] =
getVersionIdResHeader(verCfg, objectMD);
}
return callback(err, additionalResHeaders);
});
}
module.exports = objectPutRetention;

View File

@ -80,3 +80,17 @@ this._lifecycleConfiguration = lifecycleConfiguration || null;
### Usage
Used to store the bucket lifecycle configuration info
## Model version 7
### Properties Added
```javascript
this._objectLockEnabled = objectLockEnabled || false;
this._objectLockConfiguration = objectLockConfiguration || null;
```
### Usage
Used to determine whether object lock capabilities are enabled on a bucket and
to store the object lock configuration of the bucket

View File

@ -95,7 +95,7 @@ const services = {
contentType, cacheControl, contentDisposition, contentEncoding,
expires, multipart, headers, overrideMetadata, log,
lastModifiedDate, versioning, versionId, tagging, taggingCopy,
replicationInfo, dataStoreName } = params;
replicationInfo, dataStoreName, retentionInfo } = params;
log.trace('storing object in metadata');
assert.strictEqual(typeof bucketName, 'string');
const md = new ObjectMD();
@ -179,6 +179,10 @@ const services = {
md.overrideMetadataValues(overrideMetadata);
}
if (retentionInfo) {
md.setRetentionInfo(retentionInfo);
}
log.trace('object metadata', { omVal: md.getValue() });
// If this is not the completion of a multipart upload or
// the creation of a delete marker, parse the headers to

View File

@ -19,7 +19,7 @@
},
"homepage": "https://github.com/scality/S3#readme",
"dependencies": {
"arsenal": "github:scality/Arsenal#f988270",
"arsenal": "github:scality/Arsenal#ef4a2dc",
"async": "~2.5.0",
"aws-sdk": "2.363.0",
"azure-storage": "^2.1.0",

View File

@ -0,0 +1,37 @@
const assert = require('assert');
const BucketInfo = require('arsenal').models.BucketInfo;
const getRetentionInfo =
require('../../../../lib/api/apiUtils/object/getRetentionInfo');
function _getRetentionInfo(objectLockConfig) {
const bucketInfo = new BucketInfo(
'testbucket', 'someCanonicalId', 'accountDisplayName',
new Date().toJSON(),
null, null, null, null, null, null, null, null, null,
null, true, objectLockConfig);
return getRetentionInfo(bucketInfo);
}
describe.only('getRetentionInfo helper', () => {
it('should get retention info', () => {
const objectLockConfig = {
rule: {
mode: 'COMPLIANCE',
days: 1,
},
};
const retentionInfo = _getRetentionInfo(objectLockConfig);
const date = new Date();
assert.deepStrictEqual(retentionInfo, {
mode: 'COMPLIANCE',
retainUntilDate: date.getDate() + 1,
});
});
it('should not get retention info when no objectno object lock ' +
'configuration is set', () => {
const retentionInfo = _getRetentionInfo({});
assert.deepStrictEqual(retentionInfo, undefined);
});
});

View File

@ -0,0 +1,84 @@
const assert = require('assert');
const { bucketPut } = require('../../../lib/api/bucketPut');
const bucketPutObjectLock = require('../../../lib/api/bucketPutObjectLock');
const { cleanup,
DummyRequestLogger,
makeAuthInfo,
} = require('../helpers');
const metadata = require('../../../lib/metadata/wrapper');
const log = new DummyRequestLogger();
const authInfo = makeAuthInfo('accessKey1');
const bucketName = 'bucketputobjectlockbucket';
const bucketPutRequest = {
bucketName,
headers: { host: `${bucketName}.s3.amazonaws.com` },
url: '/',
};
const objectLockXml = '<ObjectLockConfiguration ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
'<ObjectLockEnabled>Enabled</ObjectLockEnabled>' +
'<Rule><DefaultRetention>' +
'<Mode>GOVERNANCE</Mode>' +
'<Days>1</Days>' +
'</DefaultRetention></Rule>' +
'</ObjectLockConfiguration>';
const putObjLockRequest = {
bucketName,
headers: { host: `${bucketName}.s3.amazonaws.com` },
post: objectLockXml,
};
const expectedObjectLockConfig = {
rule: {
mode: 'GOVERNANCE',
days: 1,
},
};
describe('putBucketObjectLock API', () => {
before(() => cleanup());
describe('without Object Lock enabled on bucket', () => {
beforeEach(done => bucketPut(authInfo, bucketPutRequest, log, done));
afterEach(() => cleanup());
it('should return InvalidBucketState error', done => {
bucketPutObjectLock(authInfo, putObjLockRequest, log, err => {
assert.strictEqual(err.InvalidBucketState, true);
done();
});
});
});
describe('with Object Lock enabled on bucket', () => {
const bucketObjLockRequest = Object.assign({}, bucketPutRequest,
{ headers: { 'x-amz-bucket-object-lock-enabled': true } });
beforeEach(done => bucketPut(authInfo, bucketObjLockRequest, log, done));
afterEach(() => cleanup());
it('should update a bucket\'s metadata with object lock config', done => {
bucketPutObjectLock(authInfo, putObjLockRequest, log, err => {
if (err) {
process.stdout.write(`Err putting lifecycle config ${err}`);
return done(err);
}
return metadata.getBucket(bucketName, log, (err, bucket) => {
if (err) {
process.stdout.write(`Err retrieving bucket MD ${err}`);
return done(err);
}
const bucketObjectLockConfig = bucket.
getObjectLockConfiguration();
assert.deepStrictEqual(
bucketObjectLockConfig, expectedObjectLockConfig);
return done();
});
});
});
});
});

View File

@ -210,9 +210,9 @@ arraybuffer.slice@0.0.6:
resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.6.tgz#f33b2159f0532a3f3107a272c0ccfbd1ad2979ca"
integrity sha1-8zshWfBTKj8xB6JywMz70a0peco=
"arsenal@github:scality/Arsenal#f988270":
"arsenal@github:scality/Arsenal#ef4a2dc":
version "7.5.0"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/f988270a0c557f0d663cae5a6d3dc5b840ccc4ff"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/ef4a2dc0778c71ac2acf3fe50cc71378f86454e4"
dependencies:
"@hapi/joi" "^15.1.0"
JSONStream "^1.0.0"
@ -261,10 +261,9 @@ arsenal@scality/Arsenal#32c895b:
ioctl "2.0.0"
arsenal@scality/Arsenal#9f2e74e:
version "7.5.0"
version "7.4.3"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/9f2e74ec6972527c2a9ca6ecb4155618f123fc19"
dependencies:
"@hapi/joi" "^15.1.0"
JSONStream "^1.0.0"
ajv "4.10.0"
async "~2.1.5"
@ -272,6 +271,7 @@ arsenal@scality/Arsenal#9f2e74e:
diskusage "^1.1.1"
ioredis "4.9.5"
ipaddr.js "1.2.0"
joi "^10.6"
level "~5.0.1"
level-sublevel "~6.6.5"
node-forge "^0.7.1"
@ -379,9 +379,9 @@ aws-sdk@2.363.0:
xml2js "0.4.19"
aws-sdk@^2.2.23:
version "2.671.0"
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.671.0.tgz#2c6e164a0f540d6fc428c123f2994ac081663ff5"
integrity sha512-i83+/TIOLlhAxvV2xVLz5+XGtNqJgQJwP/e8J49rzDkyMV6OE2FgxU8utujGrComrSJFpITqMFqug+ZfdHoLIQ==
version "2.678.0"
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.678.0.tgz#b16230f4894d40ead50f9e23805c874f4ca62549"
integrity sha512-i8t7+1/C6maQzUYUFRQXPAsUPT0YdpNsf/oHZKmmZrsOX+epnn2jmAGIBTZgUakY8jRrZxCJka+QokUIadUVQg==
dependencies:
buffer "4.9.1"
events "1.1.1"
@ -3143,9 +3143,9 @@ s3blaster@scality/s3blaster#7a836b6:
utf8 "~2.1.1"
safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0:
version "5.2.0"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519"
integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==
version "5.2.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
safe-buffer@~5.1.0, safe-buffer@~5.1.1:
version "5.1.2"
@ -3640,9 +3640,9 @@ uc.micro@^1.0.0, uc.micro@^1.0.1:
integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==
uglify-js@^3.1.4:
version "3.9.2"
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.9.2.tgz#012b74fb6a2e440d9ba1f79110a479d3b1f2d48d"
integrity sha512-zGVwKslUAD/EeqOrD1nQaBmXIHl1Vw371we8cvS8I6mYK9rmgX5tv8AAeJdfsQ3Kk5mGax2SVV/AizxdNGhl7Q==
version "3.9.3"
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.9.3.tgz#4a285d1658b8a2ebaef9e51366b3a0f7acd79ec2"
integrity sha512-r5ImcL6QyzQGVimQoov3aL2ZScywrOgBXGndbWrdehKoSvGe/RmiE5Jpw/v+GvxODt6l2tpBXwA7n+qZVlHBMA==
dependencies:
commander "~2.20.3"