Compare commits

...

12 Commits

Author SHA1 Message Date
williamlardier a155026abe
CLDSRV-???: batch calls to storage 2023-06-06 10:50:45 +02:00
williamlardier 36e69648fd
CLDSRV-???: increase multi object delete parallelism 2023-06-06 09:42:28 +02:00
Jonathan Gramain 20f9662f54 CLDSRV-321 bump hotfix version to 7.70.11-2 2022-12-21 22:04:13 -08:00
Taylor McKinnon a77b40a2a2 bf(CLDSRV-321): Fix retention extension check to consider same date as extended
(cherry picked from commit 9f5ae852bf)
2022-12-21 22:03:39 -08:00
Jonathan Gramain 75410f0ba4 CLDSRV-173 CLDSRV-170 CLDSRV-177 S3C-5390 bump hotfix version 2022-12-15 12:24:20 -08:00
Artem Bakalov 73c364acc8 remove .only
(cherry picked from commit b4725aa032)
(cherry picked from commit af95fea311)
(cherry picked from commit e6bda3460b)
2022-12-15 12:23:27 -08:00
Artem Bakalov c37007782c S3C-5390 s3api head-object with part-number 1 on empty file fails: httpCode 416
(cherry picked from commit 4f3195a6ca)
(cherry picked from commit 8f3e737664)
(cherry picked from commit 64334db65a)
2022-12-15 12:23:27 -08:00
Jonathan Gramain 08c8599e1d CLDSRV-177 fix backbeat func tests for CLDSRV-290
Since CLDSRV-290, backbeat routes functional tests need to pass a
versionId in the query string in the `/_/backbeat/metadata` route,
hence the original tests from CLDSRV-177 needed to be updated.

(cherry picked from commit fa562ae85a)
2022-12-15 12:23:27 -08:00
Jonathan Gramain 9bdab63401 [7.10] CLDSRV-177 add missing test helper checkObjectData
(cherry picked from commit 413ebe743c)
(cherry picked from commit a1c4420eab)
(cherry picked from commit 6f32ebb2ce)
2022-12-15 12:23:27 -08:00
Jonathan Gramain 189f3cf41b bugfix: CLDSRV-177 fix crash with empty object replication
Fix a case of crash when a replication occurs with an empty object
over a non-empty object.

It is not clear how this happens in practice but there can be some
corner cases with race conditions between object replication and
versioning getting suspended on the target bucket at the same time,
that could lead to this situation, as the check between replication
configuration and actual replication is not atomic.

(cherry picked from commit a4e8cbebe6)
(cherry picked from commit 108d1c920f)
(cherry picked from commit 2b32ec6163)
2022-12-15 12:23:27 -08:00
Jonathan Gramain 30067dfc73 bugfix: CLDSRV-170 skip orphan cleanup in UploadPart[Copy]
Do not delete orphan data in UploadPart/UploadPartCopy on overwrite
iff a CompleteMPU of the target MPU is already in progress.

This is to prevent a race condition where a CompleteMPU is running
while UploadPart is uploading a part for the same MPU.

It leaves an orphan in storage since only one of the upload data will
be present in the finished MPU, but the window is limited to the
CompleteMPU execution and should only occur when there are retries of
UploadPart due to prior stuck requests, or with broken clients
misusing the MPU API, so it should be acceptable.

Implementation details:

- set a flag in the MPU overview key when starting the CompleteMPU
  process, before listing the parts from metadata to construct the
  manifest

- in UploadPart/UploadPartCopy, after the part metadata is written and
  if the same part already existed, re-fetch the MPU overview key to
  check the flag: if set, skip the deletion of the old data of this
  part, since the CompleteMPU process in progress may choose either
  part data depending on the exact timing of the listing vs. the
  part overwrite.

(cherry picked from commit 8496111518)
(cherry picked from commit c2dbbfa008)
(cherry picked from commit d4063e157a)
2022-12-15 12:23:26 -08:00
Nicolas Humbert ad3d9a9f5f CLDSRV-173 DeleteMarkers created by Lifecycle should not be replicated
(cherry picked from commit d30430a81c)
(cherry picked from commit a481384538)
2022-12-15 12:23:26 -08:00
25 changed files with 991 additions and 183 deletions

View File

@ -173,6 +173,10 @@ const constants = {
'bucket', 'bucket',
], ],
allowedUtapiEventFilterStates: ['allow', 'deny'], allowedUtapiEventFilterStates: ['allow', 'deny'],
// The AWS assumed Role resource type
assumedRoleArnResourceType: 'assumed-role',
// Session name of the backbeat lifecycle assumed role session.
backbeatLifecycleSessionName: 'backbeat-lifecycle',
}; };
module.exports = constants; module.exports = constants;

View File

@ -1,7 +1,8 @@
const { evaluators, actionMaps, RequestContext } = require('arsenal').policies; const { evaluators, actionMaps, RequestContext } = require('arsenal').policies;
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { allAuthedUsersId, bucketOwnerActions, logId, publicId } = constants; const { allAuthedUsersId, bucketOwnerActions, logId, publicId,
assumedRoleArnResourceType, backbeatLifecycleSessionName } = constants;
// whitelist buckets to allow public read on objects // whitelist buckets to allow public read on objects
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ? const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ?
@ -364,10 +365,34 @@ function validatePolicyResource(bucketName, policy) {
}); });
} }
/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
* @param {string} arn - Amazon resource name - example:
* arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
* @return {boolean} true if Lifecycle assumed role session arn, false if not.
*/
function isLifecycleSession(arn) {
if (!arn) {
return false;
}
const arnSplits = arn.split(':');
const service = arnSplits[2];
const resourceNames = arnSplits[arnSplits.length - 1].split('/');
const resourceType = resourceNames[0];
const sessionName = resourceNames[resourceNames.length - 1];
return (service === 'sts' &&
resourceType === assumedRoleArnResourceType &&
sessionName === backbeatLifecycleSessionName);
}
module.exports = { module.exports = {
isBucketAuthorized, isBucketAuthorized,
isObjAuthorized, isObjAuthorized,
checkBucketAcls, checkBucketAcls,
checkObjectAcls, checkObjectAcls,
validatePolicyResource, validatePolicyResource,
isLifecycleSession,
}; };

View File

@ -136,9 +136,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
size, size,
headers, headers,
isDeleteMarker, isDeleteMarker,
replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size), replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
log, log,
}; };
if (!isDeleteMarker) { if (!isDeleteMarker) {
metadataStoreParams.contentType = request.headers['content-type']; metadataStoreParams.contentType = request.headers['content-type'];
metadataStoreParams.cacheControl = request.headers['cache-control']; metadataStoreParams.cacheControl = request.headers['cache-control'];

View File

@ -1,4 +1,5 @@
const s3config = require('../../../Config').config; const s3config = require('../../../Config').config;
const { isLifecycleSession } = require('../authorization/permissionChecks.js');
function _getBackend(objectMD, site) { function _getBackend(objectMD, site) {
const backends = objectMD ? objectMD.replicationInfo.backends : []; const backends = objectMD ? objectMD.replicationInfo.backends : [];
@ -63,14 +64,22 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
* @param {boolean} objSize - The size, in bytes, of the object being PUT * @param {boolean} objSize - The size, in bytes, of the object being PUT
* @param {string} operationType - The type of operation to replicate * @param {string} operationType - The type of operation to replicate
* @param {object} objectMD - The object metadata * @param {object} objectMD - The object metadata
* @param {AuthInfo} [authInfo] - authentication info of object owner
* @param {boolean} [isDeleteMarker] - whether creating a delete marker
* @return {undefined} * @return {undefined}
*/ */
function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType, function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
objectMD) { objectMD, authInfo, isDeleteMarker) {
const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA']; const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
const config = bucketMD.getReplicationConfiguration(); const config = bucketMD.getReplicationConfiguration();
// If bucket does not have a replication configuration, do not replicate. // If bucket does not have a replication configuration, do not replicate.
if (config) { if (config) {
// If delete an object due to a lifecycle action,
// the delete marker is not replicated to the destination buckets.
if (isDeleteMarker && authInfo && isLifecycleSession(authInfo.getArn())) {
return undefined;
}
const rule = config.rules.find(rule => const rule = config.rules.find(rule =>
(objKey.startsWith(rule.prefix) && rule.enabled)); (objKey.startsWith(rule.prefix) && rule.enabled));
if (rule) { if (rule) {

View File

@ -8,12 +8,13 @@
* *
* @param {array|string|null} prev - list of keys from the object being * @param {array|string|null} prev - list of keys from the object being
* overwritten * overwritten
* @param {array} curr - list of keys to be used in composing current object * @param {array|null} curr - list of keys to be used in composing
* current object
* @returns {boolean} true if no key in `curr` is present in `prev`, * @returns {boolean} true if no key in `curr` is present in `prev`,
* false otherwise * false otherwise
*/ */
function locationKeysHaveChanged(prev, curr) { function locationKeysHaveChanged(prev, curr) {
if (!prev || prev.length === 0) { if (!prev || prev.length === 0 || !curr) {
return true; return true;
} }
// backwards compatibility check if object is of model version 2 // backwards compatibility check if object is of model version 2

View File

@ -193,7 +193,7 @@ class ObjectLockInfo {
* @returns {bool} - True if the given timestamp is after the policy expiration date or if no expiration date is set * @returns {bool} - True if the given timestamp is after the policy expiration date or if no expiration date is set
*/ */
isExtended(timestamp) { isExtended(timestamp) {
return timestamp !== undefined && (this.date === null || moment(timestamp).isAfter(this.date)); return timestamp !== undefined && (this.date === null || moment(timestamp).isSameOrAfter(this.date));
} }
/** /**

View File

@ -157,6 +157,22 @@ function completeMultipartUpload(authInfo, request, log, callback) {
} }
return next(errors.MalformedXML, destBucket); return next(errors.MalformedXML, destBucket);
}, },
function markOverviewForCompletion(destBucket, objMD, mpuBucket, jsonList,
storedMetadata, location, mpuOverviewKey, next) {
return services.metadataMarkMPObjectForCompletion({
bucketName: mpuBucket.getName(),
objectKey,
uploadId,
splitter,
storedMetadata,
}, log, err => {
if (err) {
return next(err);
}
return next(null, destBucket, objMD, mpuBucket,
jsonList, storedMetadata, location, mpuOverviewKey);
});
},
function retrieveParts(destBucket, objMD, mpuBucket, jsonList, function retrieveParts(destBucket, objMD, mpuBucket, jsonList,
storedMetadata, location, mpuOverviewKey, next) { storedMetadata, location, mpuOverviewKey, next) {
return services.getMPUparts(mpuBucket.getName(), uploadId, log, return services.getMPUparts(mpuBucket.getName(), uploadId, log,

View File

@ -3,6 +3,8 @@ const crypto = require('crypto');
const async = require('async'); const async = require('async');
const { parseString } = require('xml2js'); const { parseString } = require('xml2js');
const { auth, errors, versioning, s3middleware, policies } = require('arsenal'); const { auth, errors, versioning, s3middleware, policies } = require('arsenal');
const { data } = require('../data/wrapper');
const logger = require('../utilities/logger');
const escapeForXml = s3middleware.escapeForXml; const escapeForXml = s3middleware.escapeForXml;
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
@ -191,9 +193,11 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
const skipError = new Error('skip'); const skipError = new Error('skip');
const objectLockedError = new Error('object locked'); const objectLockedError = new Error('object locked');
let deleteFromStorage = [];
// doing 5 requests at a time. note that the data wrapper // doing 5 requests at a time. note that the data wrapper
// will do 5 parallel requests to data backend to delete parts // will do 50 parallel requests to data backend to delete parts
return async.forEachLimit(inPlay, 5, (entry, moveOn) => { return async.forEachLimit(inPlay, 50, (entry, moveOn) => {
async.waterfall([ async.waterfall([
callback => { callback => {
let decodedVersionId; let decodedVersionId;
@ -287,9 +291,14 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
const deleteInfo = {}; const deleteInfo = {};
if (options && options.deleteData) { if (options && options.deleteData) {
deleteInfo.deleted = true; deleteInfo.deleted = true;
return services.deleteObject(bucketName, objMD, return services.deleteObjectWithDeferredStorage(bucketName, objMD,
entry.key, options, log, err => entry.key, options, log, (err, res, objectsToDelete) => {
callback(err, objMD, deleteInfo)); if (err) {
return callback(err);
}
deleteFromStorage = deleteFromStorage.concat(objectsToDelete);
return callback(null, objMD, deleteInfo);
});
} }
deleteInfo.newDeleteMarker = true; deleteInfo.newDeleteMarker = true;
// This call will create a delete-marker // This call will create a delete-marker
@ -340,9 +349,25 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
}, },
// end of forEach func // end of forEach func
err => { err => {
log.trace('finished deleting objects', { numOfObjectsRemoved }); // Batch delete all objects
return next(err, quietSetting, errorResults, numOfObjectsRemoved, const onDone = () => next(err, quietSetting, errorResults, numOfObjectsRemoved,
successfullyDeleted, totalContentLengthDeleted, bucket); successfullyDeleted, totalContentLengthDeleted, bucket);
if (err && deleteFromStorage.length === 0) {
log.trace('no objects to delete from data backend');
return onDone();
}
// If error but we have objects in the list, delete them to ensure
// consistent state.
log.trace('deleting objects from data backend');
const deleteLog = logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
return data.batchDelete(deleteFromStorage, null, null, deleteLog, err => {
if (err) {
log.error('error deleting objects from data backend', { error: err });
return onDone();
}
return onDone();
});
}); });
} }

View File

@ -130,10 +130,12 @@ function objectHead(authInfo, request, log, callback) {
return callback(errors.BadRequest, corsHeaders); return callback(errors.BadRequest, corsHeaders);
} }
const partSize = getPartSize(objMD, partNumber); const partSize = getPartSize(objMD, partNumber);
if (!partSize) { const isEmptyObject = objLength === 0;
if (!partSize && !isEmptyObject) {
return callback(errors.InvalidRange, corsHeaders); return callback(errors.InvalidRange, corsHeaders);
} }
responseHeaders['content-length'] = partSize;
responseHeaders['content-length'] = isEmptyObject ? 0 : partSize;
const partsCount = getPartCountFromMd5(objMD); const partsCount = getPartCountFromMd5(objMD);
if (partsCount) { if (partsCount) {
responseHeaders['x-amz-mp-parts-count'] = partsCount; responseHeaders['x-amz-mp-parts-count'] = partsCount;

View File

@ -236,7 +236,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
res.controllingLocationConstraint; res.controllingLocationConstraint;
return next(null, dataLocator, destBucketMD, return next(null, dataLocator, destBucketMD,
destObjLocationConstraint, copyObjectSize, destObjLocationConstraint, copyObjectSize,
sourceVerId, sourceLocationConstraintName); sourceVerId, sourceLocationConstraintName, splitter);
}); });
}, },
function goGetData( function goGetData(
@ -246,6 +246,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
copyObjectSize, copyObjectSize,
sourceVerId, sourceVerId,
sourceLocationConstraintName, sourceLocationConstraintName,
splitter,
next, next,
) { ) {
data.uploadPartCopy( data.uploadPartCopy(
@ -268,12 +269,12 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
} }
return next(null, destBucketMD, locations, eTag, return next(null, destBucketMD, locations, eTag,
copyObjectSize, sourceVerId, serverSideEncryption, copyObjectSize, sourceVerId, serverSideEncryption,
lastModified); lastModified, splitter);
}); });
}, },
function getExistingPartInfo(destBucketMD, locations, totalHash, function getExistingPartInfo(destBucketMD, locations, totalHash,
copyObjectSize, sourceVerId, serverSideEncryption, lastModified, copyObjectSize, sourceVerId, serverSideEncryption, lastModified,
next) { splitter, next) {
const partKey = const partKey =
`${uploadId}${constants.splitter}${paddedPartNumber}`; `${uploadId}${constants.splitter}${paddedPartNumber}`;
metadata.getObjectMD(mpuBucketName, partKey, {}, log, metadata.getObjectMD(mpuBucketName, partKey, {}, log,
@ -298,12 +299,12 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
} }
return next(null, destBucketMD, locations, totalHash, return next(null, destBucketMD, locations, totalHash,
prevObjectSize, copyObjectSize, sourceVerId, prevObjectSize, copyObjectSize, sourceVerId,
serverSideEncryption, lastModified, oldLocations); serverSideEncryption, lastModified, oldLocations, splitter);
}); });
}, },
function storeNewPartMetadata(destBucketMD, locations, totalHash, function storeNewPartMetadata(destBucketMD, locations, totalHash,
prevObjectSize, copyObjectSize, sourceVerId, serverSideEncryption, prevObjectSize, copyObjectSize, sourceVerId, serverSideEncryption,
lastModified, oldLocations, next) { lastModified, oldLocations, splitter, next) {
const metaStoreParams = { const metaStoreParams = {
partNumber: paddedPartNumber, partNumber: paddedPartNumber,
contentMD5: totalHash, contentMD5: totalHash,
@ -319,20 +320,58 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
{ error: err, method: 'storeNewPartMetadata' }); { error: err, method: 'storeNewPartMetadata' });
return next(err); return next(err);
} }
return next(null, oldLocations, destBucketMD, totalHash, return next(null, locations, oldLocations, destBucketMD, totalHash,
lastModified, sourceVerId, serverSideEncryption, lastModified, sourceVerId, serverSideEncryption,
prevObjectSize, copyObjectSize); prevObjectSize, copyObjectSize, splitter);
}); });
}, },
function cleanupExistingData(oldLocations, destBucketMD, totalHash, function checkCanDeleteOldLocations(partLocations, oldLocations, destBucketMD,
totalHash, lastModified, sourceVerId, serverSideEncryption,
prevObjectSize, copyObjectSize, splitter, next) {
if (!oldLocations) {
return next(null, oldLocations, destBucketMD, totalHash,
lastModified, sourceVerId, serverSideEncryption,
prevObjectSize, copyObjectSize);
}
return services.isCompleteMPUInProgress({
bucketName: destBucketName,
objectKey: destObjectKey,
uploadId,
splitter,
}, log, (err, completeInProgress) => {
if (err) {
return next(err, destBucketMD);
}
let oldLocationsToDelete = oldLocations;
// Prevent deletion of old data if a completeMPU
// is already in progress because then there is no
// guarantee that the old location will not be the
// committed one.
if (completeInProgress) {
log.warn('not deleting old locations because CompleteMPU is in progress', {
method: 'objectPutCopyPart::checkCanDeleteOldLocations',
bucketName: destBucketName,
objectKey: destObjectKey,
uploadId,
partLocations,
oldLocations,
});
oldLocationsToDelete = null;
}
return next(null, oldLocationsToDelete, destBucketMD, totalHash,
lastModified, sourceVerId, serverSideEncryption,
prevObjectSize, copyObjectSize);
});
},
function cleanupExistingData(oldLocationsToDelete, destBucketMD, totalHash,
lastModified, sourceVerId, serverSideEncryption, lastModified, sourceVerId, serverSideEncryption,
prevObjectSize, copyObjectSize, next) { prevObjectSize, copyObjectSize, next) {
// Clean up the old data now that new metadata (with new // Clean up the old data now that new metadata (with new
// data locations) has been stored // data locations) has been stored
if (oldLocations) { if (oldLocationsToDelete) {
const delLog = logger.newRequestLoggerFromSerializedUids( const delLog = logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids()); log.getSerializedUids());
return data.batchDelete(oldLocations, request.method, null, return data.batchDelete(oldLocationsToDelete, request.method, null,
delLog, err => { delLog, err => {
if (err) { if (err) {
// if error, log the error and move on as it is not // if error, log the error and move on as it is not

View File

@ -13,6 +13,7 @@ const kms = require('../kms/wrapper');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const logger = require('../utilities/logger'); const logger = require('../utilities/logger');
const services = require('../services');
const locationConstraintCheck const locationConstraintCheck
= require('./apiUtils/object/locationConstraintCheck'); = require('./apiUtils/object/locationConstraintCheck');
const writeContinue = require('../utilities/writeContinue'); const writeContinue = require('../utilities/writeContinue');
@ -243,19 +244,19 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
} }
return next(null, destinationBucket, return next(null, destinationBucket,
objectLocationConstraint, cipherBundle, objectLocationConstraint, cipherBundle,
partKey, prevObjectSize, oldLocations, partInfo); partKey, prevObjectSize, oldLocations, partInfo, splitter);
}); });
}, },
// Store in data backend. // Store in data backend.
(destinationBucket, objectLocationConstraint, cipherBundle, (destinationBucket, objectLocationConstraint, cipherBundle,
partKey, prevObjectSize, oldLocations, partInfo, next) => { partKey, prevObjectSize, oldLocations, partInfo, splitter, next) => {
// NOTE: set oldLocations to null so we do not batchDelete for now // NOTE: set oldLocations to null so we do not batchDelete for now
if (partInfo && partInfo.dataStoreType === 'azure') { if (partInfo && partInfo.dataStoreType === 'azure') {
// skip to storing metadata // skip to storing metadata
return next(null, destinationBucket, partInfo, return next(null, destinationBucket, partInfo,
partInfo.dataStoreETag, partInfo.dataStoreETag,
cipherBundle, partKey, prevObjectSize, null, cipherBundle, partKey, prevObjectSize, null,
objectLocationConstraint); objectLocationConstraint, splitter);
} }
const objectContext = { const objectContext = {
bucketName, bucketName,
@ -275,12 +276,13 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
} }
return next(null, destinationBucket, dataGetInfo, hexDigest, return next(null, destinationBucket, dataGetInfo, hexDigest,
cipherBundle, partKey, prevObjectSize, oldLocations, cipherBundle, partKey, prevObjectSize, oldLocations,
objectLocationConstraint); objectLocationConstraint, splitter);
}); });
}, },
// Store data locations in metadata and delete any overwritten data. // Store data locations in metadata and delete any overwritten
// data if completeMPU hasn't been initiated yet.
(destinationBucket, dataGetInfo, hexDigest, cipherBundle, partKey, (destinationBucket, dataGetInfo, hexDigest, cipherBundle, partKey,
prevObjectSize, oldLocations, objectLocationConstraint, next) => { prevObjectSize, oldLocations, objectLocationConstraint, splitter, next) => {
// Use an array to be consistent with objectPutCopyPart where there // Use an array to be consistent with objectPutCopyPart where there
// could be multiple locations. // could be multiple locations.
const partLocations = [dataGetInfo]; const partLocations = [dataGetInfo];
@ -310,19 +312,54 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
}); });
return next(err, destinationBucket); return next(err, destinationBucket);
} }
return next(null, oldLocations, objectLocationConstraint, return next(null, partLocations, oldLocations, objectLocationConstraint,
destinationBucket, hexDigest, prevObjectSize); destinationBucket, hexDigest, prevObjectSize, splitter);
}); });
}, },
(partLocations, oldLocations, objectLocationConstraint, destinationBucket,
hexDigest, prevObjectSize, splitter, next) => {
if (!oldLocations) {
return next(null, oldLocations, objectLocationConstraint,
destinationBucket, hexDigest, prevObjectSize);
}
return services.isCompleteMPUInProgress({
bucketName,
objectKey,
uploadId,
splitter,
}, log, (err, completeInProgress) => {
if (err) {
return next(err, destinationBucket);
}
let oldLocationsToDelete = oldLocations;
// Prevent deletion of old data if a completeMPU
// is already in progress because then there is no
// guarantee that the old location will not be the
// committed one.
if (completeInProgress) {
log.warn('not deleting old locations because CompleteMPU is in progress', {
method: 'objectPutPart::metadata.getObjectMD',
bucketName,
objectKey,
uploadId,
partLocations,
oldLocations,
});
oldLocationsToDelete = null;
}
return next(null, oldLocationsToDelete, objectLocationConstraint,
destinationBucket, hexDigest, prevObjectSize);
});
},
// Clean up any old data now that new metadata (with new // Clean up any old data now that new metadata (with new
// data locations) has been stored. // data locations) has been stored.
(oldLocations, objectLocationConstraint, destinationBucket, hexDigest, (oldLocationsToDelete, objectLocationConstraint, destinationBucket, hexDigest,
prevObjectSize, next) => { prevObjectSize, next) => {
if (oldLocations) { if (oldLocationsToDelete) {
log.trace('overwriting mpu part, deleting data'); log.trace('overwriting mpu part, deleting data');
const delLog = logger.newRequestLoggerFromSerializedUids( const delLog = logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids()); log.getSerializedUids());
return data.batchDelete(oldLocations, request.method, return data.batchDelete(oldLocationsToDelete, request.method,
objectLocationConstraint, delLog, err => { objectLocationConstraint, delLog, err => {
if (err) { if (err) {
// if error, log the error and move on as it is not // if error, log the error and move on as it is not

View File

@ -315,6 +315,57 @@ const services = {
}); });
}, },
/**
* Deletes objects from a bucket, but do not delete it from storage yet
* @param {string} bucketName - bucket in which objectMD is stored
* @param {object} objectMD - object's metadata
* @param {string} objectKey - object key name
* @param {object} options - other instructions, such as { versionId } to
* delete a specific version of the object
* @param {Log} log - logger instance
* @param {function} cb - callback from async.waterfall in objectGet
* @return {undefined}
*/
deleteObjectWithDeferredStorage(bucketName, objectMD, objectKey, options, log, cb) {
log.trace('deleting object from bucket');
assert.strictEqual(typeof bucketName, 'string');
assert.strictEqual(typeof objectMD, 'object');
function deleteMDandData() {
return metadata.deleteObjectMD(bucketName, objectKey, options, log,
(err, res) => {
if (err) {
return cb(err, res);
}
log.trace('deleteObject: metadata delete OK');
if (objectMD.location === null) {
return cb(null, res);
}
// We store all objects to delete, for each location
// separately.
let objectToDeleteFromStorage = [];
if (!Array.isArray(objectMD.location)) {
objectToDeleteFromStorage.push(objectMD.location);
} else {
objectToDeleteFromStorage = objectMD.location;
}
return cb(null, res, objectToDeleteFromStorage);
});
}
const objGetInfo = objectMD.location;
// special case that prevents azure blocks from unecessary deletion
// will return null if no need
return data.protectAzureBlocks(bucketName, objectKey, objGetInfo,
log, err => {
if (err) {
return cb(err);
}
return deleteMDandData();
});
},
/** /**
* Gets list of objects in bucket * Gets list of objects in bucket
* @param {object} bucketName - bucket in which objectMetadata is stored * @param {object} bucketName - bucket in which objectMetadata is stored
@ -451,6 +502,80 @@ const services = {
}); });
}, },
/**
* Mark the MPU overview key with a flag when starting the
* CompleteMPU operation, to be checked by "put part" operations
*
* @param {object} params - params object
* @param {string} params.bucketName - name of MPU bucket
* @param {string} params.objectKey - object key
* @param {string} params.uploadId - upload ID
* @param {string} params.splitter - splitter for this overview key
* @param {object} params.storedMetadata - original metadata of the overview key
* @param {Logger} log - Logger object
* @param {function} cb - callback(err)
* @return {undefined}
*/
metadataMarkMPObjectForCompletion(params, log, cb) {
assert.strictEqual(typeof params, 'object');
assert.strictEqual(typeof params.bucketName, 'string');
assert.strictEqual(typeof params.objectKey, 'string');
assert.strictEqual(typeof params.uploadId, 'string');
assert.strictEqual(typeof params.splitter, 'string');
assert.strictEqual(typeof params.storedMetadata, 'object');
const splitter = params.splitter;
const longMPUIdentifier =
`overview${splitter}${params.objectKey}${splitter}${params.uploadId}`;
const multipartObjectMD = Object.assign({}, params.storedMetadata);
multipartObjectMD.completeInProgress = true;
metadata.putObjectMD(params.bucketName, longMPUIdentifier, multipartObjectMD,
{}, log, err => {
if (err) {
log.error('error from metadata', { error: err });
return cb(err);
}
return cb();
});
},
/**
* Returns if a CompleteMPU operation is in progress for this
* object, by looking at the `completeInProgress` flag stored in
* the overview key
*
* @param {object} params - params object
* @param {string} params.bucketName - bucket name where object should be stored
* @param {string} params.objectKey - object key
* @param {string} params.uploadId - upload ID
* @param {string} params.splitter - splitter for this overview key
* @param {object} log - request logger instance
* @param {function} cb - callback(err, {bool} completeInProgress)
* @return {undefined}
*/
isCompleteMPUInProgress(params, log, cb) {
assert.strictEqual(typeof params, 'object');
assert.strictEqual(typeof params.bucketName, 'string');
assert.strictEqual(typeof params.objectKey, 'string');
assert.strictEqual(typeof params.uploadId, 'string');
assert.strictEqual(typeof params.splitter, 'string');
const mpuBucketName = `${constants.mpuBucketPrefix}${params.bucketName}`;
const splitter = params.splitter;
const mpuOverviewKey =
`overview${splitter}${params.objectKey}${splitter}${params.uploadId}`;
return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log,
(err, res) => {
if (err) {
log.error('error getting the overview object from mpu bucket', {
error: err,
method: 'services.isCompleteMPUInProgress',
params,
});
return cb(err);
}
return cb(null, Boolean(res.completeInProgress));
});
},
/** /**
* Checks whether bucket exists, multipart upload * Checks whether bucket exists, multipart upload

View File

@ -1,6 +1,6 @@
{ {
"name": "s3", "name": "s3",
"version": "7.70.11", "version": "7.70.11-2",
"description": "S3 connector", "description": "S3 connector",
"main": "index.js", "main": "index.js",
"engines": { "engines": {

View File

@ -210,5 +210,39 @@ describe('Complete MPU', () => {
}); });
}); });
}); });
describe('with re-upload of part during CompleteMPU execution', () => {
let uploadId;
let eTag;
beforeEach(() => _initiateMpuAndPutOnePart()
.then(result => {
uploadId = result.uploadId;
eTag = result.eTag;
})
);
it('should complete the MPU successfully and leave a readable object', done => {
async.parallel([
doneReUpload => s3.uploadPart({
Bucket: bucket,
Key: key,
PartNumber: 1,
UploadId: uploadId,
Body: 'foo',
}, err => {
// in case the CompleteMPU finished earlier,
// we may get a NoSuchKey error, so just
// ignore it
if (err && err.code === 'NoSuchKey') {
return doneReUpload();
}
return doneReUpload(err);
}),
doneComplete => _completeMpuAndCheckVid(
uploadId, eTag, undefined, doneComplete),
], done);
});
});
}); });
}); });

View File

@ -577,6 +577,72 @@ describe('Object Part Copy', () => {
checkNoError(err); checkNoError(err);
}); });
}); });
it('should not corrupt object if overwriting an existing part by copying a part ' +
'while the MPU is being completed', () => {
// AWS response etag for this completed MPU
const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"';
process.stdout.write('Putting first part in MPU test');
return s3.uploadPartCopy({ Bucket: destBucketName,
Key: destObjName,
CopySource: `${sourceBucketName}/${sourceObjName}`,
PartNumber: 1,
UploadId: uploadId,
}).promise().then(res => {
assert.strictEqual(res.ETag, etag);
assert(res.LastModified);
}).then(() => {
process.stdout.write('Overwriting first part in MPU test and completing MPU ' +
'at the same time');
return Promise.all([
s3.uploadPartCopy({
Bucket: destBucketName,
Key: destObjName,
CopySource: `${sourceBucketName}/${sourceObjName}`,
PartNumber: 1,
UploadId: uploadId,
}).promise().catch(err => {
// in case the CompleteMPU finished
// earlier, we may get a NoSuchKey error,
// so just ignore it and resolve with a
// special value, otherwise re-throw the
// error
if (err && err.code === 'NoSuchKey') {
return Promise.resolve(null);
}
throw err;
}),
s3.completeMultipartUpload({
Bucket: destBucketName,
Key: destObjName,
UploadId: uploadId,
MultipartUpload: {
Parts: [
{ ETag: etag, PartNumber: 1 },
],
},
}).promise(),
]);
}).then(([uploadRes, completeRes]) => {
// if upload succeeded before CompleteMPU finished
if (uploadRes !== null) {
assert.strictEqual(uploadRes.ETag, etag);
assert(uploadRes.LastModified);
}
assert.strictEqual(completeRes.Bucket, destBucketName);
assert.strictEqual(completeRes.Key, destObjName);
assert.strictEqual(completeRes.ETag, finalObjETag);
}).then(() => {
process.stdout.write('Getting object put by MPU with ' +
'overwrite part');
return s3.getObject({
Bucket: destBucketName,
Key: destObjName,
}).promise();
}).then(res => {
assert.strictEqual(res.ETag, finalObjETag);
});
});
}); });
it('should return an error if no such upload initiated', it('should return an error if no such upload initiated',

View File

@ -3,18 +3,7 @@ const async = require('async');
const withV4 = require('../support/withV4'); const withV4 = require('../support/withV4');
const BucketUtility = require('../../lib/utility/bucket-util'); const BucketUtility = require('../../lib/utility/bucket-util');
const { maximumAllowedPartCount } = require('../../../../../constants'); const objectConfigs = require('../support/objectConfigs');
const bucket = 'mpu-test-bucket';
const object = 'mpu-test-object';
const bodySize = 1024 * 1024 * 5;
const bodyContent = 'a';
const howManyParts = 3;
const partNumbers = Array.from(Array(howManyParts).keys());
const invalidPartNumbers = [-1, 0, maximumAllowedPartCount + 1];
let ETags = [];
function checkError(err, statusCode, code) { function checkError(err, statusCode, code) {
assert.strictEqual(err.statusCode, statusCode); assert.strictEqual(err.statusCode, statusCode);
@ -26,128 +15,154 @@ function checkNoError(err) {
`Expected success, got error ${JSON.stringify(err)}`); `Expected success, got error ${JSON.stringify(err)}`);
} }
function generateContent(partNumber) { function generateContent(size, bodyContent) {
return Buffer.alloc(bodySize + partNumber, bodyContent); return Buffer.alloc(size, bodyContent);
} }
describe('Part size tests with object head', () => { describe('Part size tests with object head', () => {
withV4(sigCfg => { objectConfigs.forEach(config => {
let bucketUtil; describe(config.signature, () => {
let s3; let ETags = [];
function headObject(fields, cb) { const {
s3.headObject(Object.assign({ bucket,
Bucket: bucket, object,
Key: object, bodySize,
}, fields), cb); bodyContent,
} partNumbers,
invalidPartNumbers,
} = config;
beforeEach(function beforeF(done) { withV4(sigCfg => { //eslint-disable-line
bucketUtil = new BucketUtility('default', sigCfg); let bucketUtil;
s3 = bucketUtil.s3; let s3;
async.waterfall([ beforeEach(function beforeF(done) {
next => s3.createBucket({ Bucket: bucket }, err => next(err)), bucketUtil = new BucketUtility('default', sigCfg);
next => s3.createMultipartUpload({ Bucket: bucket, s3 = bucketUtil.s3;
Key: object }, (err, data) => {
checkNoError(err);
this.currentTest.UploadId = data.UploadId;
return next();
}),
next => async.mapSeries(partNumbers, (partNumber, callback) => {
const uploadPartParams = {
Bucket: bucket,
Key: object,
PartNumber: partNumber + 1,
UploadId: this.currentTest.UploadId,
Body: generateContent(partNumber + 1),
};
return s3.uploadPart(uploadPartParams, async.waterfall([
(err, data) => { next => s3.createBucket({ Bucket: bucket }, err => next(err)),
if (err) { next => s3.createMultipartUpload({ Bucket: bucket,
return callback(err); Key: object }, (err, data) => {
checkNoError(err);
this.currentTest.UploadId = data.UploadId;
return next();
}),
next => async.mapSeries(partNumbers, (partNumber, callback) => {
let allocAmount = bodySize + partNumber + 1;
if (config.signature === 'for empty object') {
allocAmount = 0;
} }
return callback(null, data.ETag); const uploadPartParams = {
}); Bucket: bucket,
}, (err, results) => { Key: object,
checkNoError(err);
ETags = results;
return next();
}),
next => {
const params = {
Bucket: bucket,
Key: object,
MultipartUpload: {
Parts: partNumbers.map(partNumber => ({
ETag: ETags[partNumber],
PartNumber: partNumber + 1, PartNumber: partNumber + 1,
})), UploadId: this.currentTest.UploadId,
Body: generateContent(allocAmount, bodyContent),
};
return s3.uploadPart(uploadPartParams,
(err, data) => {
if (err) {
return callback(err);
}
return callback(null, data.ETag);
});
}, (err, results) => {
checkNoError(err);
ETags = results;
return next();
}),
next => {
const params = {
Bucket: bucket,
Key: object,
MultipartUpload: {
Parts: partNumbers.map(partNumber => ({
ETag: ETags[partNumber],
PartNumber: partNumber + 1,
})),
},
UploadId: this.currentTest.UploadId,
};
return s3.completeMultipartUpload(params, next);
}, },
UploadId: this.currentTest.UploadId, ], err => {
}; checkNoError(err);
return s3.completeMultipartUpload(params, next); done();
}, });
], err => { });
checkNoError(err);
done();
});
});
afterEach(done => { afterEach(done => {
async.waterfall([ async.waterfall([
next => s3.deleteObject({ Bucket: bucket, Key: object }, next => s3.deleteObject({ Bucket: bucket, Key: object },
err => next(err)), err => next(err)),
next => s3.deleteBucket({ Bucket: bucket }, err => next(err)), next => s3.deleteBucket({ Bucket: bucket }, err => next(err)),
], done); ], done);
}); });
it('should return the total size of the object ' + it('should return the total size of the object ' +
'when --part-number is not used', done => { 'when --part-number is not used', done => {
const totalSize = partNumbers.reduce((total, current) => const totalSize = config.meta.computeTotalSize(partNumbers, bodySize);
total + (bodySize + current + 1), 0);
headObject({}, (err, data) => {
checkNoError(err);
assert.equal(totalSize, data.ContentLength);
done();
});
});
partNumbers.forEach(part => { s3.headObject({ Bucket: bucket, Key: object }, (err, data) => {
it(`should return the size of part ${part + 1} ` + checkNoError(err);
`when --part-number is set to ${part + 1}`, done => {
const partNumber = Number.parseInt(part, 0) + 1; assert.equal(totalSize, data.ContentLength);
const partSize = bodySize + partNumber; done();
headObject({ PartNumber: partNumber }, (err, data) => { });
checkNoError(err); });
assert.equal(partSize, data.ContentLength);
done(); partNumbers.forEach(part => {
it(`should return the size of part ${part + 1} ` +
`when --part-number is set to ${part + 1}`, done => {
const partNumber = Number.parseInt(part, 0) + 1;
const partSize = bodySize + partNumber;
s3.headObject({ Bucket: bucket, Key: object, PartNumber: partNumber }, (err, data) => {
checkNoError(err);
if (data.ContentLength === 0) {
done();
}
assert.equal(partSize, data.ContentLength);
done();
});
});
});
invalidPartNumbers.forEach(part => {
it(`should return an error when --part-number is set to ${part}`,
done => {
s3.headObject({ Bucket: bucket, Key: object, PartNumber: part }, (err, data) => {
checkError(err, 400, 'BadRequest');
assert.strictEqual(data, null);
done();
});
});
});
it('when incorrect --part-number is used', done => {
bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3;
s3.headObject({ Bucket: bucket, Key: object, PartNumber: partNumbers.length + 1 },
(err, data) => {
if (config.meta.objectIsEmpty) {
// returns metadata for the only empty part
checkNoError(err);
assert.strictEqual(data.ContentLength, 0);
done();
} else {
// returns a 416 error
// the error response does not contain the actual
// statusCode instead it has '416'
checkError(err, 416, 416);
assert.strictEqual(data, null);
done();
}
});
}); });
}); });
}); });
invalidPartNumbers.forEach(part => {
it(`should return an error when --part-number is set to ${part}`,
done => {
headObject({ PartNumber: part }, (err, data) => {
checkError(err, 400, 'BadRequest');
assert.strictEqual(data, null);
done();
});
});
});
it('should return an error when incorrect --part-number is used',
done => {
headObject({ PartNumber: partNumbers.length + 1 },
(err, data) => {
// the error response does not contain the actual
// statusCode instead it has '416'
checkError(err, 416, 416);
assert.strictEqual(data, null);
done();
});
});
}); });
}); });

View File

@ -0,0 +1,40 @@
const { maximumAllowedPartCount } = require('../../../../../constants');
const canonicalObjectConfig = {
bucket: 'mpu-test-bucket-canonical-object',
object: 'mpu-test-object-canonical',
bodySize: 1024 * 1024 * 5,
bodyContent: 'a',
howManyParts: 3,
partNumbers: Array.from(Array(3).keys()), // 3 corresponds to howManyParts
invalidPartNumbers: [-1, 0, maximumAllowedPartCount + 1],
signature: 'for canonical object',
meta: {
computeTotalSize: (partNumbers, bodySize) => partNumbers.reduce((total, current) =>
total + bodySize + current + 1
, 0),
objectIsEmpty: false,
},
};
const emptyObjectConfig = {
bucket: 'mpu-test-bucket-empty-object',
object: 'mpu-test-object-empty',
bodySize: 0,
bodyContent: null,
howManyParts: 1,
partNumbers: Array.from(Array(1).keys()), // 1 corresponds to howManyParts
invalidPartNumbers: [-1, 0, maximumAllowedPartCount + 1],
signature: 'for empty object',
meta: {
computeTotalSize: () => 0,
objectIsEmpty: true,
},
};
const objectConfigs = [
canonicalObjectConfig,
emptyObjectConfig,
];
module.exports = objectConfigs;

View File

@ -27,6 +27,7 @@ const testData = 'testkey data';
const testDataMd5 = crypto.createHash('md5') const testDataMd5 = crypto.createHash('md5')
.update(testData, 'utf-8') .update(testData, 'utf-8')
.digest('hex'); .digest('hex');
const emptyContentsMd5 = 'd41d8cd98f00b204e9800998ecf8427e';
const testMd = { const testMd = {
'md-model-version': 2, 'md-model-version': 2,
'owner-display-name': 'Bart', 'owner-display-name': 'Bart',
@ -60,6 +61,17 @@ const testMd = {
}, },
}; };
function checkObjectData(s3, objectKey, dataValue, done) {
s3.getObject({
Bucket: TEST_BUCKET,
Key: objectKey,
}, (err, data) => {
assert.ifError(err);
assert.strictEqual(data.Body.toString(), dataValue);
done();
});
}
/** makeBackbeatRequest - utility function to generate a request going /** makeBackbeatRequest - utility function to generate a request going
* through backbeat route * through backbeat route
* @param {object} params - params for making request * @param {object} params - params for making request
@ -416,7 +428,8 @@ describeSkipIfAWS('backbeat routes', () => {
}); });
}); });
it('should remove old object data locations if version is overwritten', done => { it('should remove old object data locations if version is overwritten ' +
'with same contents', done => {
let oldLocation; let oldLocation;
const testKeyOldData = `${testKey}-old-data`; const testKeyOldData = `${testKey}-old-data`;
async.waterfall([next => { async.waterfall([next => {
@ -496,14 +509,96 @@ describeSkipIfAWS('backbeat routes', () => {
}, (response, next) => { }, (response, next) => {
assert.strictEqual(response.statusCode, 200); assert.strictEqual(response.statusCode, 200);
// give some time for the async deletes to complete // give some time for the async deletes to complete
setTimeout(() => s3.getObject({ setTimeout(() => checkObjectData(s3, testKey, testData, next),
1000);
}, next => {
// check that the object copy referencing the old data
// locations is unreadable, confirming that the old
// data locations have been deleted
s3.getObject({
Bucket: TEST_BUCKET, Bucket: TEST_BUCKET,
Key: testKey, Key: testKeyOldData,
}, (err, data) => { }, err => {
assert.ifError(err); assert(err, 'expected error to get object with old data ' +
assert.strictEqual(data.Body.toString(), testData); 'locations, got success');
next(); next();
}), 1000); });
}], err => {
assert.ifError(err);
done();
});
});
it('should remove old object data locations if version is overwritten ' +
'with empty contents', done => {
let oldLocation;
const testKeyOldData = `${testKey}-old-data`;
async.waterfall([next => {
// put object's data locations
makeBackbeatRequest({
method: 'PUT', bucket: TEST_BUCKET,
objectKey: testKey,
resourceType: 'data',
headers: {
'content-length': testData.length,
'content-md5': testDataMd5,
'x-scal-canonical-id': testArn,
},
authCredentials: backbeatAuthCredentials,
requestBody: testData }, next);
}, (response, next) => {
assert.strictEqual(response.statusCode, 200);
// put object metadata
const newMd = Object.assign({}, testMd);
newMd.location = JSON.parse(response.body);
oldLocation = newMd.location;
makeBackbeatRequest({
method: 'PUT', bucket: TEST_BUCKET,
objectKey: testKey,
resourceType: 'metadata',
queryObj: {
versionId: versionIdUtils.encode(testMd.versionId),
},
authCredentials: backbeatAuthCredentials,
requestBody: JSON.stringify(newMd),
}, next);
}, (response, next) => {
assert.strictEqual(response.statusCode, 200);
// put another object which metadata reference the
// same data locations, we will attempt to retrieve
// this object at the end of the test to confirm that
// its locations have been deleted
const oldDataMd = Object.assign({}, testMd);
oldDataMd.location = oldLocation;
makeBackbeatRequest({
method: 'PUT', bucket: TEST_BUCKET,
objectKey: testKeyOldData,
resourceType: 'metadata',
authCredentials: backbeatAuthCredentials,
requestBody: JSON.stringify(oldDataMd),
}, next);
}, (response, next) => {
assert.strictEqual(response.statusCode, 200);
// overwrite the original object version with an empty location
const newMd = Object.assign({}, testMd);
newMd['content-length'] = 0;
newMd['content-md5'] = emptyContentsMd5;
newMd.location = null;
makeBackbeatRequest({
method: 'PUT', bucket: TEST_BUCKET,
objectKey: testKey,
resourceType: 'metadata',
queryObj: {
versionId: versionIdUtils.encode(testMd.versionId),
},
authCredentials: backbeatAuthCredentials,
requestBody: JSON.stringify(newMd),
}, next);
}, (response, next) => {
assert.strictEqual(response.statusCode, 200);
// give some time for the async deletes to complete
setTimeout(() => checkObjectData(s3, testKey, '', next),
1000);
}, next => { }, next => {
// check that the object copy referencing the old data // check that the object copy referencing the old data
// locations is unreadable, confirming that the old // locations is unreadable, confirming that the old

View File

@ -3,14 +3,15 @@ const assert = require('assert');
const BucketInfo = require('arsenal').models.BucketInfo; const BucketInfo = require('arsenal').models.BucketInfo;
const getReplicationInfo = const getReplicationInfo =
require('../../../../lib/api/apiUtils/object/getReplicationInfo'); require('../../../../lib/api/apiUtils/object/getReplicationInfo');
const { makeAuthInfo } = require('../../helpers');
function _getObjectReplicationInfo(replicationConfig) { function _getObjectReplicationInfo(replicationConfig, authInfo, isDeleteMarker) {
const bucketInfo = new BucketInfo( const bucketInfo = new BucketInfo(
'testbucket', 'someCanonicalId', 'accountDisplayName', 'testbucket', 'someCanonicalId', 'accountDisplayName',
new Date().toJSON(), new Date().toJSON(),
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
replicationConfig); replicationConfig);
return getReplicationInfo('fookey', bucketInfo, true, 123, null, null); return getReplicationInfo('fookey', bucketInfo, true, 123, null, null, authInfo, isDeleteMarker);
} }
describe('getReplicationInfo helper', () => { describe('getReplicationInfo helper', () => {
@ -40,6 +41,65 @@ describe('getReplicationInfo helper', () => {
}); });
}); });
it('should get replication info when action comming from a non-lifecycle session', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
rules: [{
prefix: '',
enabled: true,
storageClass: 'awsbackend',
}],
destination: 'tosomewhere',
};
const authInfo = makeAuthInfo('accessKey1', null, 'another-session');
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, true);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
backends: [{
site: 'awsbackend',
status: 'PENDING',
dataStoreVersionId: '',
}],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3',
});
});
it('should get replication info when action comming from a lifecycle session ' +
'but action is not delete marker', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
rules: [{
prefix: '',
enabled: true,
storageClass: 'awsbackend',
}],
destination: 'tosomewhere',
};
const authInfo = makeAuthInfo('accessKey1', null, 'backbeat-lifecycle');
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, false);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
backends: [{
site: 'awsbackend',
status: 'PENDING',
dataStoreVersionId: '',
}],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3',
});
});
it('should not get replication info when rules are disabled', () => { it('should not get replication info when rules are disabled', () => {
const replicationConfig = { const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role', role: 'arn:aws:iam::root:role/s3-replication-role',
@ -53,4 +113,21 @@ describe('getReplicationInfo helper', () => {
const replicationInfo = _getObjectReplicationInfo(replicationConfig); const replicationInfo = _getObjectReplicationInfo(replicationConfig);
assert.deepStrictEqual(replicationInfo, undefined); assert.deepStrictEqual(replicationInfo, undefined);
}); });
it('should not get replication info when action comming from lifecycle session', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
rules: [{
prefix: '',
enabled: true,
storageClass: 'awsbackend',
}],
destination: 'tosomewhere',
};
const authInfo = makeAuthInfo('accessKey1', null, 'backbeat-lifecycle');
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, true);
assert.deepStrictEqual(replicationInfo, undefined);
});
}); });

View File

@ -38,4 +38,16 @@ describe('Check if location keys have changed between object locations', () => {
const curr = [{ key: 'ddd' }, { key: 'eee' }, { key: 'fff' }]; const curr = [{ key: 'ddd' }, { key: 'eee' }, { key: 'fff' }];
assert.strictEqual(locationKeysHaveChanged(prev, curr), true); assert.strictEqual(locationKeysHaveChanged(prev, curr), true);
}); });
it('should return true if curr location is null', () => {
const prev = [{ key: 'ddd' }, { key: 'eee' }, { key: 'fff' }];
const curr = null;
assert.strictEqual(locationKeysHaveChanged(prev, curr), true);
});
it('should return true if both prev and curr locations are null', () => {
const prev = null;
const curr = null;
assert.strictEqual(locationKeysHaveChanged(prev, curr), true);
});
}); });

View File

@ -371,6 +371,19 @@ const policyChangeTestCases = [
allowed: false, allowed: false,
allowedWithBypass: true, allowedWithBypass: true,
}, },
{
desc: 'extending governance policy using same date',
from: {
mode: 'GOVERNANCE',
date: futureDate.toISOString(),
},
to: {
mode: 'GOVERNANCE',
date: futureDate.toISOString(),
},
allowed: true,
allowedWithBypass: true,
},
{ {
desc: 'removing governance policy', desc: 'removing governance policy',
from: { from: {
@ -443,6 +456,19 @@ const policyChangeTestCases = [
allowed: false, allowed: false,
allowedWithBypass: false, allowedWithBypass: false,
}, },
{
desc: 'extending compliance policy with the same date',
from: {
mode: 'COMPLIANCE',
date: futureDate.toISOString(),
},
to: {
mode: 'COMPLIANCE',
date: futureDate.toISOString(),
},
allowed: true,
allowedWithBypass: true,
},
{ {
desc: 'removing compliance policy', desc: 'removing compliance policy',
from: { from: {

View File

@ -0,0 +1,41 @@
const assert = require('assert');
const { isLifecycleSession } =
require('../../../../lib/api/apiUtils/authorization/permissionChecks.js');
const tests = [
{
arn: 'arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle',
description: 'a role assumed by lifecycle service',
expectedResult: true,
},
{
arn: undefined,
description: 'undefined',
expectedResult: false,
},
{
arn: '',
description: 'empty',
expectedResult: false,
},
{
arn: 'arn:aws:iam::257038443293:user/bart',
description: 'a user',
expectedResult: false,
},
{
arn: 'arn:aws:sts::257038443293:assumed-role/rolename/other-service',
description: 'a role assumed by another service',
expectedResult: false,
},
];
describe('authInfoHelper', () => {
tests.forEach(t => {
it(`should return ${t.expectedResult} if arn is ${t.description}`, () => {
const result = isLifecycleSession(t.arn);
assert.equal(result, t.expectedResult);
});
});
});

View File

@ -1641,6 +1641,78 @@ describe('Multipart Upload API', () => {
}); });
}); });
it('should leave orphaned data when overwriting an object part during completeMPU',
done => {
const fullSizedPart = crypto.randomBytes(5 * 1024 * 1024);
const overWritePart = Buffer.from('Overwrite content', 'utf8');
let uploadId;
async.waterfall([
next => bucketPut(authInfo, bucketPutRequest, log, next),
(corsHeaders, next) => initiateMultipartUpload(authInfo,
initiateRequest, log, next),
(result, corsHeaders, next) => parseString(result, next),
(json, next) => {
uploadId = json.InitiateMultipartUploadResult.UploadId[0];
const requestObj = {
bucketName,
namespace,
objectKey,
headers: { host: `${bucketName}.s3.amazonaws.com` },
url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`,
query: {
partNumber: '1',
uploadId,
},
};
const partRequest = new DummyRequest(requestObj, fullSizedPart);
objectPutPart(authInfo, partRequest, undefined, log, (err, partCalculatedHash) => {
assert.deepStrictEqual(err, null);
next(null, requestObj, partCalculatedHash);
});
},
(requestObj, partCalculatedHash, next) => {
assert.deepStrictEqual(ds[1].value, fullSizedPart);
async.parallel([
done => {
const partRequest = new DummyRequest(requestObj, overWritePart);
objectPutPart(authInfo, partRequest, undefined, log, err => {
assert.deepStrictEqual(err, null);
done();
});
},
done => {
const completeBody = '<CompleteMultipartUpload>' +
'<Part>' +
'<PartNumber>1</PartNumber>' +
`<ETag>"${partCalculatedHash}"</ETag>` +
'</Part>' +
'</CompleteMultipartUpload>';
const completeRequest = {
bucketName,
namespace,
objectKey,
parsedHost: 's3.amazonaws.com',
url: `/${objectKey}?uploadId=${uploadId}`,
headers: { host: `${bucketName}.s3.amazonaws.com` },
query: { uploadId },
post: completeBody,
};
completeMultipartUpload(authInfo, completeRequest, log, done);
},
], err => next(err));
},
],
err => {
assert.deepStrictEqual(err, null);
assert.strictEqual(ds[0], undefined);
assert.deepStrictEqual(ds[1].value, fullSizedPart);
assert.deepStrictEqual(ds[2].value, overWritePart);
done();
});
});
it('should throw an error on put of an object part with an invalid ' + it('should throw an error on put of an object part with an invalid ' +
'uploadId', done => { 'uploadId', done => {
const testUploadId = 'invalidUploadID'; const testUploadId = 'invalidUploadID';
@ -1841,12 +1913,22 @@ describe('complete mpu with versioning', () => {
}, },
(eTag, testUploadId, next) => { (eTag, testUploadId, next) => {
const origPutObject = metadataBackend.putObject; const origPutObject = metadataBackend.putObject;
let callCount = 0;
metadataBackend.putObject = metadataBackend.putObject =
(bucketName, objName, objVal, params, log, cb) => { (putBucketName, objName, objVal, params, log, cb) => {
assert.strictEqual(params.replayId, testUploadId); if (callCount === 0) {
metadataBackend.putObject = origPutObject; // first putObject sets the completeInProgress flag in the overview key
metadataBackend.putObject( assert.strictEqual(putBucketName, `${constants.mpuBucketPrefix}${bucketName}`);
bucketName, objName, objVal, params, log, cb); assert.strictEqual(
objName, `overview${splitter}${objectKey}${splitter}${testUploadId}`);
assert.strictEqual(objVal.completeInProgress, true);
} else {
assert.strictEqual(params.replayId, testUploadId);
metadataBackend.putObject = origPutObject;
}
origPutObject(
putBucketName, objName, objVal, params, log, cb);
callCount += 1;
}; };
const parts = [{ partNumber: 1, eTag }]; const parts = [{ partNumber: 1, eTag }];
const completeRequest = _createCompleteMpuRequest(testUploadId, const completeRequest = _createCompleteMpuRequest(testUploadId,
@ -1903,12 +1985,22 @@ describe('complete mpu with versioning', () => {
}, },
(eTag, testUploadId, next) => { (eTag, testUploadId, next) => {
const origPutObject = metadataBackend.putObject; const origPutObject = metadataBackend.putObject;
let callCount = 0;
metadataBackend.putObject = metadataBackend.putObject =
(bucketName, objName, objVal, params, log, cb) => { (putBucketName, objName, objVal, params, log, cb) => {
assert.strictEqual(params.replayId, testUploadId); if (callCount === 0) {
metadataBackend.putObject = origPutObject; // first putObject sets the completeInProgress flag in the overview key
metadataBackend.putObject( assert.strictEqual(putBucketName, `${constants.mpuBucketPrefix}${bucketName}`);
bucketName, objName, objVal, params, log, cb); assert.strictEqual(
objName, `overview${splitter}${objectKey}${splitter}${testUploadId}`);
assert.strictEqual(objVal.completeInProgress, true);
} else {
assert.strictEqual(params.replayId, testUploadId);
metadataBackend.putObject = origPutObject;
}
origPutObject(
putBucketName, objName, objVal, params, log, cb);
callCount += 1;
}; };
const parts = [{ partNumber: 1, eTag }]; const parts = [{ partNumber: 1, eTag }];
const completeRequest = _createCompleteMpuRequest(testUploadId, const completeRequest = _createCompleteMpuRequest(testUploadId,

View File

@ -16,8 +16,8 @@ const bucketName = 'bucketname';
const objectName = 'objectName'; const objectName = 'objectName';
const postBody = Buffer.from('I am a body', 'utf8'); const postBody = Buffer.from('I am a body', 'utf8');
const date = new Date(); const expectedMode = 'GOVERNANCE';
date.setDate(date.getDate() + 1); const expectedDate = moment().add(2, 'days').toISOString();
const bucketPutRequest = { const bucketPutRequest = {
bucketName, bucketName,
@ -36,13 +36,13 @@ const putObjectRequest = new DummyRequest({
const objectRetentionXmlGovernance = '<Retention ' + const objectRetentionXmlGovernance = '<Retention ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' + 'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
'<Mode>GOVERNANCE</Mode>' + '<Mode>GOVERNANCE</Mode>' +
`<RetainUntilDate>${date.toISOString()}</RetainUntilDate>` + `<RetainUntilDate>${expectedDate}</RetainUntilDate>` +
'</Retention>'; '</Retention>';
const objectRetentionXmlCompliance = '<Retention ' + const objectRetentionXmlCompliance = '<Retention ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' + 'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
'<Mode>COMPLIANCE</Mode>' + '<Mode>COMPLIANCE</Mode>' +
`<RetainUntilDate>${moment().add(2, 'days').toISOString()}</RetainUntilDate>` + `<RetainUntilDate>${expectedDate}</RetainUntilDate>` +
'</Retention>'; '</Retention>';
const objectRetentionXmlGovernanceLonger = '<Retention ' + const objectRetentionXmlGovernanceLonger = '<Retention ' +
@ -51,6 +51,12 @@ const objectRetentionXmlGovernanceLonger = '<Retention ' +
`<RetainUntilDate>${moment().add(5, 'days').toISOString()}</RetainUntilDate>` + `<RetainUntilDate>${moment().add(5, 'days').toISOString()}</RetainUntilDate>` +
'</Retention>'; '</Retention>';
const objectRetentionXmlGovernanceShorter = '<Retention ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
'<Mode>GOVERNANCE</Mode>' +
`<RetainUntilDate>${moment().add(1, 'days').toISOString()}</RetainUntilDate>` +
'</Retention>';
const objectRetentionXmlComplianceShorter = '<Retention ' + const objectRetentionXmlComplianceShorter = '<Retention ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' + 'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
'<Mode>COMPLIANCE</Mode>' + '<Mode>COMPLIANCE</Mode>' +
@ -95,8 +101,12 @@ const putObjRetRequestGovernanceLonger = {
post: objectRetentionXmlGovernanceLonger, post: objectRetentionXmlGovernanceLonger,
}; };
const expectedMode = 'GOVERNANCE'; const putObjRetRequestGovernanceShorter = {
const expectedDate = date.toISOString(); bucketName,
objectKey: objectName,
headers: { host: `${bucketName}.s3.amazonaws.com` },
post: objectRetentionXmlGovernanceShorter,
};
describe('putObjectRetention API', () => { describe('putObjectRetention API', () => {
before(() => cleanup()); before(() => cleanup());
@ -178,13 +188,24 @@ describe('putObjectRetention API', () => {
+ 'GOVERNANCE mode is enabled', done => { + 'GOVERNANCE mode is enabled', done => {
objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => {
assert.ifError(err); assert.ifError(err);
return objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { return objectPutRetention(authInfo, putObjRetRequestGovernanceShorter, log, err => {
assert.deepStrictEqual(err, errors.AccessDenied); assert.deepStrictEqual(err, errors.AccessDenied);
done(); done();
}); });
}); });
}); });
it('should allow update if the x-amz-bypass-governance-retention header is missing and '
+ 'GOVERNANCE mode is enabled and the same date is used', done => {
objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => {
assert.ifError(err);
return objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => {
assert.ifError(err);
done();
});
});
});
it('should allow update if the x-amz-bypass-governance-retention header is present and ' it('should allow update if the x-amz-bypass-governance-retention header is present and '
+ 'GOVERNANCE mode is enabled', done => { + 'GOVERNANCE mode is enabled', done => {
objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => {

View File

@ -66,7 +66,7 @@ function timeDiff(startTime) {
return milliseconds; return milliseconds;
} }
function makeAuthInfo(accessKey, userName) { function makeAuthInfo(accessKey, userName, sessionName) {
const canIdMap = { const canIdMap = {
accessKey1: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' accessKey1: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7'
+ 'cd47ef2be', + 'cd47ef2be',
@ -94,6 +94,11 @@ function makeAuthInfo(accessKey, userName) {
params.arn = `arn:aws:iam::${shortid}:user/${userName}`; params.arn = `arn:aws:iam::${shortid}:user/${userName}`;
} }
if (sessionName) {
params.IAMdisplayName = `[assumedRole] rolename:${sessionName}`;
params.arn = `arn:aws:sts::${shortid}:assumed-role/rolename/${sessionName}`;
}
return new AuthInfo(params); return new AuthInfo(params);
} }