Compare commits

...

3 Commits

Author SHA1 Message Date
Nicolas Humbert 4a55b3cdcd MPU version id 2022-05-15 14:24:37 -07:00
Nicolas Humbert 0adb3c495f ++ 2022-05-15 08:51:01 -07:00
Nicolas Humbert 590dfc7ec6 CLDSRV-202 Put object version with x-scal-s3-version-id 2022-05-13 14:59:25 -07:00
8 changed files with 1577 additions and 26 deletions

View File

@ -8,7 +8,7 @@ const services = require('../../../services');
const logger = require('../../../utilities/logger'); const logger = require('../../../utilities/logger');
const { dataStore } = require('./storeObject'); const { dataStore } = require('./storeObject');
const locationConstraintCheck = require('./locationConstraintCheck'); const locationConstraintCheck = require('./locationConstraintCheck');
const { versioningPreprocessing } = require('./versioning'); const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
const removeAWSChunked = require('./removeAWSChunked'); const removeAWSChunked = require('./removeAWSChunked');
const getReplicationInfo = require('./getReplicationInfo'); const getReplicationInfo = require('./getReplicationInfo');
const { config } = require('../../../Config'); const { config } = require('../../../Config');
@ -60,6 +60,9 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params, canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
log, callback) { log, callback) {
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
const size = isDeleteMarker ? 0 : request.parsedContentLength; const size = isDeleteMarker ? 0 : request.parsedContentLength;
// although the request method may actually be 'DELETE' if creating a // although the request method may actually be 'DELETE' if creating a
// delete marker, for our purposes we consider this to be a 'PUT' // delete marker, for our purposes we consider this to be a 'PUT'
@ -257,6 +260,11 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
return next(null, dataGetInfoArr); return next(null, dataGetInfoArr);
}, },
function getVersioningInfo(infoArr, next) { function getVersioningInfo(infoArr, next) {
// if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
if (isPutVersion) {
const options = overwritingVersioning(bucketName, putVersionId, objMD, metadataStoreParams);
return process.nextTick(() => next(null, options, infoArr));
}
return versioningPreprocessing(bucketName, bucketMD, return versioningPreprocessing(bucketName, bucketMD,
metadataStoreParams.objectKey, objMD, log, (err, options) => { metadataStoreParams.objectKey, objMD, log, (err, options) => {
if (err) { if (err) {
@ -278,6 +286,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
metadataStoreParams.isNull = options.isNull; metadataStoreParams.isNull = options.isNull;
metadataStoreParams.nullVersionId = options.nullVersionId; metadataStoreParams.nullVersionId = options.nullVersionId;
metadataStoreParams.nullUploadId = options.nullUploadId; metadataStoreParams.nullUploadId = options.nullUploadId;
metadataStoreParams.masterVersionId = options.masterVersionId;
return _storeInMDandDeleteData(bucketName, infoArr, return _storeInMDandDeleteData(bucketName, infoArr,
cipherBundle, metadataStoreParams, cipherBundle, metadataStoreParams,
options.dataToDelete, requestLogger, requestMethod, next); options.dataToDelete, requestLogger, requestMethod, next);

View File

@ -10,7 +10,32 @@ const versionIdUtils = versioning.VersionID;
const nonVersionedObjId = const nonVersionedObjId =
versionIdUtils.getInfVid(config.replicationGroupId); versionIdUtils.getInfVid(config.replicationGroupId);
/** decodedVidResult - decode the version id from a query object /** decodeVID - decode the version id
* @param {string} versionId - version ID
* @return {(Error|string|undefined)} - return Invalid Argument if decryption
* fails due to improper format, otherwise undefined or the decoded version id
*/
function decodeVID(versionId) {
if (versionId === 'null') {
return versionId;
}
let decoded;
const invalidErr = errors.InvalidArgument.customizeDescription('Invalid version id specified');
try {
decoded = versionIdUtils.decode(versionId);
} catch (err) {
return invalidErr;
}
if (decoded instanceof Error) {
return invalidErr;
}
return decoded;
}
/** decodeVersionId - decode the version id from a query object
* @param {object} [reqQuery] - request query object * @param {object} [reqQuery] - request query object
* @param {string} [reqQuery.versionId] - version ID sent in request query * @param {string} [reqQuery.versionId] - version ID sent in request query
* @return {(Error|string|undefined)} - return Invalid Argument if decryption * @return {(Error|string|undefined)} - return Invalid Argument if decryption
@ -20,16 +45,7 @@ function decodeVersionId(reqQuery) {
if (!reqQuery || !reqQuery.versionId) { if (!reqQuery || !reqQuery.versionId) {
return undefined; return undefined;
} }
let versionId = reqQuery.versionId; return decodeVID(reqQuery.versionId);
if (versionId === 'null') {
return versionId;
}
versionId = versionIdUtils.decode(versionId);
if (versionId instanceof Error) {
return errors.InvalidArgument
.customizeDescription('Invalid version id specified');
}
return versionId;
} }
/** getVersionIdResHeader - return encrypted version ID if appropriate /** getVersionIdResHeader - return encrypted version ID if appropriate
@ -370,6 +386,33 @@ function preprocessingVersioningDelete(bucketName, bucketMD, objectMD,
return callback(null, options); return callback(null, options);
} }
/** overwritingVersioning - return versioning information for S3 to handle
* storing version metadata with a specific version id.
* @param {string} bucketName - name of the bucket.
* @param {string} putVersionId - version id from x-scal-s3-version-id header
* @param {object} objMD - obj metadata
* @param {object} metadataStoreParams - custom built object containing resource details.
* @return {object} options
* options.versionId - specific versionId to overwrite in metadata
* options.isNull - (true/undefined) whether new version is null or not
* options.nullVersionId - if storing a null version in version history, the
* version id of the null version
*/
function overwritingVersioning(bucketName, putVersionId, objMD, metadataStoreParams) {
/* eslint-disable no-param-reassign */
metadataStoreParams.creationTime = objMD['creation-time'];
metadataStoreParams.lastModifiedDate = objMD['last-modified'];
/* eslint-enable no-param-reassign */
const options = objMD.versionId ? { versionId:
objMD.versionId } : {};
options.isNull = objMD.isNull;
options.nullVersionId = objMD.nullVersionId;
return options;
}
module.exports = { module.exports = {
decodeVersionId, decodeVersionId,
getVersionIdResHeader, getVersionIdResHeader,
@ -378,4 +421,6 @@ module.exports = {
getMasterState, getMasterState,
versioningPreprocessing, versioningPreprocessing,
preprocessingVersioningDelete, preprocessingVersioningDelete,
overwritingVersioning,
decodeVID,
}; };

View File

@ -9,7 +9,7 @@ const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
const { data } = require('../data/wrapper'); const { data } = require('../data/wrapper');
const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const constants = require('../../constants'); const constants = require('../../constants');
const { versioningPreprocessing, checkQueryVersionId } const { versioningPreprocessing, checkQueryVersionId, decodeVID, overwritingVersioning }
= require('./apiUtils/object/versioning'); = require('./apiUtils/object/versioning');
const services = require('../services'); const services = require('../services');
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
@ -94,6 +94,21 @@ function completeMultipartUpload(authInfo, request, log, callback) {
let oldByteLength = null; let oldByteLength = null;
const responseHeaders = {}; const responseHeaders = {};
let versionId;
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
if (putVersionId) {
const decodedVidResult = decodeVID(putVersionId);
if (decodedVidResult instanceof Error) {
log.trace('invalid x-scal-s3-version-id header', {
versionId: putVersionId,
error: decodedVidResult,
});
return process.nextTick(() => callback(decodedVidResult));
}
versionId = decodedVidResult;
}
const queryContainsVersionId = checkQueryVersionId(request.query); const queryContainsVersionId = checkQueryVersionId(request.query);
if (queryContainsVersionId instanceof Error) { if (queryContainsVersionId instanceof Error) {
return callback(queryContainsVersionId); return callback(queryContainsVersionId);
@ -119,6 +134,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
// Required permissions for this action // Required permissions for this action
// at the destinationBucket level are same as objectPut // at the destinationBucket level are same as objectPut
requestType: 'objectPut', requestType: 'objectPut',
versionId,
}; };
metadataValidateBucketAndObj(metadataValParams, log, next); metadataValidateBucketAndObj(metadataValParams, log, next);
}, },
@ -126,6 +142,13 @@ function completeMultipartUpload(authInfo, request, log, callback) {
if (objMD) { if (objMD) {
oldByteLength = objMD['content-length']; oldByteLength = objMD['content-length'];
} }
if (isPutVersion && !objMD) {
const err = putVersionId ? errors.NoSuchVersion : errors.NoSuchKey;
log.trace('error no object metadata found', { method: 'objectPut', putVersionId });
return callback(err);
}
services.metadataValidateMultipart(metadataValParams, services.metadataValidateMultipart(metadataValParams,
(err, mpuBucket, mpuOverview, storedMetadata) => { (err, mpuBucket, mpuOverview, storedMetadata) => {
if (err) { if (err) {
@ -297,6 +320,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
contentMD5: aggregateETag, contentMD5: aggregateETag,
size: calculatedSize, size: calculatedSize,
multipart: true, multipart: true,
isDeleteMarker: false,
replicationInfo: getReplicationInfo(objectKey, destBucket, replicationInfo: getReplicationInfo(objectKey, destBucket,
false, calculatedSize, REPLICATION_ACTION), false, calculatedSize, REPLICATION_ACTION),
originOp: 's3:ObjectCreated:CompleteMultipartUpload', originOp: 's3:ObjectCreated:CompleteMultipartUpload',
@ -334,6 +358,14 @@ function completeMultipartUpload(authInfo, request, log, callback) {
masterKeyId: destBucket.getSseMasterKeyId(), masterKeyId: destBucket.getSseMasterKeyId(),
}; };
} }
// if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
if (isPutVersion) {
const options = overwritingVersioning(bucketName, putVersionId, objMD, metaStoreParams);
return process.nextTick(() => next(null, destBucket, dataLocations,
metaStoreParams, mpuBucket, keysToDelete, aggregateETag,
objMD, extraPartLocations, pseudoCipherBundle,
completeObjData, options));
}
return versioningPreprocessing(bucketName, return versioningPreprocessing(bucketName,
destBucket, objectKey, objMD, log, (err, options) => { destBucket, objectKey, objMD, log, (err, options) => {
if (err) { if (err) {
@ -347,22 +379,30 @@ function completeMultipartUpload(authInfo, request, log, callback) {
}); });
return next(err, destBucket); return next(err, destBucket);
} }
// const dataToDelete = options.dataToDelete;
// metaStoreParams.versionId = options.versionId;
// metaStoreParams.versioning = options.versioning;
// metaStoreParams.isNull = options.isNull;
// metaStoreParams.nullVersionId = options.nullVersionId;
// metaStoreParams.nullUploadId = options.nullUploadId;
return next(null, destBucket, dataLocations,
metaStoreParams, mpuBucket, keysToDelete, aggregateETag,
objMD, extraPartLocations, pseudoCipherBundle,
completeObjData, options);
});
},
function storeAsNewObj(destinationBucket, dataLocations,
metaStoreParams, mpuBucket, keysToDelete, aggregateETag, objMD,
extraPartLocations, pseudoCipherBundle,
completeObjData, options, next) {
const dataToDelete = options.dataToDelete; const dataToDelete = options.dataToDelete;
metaStoreParams.versionId = options.versionId; metaStoreParams.versionId = options.versionId;
metaStoreParams.versioning = options.versioning; metaStoreParams.versioning = options.versioning;
metaStoreParams.isNull = options.isNull; metaStoreParams.isNull = options.isNull;
metaStoreParams.nullVersionId = options.nullVersionId; metaStoreParams.nullVersionId = options.nullVersionId;
metaStoreParams.nullUploadId = options.nullUploadId; metaStoreParams.nullUploadId = options.nullUploadId;
return next(null, destBucket, dataLocations,
metaStoreParams, mpuBucket, keysToDelete, aggregateETag,
objMD, extraPartLocations, pseudoCipherBundle,
dataToDelete, completeObjData);
});
},
function storeAsNewObj(destinationBucket, dataLocations,
metaStoreParams, mpuBucket, keysToDelete, aggregateETag, objMD,
extraPartLocations, pseudoCipherBundle, dataToDelete,
completeObjData, next) {
// For external backends (where completeObjData is not // For external backends (where completeObjData is not
// null), the backend key does not change for new versions // null), the backend key does not change for new versions
// of the same object (or rewrites for nonversioned // of the same object (or rewrites for nonversioned
@ -393,6 +433,9 @@ function completeMultipartUpload(authInfo, request, log, callback) {
// metadata keys, which are likely to have failed in // metadata keys, which are likely to have failed in
// the previous MPU completion attempt // the previous MPU completion attempt
// //
console.log('objMD.location!!!', objMD.location);
console.log('dataLocations!!!', dataLocations);
console.log('locationKeysHaveChanged(objMD.location, dataLocations)!!!', locationKeysHaveChanged(objMD.location, dataLocations));
if (!locationKeysHaveChanged(objMD.location, dataLocations)) { if (!locationKeysHaveChanged(objMD.location, dataLocations)) {
log.info('MPU complete request replay detected', { log.info('MPU complete request replay detected', {
method: 'completeMultipartUpload.storeAsNewObj', method: 'completeMultipartUpload.storeAsNewObj',

View File

@ -7,7 +7,7 @@ const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption'); const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const createAndStoreObject = require('./apiUtils/object/createAndStoreObject'); const createAndStoreObject = require('./apiUtils/object/createAndStoreObject');
const { checkQueryVersionId } = require('./apiUtils/object/versioning'); const { checkQueryVersionId, decodeVID } = require('./apiUtils/object/versioning');
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const { validateHeaders } = require('./apiUtils/object/objectLockHelpers'); const { validateHeaders } = require('./apiUtils/object/objectLockHelpers');
@ -41,6 +41,24 @@ const versionIdUtils = versioning.VersionID;
*/ */
function objectPut(authInfo, request, streamingV4Params, log, callback) { function objectPut(authInfo, request, streamingV4Params, log, callback) {
log.debug('processing request', { method: 'objectPut' }); log.debug('processing request', { method: 'objectPut' });
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
let versionId;
if (putVersionId) {
const decodedVidResult = decodeVID(putVersionId);
if (decodedVidResult instanceof Error) {
log.trace('invalid x-scal-s3-version-id header', {
versionId: putVersionId,
error: decodedVidResult,
});
return process.nextTick(() => callback(decodedVidResult));
}
versionId = decodedVidResult;
}
const { const {
bucketName, bucketName,
headers, headers,
@ -69,7 +87,8 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
const invalidSSEError = errors.InvalidArgument.customizeDescription( const invalidSSEError = errors.InvalidArgument.customizeDescription(
'The encryption method specified is not supported'); 'The encryption method specified is not supported');
const requestType = 'objectPut'; const requestType = 'objectPut';
const valParams = { authInfo, bucketName, objectKey, requestType, request }; const valParams = { authInfo, bucketName, objectKey, versionId,
requestType, request };
const canonicalID = authInfo.getCanonicalID(); const canonicalID = authInfo.getCanonicalID();
if (hasNonPrintables(objectKey)) { if (hasNonPrintables(objectKey)) {
@ -99,6 +118,12 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
return callback(errors.NoSuchBucket); return callback(errors.NoSuchBucket);
} }
if (isPutVersion && !objMD) {
const err = putVersionId ? errors.NoSuchVersion : errors.NoSuchKey;
log.trace('error no object metadata found', { method: 'objectPut', putVersionId });
return callback(err);
}
return async.waterfall([ return async.waterfall([
function handleTransientOrDeleteBuckets(next) { function handleTransientOrDeleteBuckets(next) {
if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) {

View File

@ -97,7 +97,7 @@ const services = {
lastModifiedDate, versioning, versionId, uploadId, lastModifiedDate, versioning, versionId, uploadId,
tagging, taggingCopy, replicationInfo, defaultRetention, tagging, taggingCopy, replicationInfo, defaultRetention,
dataStoreName, creationTime, retentionMode, retentionDate, dataStoreName, creationTime, retentionMode, retentionDate,
legalHold, originOp } = params; legalHold, originOp, masterVersionId } = params;
log.trace('storing object in metadata'); log.trace('storing object in metadata');
assert.strictEqual(typeof bucketName, 'string'); assert.strictEqual(typeof bucketName, 'string');
const md = new ObjectMD(); const md = new ObjectMD();
@ -163,6 +163,10 @@ const services = {
md.setUploadId(uploadId); md.setUploadId(uploadId);
options.replayId = uploadId; options.replayId = uploadId;
} }
// used to overwrite master version.
if (masterVersionId || masterVersionId === '') {
options.masterVersionId = masterVersionId;
}
// information to store about the version and the null version id // information to store about the version and the null version id
// in the object metadata // in the object metadata
const { isNull, nullVersionId, nullUploadId, isDeleteMarker } = params; const { isNull, nullVersionId, nullUploadId, isDeleteMarker } = params;

View File

@ -71,7 +71,7 @@
}, },
"scripts": { "scripts": {
"cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server", "cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server",
"ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/object/mpuVersion.js",
"ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/",
"ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket", "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket",
"ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support", "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support",

View File

@ -0,0 +1,826 @@
const assert = require('assert');
const async = require('async');
const { versioning } = require('arsenal');
const { config } = require('../../../../../lib/Config');
const withV4 = require('../support/withV4');
const BucketUtility = require('../../lib/utility/bucket-util');
const metadata = require('../../../../../lib/metadata/wrapper');
const { DummyRequestLogger } = require('../../../../unit/helpers');
const checkError = require('../../lib/utility/checkError');
const versionIdUtils = versioning.VersionID;
const log = new DummyRequestLogger();
const nonVersionedObjId =
versionIdUtils.getInfVid(config.replicationGroupId);
const bucketName = 'bucket1putversion30';
const objectName = 'object1putversion';
const mdListingParams = { listingType: 'DelimiterVersions', maxKeys: 1000 };
function _getMetadata(bucketName, objectName, versionId, cb) {
let decodedVersionId;
if (versionId) {
if (versionId === 'null') {
decodedVersionId = nonVersionedObjId;
} else {
decodedVersionId = versionIdUtils.decode(versionId);
}
if (decodedVersionId instanceof Error) {
return cb(new Error('Invalid version id specified'));
}
}
return metadata.getObjectMD(bucketName, objectName, { versionId: decodedVersionId },
log, (err, objMD) => {
if (err) {
assert.equal(err, null, 'Getting object metadata: expected success, ' +
`got error ${JSON.stringify(err)}`);
}
return cb(null, objMD);
});
}
function putMPUVersion(s3, bucketName, objectName, vId, cb) {
async.waterfall([
next => {
const params = { Bucket: bucketName, Key: objectName };
const request = s3.createMultipartUpload(params);
if (vId !== undefined) {
request.on('build', () => {
request.httpRequest.headers['x-scal-s3-version-id'] = vId;
});
}
return request.send(next);
},
(resCreation, next) => {
const uploadId = resCreation.UploadId;
const params = {
Body: 'okok',
Bucket: bucketName,
Key: objectName,
PartNumber: 1,
UploadId: uploadId,
};
const request = s3.uploadPart(params);
if (vId !== undefined) {
request.on('build', () => {
request.httpRequest.headers['x-scal-s3-version-id'] = vId;
});
}
return request.send((err, res) => next(err, res, uploadId));
},
(res, uploadId, next) => {
const params = {
Bucket: bucketName,
Key: objectName,
MultipartUpload: {
Parts: [
{
ETag: res.ETag,
PartNumber: 1
},
]
},
UploadId: uploadId,
};
const request = s3.completeMultipartUpload(params);
if (vId !== undefined) {
request.on('build', () => {
request.httpRequest.headers['x-scal-s3-version-id'] = vId;
});
}
return request.send(err => next(err));
},
], err => cb(err));
}
function putMPU(s3, bucketName, objectName, cb) {
return putMPUVersion(s3, bucketName, objectName, undefined, cb);
}
// function checkVersions(versionsBefore, versionsAfter) {
// versionsAfter.forEach((v, i) => {
// // assert.notEqual(v.value.Size, versionsBefore[i].value.Size);
// assert.notEqual(v.value.ETag, versionsBefore[i].value.ETag);
// // v.value.Size = versionsBefore[i].value.Size;
// v.value.ETag = versionsBefore[i].value.ETag;
// });
// assert.deepStrictEqual(versionsAfter, versionsBefore);
// }
function checkNotEqualAndUpdate(objMDBefore, objMDAfter, props) {
props.forEach(p => {
assert.notEqual(objMDAfter[p], objMDBefore[p]);
objMDAfter[p] = objMDBefore[p];
});
}
describe('PUT object with x-scal-s3-version-id header', () => {
withV4(sigCfg => {
let bucketUtil;
let s3;
beforeEach(done => {
bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3;
return metadata.setup(() =>
s3.createBucket({ Bucket: bucketName }, err => {
if (err) {
assert.equal(err, null, 'Creating bucket: Expected success, ' +
`got error ${JSON.stringify(err)}`);
}
done();
}));
});
afterEach(() => {
process.stdout.write('Emptying bucket');
return bucketUtil.empty(bucketName)
.then(() => {
process.stdout.write('Deleting bucket');
return bucketUtil.deleteOne(bucketName);
})
.catch(err => {
process.stdout.write('Error in afterEach');
throw err;
});
});
it('should overwrite an MPU object', done => {
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
async.waterfall([
next => putMPU(s3, bucketName, objectName, err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, '', next),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'uploadId']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite an object', done => {
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, '', next),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
versionsAfter.forEach((v, i) => {
assert.notEqual(v.value.Size, versionsBefore[i].value.Size);
assert.notEqual(v.value.ETag, versionsBefore[i].value.ETag);
v.value.Size = versionsBefore[i].value.Size;
v.value.ETag = versionsBefore[i].value.ETag;
});
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite a versioned object', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, vId, next),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
versionsAfter.forEach((v, i) => {
assert.notEqual(v.value.Size, versionsBefore[i].value.Size);
assert.notEqual(v.value.ETag, versionsBefore[i].value.ETag);
v.value.Size = versionsBefore[i].value.Size;
v.value.ETag = versionsBefore[i].value.ETag;
});
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current version if empty version id header', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, '', next),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
versionsAfter.forEach((v, i) => {
assert.notEqual(v.value.Size, versionsBefore[i].value.Size);
assert.notEqual(v.value.ETag, versionsBefore[i].value.ETag);
v.value.Size = versionsBefore[i].value.Size;
v.value.ETag = versionsBefore[i].value.ETag;
});
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should fail if version specified is invalid', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => putMPUVersion(s3, bucketName, objectName, 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P', err => {
checkError(err, 'InvalidArgument', 400);
return next();
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
return done();
});
});
it('should fail if key specified does not exist', done => {
const params = { Bucket: bucketName, Key: objectName };
async.waterfall([
next => putMPUVersion(s3, bucketName, objectName, '', err => {
checkError(err, 'NoSuchKey', 404);
return next();
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
return done();
});
});
it('should fail if version specified does not exist', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => putMPUVersion(s3, bucketName, objectName, '393833343735313131383832343239393939393952473030312020313031', err => {
checkError(err, 'NoSuchVersion', 404);
return next();
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
return done();
});
});
it('should overwrite a non-current null version', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let versionsBefore;
let versionsAfter;
let objMDBefore;
let objMDAfter;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => _getMetadata(bucketName, objectName, 'null', (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, 'null', next),
next => _getMetadata(bucketName, objectName, 'null', (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.notEqual(versionsAfter[1].value.Size, versionsBefore[1].value.Size);
assert.notEqual(versionsAfter[1].value.ETag, versionsBefore[1].value.ETag);
versionsAfter[1].value.Size = versionsBefore[1].value.Size;
versionsAfter[1].value.ETag = versionsBefore[1].value.ETag;
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the lastest version and keep nullVersionId', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let versionsBefore;
let versionsAfter;
let objMDBefore;
let objMDAfter;
let vId;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, vId, next),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.notEqual(versionsAfter[0].value.Size, versionsBefore[0].value.Size);
assert.notEqual(versionsAfter[0].value.ETag, versionsBefore[0].value.ETag);
versionsAfter[0].value.Size = versionsBefore[0].value.Size;
versionsAfter[0].value.ETag = versionsBefore[0].value.ETag;
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite a current null version', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const sParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Suspended',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putBucketVersioning(sParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, '', next),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.notEqual(versionsAfter[0].value.Size, versionsBefore[0].value.Size);
assert.notEqual(versionsAfter[0].value.ETag, versionsBefore[0].value.ETag);
versionsAfter[0].value.Size = versionsBefore[0].value.Size;
versionsAfter[0].value.ETag = versionsBefore[0].value.ETag;
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite a non-current versioned object', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => s3.putObject(params, err => next(err)),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, vId, next),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.notEqual(versionsAfter[1].value.Size, versionsBefore[1].value.Size);
assert.notEqual(versionsAfter[1].value.ETag, versionsBefore[1].value.ETag);
versionsAfter[1].value.Size = versionsBefore[1].value.Size;
versionsAfter[1].value.ETag = versionsBefore[1].value.ETag;
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current versioned object', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putMPUVersion(s3, bucketName, objectName, vId, next),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.notEqual(versionsAfter[0].value.Size, versionsBefore[0].value.Size);
assert.notEqual(versionsAfter[0].value.ETag, versionsBefore[0].value.ETag);
versionsAfter[0].value.Size = versionsBefore[0].value.Size;
versionsAfter[0].value.ETag = versionsBefore[0].value.ETag;
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current versioned object after bucket version suspended', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const sParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Suspended',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => s3.putBucketVersioning(sParams, err => next(err)),
next => putMPUVersion(s3, bucketName, objectName, vId, next),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.notEqual(versionsAfter[0].value.Size, versionsBefore[0].value.Size);
assert.notEqual(versionsAfter[0].value.ETag, versionsBefore[0].value.ETag);
versionsAfter[0].value.Size = versionsBefore[0].value.Size;
versionsAfter[0].value.ETag = versionsBefore[0].value.ETag;
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current null object after bucket version enabled', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => s3.putBucketVersioning(vParams, err => next(err)),
next => putMPUVersion(s3, bucketName, objectName, 'null', next),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.notEqual(versionsAfter[0].value.Size, versionsBefore[0].value.Size);
assert.notEqual(versionsAfter[0].value.ETag, versionsBefore[0].value.ETag);
versionsAfter[0].value.Size = versionsBefore[0].value.Size;
versionsAfter[0].value.ETag = versionsBefore[0].value.ETag;
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.notEqual(objMDAfter.uploadId, objMDBefore.uploadId);
delete(objMDAfter.uploadId);
checkNotEqualAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
});
});

View File

@ -0,0 +1,599 @@
const assert = require('assert');
const async = require('async');
const { versioning } = require('arsenal');
const { config } = require('../../../../../lib/Config');
const withV4 = require('../support/withV4');
const BucketUtility = require('../../lib/utility/bucket-util');
const metadata = require('../../../../../lib/metadata/wrapper');
const { DummyRequestLogger } = require('../../../../unit/helpers');
const checkError = require('../../lib/utility/checkError');
const versionIdUtils = versioning.VersionID;
const log = new DummyRequestLogger();
const nonVersionedObjId =
versionIdUtils.getInfVid(config.replicationGroupId);
const bucketName = 'bucket1putversion30';
const objectName = 'object1putversion';
const mdListingParams = { listingType: 'DelimiterVersions', maxKeys: 1000 };
function _getMetadata(bucketName, objectName, versionId, cb) {
let decodedVersionId;
if (versionId) {
if (versionId === 'null') {
decodedVersionId = nonVersionedObjId;
} else {
decodedVersionId = versionIdUtils.decode(versionId);
}
if (decodedVersionId instanceof Error) {
return cb(new Error('Invalid version id specified'));
}
}
return metadata.getObjectMD(bucketName, objectName, { versionId: decodedVersionId },
log, (err, objMD) => {
if (err) {
assert.equal(err, null, 'Getting object metadata: expected success, ' +
`got error ${JSON.stringify(err)}`);
}
return cb(null, objMD);
});
}
function putObjectVersion(s3, params, vid, next) {
const request = s3.putObject(params);
request.on('build', () => {
request.httpRequest.headers['x-scal-s3-version-id'] = vid;
});
return request.send(next);
}
describe('PUT object with x-scal-s3-version-id header', () => {
withV4(sigCfg => {
let bucketUtil;
let s3;
beforeEach(done => {
bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3;
return metadata.setup(() =>
s3.createBucket({ Bucket: bucketName }, err => {
if (err) {
assert.equal(err, null, 'Creating bucket: Expected success, ' +
`got error ${JSON.stringify(err)}`);
}
done();
}));
});
afterEach(() => {
process.stdout.write('Emptying bucket');
return bucketUtil.empty(bucketName)
.then(() => {
process.stdout.write('Deleting bucket');
return bucketUtil.deleteOne(bucketName);
})
.catch(err => {
process.stdout.write('Error in afterEach');
throw err;
});
});
it('should overwrite an object', done => {
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putObjectVersion(s3, params, '', err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite a versioned object', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putObjectVersion(s3, params, vId, err => next(err)),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current version if empty version id header', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putObjectVersion(s3, params, '', err => next(err)),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should fail if version specified is invalid', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => putObjectVersion(s3, params, 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P', err => {
checkError(err, 'InvalidArgument', 400);
return next();
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
return done();
});
});
it('should fail if key specified does not exist', done => {
const params = { Bucket: bucketName, Key: objectName };
async.waterfall([
next => putObjectVersion(s3, params, '', err => {
checkError(err, 'NoSuchKey', 404);
return next();
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
return done();
});
});
it('should fail if version specified does not exist', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => putObjectVersion(s3, params,
'393833343735313131383832343239393939393952473030312020313031', err => {
checkError(err, 'NoSuchVersion', 404);
return next();
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
return done();
});
});
it('should overwrite a non-current null version', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let versionsBefore;
let versionsAfter;
let objMDBefore;
let objMDAfter;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => _getMetadata(bucketName, objectName, 'null', (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putObjectVersion(s3, params, 'null', err => next(err)),
next => _getMetadata(bucketName, objectName, 'null', (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the lastest version and keep nullVersionId', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let versionsBefore;
let versionsAfter;
let objMDBefore;
let objMDAfter;
let vId;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putObjectVersion(s3, params, vId, err => next(err)),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite a current null version', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const sParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Suspended',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putBucketVersioning(sParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => putObjectVersion(s3, params, '', err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite a non-current versioned object', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => s3.putObject(params, err => next(err)),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putObjectVersion(s3, params, vId, err => next(err)),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current versioned object', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => putObjectVersion(s3, params, vId, err => next(err)),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current versioned object after bucket version suspended', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const sParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Suspended',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
let vId;
async.waterfall([
next => s3.putBucketVersioning(vParams, err => next(err)),
next => s3.putObject(params, err => next(err)),
next => s3.putObject(params, (err, res) => {
vId = res.VersionId;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => s3.putBucketVersioning(sParams, err => next(err)),
next => putObjectVersion(s3, params, vId, err => next(err)),
next => _getMetadata(bucketName, objectName, vId, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
it('should overwrite the current null object after bucket version enabled', done => {
const vParams = {
Bucket: bucketName,
VersioningConfiguration: {
Status: 'Enabled',
}
};
const params = { Bucket: bucketName, Key: objectName };
let objMDBefore;
let objMDAfter;
let versionsBefore;
let versionsAfter;
async.waterfall([
next => s3.putObject(params, err => next(err)),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
next(err);
}),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDBefore = objMD;
return next(err);
}),
next => s3.putBucketVersioning(vParams, err => next(err)),
next => putObjectVersion(s3, params, 'null', err => next(err)),
next => _getMetadata(bucketName, objectName, undefined, (err, objMD) => {
objMDAfter = objMD;
return next(err);
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsAfter = res.Versions;
next(err);
}),
], err => {
assert.equal(err, null, `Expected success got error ${JSON.stringify(err)}`);
assert.deepStrictEqual(versionsAfter, versionsBefore);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
});
});
});