Compare commits

...

11 Commits

Author SHA1 Message Date
Electra Chong 0a7bf3a96f finally + update test 2017-10-21 00:50:07 -07:00
Electra Chong 27e34c198f more tests, tweaks 2017-10-20 22:18:32 -07:00
Electra Chong 04a1f7e8c0 wip tests 2 2017-10-20 20:15:31 -07:00
Electra Chong 37df1c5fae wip tests 2017-10-20 14:29:31 -07:00
Electra Chong 5b9b06b649 wip 2017-10-19 17:11:12 -07:00
Electra Chong 0b8921877c wip4 2017-10-19 15:31:54 -07:00
Electra Chong 1bb03a09a0 wip3 2017-10-19 12:57:13 -07:00
Electra Chong 6a89fe74ee wip2 2017-10-17 13:55:47 -07:00
Electra Chong 1a93e88c5a wip enabling complete mpu stuff 2017-10-17 12:57:48 -07:00
Electra Chong 018ff208ba fix obj tagging imports 2017-10-17 11:25:35 -07:00
Electra Chong 2c8266ff29 [dropme] delete + tag commits 2017-10-17 10:59:22 -07:00
31 changed files with 2121 additions and 301 deletions

View File

@ -61,6 +61,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params, canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
log, callback) { log, callback) {
const size = isDeleteMarker ? 0 : request.parsedContentLength; const size = isDeleteMarker ? 0 : request.parsedContentLength;
// although the request method may actually be 'DELETE' if creating a
// delete marker, for our purposes we consider this to be a 'PUT'
// operation
const requestMethod = 'PUT';
const websiteRedirectHeader = const websiteRedirectHeader =
request.headers['x-amz-website-redirect-location']; request.headers['x-amz-website-redirect-location'];
if (!validateWebsiteHeader(websiteRedirectHeader)) { if (!validateWebsiteHeader(websiteRedirectHeader)) {
@ -70,8 +74,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
return callback(err); return callback(err);
} }
const metaHeaders = isDeleteMarker ? [] : const metaHeaders = isDeleteMarker ? [] : getMetaHeaders(request.headers);
getMetaHeaders(request.headers);
if (metaHeaders instanceof Error) { if (metaHeaders instanceof Error) {
log.debug('user metadata validation failed', { log.debug('user metadata validation failed', {
error: metaHeaders, error: metaHeaders,
@ -79,6 +82,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
}); });
return process.nextTick(() => callback(metaHeaders)); return process.nextTick(() => callback(metaHeaders));
} }
log.trace('meta headers', { metaHeaders, method: 'objectPut' }); log.trace('meta headers', { metaHeaders, method: 'objectPut' });
const objectKeyContext = { const objectKeyContext = {
bucketName, bucketName,
@ -87,6 +91,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
objectKey, objectKey,
metaHeaders, metaHeaders,
tagging: request.headers['x-amz-tagging'], tagging: request.headers['x-amz-tagging'],
isDeleteMarker,
}; };
// If the request was made with a pre-signed url, the x-amz-acl 'header' // If the request was made with a pre-signed url, the x-amz-acl 'header'
// might be in the query string rather than the actual headers so include // might be in the query string rather than the actual headers so include
@ -116,6 +121,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
metadataStoreParams.tagging = request.headers['x-amz-tagging']; metadataStoreParams.tagging = request.headers['x-amz-tagging'];
} }
// if creating new delete marker and there is an existing object, copy
// the object's location constraint metaheader to determine backend info
if (isDeleteMarker && objMD) {
// eslint-disable-next-line no-param-reassign
request.headers[constants.objectLocationConstraintHeader] =
objMD[constants.objectLocationConstraintHeader];
}
const backendInfoObj = const backendInfoObj =
locationConstraintCheck(request, null, bucketMD, log); locationConstraintCheck(request, null, bucketMD, log);
if (backendInfoObj.err) { if (backendInfoObj.err) {
@ -205,7 +218,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
metadataStoreParams.nullVersionId = options.nullVersionId; metadataStoreParams.nullVersionId = options.nullVersionId;
return _storeInMDandDeleteData(bucketName, infoArr, return _storeInMDandDeleteData(bucketName, infoArr,
cipherBundle, metadataStoreParams, cipherBundle, metadataStoreParams,
options.dataToDelete, requestLogger, request.method, next); options.dataToDelete, requestLogger, requestMethod, next);
}, },
], callback); ], callback);
} }

View File

@ -71,22 +71,29 @@ function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
}); });
} }
/** get location of data for deletion /** get location of null version data for deletion
* @param {string} bucketName - name of bucket * @param {string} bucketName - name of bucket
* @param {string} objKey - name of object key * @param {string} objKey - name of object key
* @param {object} options - metadata options for getting object MD * @param {object} options - metadata options for getting object MD
* @param {string} options.versionId - version to get from metadata * @param {string} options.versionId - version to get from metadata
* @param {object} mst - info about the master version
* @param {string} mst.versionId - the master version's version id
* @param {RequestLogger} log - logger instanceof * @param {RequestLogger} log - logger instanceof
* @param {function} cb - callback * @param {function} cb - callback
* @return {undefined} - and call callback with (err, dataToDelete) * @return {undefined} - and call callback with (err, dataToDelete)
*/ */
function _getDeleteLocations(bucketName, objKey, options, log, cb) { function _getNullVersionsToDelete(bucketName, objKey, options, mst, log, cb) {
if (options.versionId === mst.versionId) {
// no need to get delete location, we already have the master's metadata
const dataToDelete = mst.objLocation;
return process.nextTick(cb, null, dataToDelete);
}
return metadata.getObjectMD(bucketName, objKey, options, log, return metadata.getObjectMD(bucketName, objKey, options, log,
(err, versionMD) => { (err, versionMD) => {
if (err) { if (err) {
log.debug('err from metadata getting specified version', { log.debug('err from metadata getting specified version', {
error: err, error: err,
method: '_getDeleteLocations', method: '_getNullVersionsToDelete',
}); });
return cb(err); return cb(err);
} }
@ -99,9 +106,8 @@ function _getDeleteLocations(bucketName, objKey, options, log, cb) {
}); });
} }
function _deleteNullVersionMD(bucketName, objKey, options, log, cb) { function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
// before deleting null version md, retrieve location of data to delete return _getNullVersionsToDelete(bucketName, objKey, options, mst, log,
return _getDeleteLocations(bucketName, objKey, options, log,
(err, nullDataToDelete) => { (err, nullDataToDelete) => {
if (err) { if (err) {
log.warn('could not find null version metadata', { log.warn('could not find null version metadata', {
@ -236,15 +242,16 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
if (!delOptions) { if (!delOptions) {
return process.nextTick(next, null, options); return process.nextTick(next, null, options);
} }
return _deleteNullVersionMD(bucketName, objectKey, delOptions, log, return _deleteNullVersionMD(bucketName, objectKey, delOptions, mst,
(err, nullDataToDelete) => { log, (err, nullDataToDelete) => {
if (err) { if (err) {
log.warn('unexpected error deleting null version md', { log.warn('unexpected error deleting null version md', {
error: err, error: err,
method: 'versioningPreprocessing', method: 'versioningPreprocessing',
}); });
// it's possible there was a concurrent request to delete // it's possible there was a concurrent request to
// the null version, so proceed with putting a new version // delete the null version, so proceed with putting a
// new version
if (err === errors.NoSuchKey) { if (err === errors.NoSuchKey) {
return next(null, options); return next(null, options);
} }

View File

@ -18,8 +18,7 @@ const { versioningPreprocessing, checkQueryVersionId }
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const services = require('../services'); const services = require('../services');
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { checkAzureBackendMatch, skipMpuPartProcessing } = const { skipMpuPartProcessing } = require('../data/external/utils');
require('../data/external/utils');
const logger = require('../utilities/logger'); const logger = require('../utilities/logger');
@ -214,6 +213,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
key: completeObjData.key, key: completeObjData.key,
size: completeObjData.contentLength, size: completeObjData.contentLength,
start: 0, start: 0,
dataStoreVersionId: completeObjData.dataStoreVersionId,
dataStoreName: storedMetadata.dataStoreName, dataStoreName: storedMetadata.dataStoreName,
dataStoreETag: completeObjData.eTag, dataStoreETag: completeObjData.eTag,
dataStoreType: completeObjData.dataStoreType, dataStoreType: completeObjData.dataStoreType,
@ -333,15 +333,16 @@ function completeMultipartUpload(authInfo, request, log, callback) {
// null version when versioning is suspended or versioning // null version when versioning is suspended or versioning
// is not enabled, need to delete pre-existing data // is not enabled, need to delete pre-existing data
// unless the preexisting object and the completed mpu // unless the preexisting object and the completed mpu
// are on Azure data backend // are on external backends
if (dataToDelete) { if (dataToDelete) {
if (!checkAzureBackendMatch(dataToDelete[0], const newDataStoreName =
completeObjData)) { Array.isArray(dataLocations) && dataLocations[0] ?
data.batchDelete(dataToDelete, request.method, null, dataLocations[0].dataStoreName : null;
data.batchDelete(dataToDelete, request.method,
newDataStoreName,
logger.newRequestLoggerFromSerializedUids(log logger.newRequestLoggerFromSerializedUids(log
.getSerializedUids())); .getSerializedUids()));
} }
}
return next(null, mpuBucket, keysToDelete, aggregateETag, return next(null, mpuBucket, keysToDelete, aggregateETag,
extraPartLocations, destinationBucket, extraPartLocations, destinationBucket,
generatedVersionId); generatedVersionId);

View File

@ -185,19 +185,17 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
if (err) { if (err) {
return callback(err); return callback(err);
} }
// NOTE: remove the following when we will support putting a
// versioned object to a location-constraint of type AWS or Azure.
if (locConstraint && if (locConstraint &&
config.locationConstraints[locConstraint] && config.locationConstraints[locConstraint] &&
config.locationConstraints[locConstraint].type && config.locationConstraints[locConstraint].type &&
constants.externalBackends[config constants.versioningNotImplBackends[config
.locationConstraints[locConstraint].type] .locationConstraints[locConstraint].type]
) { ) {
const vcfg = destinationBucket.getVersioningConfiguration(); const vcfg = destinationBucket.getVersioningConfiguration();
const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
if (isVersionedObj) { if (isVersionedObj) {
log.debug(externalVersioningErrorMessage, log.debug(externalVersioningErrorMessage,
{ method: 'multipleBackendGateway', { method: 'initiateMultipartUpload',
error: errors.NotImplemented }); error: errors.NotImplemented });
return callback(errors.NotImplemented return callback(errors.NotImplemented
.customizeDescription(externalVersioningErrorMessage)); .customizeDescription(externalVersioningErrorMessage));

View File

@ -23,6 +23,7 @@ const { config } = require('../Config');
const versionIdUtils = versioning.VersionID; const versionIdUtils = versioning.VersionID;
const locationHeader = constants.objectLocationConstraintHeader; const locationHeader = constants.objectLocationConstraintHeader;
const versioningNotImplBackends = constants.versioningNotImplBackends;
const externalVersioningErrorMessage = 'We do not currently support putting ' + const externalVersioningErrorMessage = 'We do not currently support putting ' +
'a versioned object to a location-constraint of type AWS or Azure.'; 'a versioned object to a location-constraint of type AWS or Azure.';
@ -321,9 +322,7 @@ function objectCopy(authInfo, request, sourceBucket,
sourceLocationConstraintType = config.getLocationConstraintType( sourceLocationConstraintType = config.getLocationConstraintType(
storeMetadataParams.metaHeaders[locationHeader]); storeMetadataParams.metaHeaders[locationHeader]);
} }
// NOTE: remove the following when we will support putting a if (versioningNotImplBackends[sourceLocationConstraintType]) {
// versioned object to a location-constraint of type AWS or Azure.
if (constants.externalBackends[sourceLocationConstraintType]) {
const vcfg = destBucketMD.getVersioningConfiguration(); const vcfg = destBucketMD.getVersioningConfiguration();
const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
if (isVersionedObj) { if (isVersionedObj) {
@ -337,6 +336,10 @@ function objectCopy(authInfo, request, sourceBucket,
if (dataLocator.length === 0) { if (dataLocator.length === 0) {
if (!storeMetadataParams.locationMatch && if (!storeMetadataParams.locationMatch &&
constants.externalBackends[sourceLocationConstraintType]) { constants.externalBackends[sourceLocationConstraintType]) {
console.log('============================');
console.log('detected zero byte object')
console.log('location does not match and source ' +
'is an external backend; about to use data.put for copy')
return data.put(null, null, storeMetadataParams.size, return data.put(null, null, storeMetadataParams.size,
dataStoreContext, backendInfo, dataStoreContext, backendInfo,
log, (error, objectRetrievalInfo) => { log, (error, objectRetrievalInfo) => {
@ -359,7 +362,7 @@ function objectCopy(authInfo, request, sourceBucket,
return next(null, storeMetadataParams, dataLocator, destObjMD, return next(null, storeMetadataParams, dataLocator, destObjMD,
serverSideEncryption, destBucketMD); serverSideEncryption, destBucketMD);
} }
console.log('about to call data wrapper copy object')
return data.copyObject(request, sourceLocationConstraintType, return data.copyObject(request, sourceLocationConstraintType,
sourceLocationConstraintName, storeMetadataParams, dataLocator, sourceLocationConstraintName, storeMetadataParams, dataLocator,
dataStoreContext, backendInfo, serverSideEncryption, log, dataStoreContext, backendInfo, serverSideEncryption, log,

View File

@ -73,6 +73,13 @@ function objectDelete(authInfo, request, log, cb) {
// versioning has been configured // versioning has been configured
return next(null, bucketMD, objMD); return next(null, bucketMD, objMD);
} }
console.log('============== objMD.location? in objDelete', objMD.location)
if (reqVersionId && objMD.location &&
Array.isArray(objMD.location) && objMD.location[0]) {
// we need this information for data deletes to AWS
// eslint-disable-next-line no-param-reassign
objMD.location[0].deleteVersion = true;
}
if (objMD['content-length'] !== undefined) { if (objMD['content-length'] !== undefined) {
log.end().addDefaultFields({ log.end().addDefaultFields({
bytesDeleted: objMD['content-length'], bytesDeleted: objMD['content-length'],

View File

@ -7,6 +7,11 @@ const { prepareStream } = require('../../api/apiUtils/object/prepareStream');
const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils'); const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils');
const { config } = require('../../Config'); const { config } = require('../../Config');
const missingVerIdInternalError = errors.InternalError.customizeDescription(
'Invalid state. Please ensure versioning is enabled ' +
'in AWS for the location constraint and try again.'
);
class AwsClient { class AwsClient {
constructor(config) { constructor(config) {
this._s3Params = config.s3Params; this._s3Params = config.s3Params;
@ -25,24 +30,15 @@ class AwsClient {
return `${requestBucketName}/${requestObjectKey}`; return `${requestBucketName}/${requestObjectKey}`;
} }
put(stream, size, keyContext, reqUids, callback) { put(stream, size, keyContext, reqUids, callback) {
console.log('=========================')
console.log('got to AwsClient put')
const awsKey = this._createAwsKey(keyContext.bucketName, const awsKey = this._createAwsKey(keyContext.bucketName,
keyContext.objectKey, this._bucketMatch); keyContext.objectKey, this._bucketMatch);
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders); const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
const log = createLogger(reqUids); const log = createLogger(reqUids);
const uploadParams = {
Bucket: this._awsBucketName,
Key: awsKey,
Metadata: metaHeaders,
ContentLength: size,
};
if (this._serverSideEncryption) {
uploadParams.ServerSideEncryption = 'AES256';
}
if (keyContext.tagging) {
uploadParams.Tagging = keyContext.tagging;
}
const putCb = (err, data) => { const putCb = (err, data) => {
console.log('result from aws put', data)
if (err) { if (err) {
logHelper(log, 'error', 'err from data backend', logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName); err, this._dataStoreName);
@ -52,21 +48,38 @@ class AwsClient {
); );
} }
if (!data.VersionId) { if (!data.VersionId) {
const error = errors.InternalError.customizeDescription(
'Invalid state. Please ensure versioning is enabled ' +
'in AWS for the location constraint and try again.'
);
logHelper(log, 'error', 'missing version id for data ' + logHelper(log, 'error', 'missing version id for data ' +
'backend object', error, this._dataStoreName); 'backend object', missingVerIdInternalError,
return callback(error); this._dataStoreName);
return callback(missingVerIdInternalError);
} }
const dataStoreVersionId = data.VersionId; const dataStoreVersionId = data.VersionId;
return callback(null, awsKey, dataStoreVersionId); return callback(null, awsKey, dataStoreVersionId);
}; };
const params = {
Bucket: this._awsBucketName,
Key: awsKey,
};
// we call data.put to create a delete marker, but it's actually a
// delete request in call to AWS
if (keyContext.isDeleteMarker) {
return this._client.deleteObject(params, putCb);
}
const uploadParams = params;
uploadParams.Metadata = metaHeaders;
uploadParams.ContentLength = size;
if (this._serverSideEncryption) {
uploadParams.ServerSideEncryption = 'AES256';
}
if (keyContext.tagging) {
uploadParams.Tagging = keyContext.tagging;
}
if (!stream) { if (!stream) {
return this._client.putObject(uploadParams, putCb); return this._client.putObject(uploadParams, putCb);
} }
console.log('uploadParams', uploadParams)
uploadParams.Body = stream; uploadParams.Body = stream;
return this._client.upload(uploadParams, putCb); return this._client.upload(uploadParams, putCb);
@ -108,6 +121,14 @@ class AwsClient {
}).on('success', response => { }).on('success', response => {
log.trace('AWS GET request response headers', log.trace('AWS GET request response headers',
{ responseHeaders: response.httpResponse.headers }); { responseHeaders: response.httpResponse.headers });
console.log('====================')
console.log('params sent for aws get request', {
Bucket: this._awsBucketName,
Key: key,
VersionId: dataStoreVersionId,
Range: range,
});
console.log('AWS GET request response headers', response.httpResponse.headers)
}); });
const stream = request.createReadStream().on('error', err => { const stream = request.createReadStream().on('error', err => {
logHelper(log, 'error', 'error streaming data from AWS', logHelper(log, 'error', 'error streaming data from AWS',
@ -117,18 +138,29 @@ class AwsClient {
return callback(null, stream); return callback(null, stream);
} }
delete(objectGetInfo, reqUids, callback) { delete(objectGetInfo, reqUids, callback) {
// for backwards compatibility const { key, dataStoreVersionId, deleteVersion } = objectGetInfo;
const key = typeof objectGetInfo === 'string' ? objectGetInfo : const log = createLogger(reqUids);
objectGetInfo.key;
const params = { const params = {
Bucket: this._awsBucketName, Bucket: this._awsBucketName,
Key: key, Key: key,
}; };
return this._client.deleteObject(params, err => { if (deleteVersion) {
params.VersionId = dataStoreVersionId;
}
console.log('=====================');
console.log('objectGetInfo', { key, dataStoreVersionId, deleteVersion })
console.log('params sending to AWS for delete', params)
return this._client.deleteObject(params, (err, data) => {
console.log('response from aws for delete', data)
if (err) { if (err) {
const log = createLogger(reqUids);
logHelper(log, 'error', 'error deleting object from ' + logHelper(log, 'error', 'error deleting object from ' +
'datastore', err, this._dataStoreName); 'datastore', err, this._dataStoreName);
if (err.code === 'NoSuchVersion') {
// data may have been deleted directly from the AWS backend
// don't want to retry the delete and errors are not
// sent back to client anyway, so no need to return err
return callback();
}
return callback(errors.InternalError return callback(errors.InternalError
.customizeDescription('Error returned from ' + .customizeDescription('Error returned from ' +
`AWS: ${err.message}`) `AWS: ${err.message}`)
@ -300,6 +332,9 @@ class AwsClient {
const completeObjData = { key: awsKey }; const completeObjData = { key: awsKey };
return this._client.completeMultipartUpload(mpuParams, return this._client.completeMultipartUpload(mpuParams,
(err, completeMpuRes) => { (err, completeMpuRes) => {
console.log('==============================')
console.log('mpuParams', mpuParams);
console.log('completeMpuRes from AWS', completeMpuRes)
if (err) { if (err) {
if (mpuError[err.code]) { if (mpuError[err.code]) {
logHelper(log, 'trace', 'err from data backend on ' + logHelper(log, 'trace', 'err from data backend on ' +
@ -313,6 +348,12 @@ class AwsClient {
`AWS: ${err.message}`) `AWS: ${err.message}`)
); );
} }
if (!completeMpuRes.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName);
return callback(missingVerIdInternalError);
}
// need to get content length of new object to store // need to get content length of new object to store
// in our metadata // in our metadata
return this._client.headObject({ Bucket: awsBucket, Key: awsKey }, return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
@ -328,6 +369,7 @@ class AwsClient {
// remove quotes from eTag because they're added later // remove quotes from eTag because they're added later
completeObjData.eTag = completeMpuRes.ETag completeObjData.eTag = completeMpuRes.ETag
.substring(1, completeMpuRes.ETag.length - 1); .substring(1, completeMpuRes.ETag.length - 1);
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
completeObjData.contentLength = objHeaders.ContentLength; completeObjData.contentLength = objHeaders.ContentLength;
return callback(null, completeObjData); return callback(null, completeObjData);
}); });
@ -357,10 +399,12 @@ class AwsClient {
objectPutTagging(key, bucket, objectMD, log, callback) { objectPutTagging(key, bucket, objectMD, log, callback) {
const awsBucket = this._awsBucketName; const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch); const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
const tagParams = { Bucket: awsBucket, Key: awsKey }; const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
if (objectMD.versionId) { const tagParams = {
tagParams.VersionId = objectMD.versionId; Bucket: awsBucket,
} Key: awsKey,
VersionId: dataStoreVersionId,
};
const keyArray = Object.keys(objectMD.tags); const keyArray = Object.keys(objectMD.tags);
tagParams.Tagging = {}; tagParams.Tagging = {};
tagParams.Tagging.TagSet = keyArray.map(key => { tagParams.Tagging.TagSet = keyArray.map(key => {
@ -383,10 +427,12 @@ class AwsClient {
objectDeleteTagging(key, bucket, objectMD, log, callback) { objectDeleteTagging(key, bucket, objectMD, log, callback) {
const awsBucket = this._awsBucketName; const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch); const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
const tagParams = { Bucket: awsBucket, Key: awsKey }; const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
if (objectMD.versionId) { const tagParams = {
tagParams.VersionId = objectMD.versionId; Bucket: awsBucket,
} Key: awsKey,
VersionId: dataStoreVersionId,
};
return this._client.deleteObjectTagging(tagParams, err => { return this._client.deleteObjectTagging(tagParams, err => {
if (err) { if (err) {
logHelper(log, 'error', 'error from data backend on ' + logHelper(log, 'error', 'error from data backend on ' +
@ -417,7 +463,7 @@ class AwsClient {
CopySource: `${sourceAwsBucketName}/${sourceKey}`, CopySource: `${sourceAwsBucketName}/${sourceKey}`,
Metadata: metaHeaders, Metadata: metaHeaders,
MetadataDirective: metadataDirective, MetadataDirective: metadataDirective,
}, err => { }, (err, copyResult) => {
if (err) { if (err) {
if (err.code === 'AccessDenied') { if (err.code === 'AccessDenied') {
logHelper(log, 'error', 'Unable to access ' + logHelper(log, 'error', 'Unable to access ' +
@ -435,7 +481,15 @@ class AwsClient {
`AWS: ${err.message}`) `AWS: ${err.message}`)
); );
} }
return callback(null, destAwsKey); if (!copyResult.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName);
return callback(missingVerIdInternalError);
}
console.log('destAwsKey in AwsClient', destAwsKey);
console.log('callback??', callback)
return callback(null, destAwsKey, copyResult.VersionId);
}); });
} }
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName, uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,

View File

@ -247,11 +247,13 @@ const multipleBackendGateway = {
const client = clients[location]; const client = clients[location];
if (client.copyObject) { if (client.copyObject) {
return client.copyObject(request, externalSourceKey, return client.copyObject(request, externalSourceKey,
sourceLocationConstraintName, log, (err, key) => { sourceLocationConstraintName, log, (err, key,
dataStoreVersionId) => {
const dataRetrievalInfo = { const dataRetrievalInfo = {
key, key,
dataStoreName: location, dataStoreName: location,
dataStoreType: client.clientType, dataStoreType: client.clientType,
dataStoreVersionId,
}; };
cb(err, dataRetrievalInfo); cb(err, dataRetrievalInfo);
}); });

View File

@ -56,6 +56,33 @@ if (config.backends.data === 'mem') {
*/ */
const MAX_RETRY = 2; const MAX_RETRY = 2;
// This check is done because on a put, complete mpu or copy request to AWS,
// if the object already exists on AWS, the existing object should not be
// deleted, which is the functionality for all other backends
function _shouldSkipDelete(locations, requestMethod, newObjDataStoreName) {
const skipBackend = externalBackends;
const skipMethods = { PUT: true, POST: true };
if (!Array.isArray(locations) || !locations[0] ||
!locations[0].dataStoreType) {
return false;
}
const isSkipBackend = skipBackend[locations[0].dataStoreType];
const isMatchingBackends =
locations[0].dataStoreName === newObjDataStoreName;
const isSkipMethod = skipMethods[requestMethod];
console.log('============================')
console.log('requestMethod', requestMethod)
console.log('isSkipMethod', isSkipMethod)
console.log('isSkipBackend', isSkipBackend);
console.log('isMatchingBackends', isMatchingBackends)
console.log('locations[0].dataStoreName', locations[0].dataStoreName);
console.log('newObjDataStoreName', newObjDataStoreName)
if (!isSkipBackend || !isMatchingBackends || !isSkipMethod) {
return false;
}
return true;
}
function _retryDelete(objectGetInfo, log, count, cb) { function _retryDelete(objectGetInfo, log, count, cb) {
if (count > MAX_RETRY) { if (count > MAX_RETRY) {
return cb(errors.InternalError); return cb(errors.InternalError);
@ -216,7 +243,6 @@ const data = {
return callback(err); return callback(err);
}); });
}, },
// It would be preferable to have an sproxyd batch delete route to // It would be preferable to have an sproxyd batch delete route to
// replace this // replace this
batchDelete: (locations, requestMethod, newObjDataStoreName, log) => { batchDelete: (locations, requestMethod, newObjDataStoreName, log) => {
@ -224,20 +250,12 @@ const data = {
// be finalized; refer Issue #312 for the discussion. In the // be finalized; refer Issue #312 for the discussion. In the
// meantime, we at least log the location of the data we are // meantime, we at least log the location of the data we are
// about to delete before attempting its deletion. // about to delete before attempting its deletion.
/* eslint-disable camelcase */ console.log('got to batchDelete... requestMethod', requestMethod)
const skipBackend = externalBackends; console.log('should skip method?', _shouldSkipDelete(locations, requestMethod, newObjDataStoreName))
/* eslint-enable camelcase */ if (_shouldSkipDelete(locations, requestMethod, newObjDataStoreName)) {
const isSkipBackend = (locations[0] && locations[0].dataStoreType) ?
skipBackend[locations[0].dataStoreType] : false;
const isMatchingBackends = locations[0] ?
locations[0].dataStoreName === newObjDataStoreName : false;
// This check is done because on a PUT request to AWS, if the object
// already exists on AWS, the existing object should not be deleted,
// which is the functionality for all other backends
// TODO: update for mpu and object copy
if (requestMethod === 'PUT' && isSkipBackend && isMatchingBackends) {
return; return;
} }
console.log('are we continuing any way? )!(@)(!)')
log.trace('initiating batch delete', { log.trace('initiating batch delete', {
keys: locations, keys: locations,
implName, implName,
@ -326,6 +344,7 @@ const data = {
copyObject: (request, sourceLocationConstraintType, copyObject: (request, sourceLocationConstraintType,
sourceLocationConstraintName, storeMetadataParams, dataLocator, sourceLocationConstraintName, storeMetadataParams, dataLocator,
dataStoreContext, destBackendInfo, serverSideEncryption, log, cb) => { dataStoreContext, destBackendInfo, serverSideEncryption, log, cb) => {
console.log('============ data wrapper copy object')
// NOTE: using copyObject only if copying object from one external // NOTE: using copyObject only if copying object from one external
// backend to the same external backend // backend to the same external backend
// and for Azure if it is the same account since Azure copy outside // and for Azure if it is the same account since Azure copy outside
@ -338,22 +357,26 @@ const data = {
(sourceLocationConstraintType === 'aws_s3' || (sourceLocationConstraintType === 'aws_s3' ||
(sourceLocationConstraintType === 'azure' && config.isSameAzureAccount( (sourceLocationConstraintType === 'azure' && config.isSameAzureAccount(
sourceLocationConstraintName, request.headers[locationHeader])))) { sourceLocationConstraintName, request.headers[locationHeader])))) {
console.log('detected location match and external backend')
const location = storeMetadataParams const location = storeMetadataParams
.metaHeaders[locationHeader]; .metaHeaders[locationHeader];
const objectGetInfo = dataLocator[0]; const objectGetInfo = dataLocator[0];
const externalSourceKey = objectGetInfo.key; const externalSourceKey = objectGetInfo.key;
console.log('about to use client.copyObject')
return client.copyObject(request, location, return client.copyObject(request, location,
externalSourceKey, sourceLocationConstraintName, log, externalSourceKey, sourceLocationConstraintName, log,
(error, objectRetrievalInfo) => { (error, objectRetrievalInfo) => {
if (error) { if (error) {
return cb(error); return cb(error);
} }
console.log('objectRetrievalInfo - awsClient', objectRetrievalInfo)
const putResult = { const putResult = {
key: objectRetrievalInfo.key, key: objectRetrievalInfo.key,
dataStoreName: objectRetrievalInfo. dataStoreName: objectRetrievalInfo.
dataStoreName, dataStoreName,
dataStoreType: objectRetrievalInfo. dataStoreType: objectRetrievalInfo.
dataStoreType, dataStoreType,
dataStoreVersionId: objectRetrievalInfo.dataStoreVersionId,
size: storeMetadataParams.size, size: storeMetadataParams.size,
dataStoreETag: objectGetInfo.dataStoreETag, dataStoreETag: objectGetInfo.dataStoreETag,
start: objectGetInfo.start, start: objectGetInfo.start,
@ -362,7 +385,6 @@ const data = {
return cb(null, putResultArr); return cb(null, putResultArr);
}); });
} }
// dataLocator is an array. need to get and put all parts // dataLocator is an array. need to get and put all parts
// For now, copy 1 part at a time. Could increase the second // For now, copy 1 part at a time. Could increase the second
// argument here to increase the number of parts // argument here to increase the number of parts
@ -438,6 +460,7 @@ const data = {
} }
// Copied object is not encrypted so just put it // Copied object is not encrypted so just put it
// without a cipherBundle // without a cipherBundle
console.log('about to call data.put?');
return data.put(null, stream, part.size, return data.put(null, stream, part.size,
dataStoreContext, destBackendInfo, dataStoreContext, destBackendInfo,
log, (error, partRetrievalInfo) => { log, (error, partRetrievalInfo) => {
@ -451,6 +474,8 @@ const data = {
dataStoreName: partRetrievalInfo. dataStoreName: partRetrievalInfo.
dataStoreName, dataStoreName,
dataStoreETag: part.dataStoreETag, dataStoreETag: part.dataStoreETag,
dataStoreVersionId: partRetrievalInfo.
dataStoreVersionId,
start: part.start, start: part.start,
size: part.size, size: part.size,
}; };

View File

@ -2,9 +2,13 @@ const assert = require('assert');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { config } = require('../../../../../../lib/Config'); const {
const { memLocation, fileLocation, awsLocation, awsLocationMismatch } describeSkipIfNotMultiple,
= require('../utils'); memLocation,
fileLocation,
awsLocation,
awsLocationMismatch,
} = require('../utils');
const bucket = 'buckettestmultiplebackenddelete'; const bucket = 'buckettestmultiplebackenddelete';
const memObject = `memObject-${Date.now()}`; const memObject = `memObject-${Date.now()}`;
@ -16,9 +20,6 @@ const mismatchObject = `mismatchOjbect-${Date.now()}`;
const body = Buffer.from('I am a body', 'utf8'); const body = Buffer.from('I am a body', 'utf8');
const bigBody = Buffer.alloc(10485760); const bigBody = Buffer.alloc(10485760);
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
describeSkipIfNotMultiple('Multiple backend delete', () => { describeSkipIfNotMultiple('Multiple backend delete', () => {
withV4(sigCfg => { withV4(sigCfg => {
let bucketUtil; let bucketUtil;

View File

@ -0,0 +1,560 @@
const assert = require('assert');
const async = require('async');
const { errors } = require('arsenal');
const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util');
const {
describeSkipIfNotMultiple,
awsS3,
awsLocation,
awsBucket,
putToAwsBackend,
enableVersioning,
suspendVersioning,
mapToAwsPuts,
putNullVersionsToAws,
putVersionsToAws,
getAndAssertResult,
awsGetLatestVerId,
} = require('../utils');
const someBody = 'testbody';
const bucket = 'buckettestmultiplebackenddeleteversioning';
// order of items by index:
// 0 - whether to expect a version id
// 1 - whether version id should match request version id (undef if n/a)
// 2 - whether x-amz-delete-marker response header should be true
const _deleteResultSchema = {
nonVersionedDelete: [false, undefined, false],
newDeleteMarker: [true, false, true],
deleteVersion: [true, true, false],
deleteDeleteMarker: [true, true, true],
};
const [nonVersionedDelete, newDeleteMarker, deleteVersion, deleteDeleteMarker]
= Object.keys(_deleteResultSchema);
function _assertDeleteResult(result, resultType, requestVersionId) {
if (!_deleteResultSchema[resultType]) {
throw new Error(`undefined result type "${resultType}"`);
}
const [expectVersionId, matchReqVersionId, expectDeleteMarker] =
_deleteResultSchema[resultType];
if (expectVersionId && matchReqVersionId) {
assert.strictEqual(result.VersionId, requestVersionId);
} else if (expectVersionId) {
assert(result.VersionId, 'expected version id in result');
} else {
assert.strictEqual(result.VersionId, undefined,
`did not expect version id in result, got "${result.VersionId}"`);
}
if (expectDeleteMarker) {
assert.strictEqual(result.DeleteMarker, 'true');
} else {
assert.strictEqual(result.DeleteMarker, undefined);
}
}
function delAndAssertResult(s3, params, cb) {
const { bucket, key, versionId, resultType, resultError } = params;
return s3.deleteObject({ Bucket: bucket, Key: key, VersionId:
versionId }, (err, result) => {
if (resultError) {
assert(err, `expected ${resultError} but found no error`);
assert.strictEqual(err.code, resultError);
assert.strictEqual(err.statusCode, errors[resultError].code);
return cb(null);
}
assert.strictEqual(err, null, 'Expected success ' +
`deleting object, got error ${err}`);
_assertDeleteResult(result, resultType, versionId);
return cb(null, result.VersionId);
});
}
function _createDeleteMarkers(s3, bucket, key, count, cb) {
return async.timesSeries(count,
(i, next) => delAndAssertResult(s3, { bucket, key,
resultType: newDeleteMarker }, next),
cb);
}
function _deleteDeleteMarkers(s3, bucket, key, deleteMarkerVids, cb) {
return async.mapSeries(deleteMarkerVids, (versionId, next) => {
delAndAssertResult(s3, { bucket, key, versionId,
resultType: deleteDeleteMarker }, next);
}, () => cb());
}
function _getAssertDeleted(s3, params, cb) {
const { key, versionId, errorCode } = params;
return s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
err => {
assert.strictEqual(err.code, errorCode);
assert.strictEqual(err.statusCode, 404);
return cb();
});
}
function _awsGetAssertDeleted(params, cb) {
const { key, versionId, errorCode } = params;
return awsS3.getObject({ Bucket: awsBucket, Key: key, VersionId:
versionId }, err => {
assert.strictEqual(err.code, errorCode);
assert.strictEqual(err.statusCode, 404);
return cb();
});
}
describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' +
'using object location constraint', function testSuite() {
this.timeout(30000);
withV4(sigCfg => {
let bucketUtil;
let s3;
beforeEach(() => {
process.stdout.write('Creating bucket\n');
bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3;
return s3.createBucketAsync({ Bucket: bucket })
.catch(err => {
process.stdout.write(`Error creating bucket: ${err}\n`);
throw err;
});
});
afterEach(() => {
process.stdout.write('Emptying bucket\n');
return bucketUtil.empty(bucket)
.then(() => {
process.stdout.write('Deleting bucket\n');
return bucketUtil.deleteOne(bucket);
})
.catch(err => {
process.stdout.write('Error emptying/deleting bucket: ' +
`${err}\n`);
throw err;
});
});
it('versioning not configured: if specifying "null" version, should ' +
'delete specific version in AWS backend', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putToAwsBackend(s3, bucket, key, someBody,
err => next(err)),
next => awsGetLatestVerId(key, someBody, next),
(awsVerId, next) => delAndAssertResult(s3, { bucket,
key, versionId: 'null', resultType: deleteVersion },
err => next(err, awsVerId)),
(awsVerId, next) => _awsGetAssertDeleted({ key,
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
], done);
});
it('versioning not configured: specifying any version id other ' +
'than null should not result in its deletion in AWS backend', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putToAwsBackend(s3, bucket, key, someBody,
err => next(err)),
next => awsGetLatestVerId(key, someBody, next),
(awsVerId, next) => delAndAssertResult(s3, { bucket,
key, versionId: 'awsVerId', resultError:
'InvalidArgument' }, err => next(err, awsVerId)),
(awsVerId, next) => awsGetLatestVerId(key, someBody,
(err, resultVid) => {
assert.strictEqual(resultVid, awsVerId);
next();
}),
], done);
});
it('versioning suspended: should delete a specific version in AWS ' +
'backend successfully', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, [someBody],
err => next(err)),
next => awsGetLatestVerId(key, someBody, next),
(awsVerId, next) => delAndAssertResult(s3, { bucket,
key, versionId: 'null', resultType: deleteVersion },
err => next(err, awsVerId)),
(awsVerId, next) => _awsGetAssertDeleted({ key,
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
], done);
});
it('versioning enabled: should delete a specific version in AWS ' +
'backend successfully', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putVersionsToAws(s3, bucket, key, [someBody],
(err, versionIds) => next(err, versionIds[0])),
(s3vid, next) => awsGetLatestVerId(key, someBody,
(err, awsVid) => next(err, s3vid, awsVid)),
(s3VerId, awsVerId, next) => delAndAssertResult(s3, { bucket,
key, versionId: s3VerId, resultType: deleteVersion },
err => next(err, awsVerId)),
(awsVerId, next) => _awsGetAssertDeleted({ key,
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
], done);
});
it('versioning not configured: deleting existing object should ' +
'not return version id or x-amz-delete-marker: true but should ' +
'create a delete marker in aws ', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putToAwsBackend(s3, bucket, key, someBody,
err => next(err)),
next => delAndAssertResult(s3, { bucket, key,
resultType: nonVersionedDelete }, err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
next),
], done);
});
it('versioning suspended: should create a delete marker in s3 ' +
'and aws successfully when deleting existing object', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, [someBody],
err => next(err)),
next => delAndAssertResult(s3, { bucket, key, resultType:
newDeleteMarker }, err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
next),
], done);
});
// NOTE: Normal deletes when versioning is suspended create a
// delete marker with the version id "null", which overwrites an
// existing null version in s3 metadata.
it('versioning suspended: creating a delete marker will overwrite an ' +
'existing null version that is the latest version in s3 metadata,' +
' but the data of the first null version will remain in AWS',
function itF(done) {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, [someBody],
err => next(err)),
next => awsGetLatestVerId(key, someBody, next),
(awsNullVid, next) => {
this.test.awsNullVid = awsNullVid;
next();
},
// following call should generate a delete marker
next => delAndAssertResult(s3, { bucket, key, resultType:
newDeleteMarker }, next),
// delete delete marker
(dmVid, next) => delAndAssertResult(s3, { bucket, key,
versionId: dmVid, resultType: deleteDeleteMarker },
err => next(err)),
// should get no such object even after deleting del marker
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
// get directly to aws however will give us first null version
next => awsGetLatestVerId(key, someBody, next),
(awsLatestVid, next) => {
assert.strictEqual(awsLatestVid, this.test.awsNullVid);
next();
},
], done);
});
// NOTE: Normal deletes when versioning is suspended create a
// delete marker with the version id "null" which is supposed to
// overwrite any existing null version.
it('versioning suspended: creating a delete marker will overwrite an ' +
'existing null version that is not the latest version in s3 metadata,' +
' but the data of the first null version will remain in AWS',
function itF(done) {
const key = `somekey-${Date.now()}`;
const data = [undefined, 'data1'];
async.waterfall([
// put null version
next => putToAwsBackend(s3, bucket, key, data[0],
err => next(err)),
next => awsGetLatestVerId(key, '', next),
(awsNullVid, next) => {
this.test.awsNullVid = awsNullVid;
next();
},
// enable versioning and put another version
next => putVersionsToAws(s3, bucket, key, [data[1]], next),
(versions, next) => {
this.test.s3vid = versions[0];
next();
},
next => suspendVersioning(s3, bucket, next),
// overwrites null version in s3 metadata but does not send
// additional delete to AWS to clean up previous "null" version
// -- see note above
next => delAndAssertResult(s3, { bucket, key,
resultType: newDeleteMarker }, next),
(s3dmVid, next) => {
this.test.s3DeleteMarkerId = s3dmVid;
next();
},
// delete delete marker
next => delAndAssertResult(s3, { bucket, key,
versionId: this.test.s3DeleteMarkerId,
resultType: deleteDeleteMarker }, err => next(err)),
// deleting latest version after del marker
next => delAndAssertResult(s3, { bucket, key,
versionId: this.test.s3vid, resultType: deleteVersion },
err => next(err)),
// should get no such object instead of null version
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
// we get the null version that should have been "overwritten"
// when getting the latest version in AWS now
next => awsGetLatestVerId(key, '', next),
(awsLatestVid, next) => {
assert.strictEqual(awsLatestVid, this.test.awsNullVid);
next();
},
], done);
});
it('versioning enabled: should create a delete marker in s3 and ' +
'aws successfully when deleting existing object', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putVersionsToAws(s3, bucket, key, [someBody],
err => next(err)),
next => delAndAssertResult(s3, { bucket, key, resultType:
newDeleteMarker }, err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
next),
], done);
});
it('versioning enabled: should delete a delete marker in s3 and ' +
'aws successfully', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putVersionsToAws(s3, bucket, key, [someBody],
(err, versionIds) => next(err, versionIds[0])),
// create a delete marker
(s3vid, next) => delAndAssertResult(s3, { bucket, key,
resultType: newDeleteMarker }, (err, delMarkerVid) =>
next(err, s3vid, delMarkerVid)),
// delete delete marker
(s3vid, dmVid, next) => delAndAssertResult(s3, { bucket, key,
versionId: dmVid, resultType: deleteDeleteMarker },
err => next(err, s3vid)),
// should be able to get object originally put from s3
(s3vid, next) => getAndAssertResult(s3, { bucket, key,
body: someBody, expectedVersionId: s3vid }, next),
// latest version in aws should now be object originally put
next => awsGetLatestVerId(key, someBody, next),
], done);
});
it('multiple delete markers: should be able to get pre-existing ' +
'versions after creating and deleting several delete markers', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putVersionsToAws(s3, bucket, key, [someBody],
(err, versionIds) => next(err, versionIds[0])),
(s3vid, next) => _createDeleteMarkers(s3, bucket, key, 3,
(err, dmVids) => next(err, s3vid, dmVids)),
(s3vid, dmVids, next) => _deleteDeleteMarkers(s3, bucket, key,
dmVids, () => next(null, s3vid)),
// should be able to get object originally put from s3
(s3vid, next) => getAndAssertResult(s3, { bucket, key,
body: someBody, expectedVersionId: s3vid }, next),
// latest version in aws should now be object originally put
next => awsGetLatestVerId(key, someBody, next),
], done);
});
it('multiple delete markers: should get NoSuchObject if only ' +
'one of the delete markers is deleted', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putVersionsToAws(s3, bucket, key, [someBody],
err => next(err)),
next => _createDeleteMarkers(s3, bucket, key, 3,
(err, dmVids) => next(err, dmVids[2])),
(lastDmVid, next) => delAndAssertResult(s3, { bucket,
key, versionId: lastDmVid, resultType: deleteDeleteMarker },
err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
next),
], done);
});
it('should get the new latest version after deleting the latest' +
'specific version', done => {
const key = `somekey-${Date.now()}`;
const data = [...Array(4).keys()].map(i => i.toString());
async.waterfall([
// put 3 null versions
next => mapToAwsPuts(s3, bucket, key, data.slice(0, 3),
err => next(err)),
// put one version
next => putVersionsToAws(s3, bucket, key, [data[3]],
(err, versionIds) => next(err, versionIds[0])),
// delete the latest version
(versionId, next) => delAndAssertResult(s3, { bucket,
key, versionId, resultType: deleteVersion },
err => next(err)),
// should get the last null version
next => getAndAssertResult(s3, { bucket, key,
body: data[2], expectedVersionId: 'null' }, next),
next => awsGetLatestVerId(key, data[2],
err => next(err)),
// delete the null version
next => delAndAssertResult(s3, { bucket,
key, versionId: 'null', resultType: deleteVersion },
err => next(err)),
// s3 metadata should report no existing versions for keyname
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
// NOTE: latest version in aws will be the second null version
next => awsGetLatestVerId(key, data[1],
err => next(err)),
], done);
});
it('should delete the correct version even if other versions or ' +
'delete markers put directly on aws', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putVersionsToAws(s3, bucket, key, [someBody],
(err, versionIds) => next(err, versionIds[0])),
(s3vid, next) => awsGetLatestVerId(key, someBody,
(err, awsVid) => next(err, s3vid, awsVid)),
// put an object in AWS
(s3vid, awsVid, next) => awsS3.putObject({ Bucket: awsBucket,
Key: key }, err => next(err, s3vid, awsVid)),
// create a delete marker in AWS
(s3vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key }, err => next(err, s3vid, awsVid)),
// delete original version in s3
(s3vid, awsVid, next) => delAndAssertResult(s3, { bucket, key,
versionId: s3vid, resultType: deleteVersion },
err => next(err, awsVid)),
(awsVid, next) => _getAssertDeleted(s3, { key,
errorCode: 'NoSuchKey' }, () => next(null, awsVid)),
(awsVerId, next) => _awsGetAssertDeleted({ key,
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
], done);
});
it('should not return an error deleting a version that was already ' +
'deleted directly from AWS backend', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putVersionsToAws(s3, bucket, key, [someBody],
(err, versionIds) => next(err, versionIds[0])),
(s3vid, next) => awsGetLatestVerId(key, someBody,
(err, awsVid) => next(err, s3vid, awsVid)),
// delete the object in AWS
(s3vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key, VersionId: awsVid }, err => next(err, s3vid)),
// then try to delete in S3
(s3vid, next) => delAndAssertResult(s3, { bucket, key,
versionId: s3vid, resultType: deleteVersion },
err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
], done);
});
});
});
describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' +
'using bucket location constraint', function testSuite() {
this.timeout(30000);
const createBucketParams = {
Bucket: bucket,
CreateBucketConfiguration: {
LocationConstraint: awsLocation,
},
};
withV4(sigCfg => {
let bucketUtil;
let s3;
beforeEach(() => {
process.stdout.write('Creating bucket\n');
bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3;
return s3.createBucketAsync(createBucketParams)
.catch(err => {
process.stdout.write(`Error creating bucket: ${err}\n`);
throw err;
});
});
afterEach(() => {
process.stdout.write('Emptying bucket\n');
return bucketUtil.empty(bucket)
.then(() => {
process.stdout.write('Deleting bucket\n');
return bucketUtil.deleteOne(bucket);
})
.catch(err => {
process.stdout.write('Error emptying/deleting bucket: ' +
`${err}\n`);
throw err;
});
});
it('versioning not configured: deleting non-existing object should ' +
'not return version id or x-amz-delete-marker: true nor create a ' +
'delete marker in aws ', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => delAndAssertResult(s3, { bucket, key,
resultType: nonVersionedDelete }, err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
next),
], done);
});
it('versioning suspended: should create a delete marker in s3 ' +
'and aws successfully when deleting non-existing object', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => suspendVersioning(s3, bucket, next),
next => delAndAssertResult(s3, { bucket, key, resultType:
newDeleteMarker }, err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
next),
], done);
});
it('versioning enabled: should create a delete marker in s3 and ' +
'aws successfully when deleting non-existing object', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => delAndAssertResult(s3, { bucket, key, resultType:
newDeleteMarker }, err => next(err)),
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
next),
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
next),
], done);
});
});
});

View File

@ -3,10 +3,15 @@ const async = require('async');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const { config } = require('../../../../../../lib/Config'); const {
const { uniqName, getAzureClient, getAzureContainerName, getAzureKeys, describeSkipIfNotMultiple,
azureLocation, azureLocationMismatch } = uniqName,
require('../utils'); getAzureClient,
getAzureContainerName,
getAzureKeys,
azureLocation,
azureLocationMismatch,
} = require('../utils');
const keyObject = 'deleteazure'; const keyObject = 'deleteazure';
const azureContainerName = getAzureContainerName(); const azureContainerName = getAzureContainerName();
@ -20,10 +25,6 @@ const nonExistingId = process.env.AWS_ON_AIR ?
'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' :
'3939393939393939393936493939393939393939756e6437'; '3939393939393939393936493939393939393939756e6437';
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
describeSkipIfNotMultiple('Multiple backend delete object from Azure', describeSkipIfNotMultiple('Multiple backend delete object from Azure',
function testSuite() { function testSuite() {
this.timeout(250000); this.timeout(250000);

View File

@ -2,9 +2,13 @@ const assert = require('assert');
const async = require('async'); const async = require('async');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { config } = require('../../../../../../lib/Config'); const {
const { memLocation, fileLocation, awsLocation, awsLocationMismatch } describeSkipIfNotMultiple,
= require('../utils'); memLocation,
fileLocation,
awsLocation,
awsLocationMismatch,
} = require('../utils');
const bucket = 'buckettestmultiplebackendget'; const bucket = 'buckettestmultiplebackendget';
const memObject = `memobject-${Date.now()}`; const memObject = `memobject-${Date.now()}`;
@ -20,9 +24,6 @@ const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
const bigMD5 = 'f1c9645dbc14efddc7d8a322685f26eb'; const bigMD5 = 'f1c9645dbc14efddc7d8a322685f26eb';
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
describe('Multiple backend get object', function testSuite() { describe('Multiple backend get object', function testSuite() {
this.timeout(30000); this.timeout(30000);
withV4(sigCfg => { withV4(sigCfg => {

View File

@ -1,46 +1,23 @@
const assert = require('assert'); const assert = require('assert');
const AWS = require('aws-sdk');
const async = require('async'); const async = require('async');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { config } = require('../../../../../../lib/Config');
const { getRealAwsConfig } = require('../../support/awsConfig');
const { const {
awsS3,
awsLocation, awsLocation,
awsBucket,
enableVersioning, enableVersioning,
suspendVersioning, suspendVersioning,
mapToAwsPuts, mapToAwsPuts,
putNullVersionsToAws, putNullVersionsToAws,
putVersionsToAws, putVersionsToAws,
expectedETag, getAndAssertResult,
describeSkipIfNotMultiple,
} = require('../utils'); } = require('../utils');
const someBody = 'testbody'; const someBody = 'testbody';
const bucket = 'buckettestmultiplebackendgetawsversioning'; const bucket = 'buckettestmultiplebackendgetawsversioning';
let awsS3;
function getAndAssertResult(s3, params, cb) {
const { bucket, key, body, versionId, expectedVersionId } = params;
s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
(err, data) => {
assert.strictEqual(err, null, 'Expected success ' +
`getting object, got error ${err}`);
if (body) {
assert(data.Body, 'expected object body in response');
const expectedMD5 = expectedETag(body, false);
const resultMD5 = expectedETag(data.Body, false);
assert.strictEqual(resultMD5, expectedMD5);
}
if (!expectedVersionId) {
assert.strictEqual(data.VersionId, undefined);
} else {
assert.strictEqual(data.VersionId, expectedVersionId);
}
cb();
});
}
function getAndAssertVersions(s3, bucket, key, versionIds, expectedData, function getAndAssertVersions(s3, bucket, key, versionIds, expectedData,
cb) { cb) {
async.mapSeries(versionIds, (versionId, next) => { async.mapSeries(versionIds, (versionId, next) => {
@ -58,16 +35,7 @@ function getAndAssertVersions(s3, bucket, key, versionIds, expectedData,
}); });
} }
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple' describeSkipIfNotMultiple('AWS backend get object with versioning',
|| process.env.S3_END_TO_END) ? describe.skip : describe;
if (describeSkipIfNotMultiple !== describe.skip) {
// can only get real aws config if not running end-to-end
const awsConfig = getRealAwsConfig(awsLocation);
awsS3 = new AWS.S3(awsConfig);
}
describeSkipIfNotMultiple('Multiple backend get object with versioning',
function testSuite() { function testSuite() {
this.timeout(30000); this.timeout(30000);
withV4(sigCfg => { withV4(sigCfg => {
@ -308,8 +276,6 @@ function testSuite() {
it('should return the correct data getting versioned object ' + it('should return the correct data getting versioned object ' +
'even if object was deleted from AWS (creating a delete marker)', 'even if object was deleted from AWS (creating a delete marker)',
done => { done => {
const awsBucket = config.locationConstraints[awsLocation].
details.bucketName;
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
async.waterfall([ async.waterfall([
next => enableVersioning(s3, bucket, next), next => enableVersioning(s3, bucket, next),
@ -327,15 +293,13 @@ function testSuite() {
it('should return the correct data getting versioned object ' + it('should return the correct data getting versioned object ' +
'even if object is put directly to AWS (creating new version)', 'even if object is put directly to AWS (creating new version)',
done => { done => {
const awsBucket = config.locationConstraints[awsLocation].
details.bucketName;
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
async.waterfall([ async.waterfall([
next => enableVersioning(s3, bucket, next), next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key, Body: someBody, next => s3.putObject({ Bucket: bucket, Key: key, Body: someBody,
Metadata: { 'scal-location-constraint': awsLocation } }, Metadata: { 'scal-location-constraint': awsLocation } },
(err, res) => next(err, res.VersionId)), (err, res) => next(err, res.VersionId)),
// create a delete marker in AWS // put an object in AWS
(versionId, next) => awsS3.putObject({ Bucket: awsBucket, (versionId, next) => awsS3.putObject({ Bucket: awsBucket,
Key: key }, err => next(err, versionId)), Key: key }, err => next(err, versionId)),
(versionId, next) => getAndAssertResult(s3, { bucket, key, (versionId, next) => getAndAssertResult(s3, { bucket, key,
@ -346,8 +310,6 @@ function testSuite() {
it('should return a InternalError if trying to get an object ' + it('should return a InternalError if trying to get an object ' +
'that was deleted in AWS but exists in s3 metadata', 'that was deleted in AWS but exists in s3 metadata',
done => { done => {
const awsBucket = config.locationConstraints[awsLocation].
details.bucketName;
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
async.waterfall([ async.waterfall([
next => enableVersioning(s3, bucket, next), next => enableVersioning(s3, bucket, next),
@ -372,8 +334,6 @@ function testSuite() {
it('should return a InternalError if trying to get a version ' + it('should return a InternalError if trying to get a version ' +
'that was deleted in AWS but exists in s3 metadata', 'that was deleted in AWS but exists in s3 metadata',
done => { done => {
const awsBucket = config.locationConstraints[awsLocation].
details.bucketName;
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
async.waterfall([ async.waterfall([
next => enableVersioning(s3, bucket, next), next => enableVersioning(s3, bucket, next),

View File

@ -3,21 +3,22 @@ const assert = require('assert');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const { uniqName, getAzureClient, getAzureContainerName, const {
getAzureKeys, azureLocation } = require('../utils'); describeSkipIfNotMultiple,
uniqName,
getAzureClient,
getAzureContainerName,
getAzureKeys,
azureLocation,
} = require('../utils');
const azureClient = getAzureClient(); const azureClient = getAzureClient();
const azureContainerName = getAzureContainerName(); const azureContainerName = getAzureContainerName();
const keys = getAzureKeys(); const keys = getAzureKeys();
const keyObject = 'getazure'; const keyObject = 'getazure';
const { config } = require('../../../../../../lib/Config');
const normalBody = Buffer.from('I am a body', 'utf8'); const normalBody = Buffer.from('I am a body', 'utf8');
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
const azureTimeout = 10000; const azureTimeout = 10000;
describeSkipIfNotMultiple('Multiple backend get object from Azure', describeSkipIfNotMultiple('Multiple backend get object from Azure',

View File

@ -1,22 +1,10 @@
const async = require('async'); const async = require('async');
const assert = require('assert'); const assert = require('assert');
const { config } = require('../../../../../../lib/Config');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { azureLocation } = require('../utils'); const { describeSkipIfNotMultiple, azureLocation, azureContainerName }
= require('../utils');
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
let azureContainerName;
if (config.locationConstraints[azureLocation] &&
config.locationConstraints[azureLocation].details &&
config.locationConstraints[azureLocation].details.azureContainerName) {
azureContainerName =
config.locationConstraints[azureLocation].details.azureContainerName;
}
const keyName = `somekey-${Date.now()}`; const keyName = `somekey-${Date.now()}`;

View File

@ -1,24 +1,14 @@
const assert = require('assert'); const assert = require('assert');
const { config } = require('../../../../../../lib/Config');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { azureLocation } = require('../utils'); const { describeSkipIfNotMultiple, azureLocation, getAzureContainerName }
= require('../utils');
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple' const azureContainerName = getAzureContainerName();
|| process.env.S3_END_TO_END) ? describe.skip : describe;
let azureContainerName;
const bodyFirstPart = Buffer.alloc(10); const bodyFirstPart = Buffer.alloc(10);
const bodySecondPart = Buffer.alloc(104857610); const bodySecondPart = Buffer.alloc(104857610);
if (config.locationConstraints[azureLocation] &&
config.locationConstraints[azureLocation].details &&
config.locationConstraints[azureLocation].details.azureContainerName) {
azureContainerName =
config.locationConstraints[azureLocation].details.azureContainerName;
}
let bucketUtil; let bucketUtil;
let s3; let s3;

View File

@ -4,9 +4,8 @@ const async = require('async');
const { s3middleware } = require('arsenal'); const { s3middleware } = require('arsenal');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { uniqName, getAzureClient, getAzureContainerName, convertMD5, const { describeSkipIfNotMultiple, uniqName, getAzureClient,
azureLocation } = require('../utils'); getAzureContainerName, convertMD5, azureLocation } = require('../utils');
const { config } = require('../../../../../../lib/Config');
const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
const maxSubPartSize = azureMpuUtils.maxSubPartSize; const maxSubPartSize = azureMpuUtils.maxSubPartSize;
@ -15,9 +14,6 @@ const azureClient = getAzureClient();
const azureContainerName = getAzureContainerName(); const azureContainerName = getAzureContainerName();
const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005'; const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
let bucketUtil; let bucketUtil;
let s3; let s3;

View File

@ -1,21 +1,23 @@
const async = require('async'); const async = require('async');
const assert = require('assert'); const assert = require('assert');
const AWS = require('aws-sdk');
const { s3middleware } = require('arsenal'); const { s3middleware } = require('arsenal');
const { config } = require('../../../../../../lib/Config');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { fileLocation, awsLocation, azureLocation, azureLocationMismatch, const {
getAzureClient, getAzureContainerName } = require('../utils'); describeSkipIfNotMultiple,
const { getRealAwsConfig } = fileLocation,
require('../../support/awsConfig'); awsS3,
awsLocation,
awsBucket,
azureLocation,
azureLocationMismatch,
getAzureClient,
getAzureContainerName,
} = require('../utils');
const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
const awsBucket = 'multitester555';
const azureContainerName = getAzureContainerName(); const azureContainerName = getAzureContainerName();
const azureClient = getAzureClient(); const azureClient = getAzureClient();
const azureTimeout = 20000; const azureTimeout = 20000;
@ -106,8 +108,7 @@ function testSuite() {
this.currentTest.key = `somekey-${Date.now()}`; this.currentTest.key = `somekey-${Date.now()}`;
bucketUtil = new BucketUtility('default', sigCfg); bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3; s3 = bucketUtil.s3;
const awsConfig = getRealAwsConfig(awsLocation); this.currentTest.awsClient = awsS3;
this.currentTest.awsClient = new AWS.S3(awsConfig);
return s3.createBucketAsync({ Bucket: azureContainerName }) return s3.createBucketAsync({ Bucket: azureContainerName })
.catch(err => { .catch(err => {
process.stdout.write(`Error creating bucket: ${err}\n`); process.stdout.write(`Error creating bucket: ${err}\n`);

View File

@ -0,0 +1,176 @@
const assert = require('assert');
const async = require('async');
const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util');
const { minimumAllowedPartSize } = require('../../../../../../constants');
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
const {
awsLocation,
enableVersioning,
suspendVersioning,
putToAwsBackend,
awsGetLatestVerId,
getAndAssertResult,
describeSkipIfNotMultiple,
} = require('../utils');
const data = ['a', 'b'].map(char => Buffer.alloc(minimumAllowedPartSize, char));
const concattedData = Buffer.concat(data);
const bucket = 'buckettestmultiplebackendmpuawsversioning';
function mpuSetup(s3, key, location, cb) {
const partArray = [];
async.waterfall([
next => {
const params = {
Bucket: bucket,
Key: key,
Metadata: { 'scal-location-constraint': location },
};
s3.createMultipartUpload(params, (err, res) => {
assert.strictEqual(err, null, `err creating mpu: ${err}`);
const uploadId = res.UploadId;
assert(uploadId);
assert.strictEqual(res.Bucket, bucket);
assert.strictEqual(res.Key, key);
next(err, uploadId);
});
},
(uploadId, next) => {
const partParams = {
Bucket: bucket,
Key: key,
PartNumber: 1,
UploadId: uploadId,
Body: data[0],
};
s3.uploadPart(partParams, (err, res) => {
assert.strictEqual(err, null, `err uploading part 1: ${err}`);
partArray.push({ ETag: res.ETag, PartNumber: 1 });
next(err, uploadId);
});
},
(uploadId, next) => {
const partParams = {
Bucket: bucket,
Key: key,
PartNumber: 2,
UploadId: uploadId,
Body: data[1],
};
s3.uploadPart(partParams, (err, res) => {
assert.strictEqual(err, null, `err uploading part 2: ${err}`);
partArray.push({ ETag: res.ETag, PartNumber: 2 });
next(err, uploadId);
});
},
], (err, uploadId) => {
process.stdout.write('Created MPU and put two parts\n');
cb(err, uploadId, partArray);
});
}
function completeAndAssertMpu(s3, params, cb) {
const { bucket, key, uploadId, partArray, expectVersionId,
expectedGetVersionId } = params;
s3.completeMultipartUpload({
Bucket: bucket,
Key: key,
UploadId: uploadId,
MultipartUpload: { Parts: partArray },
}, (err, data) => {
assert.strictEqual(err, null, `Err completing MPU: ${err}`);
if (expectVersionId) {
assert.notEqual(data.VersionId, undefined);
} else {
assert.strictEqual(data.VersionId, undefined);
}
const expectedVersionId = expectedGetVersionId || data.VersionId;
getAndAssertResult(s3, { bucket, key, body: concattedData,
expectedVersionId }, cb);
});
}
describeSkipIfNotMultiple('AWS backend complete mpu with versioning',
function testSuite() {
this.timeout(30000);
withV4(sigCfg => {
const bucketUtil = new BucketUtility('default', sigCfg);
const s3 = bucketUtil.s3;
beforeEach(done => s3.createBucket({
Bucket: bucket,
CreateBucketConfiguration: {
LocationConstraint: awsLocation,
},
}, done));
afterEach(done => {
removeAllVersions({ Bucket: bucket }, err => {
if (err) {
return done(err);
}
return s3.deleteBucket({ Bucket: bucket }, done);
});
});
it('versioning not configured: should not return version id ' +
'completing mpu', done => {
const key = `somekey-${Date.now()}`;
mpuSetup(s3, key, awsLocation, (err, uploadId, partArray) => {
completeAndAssertMpu(s3, { bucket, key, uploadId, partArray,
expectVersionId: false }, done);
});
});
it('versioning not configured: if complete mpu on already-existing ' +
'object, metadata should be overwritten but data of previous version' +
'in AWS should not be deleted', function itF(done) {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putToAwsBackend(s3, bucket, key, '', err => next(err)),
next => awsGetLatestVerId(key, '', next),
(awsVerId, next) => {
this.test.awsVerId = awsVerId;
next();
},
next => mpuSetup(s3, key, awsLocation, next),
(uploadId, partArray, next) => completeAndAssertMpu(s3,
{ bucket, key, uploadId, partArray, expectVersionId:
false }, next),
next => s3.deleteObject({ Bucket: bucket, Key: key, VersionId:
'null' }, next),
(delData, next) => getAndAssertResult(s3, { bucket, key,
expectedError: 'NoSuchKey' }, next),
next => awsGetLatestVerId(key, '', next),
(awsVerId, next) => {
assert.strictEqual(awsVerId, this.test.awsVerId);
next();
},
], done);
});
it('versioning suspended: should not return version id completing mpu',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => suspendVersioning(s3, bucket, next),
next => mpuSetup(s3, key, awsLocation, next),
(uploadId, partArray, next) => completeAndAssertMpu(s3,
{ bucket, key, uploadId, partArray, expectVersionId: false,
expectedGetVersionId: 'null' }, next),
], done);
});
it('versioning enabled: should return version id completing mpu',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => mpuSetup(s3, key, awsLocation, next),
(uploadId, partArray, next) => completeAndAssertMpu(s3,
{ bucket, key, uploadId, partArray, expectVersionId: true },
next),
], done);
});
});
});

View File

@ -4,9 +4,9 @@ const async = require('async');
const { s3middleware } = require('arsenal'); const { s3middleware } = require('arsenal');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { expectedETag, uniqName, getAzureClient, getAzureContainerName, const { describeSkipIfNotMultiple, expectedETag, uniqName, getAzureClient,
convertMD5, azureLocation } = require('../utils'); getAzureContainerName, convertMD5, azureLocation, azureLocationMismatch }
const { config } = require('../../../../../../lib/Config'); = require('../utils');
const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
const maxSubPartSize = azureMpuUtils.maxSubPartSize; const maxSubPartSize = azureMpuUtils.maxSubPartSize;
const getBlockId = azureMpuUtils.getBlockId; const getBlockId = azureMpuUtils.getBlockId;
@ -16,10 +16,6 @@ const azureClient = getAzureClient();
const azureContainerName = getAzureContainerName(); const azureContainerName = getAzureContainerName();
const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005'; const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
const azureLocationMismatch = 'azuretestmismatch';
let bucketUtil; let bucketUtil;
let s3; let s3;

View File

@ -4,10 +4,17 @@ const async = require('async');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const constants = require('../../../../../../constants'); const constants = require('../../../../../../constants');
const { config } = require('../../../../../../lib/Config'); const {
const { getAzureClient, getAzureContainerName, convertMD5, memLocation, describeSkipIfNotMultiple,
awsLocation, azureLocation, azureLocation2, azureLocationMismatch } = getAzureClient,
require('../utils'); getAzureContainerName,
convertMD5,
memLocation,
awsLocation,
azureLocation,
azureLocation2,
azureLocationMismatch,
} = require('../utils');
const { createEncryptedBucketPromise } = const { createEncryptedBucketPromise } =
require('../../../lib/utility/createEncryptedBucket'); require('../../../lib/utility/createEncryptedBucket');
@ -26,8 +33,6 @@ const azureTimeout = 40000;
let bucketUtil; let bucketUtil;
let s3; let s3;
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
function putSourceObj(key, location, objSize, cb) { function putSourceObj(key, location, objSize, cb) {
const sourceParams = { Bucket: bucket, Key: key, const sourceParams = { Bucket: bucket, Key: key,

View File

@ -6,10 +6,12 @@ const BucketUtility = require('../../../lib/utility/bucket-util');
const constants = require('../../../../../../constants'); const constants = require('../../../../../../constants');
const { config } = require('../../../../../../lib/Config'); const { config } = require('../../../../../../lib/Config');
const { getRealAwsConfig } = require('../../support/awsConfig'); const { getRealAwsConfig } = require('../../support/awsConfig');
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
const { createEncryptedBucketPromise } = const { createEncryptedBucketPromise } =
require('../../../lib/utility/createEncryptedBucket'); require('../../../lib/utility/createEncryptedBucket');
const { memLocation, awsLocation, awsLocation2, awsLocationMismatch } = const { describeSkipIfNotMultiple, awsS3, awsBucket, memLocation, awsLocation,
require('../utils'); awsLocation2, awsLocationMismatch, awsLocationEncryption }
= require('../utils');
const bucket = 'buckettestmultiplebackendobjectcopy'; const bucket = 'buckettestmultiplebackendobjectcopy';
const body = Buffer.from('I am a body', 'utf8'); const body = Buffer.from('I am a body', 'utf8');
@ -18,13 +20,8 @@ const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
const locMetaHeader = constants.objectLocationConstraintHeader.substring(11); const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
const { versioningEnabled } = require('../../../lib/utility/versioning-util'); const { versioningEnabled } = require('../../../lib/utility/versioning-util');
const awsLocationEncryption = 'aws-test-encryption';
let bucketUtil; let bucketUtil;
let s3; let s3;
let awsS3;
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
function putSourceObj(location, isEmptyObj, cb) { function putSourceObj(location, isEmptyObj, cb) {
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
@ -52,8 +49,6 @@ function putSourceObj(location, isEmptyObj, cb) {
function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey, function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey,
destBucket, destLoc, awsKey, mdDirective, isEmptyObj, awsS3, awsLocation, destBucket, destLoc, awsKey, mdDirective, isEmptyObj, awsS3, awsLocation,
callback) { callback) {
const awsBucket =
config.locationConstraints[awsLocation].details.bucketName;
const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey }; const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey };
const destGetParams = { Bucket: destBucket, Key: destKey }; const destGetParams = { Bucket: destBucket, Key: destKey };
const awsParams = { Bucket: awsBucket, Key: awsKey }; const awsParams = { Bucket: awsBucket, Key: awsKey };
@ -109,32 +104,20 @@ describeSkipIfNotMultiple('MultipleBackend object copy',
function testSuite() { function testSuite() {
this.timeout(250000); this.timeout(250000);
withV4(sigCfg => { withV4(sigCfg => {
beforeEach(() => { const bucketUtil = new BucketUtility('default', sigCfg);
bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3;
s3 = bucketUtil.s3; beforeEach(done => s3.createBucket({
const awsConfig = getRealAwsConfig(awsLocation); Bucket: bucket,
awsS3 = new AWS.S3(awsConfig); CreateBucketConfiguration: {
process.stdout.write('Creating bucket\n'); LocationConstraint: awsLocation,
if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { },
s3.createBucketAsync = createEncryptedBucketPromise; }, done));
afterEach(done => {
removeAllVersions({ Bucket: bucket }, err => {
if (err) {
return done(err);
} }
return s3.createBucketAsync({ Bucket: bucket }) return s3.deleteBucket({ Bucket: bucket }, done);
.catch(err => {
process.stdout.write(`Error creating bucket: ${err}\n`);
throw err;
});
});
afterEach(() => {
process.stdout.write('Emptying bucket\n');
return bucketUtil.empty(bucket)
.then(() => {
process.stdout.write('Deleting bucket\n');
return bucketUtil.deleteOne(bucket);
})
.catch(err => {
process.stdout.write(`Error in afterEach: ${err}\n`);
throw err;
}); });
}); });

View File

@ -0,0 +1,384 @@
const assert = require('assert');
const async = require('async');
const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util');
const constants = require('../../../../../../constants');
const {
describeSkipIfNotMultiple,
awsS3,
awsBucket,
memLocation,
fileLocation,
awsLocation,
enableVersioning,
suspendVersioning,
putToAwsBackend,
awsGetLatestVerId,
} = require('../utils');
const sourceBucketName = 'buckettestobjectcopyawsversioning-source';
const destBucketName = 'buckettestobjectcopyawsversioning-dest';
const someBody = Buffer.from('I am a body', 'utf8');
const wrongVersionBody = 'this is not the content you wanted';
const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
let bucketUtil;
let s3;
function _getTestMetadata(location) {
return {
'scal-location-constraint': location,
'test-header': 'copyme',
};
}
function putSourceObj(testParams, cb) {
const { sourceBucket, sourceLocation, isEmptyObj } = testParams;
const sourceKey = `sourcekey-${Date.now()}`;
const sourceParams = {
Bucket: sourceBucket,
Key: sourceKey,
Metadata: _getTestMetadata(sourceLocation),
};
if (!isEmptyObj) {
sourceParams.Body = someBody;
}
s3.putObject(sourceParams, (err, result) => {
assert.strictEqual(err, null,
`Error putting source object: ${err}`);
if (isEmptyObj) {
assert.strictEqual(result.ETag, `"${emptyMD5}"`);
} else {
assert.strictEqual(result.ETag, `"${correctMD5}"`);
}
Object.assign(testParams, {
sourceKey,
sourceVersionId: result.VersionId,
});
cb();
});
}
function copyObject(testParams, cb) {
const { sourceBucket, sourceKey, sourceVersionId, sourceVersioningState,
destBucket, destLocation, directive, destVersioningState, isEmptyObj }
= testParams;
const destKey = `destkey-${Date.now()}`;
const copyParams = {
Bucket: destBucket,
Key: destKey,
CopySource: `/${sourceBucket}/${sourceKey}`,
MetadataDirective: directive,
Metadata: {
'scal-location-constraint': destLocation,
},
};
if (sourceVersionId) {
copyParams.CopySource =
`${copyParams.CopySource}?versionId=${sourceVersionId}`;
} else if (sourceVersioningState === 'Suspended') {
copyParams.CopySource =
`${copyParams.CopySource}?versionId=null`;
}
console.log('===================');
console.log('params sent to object copy', copyParams)
s3.copyObject(copyParams, (err, data) => {
assert.strictEqual(err, null,
`Error copying object to destination: ${err}`);
console.log('copy object result', data)
if (destVersioningState === 'Enabled') {
console.log('got version id for dest object', data.VersionId)
assert.notEqual(data.VersionId, undefined);
} else {
assert.strictEqual(data.VersionId, undefined);
}
const expectedBody = isEmptyObj ? '' : someBody;
return awsGetLatestVerId(destKey, expectedBody, (err, awsVersionId) => {
Object.assign(testParams, {
destKey,
destVersionId: data.VersionId,
awsVersionId,
});
if (!data.VersionId && destVersioningState === 'Suspended') {
// eslint-disable-next-line no-param-reassign
testParams.destVersionId = 'null';
}
cb();
});
});
}
function assertGetObjects(testParams, cb) {
console.log('assertGetObjecs');
const {
sourceBucket,
sourceLocation,
sourceKey,
sourceVersionId,
sourceVersioningState,
destBucket,
destLocation,
destKey,
destVersionId,
destVersioningState,
awsVersionId,
isEmptyObj,
directive,
} = testParams;
console.log('testParams in assertGetObjects..', testParams)
const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey,
VersionId: sourceVersionId };
const destGetParams = { Bucket: destBucket, Key: destKey,
VersionId: destVersionId };
const awsParams = { Bucket: awsBucket, Key: destKey,
VersionId: awsVersionId };
async.series([
cb => s3.getObject(sourceGetParams, cb),
cb => s3.getObject(destGetParams, cb),
cb => awsS3.getObject(awsParams, cb),
], (err, results) => {
assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`);
const [sourceRes, destRes, awsRes] = results;
console.log('******** sourceGetParams', sourceGetParams);
console.log('==== result of getting source object', sourceRes);
console.log('******** destGetParams', destGetParams);
console.log('==== result of getting dest object', destRes);
console.log('******** awsGetParams', awsParams);
console.log('==== result of getting aws object', awsRes)
// NOTE: assert version ids?
if (isEmptyObj) {
assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`);
assert.strictEqual(destRes.ETag, `"${emptyMD5}"`);
assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`);
} else {
assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`);
assert.strictEqual(destRes.ETag, `"${correctMD5}"`);
assert.deepStrictEqual(sourceRes.Body, destRes.Body);
assert.strictEqual(awsRes.ETag, `"${correctMD5}"`);
assert.deepStrictEqual(sourceRes.Body, awsRes.Body);
}
if (directive === 'COPY') {
assert.deepStrictEqual(sourceRes.Metadata['test-header'],
destRes.Metadata['test-header']);
} else if (directive === 'REPLACE') {
assert.strictEqual(destRes.Metadata['test-header'],
undefined);
}
assert.strictEqual(awsRes.Metadata[locMetaHeader], destLocation);
if (directive === 'COPY') {
assert.deepStrictEqual(sourceRes.Metadata['test-header'],
awsRes.Metadata['test-header']);
} else if (directive === 'REPLACE') {
assert.strictEqual(awsRes.Metadata['test-header'],
undefined);
}
assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength);
assert.strictEqual(sourceRes.Metadata[locMetaHeader], sourceLocation);
assert.strictEqual(destRes.Metadata[locMetaHeader], destLocation);
cb();
});
}
/*
const testParams = {
sourceBucket: sourceBucketName,
sourceLocation: awsLocation,
sourceVersioningState: undefined,
destBucket: destBucketName,
destLocation: awsLocation,
destVersioningState: 'Enabled',
isEmptyObj: false,
directive: 'REPLACE',
};*/
// describeSkipIfNotMultiple
describe.only('AWS backend object copy with versioning',
function testSuite() {
this.timeout(250000);
withV4(sigCfg => {
beforeEach(() => {
bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3;
process.stdout.write('Creating buckets\n');
/* if (process.env.ENABLE_KMS_ENCRYPTION === 'true') {
s3.createBucketAsync = createEncryptedBucketPromise;
} */
return s3.createBucketAsync({ Bucket: sourceBucketName })
.then(() => s3.createBucketAsync({ Bucket: destBucketName }))
.catch(err => {
process.stdout.write(`Error creating bucket: ${err}\n`);
throw err;
});
});
afterEach(() => bucketUtil.empty(sourceBucketName)
.then(() => bucketUtil.deleteOne(sourceBucketName))
.catch(err => {
process.stdout.write('Error deleting source bucket ' +
`in afterEach: ${err}\n`);
throw err;
})
.then(() => bucketUtil.empty(destBucketName))
.then(() => bucketUtil.deleteOne(destBucketName))
);
[{
directive: 'REPLACE',
isEmptyObj: true,
}, {
directive: 'REPLACE',
isEmptyObj: false,
}, {
directive: 'COPY',
isEmptyObj: false,
}].forEach(testParams => {
Object.assign(testParams, {
sourceBucket: sourceBucketName,
sourceLocation: awsLocation,
destBucket: destBucketName,
destLocation: awsLocation,
});
const { isEmptyObj, directive } = testParams;
it(`should copy a${isEmptyObj ? 'n empty' : ''} object from AWS ` +
'backend non-versioned bucket to AWS backend versioned bucket ' +
`with ${directive} directive`, done => {
Object.assign(testParams, {
sourceVersioningState: undefined,
destVersioningState: 'Enabled',
});
async.waterfall([
next => putSourceObj(testParams, next),
next => enableVersioning(s3, testParams.destBucket, next),
next => copyObject(testParams, next),
// put another version to test and make sure version id from
// copy was stored to get the right version
next => putToAwsBackend(s3, destBucketName,
testParams.destKey, wrongVersionBody, () => next()),
next => assertGetObjects(testParams, next),
], done);
});
it(`should copy ${isEmptyObj ? 'an empty' : ''}version from one ` +
`AWS backend versioned bucket to another on ${directive} directive`,
done => {
Object.assign(testParams, {
sourceVersioningState: 'Enabled',
destVersioningState: 'Enabled',
});
async.waterfall([
next => enableVersioning(s3, testParams.sourceBucket, next),
next => putSourceObj(testParams, next),
next => enableVersioning(s3, testParams.destBucket, next),
next => copyObject(testParams, next),
// put another version to test and make sure version id from
// copy was stored to get the right version
next => putToAwsBackend(s3, destBucketName,
testParams.destKey, wrongVersionBody, () => next()),
next => assertGetObjects(testParams, next),
], done);
});
it(`should copy ${isEmptyObj ? 'an empty' : ''} null version ` +
'from an AWS backend versioned bucket to a non-versioned one with '
+ `${directive} directive`, done => {
Object.assign(testParams, {
sourceVersioningState: 'Suspend',
destVersioningState: 'Suspended',
});
async.waterfall([
next => suspendVersioning(s3, testParams.sourceBucket,
next),
next => putSourceObj(testParams, next),
next => suspendVersioning(s3, testParams.destBucket, next),
next => copyObject(testParams, next),
next => enableVersioning(s3, testParams.destBucket, next),
// put another version to test and make sure version id from
// copy was stored to get the right version
next => putToAwsBackend(s3, destBucketName,
testParams.destKey, wrongVersionBody, () => next()),
next => assertGetObjects(testParams, next),
], done);
});
it(`should copy a ${isEmptyObj ? 'empty ' : ''}version from a ` +
'AWS backend versioned bucket to a non-versioned one with '
+ `${directive} directive`, done => {
Object.assign(testParams, {
sourceVersioningState: 'Enabled',
destVersioningState: 'Suspended',
});
async.waterfall([
next => enableVersioning(s3, testParams.sourceBucket, next),
next => putSourceObj(testParams, next),
next => suspendVersioning(s3, testParams.destBucket, next),
next => copyObject(testParams, next),
// put another version to test and make sure version id from
// copy was stored to get the right version
next => enableVersioning(s3, testParams.destBucket, next),
next => putToAwsBackend(s3, destBucketName,
testParams.destKey, wrongVersionBody, () => next()),
next => assertGetObjects(testParams, next),
], done);
});
});
[{
sourceLocation: memLocation,
directive: 'REPLACE',
isEmptyObj: true,
}, {
sourceLocation: fileLocation,
directive: 'REPLACE',
isEmptyObj: true,
}, {
sourceLocation: memLocation,
directive: 'COPY',
isEmptyObj: false,
}, {
sourceLocation: fileLocation,
directive: 'COPY',
isEmptyObj: false,
}].forEach(testParams => {
Object.assign(testParams, {
sourceBucket: sourceBucketName,
sourceVersioningState: 'Enabled',
destBucket: sourceBucketName,
destLocation: awsLocation,
destVersioningState: 'Enabled',
});
const { sourceLocation, directive, isEmptyObj } = testParams;
it(`should copy ${isEmptyObj ? 'empty ' : ''}object from ` +
`${sourceLocation} to same bucket on AWS backend with ` +
`versioning with ${directive}`, done => {
async.waterfall([
next => putSourceObj(testParams, next),
next => enableVersioning(s3, testParams.sourceBucket, next),
next => copyObject(testParams, next),
next => assertGetObjects(testParams, next),
], done);
});
it(`should copy a ${isEmptyObj ? 'empty ' : ''}version from ` +
`${sourceLocation} to same bucket on AWS backend with ` +
`versioning with ${directive} directive`, done => {
async.waterfall([
next => enableVersioning(s3, testParams.sourceBucket, next),
// returns a version id which is added to testParams
// to be used in object copy
next => putSourceObj(testParams, next),
next => copyObject(testParams, next),
// put another version to test and make sure version id
// from copy was stored to get the right version
next => putToAwsBackend(s3, destBucketName,
testParams.destKey, wrongVersionBody, () => next()),
next => assertGetObjects(testParams, next),
], done);
});
});
});
});

View File

@ -1,13 +1,10 @@
const assert = require('assert'); const assert = require('assert');
const async = require('async'); const async = require('async');
const AWS = require('aws-sdk'); const withV4 = require('../../support/withV4');
const withV4 = require('../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util');
const BucketUtility = require('../../lib/utility/bucket-util'); const { describeSkipIfNotMultiple, awsS3, getAzureClient, getAzureContainerName,
const { config } = require('../../../../../lib/Config'); convertMD5, memLocation, fileLocation, awsLocation, azureLocation } =
const { getRealAwsConfig } = require('../support/awsConfig'); require('../utils');
const { getAzureClient, getAzureContainerName, convertMD5,
memLocation, fileLocation, awsLocation, azureLocation } =
require('./utils');
const awsBucket = 'multitester555'; const awsBucket = 'multitester555';
const azureClient = getAzureClient(); const azureClient = getAzureClient();
@ -22,9 +19,6 @@ const cloudTimeout = 10000;
let bucketUtil; let bucketUtil;
let s3; let s3;
let awsS3;
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
const putParams = { Bucket: bucket, Body: body }; const putParams = { Bucket: bucket, Body: body };
const testBackends = [memLocation, fileLocation, awsLocation, azureLocation]; const testBackends = [memLocation, fileLocation, awsLocation, azureLocation];
@ -159,8 +153,6 @@ function testSuite() {
this.timeout(80000); this.timeout(80000);
withV4(sigCfg => { withV4(sigCfg => {
beforeEach(() => { beforeEach(() => {
const awsConfig = getRealAwsConfig(awsLocation);
awsS3 = new AWS.S3(awsConfig);
bucketUtil = new BucketUtility('default', sigCfg); bucketUtil = new BucketUtility('default', sigCfg);
s3 = bucketUtil.s3; s3 = bucketUtil.s3;
return s3.createBucketAsync({ Bucket: bucket }) return s3.createBucketAsync({ Bucket: bucket })
@ -274,7 +266,8 @@ function testSuite() {
}); });
}); });
it('should return error on putting tags to object deleted from AWS', it('should not return error on putting tags to object ' +
'that has had a delete marker put directly on from AWS',
done => { done => {
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
const params = Object.assign({ Key: key, Metadata: const params = Object.assign({ Key: key, Metadata:
@ -289,7 +282,7 @@ function testSuite() {
Tagging: putTags }; Tagging: putTags };
process.stdout.write('Putting object tags\n'); process.stdout.write('Putting object tags\n');
s3.putObjectTagging(putTagParams, err => { s3.putObjectTagging(putTagParams, err => {
assert.strictEqual(err.code, 'InternalError'); assert.strictEqual(err, null);
done(); done();
}); });
}); });
@ -325,8 +318,8 @@ function testSuite() {
}); });
}); });
it('should not return error on getting tags from object deleted ' + it('should not return error on getting tags from object that has ' +
'from AWS', done => { 'had a delete marker put directly on AWS', done => {
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
const params = Object.assign({ Key: key, Tagging: tagString, const params = Object.assign({ Key: key, Tagging: tagString,
Metadata: { 'scal-location-constraint': awsLocation } }, Metadata: { 'scal-location-constraint': awsLocation } },
@ -367,8 +360,8 @@ function testSuite() {
}); });
}); });
it('should return error on deleting tags from object deleted ' + it('should not return error on deleting tags from object that ' +
'from AWS', done => { 'has had delete markers put directly on AWS', done => {
const key = `somekey-${Date.now()}`; const key = `somekey-${Date.now()}`;
const params = Object.assign({ Key: key, Tagging: tagString, const params = Object.assign({ Key: key, Tagging: tagString,
Metadata: { 'scal-location-constraint': awsLocation } }, Metadata: { 'scal-location-constraint': awsLocation } },
@ -381,7 +374,7 @@ function testSuite() {
assert.equal(err, null); assert.equal(err, null);
const tagParams = { Bucket: bucket, Key: key }; const tagParams = { Bucket: bucket, Key: key };
s3.deleteObjectTagging(tagParams, err => { s3.deleteObjectTagging(tagParams, err => {
assert.strictEqual(err.code, 'InternalError'); assert.strictEqual(err, null);
done(); done();
}); });
}); });

View File

@ -0,0 +1,223 @@
const async = require('async');
const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util');
const bucket = 'testawsbackendtaggingdeleteversioned';
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
const {
describeSkipIfNotMultiple,
awsS3,
awsBucket,
awsLocation,
enableVersioning,
putNullVersionsToAws,
putVersionsToAws,
awsGetLatestVerId,
tagging,
} = require('../utils');
const { putTaggingAndAssert, delTaggingAndAssert, awsGetAssertTags } = tagging;
const someBody = 'teststring';
describeSkipIfNotMultiple('AWS backend object tagging with versioning :: ' +
'delete', function testSuite() {
this.timeout(30000);
const tags = { key1: 'value1', key2: 'value2' };
withV4(sigCfg => {
const bucketUtil = new BucketUtility('default', sigCfg);
const s3 = bucketUtil.s3;
beforeEach(done => s3.createBucket({
Bucket: bucket,
CreateBucketConfiguration: {
LocationConstraint: awsLocation,
},
}, done));
afterEach(done => {
removeAllVersions({ Bucket: bucket }, err => {
if (err) {
return done(err);
}
return s3.deleteBucket({ Bucket: bucket }, done);
});
});
it('versioning not configured: should delete a tag set on the ' +
'latest version if no version is specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
expectedVersionId: false }, next),
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
expectedVersionId: false }, next),
next => awsGetAssertTags({ key, expectedTags: {} }, next),
], done);
});
it('versioning not configured: should delete a tag set on the ' +
'version if specified (null)', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: 'null', expectedVersionId: false }, next),
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
versionId: 'null', expectedVersionId: false }, next),
next => awsGetAssertTags({ key, expectedTags: {} }, next),
], done);
});
it('versioning suspended: should delete a tag set on the latest ' +
'version if no version is specified', done => {
const data = [undefined, 'test1', 'test2'];
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, data, next),
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
tags, expectedVersionId: 'null' }, next),
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
expectedVersionId: 'null' }, next),
next => awsGetAssertTags({ key, expectedTags: {} }, next),
], done);
});
it('versioning suspended: should delete a tag set on a specific ' +
'version (null)', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, [undefined],
next),
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
tags, versionId: 'null', expectedVersionId: 'null' }, next),
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
versionId: 'null', expectedTags: tags,
expectedVersionId: 'null' }, next),
next => awsGetAssertTags({ key, expectedTags: {} }, next),
], done);
});
it('versioning enabled then suspended: should delete a tag set on ' +
'a specific (non-null) version if specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '',
(err, awsVid) => next(err, putData.VersionId, awsVid)),
(s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key,
[someBody], () => next(null, s3Vid, awsVid)),
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
tags, versionId: s3Vid, expectedVersionId: s3Vid }, () =>
next(null, s3Vid, awsVid)),
(s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
versionId: s3Vid, expectedVersionId: s3Vid },
() => next(null, awsVid)),
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
expectedTags: {} }, next),
], done);
});
it('versioning enabled: should delete a tag set on the latest ' +
'version if no version is specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
expectedVersionId: putData.VersionId }, next),
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
expectedVersionId: versionId }, next),
next => awsGetAssertTags({ key, expectedTags: {} }, next),
], done);
});
it('versioning enabled: should delete a tag set on a specific version',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: putData.VersionId,
expectedVersionId: putData.VersionId }, next),
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
versionId, expectedVersionId: versionId }, next),
next => awsGetAssertTags({ key, expectedTags: {} }, next),
], done);
});
it('versioning enabled: should delete a tag set on a specific ' +
'version that is not the latest version', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '',
(err, awsVid) => next(err, putData.VersionId, awsVid)),
// put another version
(s3Vid, awsVid, next) => s3.putObject({ Bucket: bucket,
Key: key, Body: someBody },
err => next(err, s3Vid, awsVid)),
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
tags, versionId: s3Vid, expectedVersionId: s3Vid }, err =>
next(err, s3Vid, awsVid)),
(s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
versionId: s3Vid, expectedVersionId: s3Vid },
() => next(null, awsVid)),
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
expectedTags: {} }, next),
], done);
});
it('versioning suspended then enabled: should delete a tag set on ' +
'a specific version (null) if specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, [undefined],
() => next()),
next => awsGetLatestVerId(key, '', next),
(awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody],
() => next(null, awsVid)),
(awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: 'null', expectedVersionId: 'null' },
() => next(null, awsVid)),
(awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
versionId: 'null', expectedVersionId: 'null' },
() => next(null, awsVid)),
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
expectedTags: {} }, next),
], done);
});
it('should return an InternalError if trying to delete ' +
'tags from object that was deleted from AWS directly',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '', next),
(awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key, VersionId: awsVid }, next),
(delData, next) => delTaggingAndAssert(s3, { bucket, key,
expectedError: 'InternalError' }, next),
], done);
});
it('should return an InternalError if trying to delete ' +
'tags from object that was deleted from AWS directly',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '',
(err, awsVid) => next(err, putData.VersionId, awsVid)),
(s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
(s3Vid, next) => delTaggingAndAssert(s3, { bucket, key,
versionId: s3Vid, expectedError: 'InternalError' }, next),
], done);
});
});
});

View File

@ -0,0 +1,285 @@
const async = require('async');
const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util');
const bucket = 'testawsbackendtaggingversioned';
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
const {
describeSkipIfNotMultiple,
awsS3,
awsBucket,
awsLocation,
enableVersioning,
putNullVersionsToAws,
putVersionsToAws,
awsGetLatestVerId,
tagging,
} = require('../utils');
const { putTaggingAndAssert, getTaggingAndAssert, delTaggingAndAssert,
awsGetAssertTags } = tagging;
const someBody = 'teststring';
describeSkipIfNotMultiple('AWS backend object tagging with versioning',
function testSuite() {
this.timeout(30000);
const tags = { key1: 'value1', key2: 'value2' };
withV4(sigCfg => {
const bucketUtil = new BucketUtility('default', sigCfg);
const s3 = bucketUtil.s3;
beforeEach(done => s3.createBucket({
Bucket: bucket,
CreateBucketConfiguration: {
LocationConstraint: awsLocation,
},
}, done));
afterEach(done => {
removeAllVersions({ Bucket: bucket }, err => {
if (err) {
return done(err);
}
return s3.deleteBucket({ Bucket: bucket }, done);
});
});
it('versioning not configured: should put/get a tag set on the ' +
'latest version if no version is specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
expectedVersionId: false }, next),
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
expectedTags: tags, expectedVersionId: false }, next),
(versionId, next) => awsGetAssertTags({ key,
expectedTags: tags }, next),
], done);
});
it('versioning not configured: should put/get a tag set on a ' +
'specific version if specified (null)', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: 'null', expectedVersionId: false }, next),
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
versionId: 'null', expectedTags: tags,
expectedVersionId: false }, next),
(versionId, next) => awsGetAssertTags({ key,
expectedTags: tags }, next),
], done);
});
it('versioning suspended: should put/get a tag set on the latest ' +
'version if no version is specified', done => {
const data = [undefined, 'test1', 'test2'];
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, data, next),
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
tags, expectedVersionId: 'null' }, next),
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
expectedTags: tags, expectedVersionId: 'null' }, next),
(versionId, next) => awsGetAssertTags({ key,
expectedTags: tags }, next),
], done);
});
it('versioning suspended: should put/get a tag set on a specific ' +
'version (null)', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, [undefined],
next),
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
tags, versionId: 'null', expectedVersionId: 'null' }, next),
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
versionId: 'null', expectedTags: tags,
expectedVersionId: 'null' }, next),
(versionId, next) => awsGetAssertTags({ key,
expectedTags: tags }, next),
], done);
});
it('versioning enabled then suspended: should put/get a tag set on ' +
'a specific (non-null) version if specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '',
(err, awsVid) => next(err, putData.VersionId, awsVid)),
(s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key,
[someBody], () => next(null, s3Vid, awsVid)),
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
tags, versionId: s3Vid, expectedVersionId: s3Vid }, () =>
next(null, s3Vid, awsVid)),
(s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
versionId: s3Vid, expectedTags: tags,
expectedVersionId: s3Vid }, () => next(null, awsVid)),
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
expectedTags: tags }, next),
], done);
});
it('versioning enabled: should put/get a tag set on the latest ' +
'version if no version is specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
expectedVersionId: putData.VersionId }, next),
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
expectedTags: tags, expectedVersionId: versionId }, next),
(versionId, next) => awsGetAssertTags({ key,
expectedTags: tags }, next),
], done);
});
it('versioning enabled: should put/get a tag set on a specific version',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: putData.VersionId,
expectedVersionId: putData.VersionId }, next),
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
versionId, expectedTags: tags,
expectedVersionId: versionId }, next),
(versionId, next) => awsGetAssertTags({ key,
expectedTags: tags }, next),
], done);
});
it('versioning enabled: should put/get a tag set on a specific version',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: putData.VersionId,
expectedVersionId: putData.VersionId }, next),
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
versionId, expectedVersionId: versionId }, next),
next => awsGetAssertTags({ key, expectedTags: {} }, next),
], done);
});
it('versioning enabled: should put/get a tag set on a specific ' +
'version that is not the latest version', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '',
(err, awsVid) => next(err, putData.VersionId, awsVid)),
// put another version
(s3Vid, awsVid, next) => s3.putObject({ Bucket: bucket,
Key: key, Body: someBody },
err => next(err, s3Vid, awsVid)),
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
tags, versionId: s3Vid, expectedVersionId: s3Vid }, err =>
next(err, s3Vid, awsVid)),
(s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
versionId: s3Vid, expectedTags: tags,
expectedVersionId: s3Vid }, () => next(null, awsVid)),
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
expectedTags: tags }, next),
], done);
});
it('versioning suspended then enabled: should put/get a tag set on ' +
'a specific version (null) if specified', done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => putNullVersionsToAws(s3, bucket, key, [undefined],
() => next()),
next => awsGetLatestVerId(key, '', next),
(awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody],
() => next(null, awsVid)),
(awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: 'null', expectedVersionId: 'null' },
() => next(null, awsVid)),
(awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
versionId: 'null', expectedTags: tags,
expectedVersionId: 'null' }, () => next(null, awsVid)),
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
expectedTags: tags }, next),
], done);
});
it('should get tags for an object even if it was deleted from ' +
'AWS directly (we rely on s3 metadata)',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '', next),
(awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
expectedVersionId: false }, () => next(null, awsVid)),
(awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key, VersionId: awsVid }, next),
(delData, next) => getTaggingAndAssert(s3, { bucket, key,
expectedTags: tags, expectedVersionId: false,
getObject: false }, next),
], done);
});
it('should return an InternalError if trying to put ' +
'tags from object that was deleted from AWS directly',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '', next),
(awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key, VersionId: awsVid }, next),
(delData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
expectedError: 'InternalError' }, next),
], done);
});
it('should get tags for an version even if it was deleted from ' +
'AWS directly (we rely on s3 metadata)',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => enableVersioning(s3, bucket, next),
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '',
(err, awsVid) => next(err, putData.VersionId, awsVid)),
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
tags, versionId: s3Vid, expectedVersionId: s3Vid },
() => next(null, s3Vid, awsVid)),
(s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
(s3Vid, next) => getTaggingAndAssert(s3, { bucket, key,
versionId: s3Vid, expectedTags: tags,
expectedVersionId: s3Vid, getObject: false }, next),
], done);
});
it('should return an InternalError if trying to put ' +
'tags on version that was deleted from AWS directly',
done => {
const key = `somekey-${Date.now()}`;
async.waterfall([
next => s3.putObject({ Bucket: bucket, Key: key }, next),
(putData, next) => awsGetLatestVerId(key, '',
(err, awsVid) => next(err, putData.VersionId, awsVid)),
(s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
(s3Vid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
versionId: s3Vid, expectedError: 'InternalError' }, next),
], done);
});
});
});

View File

@ -1,17 +1,15 @@
const assert = require('assert'); const assert = require('assert');
const AWS = require('aws-sdk');
const async = require('async'); const async = require('async');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { config } = require('../../../../../../lib/Config'); const { config } = require('../../../../../../lib/Config');
const { getRealAwsConfig } = require('../../support/awsConfig');
const { createEncryptedBucketPromise } = const { createEncryptedBucketPromise } =
require('../../../lib/utility/createEncryptedBucket'); require('../../../lib/utility/createEncryptedBucket');
const { versioningEnabled } = require('../../../lib/utility/versioning-util'); const { versioningEnabled } = require('../../../lib/utility/versioning-util');
const { awsLocation, memLocation, fileLocation } = require('../utils'); const { describeSkipIfNotMultiple, awsS3, awsBucket, awsLocation,
const awsLocationEncryption = 'aws-test-encryption'; awsLocationEncryption, memLocation, fileLocation } = require('../utils');
const bucket = 'buckettestmultiplebackendput'; const bucket = 'buckettestmultiplebackendput';
const body = Buffer.from('I am a body', 'utf8'); const body = Buffer.from('I am a body', 'utf8');
const bigBody = Buffer.alloc(10485760); const bigBody = Buffer.alloc(10485760);
@ -23,13 +21,9 @@ const bigAWSMD5 = 'a7d414b9133d6483d9a1c4e04e856e3b-2';
let bucketUtil; let bucketUtil;
let s3; let s3;
let awsS3;
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
const awsTimeout = 30000; const awsTimeout = 30000;
const retryTimeout = 10000; const retryTimeout = 10000;
let awsBucket;
function awsGetCheck(objectKey, s3MD5, awsMD5, location, cb) { function awsGetCheck(objectKey, s3MD5, awsMD5, location, cb) {
process.stdout.write('Getting object\n'); process.stdout.write('Getting object\n');
@ -113,12 +107,6 @@ describe('MultipleBackend put object', function testSuite() {
if (!process.env.S3_END_TO_END) { if (!process.env.S3_END_TO_END) {
this.retries(2); this.retries(2);
} }
before(() => {
awsBucket = config.locationConstraints[awsLocation].
details.bucketName;
const awsConfig = getRealAwsConfig(awsLocation);
awsS3 = new AWS.S3(awsConfig);
});
it('should return an error to put request without a valid ' + it('should return an error to put request without a valid ' +
'location constraint', done => { 'location constraint', done => {

View File

@ -3,10 +3,17 @@ const async = require('async');
const withV4 = require('../../support/withV4'); const withV4 = require('../../support/withV4');
const BucketUtility = require('../../../lib/utility/bucket-util'); const BucketUtility = require('../../../lib/utility/bucket-util');
const { uniqName, getAzureClient, getAzureContainerName, getAzureKeys, const {
convertMD5, fileLocation, azureLocation, azureLocationMismatch } describeSkipIfNotMultiple,
= require('../utils'); uniqName,
const { config } = require('../../../../../../lib/Config'); getAzureClient,
getAzureContainerName,
getAzureKeys,
convertMD5,
fileLocation,
azureLocation,
azureLocationMismatch,
} = require('../utils');
const keyObject = 'putazure'; const keyObject = 'putazure';
const azureClient = getAzureClient(); const azureClient = getAzureClient();
@ -16,9 +23,6 @@ const { versioningEnabled } = require('../../../lib/utility/versioning-util');
const normalBody = Buffer.from('I am a body', 'utf8'); const normalBody = Buffer.from('I am a body', 'utf8');
const normalMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; const normalMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|| process.env.S3_END_TO_END) ? describe.skip : describe;
const keys = getAzureKeys(); const keys = getAzureKeys();
/* eslint-disable camelcase */ /* eslint-disable camelcase */
const azureMetadata = { const azureMetadata = {

View File

@ -1,9 +1,12 @@
const assert = require('assert'); const assert = require('assert');
const crypto = require('crypto'); const crypto = require('crypto');
const { errors } = require('arsenal');
const AWS = require('aws-sdk');
const async = require('async'); const async = require('async');
const azure = require('azure-storage'); const azure = require('azure-storage');
const { getRealAwsConfig } = require('../support/awsConfig');
const { config } = require('../../../../../lib/Config'); const { config } = require('../../../../../lib/Config');
const memLocation = 'mem-test'; const memLocation = 'mem-test';
@ -14,10 +17,35 @@ const awsLocationMismatch = 'aws-test-mismatch';
const azureLocation = 'azuretest'; const azureLocation = 'azuretest';
const azureLocation2 = 'azuretest2'; const azureLocation2 = 'azuretest2';
const azureLocationMismatch = 'azuretestmismatch'; const azureLocationMismatch = 'azuretestmismatch';
const awsLocationEncryption = 'aws-test-encryption';
const versioningEnabled = { Status: 'Enabled' }; const versioningEnabled = { Status: 'Enabled' };
const versioningSuspended = { Status: 'Suspended' }; const versioningSuspended = { Status: 'Suspended' };
let describeSkipIfNotMultiple = describe.skip;
let awsS3;
let awsBucket;
if (config.backends.data === 'multiple' && !process.env.S3_END_TO_END) {
describeSkipIfNotMultiple = describe;
// can only get real aws config if not running end-to-end
const awsConfig = getRealAwsConfig(awsLocation);
awsS3 = new AWS.S3(awsConfig);
awsBucket = config.locationConstraints[awsLocation].details.bucketName;
}
function _assertErrorResult(err, expectedError, desc) {
if (!expectedError) {
assert.strictEqual(err, null, `got error for ${desc}: ${err}`);
return;
}
assert(err, `expected ${expectedError} but found no error`);
assert.strictEqual(err.code, expectedError);
assert.strictEqual(err.statusCode, errors[expectedError].code);
}
const utils = { const utils = {
describeSkipIfNotMultiple,
awsS3,
awsBucket,
fileLocation, fileLocation,
memLocation, memLocation,
awsLocation, awsLocation,
@ -26,6 +54,7 @@ const utils = {
azureLocation, azureLocation,
azureLocation2, azureLocation2,
azureLocationMismatch, azureLocationMismatch,
awsLocationEncryption,
}; };
utils.uniqName = name => `${name}${new Date().getTime()}`; utils.uniqName = name => `${name}${new Date().getTime()}`;
@ -119,6 +148,14 @@ utils.expectedETag = (body, getStringified = true) => {
return `"${eTagValue}"`; return `"${eTagValue}"`;
}; };
utils.putToAwsBackend = (s3, bucket, key, body, cb) => {
console.log('aaaain put to aws backend')
console.log('what is cb in putToAWsBackend?', cb)
s3.putObject({ Bucket: bucket, Key: key, Body: body,
Metadata: { 'scal-location-constraint': awsLocation } },
(err, result) => cb(err, result.VersionId));
};
utils.enableVersioning = (s3, bucket, cb) => { utils.enableVersioning = (s3, bucket, cb) => {
s3.putBucketVersioning({ Bucket: bucket, s3.putBucketVersioning({ Bucket: bucket,
VersioningConfiguration: versioningEnabled }, err => { VersioningConfiguration: versioningEnabled }, err => {
@ -139,13 +176,11 @@ utils.suspendVersioning = (s3, bucket, cb) => {
utils.mapToAwsPuts = (s3, bucket, key, dataArray, cb) => { utils.mapToAwsPuts = (s3, bucket, key, dataArray, cb) => {
async.mapSeries(dataArray, (data, next) => { async.mapSeries(dataArray, (data, next) => {
s3.putObject({ Bucket: bucket, Key: key, Body: data, utils.putToAwsBackend(s3, bucket, key, data, next);
Metadata: { 'scal-location-constraint': awsLocation } },
next);
}, (err, results) => { }, (err, results) => {
assert.strictEqual(err, null, 'Expected success ' + assert.strictEqual(err, null, 'Expected success ' +
`putting object, got error ${err}`); `putting object, got error ${err}`);
cb(null, results.map(result => result.VersionId)); cb(null, results);
}); });
}; };
@ -161,4 +196,143 @@ utils.putNullVersionsToAws = (s3, bucket, key, versions, cb) => {
}); });
}; };
utils.getAndAssertResult = (s3, params, cb) => {
console.log('params', params);
const { bucket, key, body, versionId, expectedVersionId, expectedTagCount,
expectedError } = params;
s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
(err, data) => {
_assertErrorResult(err, expectedError, 'putting tags');
console.log('expected error...', expectedError)
if (expectedError) {
console.log('do we get here?', expectedError)
return cb();
}
console.log('we get to callback from get request to s3...')
assert.strictEqual(err, null, 'Expected success ' +
`getting object, got error ${err}`);
if (body) {
assert(data.Body, 'expected object body in response');
const expectedMD5 = utils.expectedETag(body, false);
const resultMD5 = utils.expectedETag(data.Body, false);
assert.strictEqual(resultMD5, expectedMD5);
}
if (!expectedVersionId) {
assert.strictEqual(data.VersionId, undefined);
} else {
assert.strictEqual(data.VersionId, expectedVersionId);
}
if (expectedTagCount && expectedTagCount === '0') {
assert.strictEqual(data.TagCount, undefined);
} else if (expectedTagCount) {
assert.strictEqual(data.TagCount, expectedTagCount);
}
return cb();
});
};
utils.awsGetLatestVerId = (key, body, cb) =>
awsS3.getObject({ Bucket: awsBucket, Key: key }, (err, result) => {
assert.strictEqual(err, null, 'Expected success ' +
`getting object from AWS, got error ${err}`);
const resultMD5 = utils.expectedETag(result.Body, false);
const expectedMD5 = utils.expectedETag(body, false);
assert.strictEqual(resultMD5, expectedMD5);
return cb(null, result.VersionId);
});
utils.tagging = {};
function _getTaggingConfig(tags) {
return {
// eslint-disable-next-line arrow-body-style
TagSet: Object.keys(tags).map(key => {
return {
Key: key,
Value: tags[key],
};
}),
};
}
utils.tagging.putTaggingAndAssert = (s3, params, cb) => {
const { bucket, key, tags, versionId, expectedVersionId,
expectedError } = params;
const taggingConfig = _getTaggingConfig(tags);
return s3.putObjectTagging({ Bucket: bucket, Key: key, VersionId: versionId,
Tagging: taggingConfig }, (err, data) => {
_assertErrorResult(err, expectedError, 'putting tags');
if (expectedError) {
return cb();
}
assert.strictEqual(err, null, `got error for putting tags: ${err}`);
if (expectedVersionId) {
assert.strictEqual(data.VersionId, expectedVersionId);
} else {
assert.strictEqual(data.VersionId, undefined);
}
return cb(null, data.VersionId);
});
};
utils.tagging.getTaggingAndAssert = (s3, params, cb) => {
const { bucket, key, expectedTags, versionId, expectedVersionId,
expectedError, getObject } = params;
s3.getObjectTagging({ Bucket: bucket, Key: key, VersionId: versionId },
(err, data) => {
_assertErrorResult(err, expectedError, 'putting tags');
if (expectedError) {
return cb();
}
const expectedTagResult = _getTaggingConfig(expectedTags);
const expectedTagCount = `${Object.keys(expectedTags).length}`;
assert.strictEqual(err, null, `got error for putting tags: ${err}`);
if (expectedVersionId) {
assert.strictEqual(data.VersionId, expectedVersionId);
} else {
assert.strictEqual(data.VersionId, undefined);
}
assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet);
if (getObject === false) {
return process.nextTick(cb, null, data.VersionId);
}
return utils.getAndAssertResult(s3, { bucket, key, versionId,
expectedVersionId, expectedTagCount },
() => cb(null, data.VersionId));
});
};
utils.tagging.delTaggingAndAssert = (s3, params, cb) => {
const { bucket, key, versionId, expectedVersionId, expectedError } = params;
return s3.deleteObjectTagging({ Bucket: bucket, Key: key,
VersionId: versionId }, (err, data) => {
_assertErrorResult(err, expectedError, 'putting tags');
if (expectedError) {
return cb();
}
assert.strictEqual(err, null, `got error for putting tags: ${err}`);
if (expectedVersionId) {
assert.strictEqual(data.VersionId, expectedVersionId);
} else {
assert.strictEqual(data.VersionId, undefined);
}
return utils.tagging.getTaggingAndAssert(s3, { bucket, key, versionId,
expectedVersionId, expectedTags: {} }, () => cb());
});
};
utils.tagging.awsGetAssertTags = (params, cb) => {
const { key, versionId, expectedTags } = params;
const expectedTagResult = _getTaggingConfig(expectedTags);
awsS3.getObjectTagging({ Bucket: awsBucket, Key: key,
VersionId: versionId }, (err, data) => {
assert.strictEqual(err, null, 'got unexpected error getting ' +
`tags directly from AWS: ${err}`);
assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet);
return cb();
});
};
module.exports = utils; module.exports = utils;

View File

@ -59,7 +59,7 @@ describe('Put object tagging with versioning', () => {
}); });
}); });
it('should not create version puting object tags on a ' + it('should not create version putting object tags on a ' +
' version-enabled bucket where no version id is specified ', done => { ' version-enabled bucket where no version id is specified ', done => {
async.waterfall([ async.waterfall([
next => s3.putBucketVersioning({ Bucket: bucketName, next => s3.putBucketVersioning({ Bucket: bucketName,