Compare commits
11 Commits
developmen
...
ft/aws-bac
Author | SHA1 | Date |
---|---|---|
Electra Chong | 0a7bf3a96f | |
Electra Chong | 27e34c198f | |
Electra Chong | 04a1f7e8c0 | |
Electra Chong | 37df1c5fae | |
Electra Chong | 5b9b06b649 | |
Electra Chong | 0b8921877c | |
Electra Chong | 1bb03a09a0 | |
Electra Chong | 6a89fe74ee | |
Electra Chong | 1a93e88c5a | |
Electra Chong | 018ff208ba | |
Electra Chong | 2c8266ff29 |
|
@ -61,6 +61,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
|
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
|
||||||
log, callback) {
|
log, callback) {
|
||||||
const size = isDeleteMarker ? 0 : request.parsedContentLength;
|
const size = isDeleteMarker ? 0 : request.parsedContentLength;
|
||||||
|
// although the request method may actually be 'DELETE' if creating a
|
||||||
|
// delete marker, for our purposes we consider this to be a 'PUT'
|
||||||
|
// operation
|
||||||
|
const requestMethod = 'PUT';
|
||||||
const websiteRedirectHeader =
|
const websiteRedirectHeader =
|
||||||
request.headers['x-amz-website-redirect-location'];
|
request.headers['x-amz-website-redirect-location'];
|
||||||
if (!validateWebsiteHeader(websiteRedirectHeader)) {
|
if (!validateWebsiteHeader(websiteRedirectHeader)) {
|
||||||
|
@ -70,8 +74,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
return callback(err);
|
return callback(err);
|
||||||
}
|
}
|
||||||
|
|
||||||
const metaHeaders = isDeleteMarker ? [] :
|
const metaHeaders = isDeleteMarker ? [] : getMetaHeaders(request.headers);
|
||||||
getMetaHeaders(request.headers);
|
|
||||||
if (metaHeaders instanceof Error) {
|
if (metaHeaders instanceof Error) {
|
||||||
log.debug('user metadata validation failed', {
|
log.debug('user metadata validation failed', {
|
||||||
error: metaHeaders,
|
error: metaHeaders,
|
||||||
|
@ -79,6 +82,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
});
|
});
|
||||||
return process.nextTick(() => callback(metaHeaders));
|
return process.nextTick(() => callback(metaHeaders));
|
||||||
}
|
}
|
||||||
|
|
||||||
log.trace('meta headers', { metaHeaders, method: 'objectPut' });
|
log.trace('meta headers', { metaHeaders, method: 'objectPut' });
|
||||||
const objectKeyContext = {
|
const objectKeyContext = {
|
||||||
bucketName,
|
bucketName,
|
||||||
|
@ -87,6 +91,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
objectKey,
|
objectKey,
|
||||||
metaHeaders,
|
metaHeaders,
|
||||||
tagging: request.headers['x-amz-tagging'],
|
tagging: request.headers['x-amz-tagging'],
|
||||||
|
isDeleteMarker,
|
||||||
};
|
};
|
||||||
// If the request was made with a pre-signed url, the x-amz-acl 'header'
|
// If the request was made with a pre-signed url, the x-amz-acl 'header'
|
||||||
// might be in the query string rather than the actual headers so include
|
// might be in the query string rather than the actual headers so include
|
||||||
|
@ -116,6 +121,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
metadataStoreParams.tagging = request.headers['x-amz-tagging'];
|
metadataStoreParams.tagging = request.headers['x-amz-tagging'];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// if creating new delete marker and there is an existing object, copy
|
||||||
|
// the object's location constraint metaheader to determine backend info
|
||||||
|
if (isDeleteMarker && objMD) {
|
||||||
|
// eslint-disable-next-line no-param-reassign
|
||||||
|
request.headers[constants.objectLocationConstraintHeader] =
|
||||||
|
objMD[constants.objectLocationConstraintHeader];
|
||||||
|
}
|
||||||
|
|
||||||
const backendInfoObj =
|
const backendInfoObj =
|
||||||
locationConstraintCheck(request, null, bucketMD, log);
|
locationConstraintCheck(request, null, bucketMD, log);
|
||||||
if (backendInfoObj.err) {
|
if (backendInfoObj.err) {
|
||||||
|
@ -205,7 +218,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
metadataStoreParams.nullVersionId = options.nullVersionId;
|
metadataStoreParams.nullVersionId = options.nullVersionId;
|
||||||
return _storeInMDandDeleteData(bucketName, infoArr,
|
return _storeInMDandDeleteData(bucketName, infoArr,
|
||||||
cipherBundle, metadataStoreParams,
|
cipherBundle, metadataStoreParams,
|
||||||
options.dataToDelete, requestLogger, request.method, next);
|
options.dataToDelete, requestLogger, requestMethod, next);
|
||||||
},
|
},
|
||||||
], callback);
|
], callback);
|
||||||
}
|
}
|
||||||
|
|
|
@ -71,22 +71,29 @@ function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** get location of data for deletion
|
/** get location of null version data for deletion
|
||||||
* @param {string} bucketName - name of bucket
|
* @param {string} bucketName - name of bucket
|
||||||
* @param {string} objKey - name of object key
|
* @param {string} objKey - name of object key
|
||||||
* @param {object} options - metadata options for getting object MD
|
* @param {object} options - metadata options for getting object MD
|
||||||
* @param {string} options.versionId - version to get from metadata
|
* @param {string} options.versionId - version to get from metadata
|
||||||
|
* @param {object} mst - info about the master version
|
||||||
|
* @param {string} mst.versionId - the master version's version id
|
||||||
* @param {RequestLogger} log - logger instanceof
|
* @param {RequestLogger} log - logger instanceof
|
||||||
* @param {function} cb - callback
|
* @param {function} cb - callback
|
||||||
* @return {undefined} - and call callback with (err, dataToDelete)
|
* @return {undefined} - and call callback with (err, dataToDelete)
|
||||||
*/
|
*/
|
||||||
function _getDeleteLocations(bucketName, objKey, options, log, cb) {
|
function _getNullVersionsToDelete(bucketName, objKey, options, mst, log, cb) {
|
||||||
|
if (options.versionId === mst.versionId) {
|
||||||
|
// no need to get delete location, we already have the master's metadata
|
||||||
|
const dataToDelete = mst.objLocation;
|
||||||
|
return process.nextTick(cb, null, dataToDelete);
|
||||||
|
}
|
||||||
return metadata.getObjectMD(bucketName, objKey, options, log,
|
return metadata.getObjectMD(bucketName, objKey, options, log,
|
||||||
(err, versionMD) => {
|
(err, versionMD) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('err from metadata getting specified version', {
|
log.debug('err from metadata getting specified version', {
|
||||||
error: err,
|
error: err,
|
||||||
method: '_getDeleteLocations',
|
method: '_getNullVersionsToDelete',
|
||||||
});
|
});
|
||||||
return cb(err);
|
return cb(err);
|
||||||
}
|
}
|
||||||
|
@ -99,9 +106,8 @@ function _getDeleteLocations(bucketName, objKey, options, log, cb) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
|
function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
|
||||||
// before deleting null version md, retrieve location of data to delete
|
return _getNullVersionsToDelete(bucketName, objKey, options, mst, log,
|
||||||
return _getDeleteLocations(bucketName, objKey, options, log,
|
|
||||||
(err, nullDataToDelete) => {
|
(err, nullDataToDelete) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log.warn('could not find null version metadata', {
|
log.warn('could not find null version metadata', {
|
||||||
|
@ -236,15 +242,16 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
|
||||||
if (!delOptions) {
|
if (!delOptions) {
|
||||||
return process.nextTick(next, null, options);
|
return process.nextTick(next, null, options);
|
||||||
}
|
}
|
||||||
return _deleteNullVersionMD(bucketName, objectKey, delOptions, log,
|
return _deleteNullVersionMD(bucketName, objectKey, delOptions, mst,
|
||||||
(err, nullDataToDelete) => {
|
log, (err, nullDataToDelete) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log.warn('unexpected error deleting null version md', {
|
log.warn('unexpected error deleting null version md', {
|
||||||
error: err,
|
error: err,
|
||||||
method: 'versioningPreprocessing',
|
method: 'versioningPreprocessing',
|
||||||
});
|
});
|
||||||
// it's possible there was a concurrent request to delete
|
// it's possible there was a concurrent request to
|
||||||
// the null version, so proceed with putting a new version
|
// delete the null version, so proceed with putting a
|
||||||
|
// new version
|
||||||
if (err === errors.NoSuchKey) {
|
if (err === errors.NoSuchKey) {
|
||||||
return next(null, options);
|
return next(null, options);
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,8 +18,7 @@ const { versioningPreprocessing, checkQueryVersionId }
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const services = require('../services');
|
const services = require('../services');
|
||||||
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
|
const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
|
||||||
const { checkAzureBackendMatch, skipMpuPartProcessing } =
|
const { skipMpuPartProcessing } = require('../data/external/utils');
|
||||||
require('../data/external/utils');
|
|
||||||
|
|
||||||
const logger = require('../utilities/logger');
|
const logger = require('../utilities/logger');
|
||||||
|
|
||||||
|
@ -214,6 +213,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
|
||||||
key: completeObjData.key,
|
key: completeObjData.key,
|
||||||
size: completeObjData.contentLength,
|
size: completeObjData.contentLength,
|
||||||
start: 0,
|
start: 0,
|
||||||
|
dataStoreVersionId: completeObjData.dataStoreVersionId,
|
||||||
dataStoreName: storedMetadata.dataStoreName,
|
dataStoreName: storedMetadata.dataStoreName,
|
||||||
dataStoreETag: completeObjData.eTag,
|
dataStoreETag: completeObjData.eTag,
|
||||||
dataStoreType: completeObjData.dataStoreType,
|
dataStoreType: completeObjData.dataStoreType,
|
||||||
|
@ -333,15 +333,16 @@ function completeMultipartUpload(authInfo, request, log, callback) {
|
||||||
// null version when versioning is suspended or versioning
|
// null version when versioning is suspended or versioning
|
||||||
// is not enabled, need to delete pre-existing data
|
// is not enabled, need to delete pre-existing data
|
||||||
// unless the preexisting object and the completed mpu
|
// unless the preexisting object and the completed mpu
|
||||||
// are on Azure data backend
|
// are on external backends
|
||||||
if (dataToDelete) {
|
if (dataToDelete) {
|
||||||
if (!checkAzureBackendMatch(dataToDelete[0],
|
const newDataStoreName =
|
||||||
completeObjData)) {
|
Array.isArray(dataLocations) && dataLocations[0] ?
|
||||||
data.batchDelete(dataToDelete, request.method, null,
|
dataLocations[0].dataStoreName : null;
|
||||||
|
data.batchDelete(dataToDelete, request.method,
|
||||||
|
newDataStoreName,
|
||||||
logger.newRequestLoggerFromSerializedUids(log
|
logger.newRequestLoggerFromSerializedUids(log
|
||||||
.getSerializedUids()));
|
.getSerializedUids()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return next(null, mpuBucket, keysToDelete, aggregateETag,
|
return next(null, mpuBucket, keysToDelete, aggregateETag,
|
||||||
extraPartLocations, destinationBucket,
|
extraPartLocations, destinationBucket,
|
||||||
generatedVersionId);
|
generatedVersionId);
|
||||||
|
|
|
@ -185,19 +185,17 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
|
||||||
if (err) {
|
if (err) {
|
||||||
return callback(err);
|
return callback(err);
|
||||||
}
|
}
|
||||||
// NOTE: remove the following when we will support putting a
|
|
||||||
// versioned object to a location-constraint of type AWS or Azure.
|
|
||||||
if (locConstraint &&
|
if (locConstraint &&
|
||||||
config.locationConstraints[locConstraint] &&
|
config.locationConstraints[locConstraint] &&
|
||||||
config.locationConstraints[locConstraint].type &&
|
config.locationConstraints[locConstraint].type &&
|
||||||
constants.externalBackends[config
|
constants.versioningNotImplBackends[config
|
||||||
.locationConstraints[locConstraint].type]
|
.locationConstraints[locConstraint].type]
|
||||||
) {
|
) {
|
||||||
const vcfg = destinationBucket.getVersioningConfiguration();
|
const vcfg = destinationBucket.getVersioningConfiguration();
|
||||||
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
|
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
|
||||||
if (isVersionedObj) {
|
if (isVersionedObj) {
|
||||||
log.debug(externalVersioningErrorMessage,
|
log.debug(externalVersioningErrorMessage,
|
||||||
{ method: 'multipleBackendGateway',
|
{ method: 'initiateMultipartUpload',
|
||||||
error: errors.NotImplemented });
|
error: errors.NotImplemented });
|
||||||
return callback(errors.NotImplemented
|
return callback(errors.NotImplemented
|
||||||
.customizeDescription(externalVersioningErrorMessage));
|
.customizeDescription(externalVersioningErrorMessage));
|
||||||
|
|
|
@ -23,6 +23,7 @@ const { config } = require('../Config');
|
||||||
|
|
||||||
const versionIdUtils = versioning.VersionID;
|
const versionIdUtils = versioning.VersionID;
|
||||||
const locationHeader = constants.objectLocationConstraintHeader;
|
const locationHeader = constants.objectLocationConstraintHeader;
|
||||||
|
const versioningNotImplBackends = constants.versioningNotImplBackends;
|
||||||
const externalVersioningErrorMessage = 'We do not currently support putting ' +
|
const externalVersioningErrorMessage = 'We do not currently support putting ' +
|
||||||
'a versioned object to a location-constraint of type AWS or Azure.';
|
'a versioned object to a location-constraint of type AWS or Azure.';
|
||||||
|
|
||||||
|
@ -321,9 +322,7 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
sourceLocationConstraintType = config.getLocationConstraintType(
|
sourceLocationConstraintType = config.getLocationConstraintType(
|
||||||
storeMetadataParams.metaHeaders[locationHeader]);
|
storeMetadataParams.metaHeaders[locationHeader]);
|
||||||
}
|
}
|
||||||
// NOTE: remove the following when we will support putting a
|
if (versioningNotImplBackends[sourceLocationConstraintType]) {
|
||||||
// versioned object to a location-constraint of type AWS or Azure.
|
|
||||||
if (constants.externalBackends[sourceLocationConstraintType]) {
|
|
||||||
const vcfg = destBucketMD.getVersioningConfiguration();
|
const vcfg = destBucketMD.getVersioningConfiguration();
|
||||||
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
|
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
|
||||||
if (isVersionedObj) {
|
if (isVersionedObj) {
|
||||||
|
@ -337,6 +336,10 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
if (dataLocator.length === 0) {
|
if (dataLocator.length === 0) {
|
||||||
if (!storeMetadataParams.locationMatch &&
|
if (!storeMetadataParams.locationMatch &&
|
||||||
constants.externalBackends[sourceLocationConstraintType]) {
|
constants.externalBackends[sourceLocationConstraintType]) {
|
||||||
|
console.log('============================');
|
||||||
|
console.log('detected zero byte object')
|
||||||
|
console.log('location does not match and source ' +
|
||||||
|
'is an external backend; about to use data.put for copy')
|
||||||
return data.put(null, null, storeMetadataParams.size,
|
return data.put(null, null, storeMetadataParams.size,
|
||||||
dataStoreContext, backendInfo,
|
dataStoreContext, backendInfo,
|
||||||
log, (error, objectRetrievalInfo) => {
|
log, (error, objectRetrievalInfo) => {
|
||||||
|
@ -359,7 +362,7 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
return next(null, storeMetadataParams, dataLocator, destObjMD,
|
return next(null, storeMetadataParams, dataLocator, destObjMD,
|
||||||
serverSideEncryption, destBucketMD);
|
serverSideEncryption, destBucketMD);
|
||||||
}
|
}
|
||||||
|
console.log('about to call data wrapper copy object')
|
||||||
return data.copyObject(request, sourceLocationConstraintType,
|
return data.copyObject(request, sourceLocationConstraintType,
|
||||||
sourceLocationConstraintName, storeMetadataParams, dataLocator,
|
sourceLocationConstraintName, storeMetadataParams, dataLocator,
|
||||||
dataStoreContext, backendInfo, serverSideEncryption, log,
|
dataStoreContext, backendInfo, serverSideEncryption, log,
|
||||||
|
|
|
@ -73,6 +73,13 @@ function objectDelete(authInfo, request, log, cb) {
|
||||||
// versioning has been configured
|
// versioning has been configured
|
||||||
return next(null, bucketMD, objMD);
|
return next(null, bucketMD, objMD);
|
||||||
}
|
}
|
||||||
|
console.log('============== objMD.location? in objDelete', objMD.location)
|
||||||
|
if (reqVersionId && objMD.location &&
|
||||||
|
Array.isArray(objMD.location) && objMD.location[0]) {
|
||||||
|
// we need this information for data deletes to AWS
|
||||||
|
// eslint-disable-next-line no-param-reassign
|
||||||
|
objMD.location[0].deleteVersion = true;
|
||||||
|
}
|
||||||
if (objMD['content-length'] !== undefined) {
|
if (objMD['content-length'] !== undefined) {
|
||||||
log.end().addDefaultFields({
|
log.end().addDefaultFields({
|
||||||
bytesDeleted: objMD['content-length'],
|
bytesDeleted: objMD['content-length'],
|
||||||
|
|
|
@ -7,6 +7,11 @@ const { prepareStream } = require('../../api/apiUtils/object/prepareStream');
|
||||||
const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils');
|
const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils');
|
||||||
const { config } = require('../../Config');
|
const { config } = require('../../Config');
|
||||||
|
|
||||||
|
const missingVerIdInternalError = errors.InternalError.customizeDescription(
|
||||||
|
'Invalid state. Please ensure versioning is enabled ' +
|
||||||
|
'in AWS for the location constraint and try again.'
|
||||||
|
);
|
||||||
|
|
||||||
class AwsClient {
|
class AwsClient {
|
||||||
constructor(config) {
|
constructor(config) {
|
||||||
this._s3Params = config.s3Params;
|
this._s3Params = config.s3Params;
|
||||||
|
@ -25,24 +30,15 @@ class AwsClient {
|
||||||
return `${requestBucketName}/${requestObjectKey}`;
|
return `${requestBucketName}/${requestObjectKey}`;
|
||||||
}
|
}
|
||||||
put(stream, size, keyContext, reqUids, callback) {
|
put(stream, size, keyContext, reqUids, callback) {
|
||||||
|
console.log('=========================')
|
||||||
|
console.log('got to AwsClient put')
|
||||||
const awsKey = this._createAwsKey(keyContext.bucketName,
|
const awsKey = this._createAwsKey(keyContext.bucketName,
|
||||||
keyContext.objectKey, this._bucketMatch);
|
keyContext.objectKey, this._bucketMatch);
|
||||||
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
|
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
|
||||||
const log = createLogger(reqUids);
|
const log = createLogger(reqUids);
|
||||||
const uploadParams = {
|
|
||||||
Bucket: this._awsBucketName,
|
|
||||||
Key: awsKey,
|
|
||||||
Metadata: metaHeaders,
|
|
||||||
ContentLength: size,
|
|
||||||
};
|
|
||||||
if (this._serverSideEncryption) {
|
|
||||||
uploadParams.ServerSideEncryption = 'AES256';
|
|
||||||
}
|
|
||||||
if (keyContext.tagging) {
|
|
||||||
uploadParams.Tagging = keyContext.tagging;
|
|
||||||
}
|
|
||||||
|
|
||||||
const putCb = (err, data) => {
|
const putCb = (err, data) => {
|
||||||
|
console.log('result from aws put', data)
|
||||||
if (err) {
|
if (err) {
|
||||||
logHelper(log, 'error', 'err from data backend',
|
logHelper(log, 'error', 'err from data backend',
|
||||||
err, this._dataStoreName);
|
err, this._dataStoreName);
|
||||||
|
@ -52,21 +48,38 @@ class AwsClient {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (!data.VersionId) {
|
if (!data.VersionId) {
|
||||||
const error = errors.InternalError.customizeDescription(
|
|
||||||
'Invalid state. Please ensure versioning is enabled ' +
|
|
||||||
'in AWS for the location constraint and try again.'
|
|
||||||
);
|
|
||||||
logHelper(log, 'error', 'missing version id for data ' +
|
logHelper(log, 'error', 'missing version id for data ' +
|
||||||
'backend object', error, this._dataStoreName);
|
'backend object', missingVerIdInternalError,
|
||||||
return callback(error);
|
this._dataStoreName);
|
||||||
|
return callback(missingVerIdInternalError);
|
||||||
}
|
}
|
||||||
const dataStoreVersionId = data.VersionId;
|
const dataStoreVersionId = data.VersionId;
|
||||||
return callback(null, awsKey, dataStoreVersionId);
|
return callback(null, awsKey, dataStoreVersionId);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const params = {
|
||||||
|
Bucket: this._awsBucketName,
|
||||||
|
Key: awsKey,
|
||||||
|
};
|
||||||
|
// we call data.put to create a delete marker, but it's actually a
|
||||||
|
// delete request in call to AWS
|
||||||
|
if (keyContext.isDeleteMarker) {
|
||||||
|
return this._client.deleteObject(params, putCb);
|
||||||
|
}
|
||||||
|
const uploadParams = params;
|
||||||
|
uploadParams.Metadata = metaHeaders;
|
||||||
|
uploadParams.ContentLength = size;
|
||||||
|
if (this._serverSideEncryption) {
|
||||||
|
uploadParams.ServerSideEncryption = 'AES256';
|
||||||
|
}
|
||||||
|
if (keyContext.tagging) {
|
||||||
|
uploadParams.Tagging = keyContext.tagging;
|
||||||
|
}
|
||||||
|
|
||||||
if (!stream) {
|
if (!stream) {
|
||||||
return this._client.putObject(uploadParams, putCb);
|
return this._client.putObject(uploadParams, putCb);
|
||||||
}
|
}
|
||||||
|
console.log('uploadParams', uploadParams)
|
||||||
|
|
||||||
uploadParams.Body = stream;
|
uploadParams.Body = stream;
|
||||||
return this._client.upload(uploadParams, putCb);
|
return this._client.upload(uploadParams, putCb);
|
||||||
|
@ -108,6 +121,14 @@ class AwsClient {
|
||||||
}).on('success', response => {
|
}).on('success', response => {
|
||||||
log.trace('AWS GET request response headers',
|
log.trace('AWS GET request response headers',
|
||||||
{ responseHeaders: response.httpResponse.headers });
|
{ responseHeaders: response.httpResponse.headers });
|
||||||
|
console.log('====================')
|
||||||
|
console.log('params sent for aws get request', {
|
||||||
|
Bucket: this._awsBucketName,
|
||||||
|
Key: key,
|
||||||
|
VersionId: dataStoreVersionId,
|
||||||
|
Range: range,
|
||||||
|
});
|
||||||
|
console.log('AWS GET request response headers', response.httpResponse.headers)
|
||||||
});
|
});
|
||||||
const stream = request.createReadStream().on('error', err => {
|
const stream = request.createReadStream().on('error', err => {
|
||||||
logHelper(log, 'error', 'error streaming data from AWS',
|
logHelper(log, 'error', 'error streaming data from AWS',
|
||||||
|
@ -117,18 +138,29 @@ class AwsClient {
|
||||||
return callback(null, stream);
|
return callback(null, stream);
|
||||||
}
|
}
|
||||||
delete(objectGetInfo, reqUids, callback) {
|
delete(objectGetInfo, reqUids, callback) {
|
||||||
// for backwards compatibility
|
const { key, dataStoreVersionId, deleteVersion } = objectGetInfo;
|
||||||
const key = typeof objectGetInfo === 'string' ? objectGetInfo :
|
const log = createLogger(reqUids);
|
||||||
objectGetInfo.key;
|
|
||||||
const params = {
|
const params = {
|
||||||
Bucket: this._awsBucketName,
|
Bucket: this._awsBucketName,
|
||||||
Key: key,
|
Key: key,
|
||||||
};
|
};
|
||||||
return this._client.deleteObject(params, err => {
|
if (deleteVersion) {
|
||||||
|
params.VersionId = dataStoreVersionId;
|
||||||
|
}
|
||||||
|
console.log('=====================');
|
||||||
|
console.log('objectGetInfo', { key, dataStoreVersionId, deleteVersion })
|
||||||
|
console.log('params sending to AWS for delete', params)
|
||||||
|
return this._client.deleteObject(params, (err, data) => {
|
||||||
|
console.log('response from aws for delete', data)
|
||||||
if (err) {
|
if (err) {
|
||||||
const log = createLogger(reqUids);
|
|
||||||
logHelper(log, 'error', 'error deleting object from ' +
|
logHelper(log, 'error', 'error deleting object from ' +
|
||||||
'datastore', err, this._dataStoreName);
|
'datastore', err, this._dataStoreName);
|
||||||
|
if (err.code === 'NoSuchVersion') {
|
||||||
|
// data may have been deleted directly from the AWS backend
|
||||||
|
// don't want to retry the delete and errors are not
|
||||||
|
// sent back to client anyway, so no need to return err
|
||||||
|
return callback();
|
||||||
|
}
|
||||||
return callback(errors.InternalError
|
return callback(errors.InternalError
|
||||||
.customizeDescription('Error returned from ' +
|
.customizeDescription('Error returned from ' +
|
||||||
`AWS: ${err.message}`)
|
`AWS: ${err.message}`)
|
||||||
|
@ -300,6 +332,9 @@ class AwsClient {
|
||||||
const completeObjData = { key: awsKey };
|
const completeObjData = { key: awsKey };
|
||||||
return this._client.completeMultipartUpload(mpuParams,
|
return this._client.completeMultipartUpload(mpuParams,
|
||||||
(err, completeMpuRes) => {
|
(err, completeMpuRes) => {
|
||||||
|
console.log('==============================')
|
||||||
|
console.log('mpuParams', mpuParams);
|
||||||
|
console.log('completeMpuRes from AWS', completeMpuRes)
|
||||||
if (err) {
|
if (err) {
|
||||||
if (mpuError[err.code]) {
|
if (mpuError[err.code]) {
|
||||||
logHelper(log, 'trace', 'err from data backend on ' +
|
logHelper(log, 'trace', 'err from data backend on ' +
|
||||||
|
@ -313,6 +348,12 @@ class AwsClient {
|
||||||
`AWS: ${err.message}`)
|
`AWS: ${err.message}`)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (!completeMpuRes.VersionId) {
|
||||||
|
logHelper(log, 'error', 'missing version id for data ' +
|
||||||
|
'backend object', missingVerIdInternalError,
|
||||||
|
this._dataStoreName);
|
||||||
|
return callback(missingVerIdInternalError);
|
||||||
|
}
|
||||||
// need to get content length of new object to store
|
// need to get content length of new object to store
|
||||||
// in our metadata
|
// in our metadata
|
||||||
return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
|
return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
|
||||||
|
@ -328,6 +369,7 @@ class AwsClient {
|
||||||
// remove quotes from eTag because they're added later
|
// remove quotes from eTag because they're added later
|
||||||
completeObjData.eTag = completeMpuRes.ETag
|
completeObjData.eTag = completeMpuRes.ETag
|
||||||
.substring(1, completeMpuRes.ETag.length - 1);
|
.substring(1, completeMpuRes.ETag.length - 1);
|
||||||
|
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
|
||||||
completeObjData.contentLength = objHeaders.ContentLength;
|
completeObjData.contentLength = objHeaders.ContentLength;
|
||||||
return callback(null, completeObjData);
|
return callback(null, completeObjData);
|
||||||
});
|
});
|
||||||
|
@ -357,10 +399,12 @@ class AwsClient {
|
||||||
objectPutTagging(key, bucket, objectMD, log, callback) {
|
objectPutTagging(key, bucket, objectMD, log, callback) {
|
||||||
const awsBucket = this._awsBucketName;
|
const awsBucket = this._awsBucketName;
|
||||||
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
|
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
|
||||||
const tagParams = { Bucket: awsBucket, Key: awsKey };
|
const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
|
||||||
if (objectMD.versionId) {
|
const tagParams = {
|
||||||
tagParams.VersionId = objectMD.versionId;
|
Bucket: awsBucket,
|
||||||
}
|
Key: awsKey,
|
||||||
|
VersionId: dataStoreVersionId,
|
||||||
|
};
|
||||||
const keyArray = Object.keys(objectMD.tags);
|
const keyArray = Object.keys(objectMD.tags);
|
||||||
tagParams.Tagging = {};
|
tagParams.Tagging = {};
|
||||||
tagParams.Tagging.TagSet = keyArray.map(key => {
|
tagParams.Tagging.TagSet = keyArray.map(key => {
|
||||||
|
@ -383,10 +427,12 @@ class AwsClient {
|
||||||
objectDeleteTagging(key, bucket, objectMD, log, callback) {
|
objectDeleteTagging(key, bucket, objectMD, log, callback) {
|
||||||
const awsBucket = this._awsBucketName;
|
const awsBucket = this._awsBucketName;
|
||||||
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
|
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
|
||||||
const tagParams = { Bucket: awsBucket, Key: awsKey };
|
const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
|
||||||
if (objectMD.versionId) {
|
const tagParams = {
|
||||||
tagParams.VersionId = objectMD.versionId;
|
Bucket: awsBucket,
|
||||||
}
|
Key: awsKey,
|
||||||
|
VersionId: dataStoreVersionId,
|
||||||
|
};
|
||||||
return this._client.deleteObjectTagging(tagParams, err => {
|
return this._client.deleteObjectTagging(tagParams, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
logHelper(log, 'error', 'error from data backend on ' +
|
logHelper(log, 'error', 'error from data backend on ' +
|
||||||
|
@ -417,7 +463,7 @@ class AwsClient {
|
||||||
CopySource: `${sourceAwsBucketName}/${sourceKey}`,
|
CopySource: `${sourceAwsBucketName}/${sourceKey}`,
|
||||||
Metadata: metaHeaders,
|
Metadata: metaHeaders,
|
||||||
MetadataDirective: metadataDirective,
|
MetadataDirective: metadataDirective,
|
||||||
}, err => {
|
}, (err, copyResult) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
if (err.code === 'AccessDenied') {
|
if (err.code === 'AccessDenied') {
|
||||||
logHelper(log, 'error', 'Unable to access ' +
|
logHelper(log, 'error', 'Unable to access ' +
|
||||||
|
@ -435,7 +481,15 @@ class AwsClient {
|
||||||
`AWS: ${err.message}`)
|
`AWS: ${err.message}`)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return callback(null, destAwsKey);
|
if (!copyResult.VersionId) {
|
||||||
|
logHelper(log, 'error', 'missing version id for data ' +
|
||||||
|
'backend object', missingVerIdInternalError,
|
||||||
|
this._dataStoreName);
|
||||||
|
return callback(missingVerIdInternalError);
|
||||||
|
}
|
||||||
|
console.log('destAwsKey in AwsClient', destAwsKey);
|
||||||
|
console.log('callback??', callback)
|
||||||
|
return callback(null, destAwsKey, copyResult.VersionId);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
|
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
|
||||||
|
|
|
@ -247,11 +247,13 @@ const multipleBackendGateway = {
|
||||||
const client = clients[location];
|
const client = clients[location];
|
||||||
if (client.copyObject) {
|
if (client.copyObject) {
|
||||||
return client.copyObject(request, externalSourceKey,
|
return client.copyObject(request, externalSourceKey,
|
||||||
sourceLocationConstraintName, log, (err, key) => {
|
sourceLocationConstraintName, log, (err, key,
|
||||||
|
dataStoreVersionId) => {
|
||||||
const dataRetrievalInfo = {
|
const dataRetrievalInfo = {
|
||||||
key,
|
key,
|
||||||
dataStoreName: location,
|
dataStoreName: location,
|
||||||
dataStoreType: client.clientType,
|
dataStoreType: client.clientType,
|
||||||
|
dataStoreVersionId,
|
||||||
};
|
};
|
||||||
cb(err, dataRetrievalInfo);
|
cb(err, dataRetrievalInfo);
|
||||||
});
|
});
|
||||||
|
|
|
@ -56,6 +56,33 @@ if (config.backends.data === 'mem') {
|
||||||
*/
|
*/
|
||||||
const MAX_RETRY = 2;
|
const MAX_RETRY = 2;
|
||||||
|
|
||||||
|
// This check is done because on a put, complete mpu or copy request to AWS,
|
||||||
|
// if the object already exists on AWS, the existing object should not be
|
||||||
|
// deleted, which is the functionality for all other backends
|
||||||
|
function _shouldSkipDelete(locations, requestMethod, newObjDataStoreName) {
|
||||||
|
const skipBackend = externalBackends;
|
||||||
|
const skipMethods = { PUT: true, POST: true };
|
||||||
|
if (!Array.isArray(locations) || !locations[0] ||
|
||||||
|
!locations[0].dataStoreType) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const isSkipBackend = skipBackend[locations[0].dataStoreType];
|
||||||
|
const isMatchingBackends =
|
||||||
|
locations[0].dataStoreName === newObjDataStoreName;
|
||||||
|
const isSkipMethod = skipMethods[requestMethod];
|
||||||
|
console.log('============================')
|
||||||
|
console.log('requestMethod', requestMethod)
|
||||||
|
console.log('isSkipMethod', isSkipMethod)
|
||||||
|
console.log('isSkipBackend', isSkipBackend);
|
||||||
|
console.log('isMatchingBackends', isMatchingBackends)
|
||||||
|
console.log('locations[0].dataStoreName', locations[0].dataStoreName);
|
||||||
|
console.log('newObjDataStoreName', newObjDataStoreName)
|
||||||
|
if (!isSkipBackend || !isMatchingBackends || !isSkipMethod) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
function _retryDelete(objectGetInfo, log, count, cb) {
|
function _retryDelete(objectGetInfo, log, count, cb) {
|
||||||
if (count > MAX_RETRY) {
|
if (count > MAX_RETRY) {
|
||||||
return cb(errors.InternalError);
|
return cb(errors.InternalError);
|
||||||
|
@ -216,7 +243,6 @@ const data = {
|
||||||
return callback(err);
|
return callback(err);
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
// It would be preferable to have an sproxyd batch delete route to
|
// It would be preferable to have an sproxyd batch delete route to
|
||||||
// replace this
|
// replace this
|
||||||
batchDelete: (locations, requestMethod, newObjDataStoreName, log) => {
|
batchDelete: (locations, requestMethod, newObjDataStoreName, log) => {
|
||||||
|
@ -224,20 +250,12 @@ const data = {
|
||||||
// be finalized; refer Issue #312 for the discussion. In the
|
// be finalized; refer Issue #312 for the discussion. In the
|
||||||
// meantime, we at least log the location of the data we are
|
// meantime, we at least log the location of the data we are
|
||||||
// about to delete before attempting its deletion.
|
// about to delete before attempting its deletion.
|
||||||
/* eslint-disable camelcase */
|
console.log('got to batchDelete... requestMethod', requestMethod)
|
||||||
const skipBackend = externalBackends;
|
console.log('should skip method?', _shouldSkipDelete(locations, requestMethod, newObjDataStoreName))
|
||||||
/* eslint-enable camelcase */
|
if (_shouldSkipDelete(locations, requestMethod, newObjDataStoreName)) {
|
||||||
const isSkipBackend = (locations[0] && locations[0].dataStoreType) ?
|
|
||||||
skipBackend[locations[0].dataStoreType] : false;
|
|
||||||
const isMatchingBackends = locations[0] ?
|
|
||||||
locations[0].dataStoreName === newObjDataStoreName : false;
|
|
||||||
// This check is done because on a PUT request to AWS, if the object
|
|
||||||
// already exists on AWS, the existing object should not be deleted,
|
|
||||||
// which is the functionality for all other backends
|
|
||||||
// TODO: update for mpu and object copy
|
|
||||||
if (requestMethod === 'PUT' && isSkipBackend && isMatchingBackends) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
console.log('are we continuing any way? )!(@)(!)')
|
||||||
log.trace('initiating batch delete', {
|
log.trace('initiating batch delete', {
|
||||||
keys: locations,
|
keys: locations,
|
||||||
implName,
|
implName,
|
||||||
|
@ -326,6 +344,7 @@ const data = {
|
||||||
copyObject: (request, sourceLocationConstraintType,
|
copyObject: (request, sourceLocationConstraintType,
|
||||||
sourceLocationConstraintName, storeMetadataParams, dataLocator,
|
sourceLocationConstraintName, storeMetadataParams, dataLocator,
|
||||||
dataStoreContext, destBackendInfo, serverSideEncryption, log, cb) => {
|
dataStoreContext, destBackendInfo, serverSideEncryption, log, cb) => {
|
||||||
|
console.log('============ data wrapper copy object')
|
||||||
// NOTE: using copyObject only if copying object from one external
|
// NOTE: using copyObject only if copying object from one external
|
||||||
// backend to the same external backend
|
// backend to the same external backend
|
||||||
// and for Azure if it is the same account since Azure copy outside
|
// and for Azure if it is the same account since Azure copy outside
|
||||||
|
@ -338,22 +357,26 @@ const data = {
|
||||||
(sourceLocationConstraintType === 'aws_s3' ||
|
(sourceLocationConstraintType === 'aws_s3' ||
|
||||||
(sourceLocationConstraintType === 'azure' && config.isSameAzureAccount(
|
(sourceLocationConstraintType === 'azure' && config.isSameAzureAccount(
|
||||||
sourceLocationConstraintName, request.headers[locationHeader])))) {
|
sourceLocationConstraintName, request.headers[locationHeader])))) {
|
||||||
|
console.log('detected location match and external backend')
|
||||||
const location = storeMetadataParams
|
const location = storeMetadataParams
|
||||||
.metaHeaders[locationHeader];
|
.metaHeaders[locationHeader];
|
||||||
const objectGetInfo = dataLocator[0];
|
const objectGetInfo = dataLocator[0];
|
||||||
const externalSourceKey = objectGetInfo.key;
|
const externalSourceKey = objectGetInfo.key;
|
||||||
|
console.log('about to use client.copyObject')
|
||||||
return client.copyObject(request, location,
|
return client.copyObject(request, location,
|
||||||
externalSourceKey, sourceLocationConstraintName, log,
|
externalSourceKey, sourceLocationConstraintName, log,
|
||||||
(error, objectRetrievalInfo) => {
|
(error, objectRetrievalInfo) => {
|
||||||
if (error) {
|
if (error) {
|
||||||
return cb(error);
|
return cb(error);
|
||||||
}
|
}
|
||||||
|
console.log('objectRetrievalInfo - awsClient', objectRetrievalInfo)
|
||||||
const putResult = {
|
const putResult = {
|
||||||
key: objectRetrievalInfo.key,
|
key: objectRetrievalInfo.key,
|
||||||
dataStoreName: objectRetrievalInfo.
|
dataStoreName: objectRetrievalInfo.
|
||||||
dataStoreName,
|
dataStoreName,
|
||||||
dataStoreType: objectRetrievalInfo.
|
dataStoreType: objectRetrievalInfo.
|
||||||
dataStoreType,
|
dataStoreType,
|
||||||
|
dataStoreVersionId: objectRetrievalInfo.dataStoreVersionId,
|
||||||
size: storeMetadataParams.size,
|
size: storeMetadataParams.size,
|
||||||
dataStoreETag: objectGetInfo.dataStoreETag,
|
dataStoreETag: objectGetInfo.dataStoreETag,
|
||||||
start: objectGetInfo.start,
|
start: objectGetInfo.start,
|
||||||
|
@ -362,7 +385,6 @@ const data = {
|
||||||
return cb(null, putResultArr);
|
return cb(null, putResultArr);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// dataLocator is an array. need to get and put all parts
|
// dataLocator is an array. need to get and put all parts
|
||||||
// For now, copy 1 part at a time. Could increase the second
|
// For now, copy 1 part at a time. Could increase the second
|
||||||
// argument here to increase the number of parts
|
// argument here to increase the number of parts
|
||||||
|
@ -438,6 +460,7 @@ const data = {
|
||||||
}
|
}
|
||||||
// Copied object is not encrypted so just put it
|
// Copied object is not encrypted so just put it
|
||||||
// without a cipherBundle
|
// without a cipherBundle
|
||||||
|
console.log('about to call data.put?');
|
||||||
return data.put(null, stream, part.size,
|
return data.put(null, stream, part.size,
|
||||||
dataStoreContext, destBackendInfo,
|
dataStoreContext, destBackendInfo,
|
||||||
log, (error, partRetrievalInfo) => {
|
log, (error, partRetrievalInfo) => {
|
||||||
|
@ -451,6 +474,8 @@ const data = {
|
||||||
dataStoreName: partRetrievalInfo.
|
dataStoreName: partRetrievalInfo.
|
||||||
dataStoreName,
|
dataStoreName,
|
||||||
dataStoreETag: part.dataStoreETag,
|
dataStoreETag: part.dataStoreETag,
|
||||||
|
dataStoreVersionId: partRetrievalInfo.
|
||||||
|
dataStoreVersionId,
|
||||||
start: part.start,
|
start: part.start,
|
||||||
size: part.size,
|
size: part.size,
|
||||||
};
|
};
|
||||||
|
|
|
@ -2,9 +2,13 @@ const assert = require('assert');
|
||||||
|
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
const {
|
||||||
const { memLocation, fileLocation, awsLocation, awsLocationMismatch }
|
describeSkipIfNotMultiple,
|
||||||
= require('../utils');
|
memLocation,
|
||||||
|
fileLocation,
|
||||||
|
awsLocation,
|
||||||
|
awsLocationMismatch,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
const bucket = 'buckettestmultiplebackenddelete';
|
const bucket = 'buckettestmultiplebackenddelete';
|
||||||
const memObject = `memObject-${Date.now()}`;
|
const memObject = `memObject-${Date.now()}`;
|
||||||
|
@ -16,9 +20,6 @@ const mismatchObject = `mismatchOjbect-${Date.now()}`;
|
||||||
const body = Buffer.from('I am a body', 'utf8');
|
const body = Buffer.from('I am a body', 'utf8');
|
||||||
const bigBody = Buffer.alloc(10485760);
|
const bigBody = Buffer.alloc(10485760);
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
describeSkipIfNotMultiple('Multiple backend delete', () => {
|
describeSkipIfNotMultiple('Multiple backend delete', () => {
|
||||||
withV4(sigCfg => {
|
withV4(sigCfg => {
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
|
|
|
@ -0,0 +1,560 @@
|
||||||
|
const assert = require('assert');
|
||||||
|
const async = require('async');
|
||||||
|
const { errors } = require('arsenal');
|
||||||
|
|
||||||
|
const withV4 = require('../../support/withV4');
|
||||||
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
|
|
||||||
|
const {
|
||||||
|
describeSkipIfNotMultiple,
|
||||||
|
awsS3,
|
||||||
|
awsLocation,
|
||||||
|
awsBucket,
|
||||||
|
putToAwsBackend,
|
||||||
|
enableVersioning,
|
||||||
|
suspendVersioning,
|
||||||
|
mapToAwsPuts,
|
||||||
|
putNullVersionsToAws,
|
||||||
|
putVersionsToAws,
|
||||||
|
getAndAssertResult,
|
||||||
|
awsGetLatestVerId,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
|
const someBody = 'testbody';
|
||||||
|
const bucket = 'buckettestmultiplebackenddeleteversioning';
|
||||||
|
|
||||||
|
// order of items by index:
|
||||||
|
// 0 - whether to expect a version id
|
||||||
|
// 1 - whether version id should match request version id (undef if n/a)
|
||||||
|
// 2 - whether x-amz-delete-marker response header should be true
|
||||||
|
const _deleteResultSchema = {
|
||||||
|
nonVersionedDelete: [false, undefined, false],
|
||||||
|
newDeleteMarker: [true, false, true],
|
||||||
|
deleteVersion: [true, true, false],
|
||||||
|
deleteDeleteMarker: [true, true, true],
|
||||||
|
};
|
||||||
|
|
||||||
|
const [nonVersionedDelete, newDeleteMarker, deleteVersion, deleteDeleteMarker]
|
||||||
|
= Object.keys(_deleteResultSchema);
|
||||||
|
|
||||||
|
function _assertDeleteResult(result, resultType, requestVersionId) {
|
||||||
|
if (!_deleteResultSchema[resultType]) {
|
||||||
|
throw new Error(`undefined result type "${resultType}"`);
|
||||||
|
}
|
||||||
|
const [expectVersionId, matchReqVersionId, expectDeleteMarker] =
|
||||||
|
_deleteResultSchema[resultType];
|
||||||
|
if (expectVersionId && matchReqVersionId) {
|
||||||
|
assert.strictEqual(result.VersionId, requestVersionId);
|
||||||
|
} else if (expectVersionId) {
|
||||||
|
assert(result.VersionId, 'expected version id in result');
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(result.VersionId, undefined,
|
||||||
|
`did not expect version id in result, got "${result.VersionId}"`);
|
||||||
|
}
|
||||||
|
if (expectDeleteMarker) {
|
||||||
|
assert.strictEqual(result.DeleteMarker, 'true');
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(result.DeleteMarker, undefined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function delAndAssertResult(s3, params, cb) {
|
||||||
|
const { bucket, key, versionId, resultType, resultError } = params;
|
||||||
|
return s3.deleteObject({ Bucket: bucket, Key: key, VersionId:
|
||||||
|
versionId }, (err, result) => {
|
||||||
|
if (resultError) {
|
||||||
|
assert(err, `expected ${resultError} but found no error`);
|
||||||
|
assert.strictEqual(err.code, resultError);
|
||||||
|
assert.strictEqual(err.statusCode, errors[resultError].code);
|
||||||
|
return cb(null);
|
||||||
|
}
|
||||||
|
assert.strictEqual(err, null, 'Expected success ' +
|
||||||
|
`deleting object, got error ${err}`);
|
||||||
|
_assertDeleteResult(result, resultType, versionId);
|
||||||
|
return cb(null, result.VersionId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function _createDeleteMarkers(s3, bucket, key, count, cb) {
|
||||||
|
return async.timesSeries(count,
|
||||||
|
(i, next) => delAndAssertResult(s3, { bucket, key,
|
||||||
|
resultType: newDeleteMarker }, next),
|
||||||
|
cb);
|
||||||
|
}
|
||||||
|
|
||||||
|
function _deleteDeleteMarkers(s3, bucket, key, deleteMarkerVids, cb) {
|
||||||
|
return async.mapSeries(deleteMarkerVids, (versionId, next) => {
|
||||||
|
delAndAssertResult(s3, { bucket, key, versionId,
|
||||||
|
resultType: deleteDeleteMarker }, next);
|
||||||
|
}, () => cb());
|
||||||
|
}
|
||||||
|
|
||||||
|
function _getAssertDeleted(s3, params, cb) {
|
||||||
|
const { key, versionId, errorCode } = params;
|
||||||
|
return s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
|
||||||
|
err => {
|
||||||
|
assert.strictEqual(err.code, errorCode);
|
||||||
|
assert.strictEqual(err.statusCode, 404);
|
||||||
|
return cb();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function _awsGetAssertDeleted(params, cb) {
|
||||||
|
const { key, versionId, errorCode } = params;
|
||||||
|
return awsS3.getObject({ Bucket: awsBucket, Key: key, VersionId:
|
||||||
|
versionId }, err => {
|
||||||
|
assert.strictEqual(err.code, errorCode);
|
||||||
|
assert.strictEqual(err.statusCode, 404);
|
||||||
|
return cb();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' +
|
||||||
|
'using object location constraint', function testSuite() {
|
||||||
|
this.timeout(30000);
|
||||||
|
withV4(sigCfg => {
|
||||||
|
let bucketUtil;
|
||||||
|
let s3;
|
||||||
|
beforeEach(() => {
|
||||||
|
process.stdout.write('Creating bucket\n');
|
||||||
|
bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
|
s3 = bucketUtil.s3;
|
||||||
|
return s3.createBucketAsync({ Bucket: bucket })
|
||||||
|
.catch(err => {
|
||||||
|
process.stdout.write(`Error creating bucket: ${err}\n`);
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
process.stdout.write('Emptying bucket\n');
|
||||||
|
return bucketUtil.empty(bucket)
|
||||||
|
.then(() => {
|
||||||
|
process.stdout.write('Deleting bucket\n');
|
||||||
|
return bucketUtil.deleteOne(bucket);
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
process.stdout.write('Error emptying/deleting bucket: ' +
|
||||||
|
`${err}\n`);
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: if specifying "null" version, should ' +
|
||||||
|
'delete specific version in AWS backend', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putToAwsBackend(s3, bucket, key, someBody,
|
||||||
|
err => next(err)),
|
||||||
|
next => awsGetLatestVerId(key, someBody, next),
|
||||||
|
(awsVerId, next) => delAndAssertResult(s3, { bucket,
|
||||||
|
key, versionId: 'null', resultType: deleteVersion },
|
||||||
|
err => next(err, awsVerId)),
|
||||||
|
(awsVerId, next) => _awsGetAssertDeleted({ key,
|
||||||
|
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: specifying any version id other ' +
|
||||||
|
'than null should not result in its deletion in AWS backend', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putToAwsBackend(s3, bucket, key, someBody,
|
||||||
|
err => next(err)),
|
||||||
|
next => awsGetLatestVerId(key, someBody, next),
|
||||||
|
(awsVerId, next) => delAndAssertResult(s3, { bucket,
|
||||||
|
key, versionId: 'awsVerId', resultError:
|
||||||
|
'InvalidArgument' }, err => next(err, awsVerId)),
|
||||||
|
(awsVerId, next) => awsGetLatestVerId(key, someBody,
|
||||||
|
(err, resultVid) => {
|
||||||
|
assert.strictEqual(resultVid, awsVerId);
|
||||||
|
next();
|
||||||
|
}),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should delete a specific version in AWS ' +
|
||||||
|
'backend successfully', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
err => next(err)),
|
||||||
|
next => awsGetLatestVerId(key, someBody, next),
|
||||||
|
(awsVerId, next) => delAndAssertResult(s3, { bucket,
|
||||||
|
key, versionId: 'null', resultType: deleteVersion },
|
||||||
|
err => next(err, awsVerId)),
|
||||||
|
(awsVerId, next) => _awsGetAssertDeleted({ key,
|
||||||
|
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should delete a specific version in AWS ' +
|
||||||
|
'backend successfully', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
(err, versionIds) => next(err, versionIds[0])),
|
||||||
|
(s3vid, next) => awsGetLatestVerId(key, someBody,
|
||||||
|
(err, awsVid) => next(err, s3vid, awsVid)),
|
||||||
|
(s3VerId, awsVerId, next) => delAndAssertResult(s3, { bucket,
|
||||||
|
key, versionId: s3VerId, resultType: deleteVersion },
|
||||||
|
err => next(err, awsVerId)),
|
||||||
|
(awsVerId, next) => _awsGetAssertDeleted({ key,
|
||||||
|
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: deleting existing object should ' +
|
||||||
|
'not return version id or x-amz-delete-marker: true but should ' +
|
||||||
|
'create a delete marker in aws ', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putToAwsBackend(s3, bucket, key, someBody,
|
||||||
|
err => next(err)),
|
||||||
|
next => delAndAssertResult(s3, { bucket, key,
|
||||||
|
resultType: nonVersionedDelete }, err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should create a delete marker in s3 ' +
|
||||||
|
'and aws successfully when deleting existing object', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
err => next(err)),
|
||||||
|
next => delAndAssertResult(s3, { bucket, key, resultType:
|
||||||
|
newDeleteMarker }, err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
// NOTE: Normal deletes when versioning is suspended create a
|
||||||
|
// delete marker with the version id "null", which overwrites an
|
||||||
|
// existing null version in s3 metadata.
|
||||||
|
it('versioning suspended: creating a delete marker will overwrite an ' +
|
||||||
|
'existing null version that is the latest version in s3 metadata,' +
|
||||||
|
' but the data of the first null version will remain in AWS',
|
||||||
|
function itF(done) {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
err => next(err)),
|
||||||
|
next => awsGetLatestVerId(key, someBody, next),
|
||||||
|
(awsNullVid, next) => {
|
||||||
|
this.test.awsNullVid = awsNullVid;
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
// following call should generate a delete marker
|
||||||
|
next => delAndAssertResult(s3, { bucket, key, resultType:
|
||||||
|
newDeleteMarker }, next),
|
||||||
|
// delete delete marker
|
||||||
|
(dmVid, next) => delAndAssertResult(s3, { bucket, key,
|
||||||
|
versionId: dmVid, resultType: deleteDeleteMarker },
|
||||||
|
err => next(err)),
|
||||||
|
// should get no such object even after deleting del marker
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
// get directly to aws however will give us first null version
|
||||||
|
next => awsGetLatestVerId(key, someBody, next),
|
||||||
|
(awsLatestVid, next) => {
|
||||||
|
assert.strictEqual(awsLatestVid, this.test.awsNullVid);
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
// NOTE: Normal deletes when versioning is suspended create a
|
||||||
|
// delete marker with the version id "null" which is supposed to
|
||||||
|
// overwrite any existing null version.
|
||||||
|
it('versioning suspended: creating a delete marker will overwrite an ' +
|
||||||
|
'existing null version that is not the latest version in s3 metadata,' +
|
||||||
|
' but the data of the first null version will remain in AWS',
|
||||||
|
function itF(done) {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
const data = [undefined, 'data1'];
|
||||||
|
async.waterfall([
|
||||||
|
// put null version
|
||||||
|
next => putToAwsBackend(s3, bucket, key, data[0],
|
||||||
|
err => next(err)),
|
||||||
|
next => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsNullVid, next) => {
|
||||||
|
this.test.awsNullVid = awsNullVid;
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
// enable versioning and put another version
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [data[1]], next),
|
||||||
|
(versions, next) => {
|
||||||
|
this.test.s3vid = versions[0];
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
next => suspendVersioning(s3, bucket, next),
|
||||||
|
// overwrites null version in s3 metadata but does not send
|
||||||
|
// additional delete to AWS to clean up previous "null" version
|
||||||
|
// -- see note above
|
||||||
|
next => delAndAssertResult(s3, { bucket, key,
|
||||||
|
resultType: newDeleteMarker }, next),
|
||||||
|
(s3dmVid, next) => {
|
||||||
|
this.test.s3DeleteMarkerId = s3dmVid;
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
// delete delete marker
|
||||||
|
next => delAndAssertResult(s3, { bucket, key,
|
||||||
|
versionId: this.test.s3DeleteMarkerId,
|
||||||
|
resultType: deleteDeleteMarker }, err => next(err)),
|
||||||
|
// deleting latest version after del marker
|
||||||
|
next => delAndAssertResult(s3, { bucket, key,
|
||||||
|
versionId: this.test.s3vid, resultType: deleteVersion },
|
||||||
|
err => next(err)),
|
||||||
|
// should get no such object instead of null version
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
// we get the null version that should have been "overwritten"
|
||||||
|
// when getting the latest version in AWS now
|
||||||
|
next => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsLatestVid, next) => {
|
||||||
|
assert.strictEqual(awsLatestVid, this.test.awsNullVid);
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should create a delete marker in s3 and ' +
|
||||||
|
'aws successfully when deleting existing object', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
err => next(err)),
|
||||||
|
next => delAndAssertResult(s3, { bucket, key, resultType:
|
||||||
|
newDeleteMarker }, err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should delete a delete marker in s3 and ' +
|
||||||
|
'aws successfully', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
(err, versionIds) => next(err, versionIds[0])),
|
||||||
|
// create a delete marker
|
||||||
|
(s3vid, next) => delAndAssertResult(s3, { bucket, key,
|
||||||
|
resultType: newDeleteMarker }, (err, delMarkerVid) =>
|
||||||
|
next(err, s3vid, delMarkerVid)),
|
||||||
|
// delete delete marker
|
||||||
|
(s3vid, dmVid, next) => delAndAssertResult(s3, { bucket, key,
|
||||||
|
versionId: dmVid, resultType: deleteDeleteMarker },
|
||||||
|
err => next(err, s3vid)),
|
||||||
|
// should be able to get object originally put from s3
|
||||||
|
(s3vid, next) => getAndAssertResult(s3, { bucket, key,
|
||||||
|
body: someBody, expectedVersionId: s3vid }, next),
|
||||||
|
// latest version in aws should now be object originally put
|
||||||
|
next => awsGetLatestVerId(key, someBody, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('multiple delete markers: should be able to get pre-existing ' +
|
||||||
|
'versions after creating and deleting several delete markers', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
(err, versionIds) => next(err, versionIds[0])),
|
||||||
|
(s3vid, next) => _createDeleteMarkers(s3, bucket, key, 3,
|
||||||
|
(err, dmVids) => next(err, s3vid, dmVids)),
|
||||||
|
(s3vid, dmVids, next) => _deleteDeleteMarkers(s3, bucket, key,
|
||||||
|
dmVids, () => next(null, s3vid)),
|
||||||
|
// should be able to get object originally put from s3
|
||||||
|
(s3vid, next) => getAndAssertResult(s3, { bucket, key,
|
||||||
|
body: someBody, expectedVersionId: s3vid }, next),
|
||||||
|
// latest version in aws should now be object originally put
|
||||||
|
next => awsGetLatestVerId(key, someBody, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('multiple delete markers: should get NoSuchObject if only ' +
|
||||||
|
'one of the delete markers is deleted', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
err => next(err)),
|
||||||
|
next => _createDeleteMarkers(s3, bucket, key, 3,
|
||||||
|
(err, dmVids) => next(err, dmVids[2])),
|
||||||
|
(lastDmVid, next) => delAndAssertResult(s3, { bucket,
|
||||||
|
key, versionId: lastDmVid, resultType: deleteDeleteMarker },
|
||||||
|
err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get the new latest version after deleting the latest' +
|
||||||
|
'specific version', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
const data = [...Array(4).keys()].map(i => i.toString());
|
||||||
|
async.waterfall([
|
||||||
|
// put 3 null versions
|
||||||
|
next => mapToAwsPuts(s3, bucket, key, data.slice(0, 3),
|
||||||
|
err => next(err)),
|
||||||
|
// put one version
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [data[3]],
|
||||||
|
(err, versionIds) => next(err, versionIds[0])),
|
||||||
|
// delete the latest version
|
||||||
|
(versionId, next) => delAndAssertResult(s3, { bucket,
|
||||||
|
key, versionId, resultType: deleteVersion },
|
||||||
|
err => next(err)),
|
||||||
|
// should get the last null version
|
||||||
|
next => getAndAssertResult(s3, { bucket, key,
|
||||||
|
body: data[2], expectedVersionId: 'null' }, next),
|
||||||
|
next => awsGetLatestVerId(key, data[2],
|
||||||
|
err => next(err)),
|
||||||
|
// delete the null version
|
||||||
|
next => delAndAssertResult(s3, { bucket,
|
||||||
|
key, versionId: 'null', resultType: deleteVersion },
|
||||||
|
err => next(err)),
|
||||||
|
// s3 metadata should report no existing versions for keyname
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
// NOTE: latest version in aws will be the second null version
|
||||||
|
next => awsGetLatestVerId(key, data[1],
|
||||||
|
err => next(err)),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete the correct version even if other versions or ' +
|
||||||
|
'delete markers put directly on aws', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
(err, versionIds) => next(err, versionIds[0])),
|
||||||
|
(s3vid, next) => awsGetLatestVerId(key, someBody,
|
||||||
|
(err, awsVid) => next(err, s3vid, awsVid)),
|
||||||
|
// put an object in AWS
|
||||||
|
(s3vid, awsVid, next) => awsS3.putObject({ Bucket: awsBucket,
|
||||||
|
Key: key }, err => next(err, s3vid, awsVid)),
|
||||||
|
// create a delete marker in AWS
|
||||||
|
(s3vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key }, err => next(err, s3vid, awsVid)),
|
||||||
|
// delete original version in s3
|
||||||
|
(s3vid, awsVid, next) => delAndAssertResult(s3, { bucket, key,
|
||||||
|
versionId: s3vid, resultType: deleteVersion },
|
||||||
|
err => next(err, awsVid)),
|
||||||
|
(awsVid, next) => _getAssertDeleted(s3, { key,
|
||||||
|
errorCode: 'NoSuchKey' }, () => next(null, awsVid)),
|
||||||
|
(awsVerId, next) => _awsGetAssertDeleted({ key,
|
||||||
|
versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not return an error deleting a version that was already ' +
|
||||||
|
'deleted directly from AWS backend', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
(err, versionIds) => next(err, versionIds[0])),
|
||||||
|
(s3vid, next) => awsGetLatestVerId(key, someBody,
|
||||||
|
(err, awsVid) => next(err, s3vid, awsVid)),
|
||||||
|
// delete the object in AWS
|
||||||
|
(s3vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key, VersionId: awsVid }, err => next(err, s3vid)),
|
||||||
|
// then try to delete in S3
|
||||||
|
(s3vid, next) => delAndAssertResult(s3, { bucket, key,
|
||||||
|
versionId: s3vid, resultType: deleteVersion },
|
||||||
|
err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' +
|
||||||
|
'using bucket location constraint', function testSuite() {
|
||||||
|
this.timeout(30000);
|
||||||
|
const createBucketParams = {
|
||||||
|
Bucket: bucket,
|
||||||
|
CreateBucketConfiguration: {
|
||||||
|
LocationConstraint: awsLocation,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
withV4(sigCfg => {
|
||||||
|
let bucketUtil;
|
||||||
|
let s3;
|
||||||
|
beforeEach(() => {
|
||||||
|
process.stdout.write('Creating bucket\n');
|
||||||
|
bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
|
s3 = bucketUtil.s3;
|
||||||
|
return s3.createBucketAsync(createBucketParams)
|
||||||
|
.catch(err => {
|
||||||
|
process.stdout.write(`Error creating bucket: ${err}\n`);
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
process.stdout.write('Emptying bucket\n');
|
||||||
|
return bucketUtil.empty(bucket)
|
||||||
|
.then(() => {
|
||||||
|
process.stdout.write('Deleting bucket\n');
|
||||||
|
return bucketUtil.deleteOne(bucket);
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
process.stdout.write('Error emptying/deleting bucket: ' +
|
||||||
|
`${err}\n`);
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: deleting non-existing object should ' +
|
||||||
|
'not return version id or x-amz-delete-marker: true nor create a ' +
|
||||||
|
'delete marker in aws ', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => delAndAssertResult(s3, { bucket, key,
|
||||||
|
resultType: nonVersionedDelete }, err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should create a delete marker in s3 ' +
|
||||||
|
'and aws successfully when deleting non-existing object', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => suspendVersioning(s3, bucket, next),
|
||||||
|
next => delAndAssertResult(s3, { bucket, key, resultType:
|
||||||
|
newDeleteMarker }, err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should create a delete marker in s3 and ' +
|
||||||
|
'aws successfully when deleting non-existing object', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => delAndAssertResult(s3, { bucket, key, resultType:
|
||||||
|
newDeleteMarker }, err => next(err)),
|
||||||
|
next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -3,10 +3,15 @@ const async = require('async');
|
||||||
|
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
const {
|
||||||
const { uniqName, getAzureClient, getAzureContainerName, getAzureKeys,
|
describeSkipIfNotMultiple,
|
||||||
azureLocation, azureLocationMismatch } =
|
uniqName,
|
||||||
require('../utils');
|
getAzureClient,
|
||||||
|
getAzureContainerName,
|
||||||
|
getAzureKeys,
|
||||||
|
azureLocation,
|
||||||
|
azureLocationMismatch,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
const keyObject = 'deleteazure';
|
const keyObject = 'deleteazure';
|
||||||
const azureContainerName = getAzureContainerName();
|
const azureContainerName = getAzureContainerName();
|
||||||
|
@ -20,10 +25,6 @@ const nonExistingId = process.env.AWS_ON_AIR ?
|
||||||
'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' :
|
'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' :
|
||||||
'3939393939393939393936493939393939393939756e6437';
|
'3939393939393939393936493939393939393939756e6437';
|
||||||
|
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
describeSkipIfNotMultiple('Multiple backend delete object from Azure',
|
describeSkipIfNotMultiple('Multiple backend delete object from Azure',
|
||||||
function testSuite() {
|
function testSuite() {
|
||||||
this.timeout(250000);
|
this.timeout(250000);
|
||||||
|
|
|
@ -2,9 +2,13 @@ const assert = require('assert');
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
const {
|
||||||
const { memLocation, fileLocation, awsLocation, awsLocationMismatch }
|
describeSkipIfNotMultiple,
|
||||||
= require('../utils');
|
memLocation,
|
||||||
|
fileLocation,
|
||||||
|
awsLocation,
|
||||||
|
awsLocationMismatch,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
const bucket = 'buckettestmultiplebackendget';
|
const bucket = 'buckettestmultiplebackendget';
|
||||||
const memObject = `memobject-${Date.now()}`;
|
const memObject = `memobject-${Date.now()}`;
|
||||||
|
@ -20,9 +24,6 @@ const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
|
||||||
const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
||||||
const bigMD5 = 'f1c9645dbc14efddc7d8a322685f26eb';
|
const bigMD5 = 'f1c9645dbc14efddc7d8a322685f26eb';
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
describe('Multiple backend get object', function testSuite() {
|
describe('Multiple backend get object', function testSuite() {
|
||||||
this.timeout(30000);
|
this.timeout(30000);
|
||||||
withV4(sigCfg => {
|
withV4(sigCfg => {
|
||||||
|
|
|
@ -1,46 +1,23 @@
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const AWS = require('aws-sdk');
|
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
|
||||||
const { getRealAwsConfig } = require('../../support/awsConfig');
|
|
||||||
const {
|
const {
|
||||||
|
awsS3,
|
||||||
awsLocation,
|
awsLocation,
|
||||||
|
awsBucket,
|
||||||
enableVersioning,
|
enableVersioning,
|
||||||
suspendVersioning,
|
suspendVersioning,
|
||||||
mapToAwsPuts,
|
mapToAwsPuts,
|
||||||
putNullVersionsToAws,
|
putNullVersionsToAws,
|
||||||
putVersionsToAws,
|
putVersionsToAws,
|
||||||
expectedETag,
|
getAndAssertResult,
|
||||||
|
describeSkipIfNotMultiple,
|
||||||
} = require('../utils');
|
} = require('../utils');
|
||||||
|
|
||||||
const someBody = 'testbody';
|
const someBody = 'testbody';
|
||||||
const bucket = 'buckettestmultiplebackendgetawsversioning';
|
const bucket = 'buckettestmultiplebackendgetawsversioning';
|
||||||
|
|
||||||
let awsS3;
|
|
||||||
|
|
||||||
function getAndAssertResult(s3, params, cb) {
|
|
||||||
const { bucket, key, body, versionId, expectedVersionId } = params;
|
|
||||||
s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
|
|
||||||
(err, data) => {
|
|
||||||
assert.strictEqual(err, null, 'Expected success ' +
|
|
||||||
`getting object, got error ${err}`);
|
|
||||||
if (body) {
|
|
||||||
assert(data.Body, 'expected object body in response');
|
|
||||||
const expectedMD5 = expectedETag(body, false);
|
|
||||||
const resultMD5 = expectedETag(data.Body, false);
|
|
||||||
assert.strictEqual(resultMD5, expectedMD5);
|
|
||||||
}
|
|
||||||
if (!expectedVersionId) {
|
|
||||||
assert.strictEqual(data.VersionId, undefined);
|
|
||||||
} else {
|
|
||||||
assert.strictEqual(data.VersionId, expectedVersionId);
|
|
||||||
}
|
|
||||||
cb();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getAndAssertVersions(s3, bucket, key, versionIds, expectedData,
|
function getAndAssertVersions(s3, bucket, key, versionIds, expectedData,
|
||||||
cb) {
|
cb) {
|
||||||
async.mapSeries(versionIds, (versionId, next) => {
|
async.mapSeries(versionIds, (versionId, next) => {
|
||||||
|
@ -58,16 +35,7 @@ function getAndAssertVersions(s3, bucket, key, versionIds, expectedData,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
describeSkipIfNotMultiple('AWS backend get object with versioning',
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
if (describeSkipIfNotMultiple !== describe.skip) {
|
|
||||||
// can only get real aws config if not running end-to-end
|
|
||||||
const awsConfig = getRealAwsConfig(awsLocation);
|
|
||||||
awsS3 = new AWS.S3(awsConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
describeSkipIfNotMultiple('Multiple backend get object with versioning',
|
|
||||||
function testSuite() {
|
function testSuite() {
|
||||||
this.timeout(30000);
|
this.timeout(30000);
|
||||||
withV4(sigCfg => {
|
withV4(sigCfg => {
|
||||||
|
@ -308,8 +276,6 @@ function testSuite() {
|
||||||
it('should return the correct data getting versioned object ' +
|
it('should return the correct data getting versioned object ' +
|
||||||
'even if object was deleted from AWS (creating a delete marker)',
|
'even if object was deleted from AWS (creating a delete marker)',
|
||||||
done => {
|
done => {
|
||||||
const awsBucket = config.locationConstraints[awsLocation].
|
|
||||||
details.bucketName;
|
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
async.waterfall([
|
async.waterfall([
|
||||||
next => enableVersioning(s3, bucket, next),
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
@ -327,15 +293,13 @@ function testSuite() {
|
||||||
it('should return the correct data getting versioned object ' +
|
it('should return the correct data getting versioned object ' +
|
||||||
'even if object is put directly to AWS (creating new version)',
|
'even if object is put directly to AWS (creating new version)',
|
||||||
done => {
|
done => {
|
||||||
const awsBucket = config.locationConstraints[awsLocation].
|
|
||||||
details.bucketName;
|
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
async.waterfall([
|
async.waterfall([
|
||||||
next => enableVersioning(s3, bucket, next),
|
next => enableVersioning(s3, bucket, next),
|
||||||
next => s3.putObject({ Bucket: bucket, Key: key, Body: someBody,
|
next => s3.putObject({ Bucket: bucket, Key: key, Body: someBody,
|
||||||
Metadata: { 'scal-location-constraint': awsLocation } },
|
Metadata: { 'scal-location-constraint': awsLocation } },
|
||||||
(err, res) => next(err, res.VersionId)),
|
(err, res) => next(err, res.VersionId)),
|
||||||
// create a delete marker in AWS
|
// put an object in AWS
|
||||||
(versionId, next) => awsS3.putObject({ Bucket: awsBucket,
|
(versionId, next) => awsS3.putObject({ Bucket: awsBucket,
|
||||||
Key: key }, err => next(err, versionId)),
|
Key: key }, err => next(err, versionId)),
|
||||||
(versionId, next) => getAndAssertResult(s3, { bucket, key,
|
(versionId, next) => getAndAssertResult(s3, { bucket, key,
|
||||||
|
@ -346,8 +310,6 @@ function testSuite() {
|
||||||
it('should return a InternalError if trying to get an object ' +
|
it('should return a InternalError if trying to get an object ' +
|
||||||
'that was deleted in AWS but exists in s3 metadata',
|
'that was deleted in AWS but exists in s3 metadata',
|
||||||
done => {
|
done => {
|
||||||
const awsBucket = config.locationConstraints[awsLocation].
|
|
||||||
details.bucketName;
|
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
async.waterfall([
|
async.waterfall([
|
||||||
next => enableVersioning(s3, bucket, next),
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
@ -372,8 +334,6 @@ function testSuite() {
|
||||||
it('should return a InternalError if trying to get a version ' +
|
it('should return a InternalError if trying to get a version ' +
|
||||||
'that was deleted in AWS but exists in s3 metadata',
|
'that was deleted in AWS but exists in s3 metadata',
|
||||||
done => {
|
done => {
|
||||||
const awsBucket = config.locationConstraints[awsLocation].
|
|
||||||
details.bucketName;
|
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
async.waterfall([
|
async.waterfall([
|
||||||
next => enableVersioning(s3, bucket, next),
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
|
|
@ -3,21 +3,22 @@ const assert = require('assert');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
|
|
||||||
const { uniqName, getAzureClient, getAzureContainerName,
|
const {
|
||||||
getAzureKeys, azureLocation } = require('../utils');
|
describeSkipIfNotMultiple,
|
||||||
|
uniqName,
|
||||||
|
getAzureClient,
|
||||||
|
getAzureContainerName,
|
||||||
|
getAzureKeys,
|
||||||
|
azureLocation,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
const azureClient = getAzureClient();
|
const azureClient = getAzureClient();
|
||||||
const azureContainerName = getAzureContainerName();
|
const azureContainerName = getAzureContainerName();
|
||||||
const keys = getAzureKeys();
|
const keys = getAzureKeys();
|
||||||
const keyObject = 'getazure';
|
const keyObject = 'getazure';
|
||||||
|
|
||||||
const { config } = require('../../../../../../lib/Config');
|
|
||||||
|
|
||||||
const normalBody = Buffer.from('I am a body', 'utf8');
|
const normalBody = Buffer.from('I am a body', 'utf8');
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
const azureTimeout = 10000;
|
const azureTimeout = 10000;
|
||||||
|
|
||||||
describeSkipIfNotMultiple('Multiple backend get object from Azure',
|
describeSkipIfNotMultiple('Multiple backend get object from Azure',
|
||||||
|
|
|
@ -1,22 +1,10 @@
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
|
|
||||||
const { config } = require('../../../../../../lib/Config');
|
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { azureLocation } = require('../utils');
|
const { describeSkipIfNotMultiple, azureLocation, azureContainerName }
|
||||||
|
= require('../utils');
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
let azureContainerName;
|
|
||||||
|
|
||||||
if (config.locationConstraints[azureLocation] &&
|
|
||||||
config.locationConstraints[azureLocation].details &&
|
|
||||||
config.locationConstraints[azureLocation].details.azureContainerName) {
|
|
||||||
azureContainerName =
|
|
||||||
config.locationConstraints[azureLocation].details.azureContainerName;
|
|
||||||
}
|
|
||||||
|
|
||||||
const keyName = `somekey-${Date.now()}`;
|
const keyName = `somekey-${Date.now()}`;
|
||||||
|
|
||||||
|
|
|
@ -1,24 +1,14 @@
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
|
|
||||||
const { config } = require('../../../../../../lib/Config');
|
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { azureLocation } = require('../utils');
|
const { describeSkipIfNotMultiple, azureLocation, getAzureContainerName }
|
||||||
|
= require('../utils');
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
const azureContainerName = getAzureContainerName();
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
let azureContainerName;
|
|
||||||
const bodyFirstPart = Buffer.alloc(10);
|
const bodyFirstPart = Buffer.alloc(10);
|
||||||
const bodySecondPart = Buffer.alloc(104857610);
|
const bodySecondPart = Buffer.alloc(104857610);
|
||||||
|
|
||||||
if (config.locationConstraints[azureLocation] &&
|
|
||||||
config.locationConstraints[azureLocation].details &&
|
|
||||||
config.locationConstraints[azureLocation].details.azureContainerName) {
|
|
||||||
azureContainerName =
|
|
||||||
config.locationConstraints[azureLocation].details.azureContainerName;
|
|
||||||
}
|
|
||||||
|
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
let s3;
|
let s3;
|
||||||
|
|
||||||
|
|
|
@ -4,9 +4,8 @@ const async = require('async');
|
||||||
const { s3middleware } = require('arsenal');
|
const { s3middleware } = require('arsenal');
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { uniqName, getAzureClient, getAzureContainerName, convertMD5,
|
const { describeSkipIfNotMultiple, uniqName, getAzureClient,
|
||||||
azureLocation } = require('../utils');
|
getAzureContainerName, convertMD5, azureLocation } = require('../utils');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
|
||||||
const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
|
const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
|
||||||
const maxSubPartSize = azureMpuUtils.maxSubPartSize;
|
const maxSubPartSize = azureMpuUtils.maxSubPartSize;
|
||||||
|
|
||||||
|
@ -15,9 +14,6 @@ const azureClient = getAzureClient();
|
||||||
const azureContainerName = getAzureContainerName();
|
const azureContainerName = getAzureContainerName();
|
||||||
const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
|
const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
let s3;
|
let s3;
|
||||||
|
|
||||||
|
|
|
@ -1,21 +1,23 @@
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const AWS = require('aws-sdk');
|
|
||||||
|
|
||||||
const { s3middleware } = require('arsenal');
|
const { s3middleware } = require('arsenal');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { fileLocation, awsLocation, azureLocation, azureLocationMismatch,
|
const {
|
||||||
getAzureClient, getAzureContainerName } = require('../utils');
|
describeSkipIfNotMultiple,
|
||||||
const { getRealAwsConfig } =
|
fileLocation,
|
||||||
require('../../support/awsConfig');
|
awsS3,
|
||||||
|
awsLocation,
|
||||||
|
awsBucket,
|
||||||
|
azureLocation,
|
||||||
|
azureLocationMismatch,
|
||||||
|
getAzureClient,
|
||||||
|
getAzureContainerName,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
|
const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
const awsBucket = 'multitester555';
|
|
||||||
const azureContainerName = getAzureContainerName();
|
const azureContainerName = getAzureContainerName();
|
||||||
const azureClient = getAzureClient();
|
const azureClient = getAzureClient();
|
||||||
const azureTimeout = 20000;
|
const azureTimeout = 20000;
|
||||||
|
@ -106,8 +108,7 @@ function testSuite() {
|
||||||
this.currentTest.key = `somekey-${Date.now()}`;
|
this.currentTest.key = `somekey-${Date.now()}`;
|
||||||
bucketUtil = new BucketUtility('default', sigCfg);
|
bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
s3 = bucketUtil.s3;
|
s3 = bucketUtil.s3;
|
||||||
const awsConfig = getRealAwsConfig(awsLocation);
|
this.currentTest.awsClient = awsS3;
|
||||||
this.currentTest.awsClient = new AWS.S3(awsConfig);
|
|
||||||
return s3.createBucketAsync({ Bucket: azureContainerName })
|
return s3.createBucketAsync({ Bucket: azureContainerName })
|
||||||
.catch(err => {
|
.catch(err => {
|
||||||
process.stdout.write(`Error creating bucket: ${err}\n`);
|
process.stdout.write(`Error creating bucket: ${err}\n`);
|
||||||
|
|
|
@ -0,0 +1,176 @@
|
||||||
|
const assert = require('assert');
|
||||||
|
const async = require('async');
|
||||||
|
const withV4 = require('../../support/withV4');
|
||||||
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
|
const { minimumAllowedPartSize } = require('../../../../../../constants');
|
||||||
|
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
|
||||||
|
const {
|
||||||
|
awsLocation,
|
||||||
|
enableVersioning,
|
||||||
|
suspendVersioning,
|
||||||
|
putToAwsBackend,
|
||||||
|
awsGetLatestVerId,
|
||||||
|
getAndAssertResult,
|
||||||
|
describeSkipIfNotMultiple,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
|
const data = ['a', 'b'].map(char => Buffer.alloc(minimumAllowedPartSize, char));
|
||||||
|
const concattedData = Buffer.concat(data);
|
||||||
|
|
||||||
|
const bucket = 'buckettestmultiplebackendmpuawsversioning';
|
||||||
|
|
||||||
|
function mpuSetup(s3, key, location, cb) {
|
||||||
|
const partArray = [];
|
||||||
|
async.waterfall([
|
||||||
|
next => {
|
||||||
|
const params = {
|
||||||
|
Bucket: bucket,
|
||||||
|
Key: key,
|
||||||
|
Metadata: { 'scal-location-constraint': location },
|
||||||
|
};
|
||||||
|
s3.createMultipartUpload(params, (err, res) => {
|
||||||
|
assert.strictEqual(err, null, `err creating mpu: ${err}`);
|
||||||
|
const uploadId = res.UploadId;
|
||||||
|
assert(uploadId);
|
||||||
|
assert.strictEqual(res.Bucket, bucket);
|
||||||
|
assert.strictEqual(res.Key, key);
|
||||||
|
next(err, uploadId);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
(uploadId, next) => {
|
||||||
|
const partParams = {
|
||||||
|
Bucket: bucket,
|
||||||
|
Key: key,
|
||||||
|
PartNumber: 1,
|
||||||
|
UploadId: uploadId,
|
||||||
|
Body: data[0],
|
||||||
|
};
|
||||||
|
s3.uploadPart(partParams, (err, res) => {
|
||||||
|
assert.strictEqual(err, null, `err uploading part 1: ${err}`);
|
||||||
|
partArray.push({ ETag: res.ETag, PartNumber: 1 });
|
||||||
|
next(err, uploadId);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
(uploadId, next) => {
|
||||||
|
const partParams = {
|
||||||
|
Bucket: bucket,
|
||||||
|
Key: key,
|
||||||
|
PartNumber: 2,
|
||||||
|
UploadId: uploadId,
|
||||||
|
Body: data[1],
|
||||||
|
};
|
||||||
|
s3.uploadPart(partParams, (err, res) => {
|
||||||
|
assert.strictEqual(err, null, `err uploading part 2: ${err}`);
|
||||||
|
partArray.push({ ETag: res.ETag, PartNumber: 2 });
|
||||||
|
next(err, uploadId);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
], (err, uploadId) => {
|
||||||
|
process.stdout.write('Created MPU and put two parts\n');
|
||||||
|
cb(err, uploadId, partArray);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function completeAndAssertMpu(s3, params, cb) {
|
||||||
|
const { bucket, key, uploadId, partArray, expectVersionId,
|
||||||
|
expectedGetVersionId } = params;
|
||||||
|
s3.completeMultipartUpload({
|
||||||
|
Bucket: bucket,
|
||||||
|
Key: key,
|
||||||
|
UploadId: uploadId,
|
||||||
|
MultipartUpload: { Parts: partArray },
|
||||||
|
}, (err, data) => {
|
||||||
|
assert.strictEqual(err, null, `Err completing MPU: ${err}`);
|
||||||
|
if (expectVersionId) {
|
||||||
|
assert.notEqual(data.VersionId, undefined);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(data.VersionId, undefined);
|
||||||
|
}
|
||||||
|
const expectedVersionId = expectedGetVersionId || data.VersionId;
|
||||||
|
getAndAssertResult(s3, { bucket, key, body: concattedData,
|
||||||
|
expectedVersionId }, cb);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describeSkipIfNotMultiple('AWS backend complete mpu with versioning',
|
||||||
|
function testSuite() {
|
||||||
|
this.timeout(30000);
|
||||||
|
withV4(sigCfg => {
|
||||||
|
const bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
|
const s3 = bucketUtil.s3;
|
||||||
|
beforeEach(done => s3.createBucket({
|
||||||
|
Bucket: bucket,
|
||||||
|
CreateBucketConfiguration: {
|
||||||
|
LocationConstraint: awsLocation,
|
||||||
|
},
|
||||||
|
}, done));
|
||||||
|
afterEach(done => {
|
||||||
|
removeAllVersions({ Bucket: bucket }, err => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
return s3.deleteBucket({ Bucket: bucket }, done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: should not return version id ' +
|
||||||
|
'completing mpu', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
mpuSetup(s3, key, awsLocation, (err, uploadId, partArray) => {
|
||||||
|
completeAndAssertMpu(s3, { bucket, key, uploadId, partArray,
|
||||||
|
expectVersionId: false }, done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: if complete mpu on already-existing ' +
|
||||||
|
'object, metadata should be overwritten but data of previous version' +
|
||||||
|
'in AWS should not be deleted', function itF(done) {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putToAwsBackend(s3, bucket, key, '', err => next(err)),
|
||||||
|
next => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsVerId, next) => {
|
||||||
|
this.test.awsVerId = awsVerId;
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
next => mpuSetup(s3, key, awsLocation, next),
|
||||||
|
(uploadId, partArray, next) => completeAndAssertMpu(s3,
|
||||||
|
{ bucket, key, uploadId, partArray, expectVersionId:
|
||||||
|
false }, next),
|
||||||
|
next => s3.deleteObject({ Bucket: bucket, Key: key, VersionId:
|
||||||
|
'null' }, next),
|
||||||
|
(delData, next) => getAndAssertResult(s3, { bucket, key,
|
||||||
|
expectedError: 'NoSuchKey' }, next),
|
||||||
|
next => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsVerId, next) => {
|
||||||
|
assert.strictEqual(awsVerId, this.test.awsVerId);
|
||||||
|
next();
|
||||||
|
},
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should not return version id completing mpu',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => suspendVersioning(s3, bucket, next),
|
||||||
|
next => mpuSetup(s3, key, awsLocation, next),
|
||||||
|
(uploadId, partArray, next) => completeAndAssertMpu(s3,
|
||||||
|
{ bucket, key, uploadId, partArray, expectVersionId: false,
|
||||||
|
expectedGetVersionId: 'null' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should return version id completing mpu',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => mpuSetup(s3, key, awsLocation, next),
|
||||||
|
(uploadId, partArray, next) => completeAndAssertMpu(s3,
|
||||||
|
{ bucket, key, uploadId, partArray, expectVersionId: true },
|
||||||
|
next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -4,9 +4,9 @@ const async = require('async');
|
||||||
const { s3middleware } = require('arsenal');
|
const { s3middleware } = require('arsenal');
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { expectedETag, uniqName, getAzureClient, getAzureContainerName,
|
const { describeSkipIfNotMultiple, expectedETag, uniqName, getAzureClient,
|
||||||
convertMD5, azureLocation } = require('../utils');
|
getAzureContainerName, convertMD5, azureLocation, azureLocationMismatch }
|
||||||
const { config } = require('../../../../../../lib/Config');
|
= require('../utils');
|
||||||
const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
|
const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
|
||||||
const maxSubPartSize = azureMpuUtils.maxSubPartSize;
|
const maxSubPartSize = azureMpuUtils.maxSubPartSize;
|
||||||
const getBlockId = azureMpuUtils.getBlockId;
|
const getBlockId = azureMpuUtils.getBlockId;
|
||||||
|
@ -16,10 +16,6 @@ const azureClient = getAzureClient();
|
||||||
const azureContainerName = getAzureContainerName();
|
const azureContainerName = getAzureContainerName();
|
||||||
const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
|
const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
const azureLocationMismatch = 'azuretestmismatch';
|
|
||||||
|
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
let s3;
|
let s3;
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,17 @@ const async = require('async');
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const constants = require('../../../../../../constants');
|
const constants = require('../../../../../../constants');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
const {
|
||||||
const { getAzureClient, getAzureContainerName, convertMD5, memLocation,
|
describeSkipIfNotMultiple,
|
||||||
awsLocation, azureLocation, azureLocation2, azureLocationMismatch } =
|
getAzureClient,
|
||||||
require('../utils');
|
getAzureContainerName,
|
||||||
|
convertMD5,
|
||||||
|
memLocation,
|
||||||
|
awsLocation,
|
||||||
|
azureLocation,
|
||||||
|
azureLocation2,
|
||||||
|
azureLocationMismatch,
|
||||||
|
} = require('../utils');
|
||||||
const { createEncryptedBucketPromise } =
|
const { createEncryptedBucketPromise } =
|
||||||
require('../../../lib/utility/createEncryptedBucket');
|
require('../../../lib/utility/createEncryptedBucket');
|
||||||
|
|
||||||
|
@ -26,8 +33,6 @@ const azureTimeout = 40000;
|
||||||
|
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
let s3;
|
let s3;
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
function putSourceObj(key, location, objSize, cb) {
|
function putSourceObj(key, location, objSize, cb) {
|
||||||
const sourceParams = { Bucket: bucket, Key: key,
|
const sourceParams = { Bucket: bucket, Key: key,
|
||||||
|
|
|
@ -6,10 +6,12 @@ const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const constants = require('../../../../../../constants');
|
const constants = require('../../../../../../constants');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
const { config } = require('../../../../../../lib/Config');
|
||||||
const { getRealAwsConfig } = require('../../support/awsConfig');
|
const { getRealAwsConfig } = require('../../support/awsConfig');
|
||||||
|
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
|
||||||
const { createEncryptedBucketPromise } =
|
const { createEncryptedBucketPromise } =
|
||||||
require('../../../lib/utility/createEncryptedBucket');
|
require('../../../lib/utility/createEncryptedBucket');
|
||||||
const { memLocation, awsLocation, awsLocation2, awsLocationMismatch } =
|
const { describeSkipIfNotMultiple, awsS3, awsBucket, memLocation, awsLocation,
|
||||||
require('../utils');
|
awsLocation2, awsLocationMismatch, awsLocationEncryption }
|
||||||
|
= require('../utils');
|
||||||
|
|
||||||
const bucket = 'buckettestmultiplebackendobjectcopy';
|
const bucket = 'buckettestmultiplebackendobjectcopy';
|
||||||
const body = Buffer.from('I am a body', 'utf8');
|
const body = Buffer.from('I am a body', 'utf8');
|
||||||
|
@ -18,13 +20,8 @@ const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
||||||
const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
|
const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
|
||||||
const { versioningEnabled } = require('../../../lib/utility/versioning-util');
|
const { versioningEnabled } = require('../../../lib/utility/versioning-util');
|
||||||
|
|
||||||
const awsLocationEncryption = 'aws-test-encryption';
|
|
||||||
|
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
let s3;
|
let s3;
|
||||||
let awsS3;
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
function putSourceObj(location, isEmptyObj, cb) {
|
function putSourceObj(location, isEmptyObj, cb) {
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
|
@ -52,8 +49,6 @@ function putSourceObj(location, isEmptyObj, cb) {
|
||||||
function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey,
|
function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey,
|
||||||
destBucket, destLoc, awsKey, mdDirective, isEmptyObj, awsS3, awsLocation,
|
destBucket, destLoc, awsKey, mdDirective, isEmptyObj, awsS3, awsLocation,
|
||||||
callback) {
|
callback) {
|
||||||
const awsBucket =
|
|
||||||
config.locationConstraints[awsLocation].details.bucketName;
|
|
||||||
const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey };
|
const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey };
|
||||||
const destGetParams = { Bucket: destBucket, Key: destKey };
|
const destGetParams = { Bucket: destBucket, Key: destKey };
|
||||||
const awsParams = { Bucket: awsBucket, Key: awsKey };
|
const awsParams = { Bucket: awsBucket, Key: awsKey };
|
||||||
|
@ -109,32 +104,20 @@ describeSkipIfNotMultiple('MultipleBackend object copy',
|
||||||
function testSuite() {
|
function testSuite() {
|
||||||
this.timeout(250000);
|
this.timeout(250000);
|
||||||
withV4(sigCfg => {
|
withV4(sigCfg => {
|
||||||
beforeEach(() => {
|
const bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
bucketUtil = new BucketUtility('default', sigCfg);
|
const s3 = bucketUtil.s3;
|
||||||
s3 = bucketUtil.s3;
|
beforeEach(done => s3.createBucket({
|
||||||
const awsConfig = getRealAwsConfig(awsLocation);
|
Bucket: bucket,
|
||||||
awsS3 = new AWS.S3(awsConfig);
|
CreateBucketConfiguration: {
|
||||||
process.stdout.write('Creating bucket\n');
|
LocationConstraint: awsLocation,
|
||||||
if (process.env.ENABLE_KMS_ENCRYPTION === 'true') {
|
},
|
||||||
s3.createBucketAsync = createEncryptedBucketPromise;
|
}, done));
|
||||||
|
afterEach(done => {
|
||||||
|
removeAllVersions({ Bucket: bucket }, err => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
}
|
}
|
||||||
return s3.createBucketAsync({ Bucket: bucket })
|
return s3.deleteBucket({ Bucket: bucket }, done);
|
||||||
.catch(err => {
|
|
||||||
process.stdout.write(`Error creating bucket: ${err}\n`);
|
|
||||||
throw err;
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
process.stdout.write('Emptying bucket\n');
|
|
||||||
return bucketUtil.empty(bucket)
|
|
||||||
.then(() => {
|
|
||||||
process.stdout.write('Deleting bucket\n');
|
|
||||||
return bucketUtil.deleteOne(bucket);
|
|
||||||
})
|
|
||||||
.catch(err => {
|
|
||||||
process.stdout.write(`Error in afterEach: ${err}\n`);
|
|
||||||
throw err;
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,384 @@
|
||||||
|
const assert = require('assert');
|
||||||
|
const async = require('async');
|
||||||
|
const withV4 = require('../../support/withV4');
|
||||||
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
|
const constants = require('../../../../../../constants');
|
||||||
|
const {
|
||||||
|
describeSkipIfNotMultiple,
|
||||||
|
awsS3,
|
||||||
|
awsBucket,
|
||||||
|
memLocation,
|
||||||
|
fileLocation,
|
||||||
|
awsLocation,
|
||||||
|
enableVersioning,
|
||||||
|
suspendVersioning,
|
||||||
|
putToAwsBackend,
|
||||||
|
awsGetLatestVerId,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
|
const sourceBucketName = 'buckettestobjectcopyawsversioning-source';
|
||||||
|
const destBucketName = 'buckettestobjectcopyawsversioning-dest';
|
||||||
|
|
||||||
|
const someBody = Buffer.from('I am a body', 'utf8');
|
||||||
|
const wrongVersionBody = 'this is not the content you wanted';
|
||||||
|
const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
|
||||||
|
const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
||||||
|
const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
|
||||||
|
|
||||||
|
let bucketUtil;
|
||||||
|
let s3;
|
||||||
|
|
||||||
|
function _getTestMetadata(location) {
|
||||||
|
return {
|
||||||
|
'scal-location-constraint': location,
|
||||||
|
'test-header': 'copyme',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function putSourceObj(testParams, cb) {
|
||||||
|
const { sourceBucket, sourceLocation, isEmptyObj } = testParams;
|
||||||
|
const sourceKey = `sourcekey-${Date.now()}`;
|
||||||
|
const sourceParams = {
|
||||||
|
Bucket: sourceBucket,
|
||||||
|
Key: sourceKey,
|
||||||
|
Metadata: _getTestMetadata(sourceLocation),
|
||||||
|
};
|
||||||
|
if (!isEmptyObj) {
|
||||||
|
sourceParams.Body = someBody;
|
||||||
|
}
|
||||||
|
s3.putObject(sourceParams, (err, result) => {
|
||||||
|
assert.strictEqual(err, null,
|
||||||
|
`Error putting source object: ${err}`);
|
||||||
|
if (isEmptyObj) {
|
||||||
|
assert.strictEqual(result.ETag, `"${emptyMD5}"`);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(result.ETag, `"${correctMD5}"`);
|
||||||
|
}
|
||||||
|
Object.assign(testParams, {
|
||||||
|
sourceKey,
|
||||||
|
sourceVersionId: result.VersionId,
|
||||||
|
});
|
||||||
|
cb();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function copyObject(testParams, cb) {
|
||||||
|
const { sourceBucket, sourceKey, sourceVersionId, sourceVersioningState,
|
||||||
|
destBucket, destLocation, directive, destVersioningState, isEmptyObj }
|
||||||
|
= testParams;
|
||||||
|
const destKey = `destkey-${Date.now()}`;
|
||||||
|
const copyParams = {
|
||||||
|
Bucket: destBucket,
|
||||||
|
Key: destKey,
|
||||||
|
CopySource: `/${sourceBucket}/${sourceKey}`,
|
||||||
|
MetadataDirective: directive,
|
||||||
|
Metadata: {
|
||||||
|
'scal-location-constraint': destLocation,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if (sourceVersionId) {
|
||||||
|
copyParams.CopySource =
|
||||||
|
`${copyParams.CopySource}?versionId=${sourceVersionId}`;
|
||||||
|
} else if (sourceVersioningState === 'Suspended') {
|
||||||
|
copyParams.CopySource =
|
||||||
|
`${copyParams.CopySource}?versionId=null`;
|
||||||
|
}
|
||||||
|
console.log('===================');
|
||||||
|
console.log('params sent to object copy', copyParams)
|
||||||
|
s3.copyObject(copyParams, (err, data) => {
|
||||||
|
assert.strictEqual(err, null,
|
||||||
|
`Error copying object to destination: ${err}`);
|
||||||
|
console.log('copy object result', data)
|
||||||
|
if (destVersioningState === 'Enabled') {
|
||||||
|
console.log('got version id for dest object', data.VersionId)
|
||||||
|
assert.notEqual(data.VersionId, undefined);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(data.VersionId, undefined);
|
||||||
|
}
|
||||||
|
const expectedBody = isEmptyObj ? '' : someBody;
|
||||||
|
return awsGetLatestVerId(destKey, expectedBody, (err, awsVersionId) => {
|
||||||
|
Object.assign(testParams, {
|
||||||
|
destKey,
|
||||||
|
destVersionId: data.VersionId,
|
||||||
|
awsVersionId,
|
||||||
|
});
|
||||||
|
if (!data.VersionId && destVersioningState === 'Suspended') {
|
||||||
|
// eslint-disable-next-line no-param-reassign
|
||||||
|
testParams.destVersionId = 'null';
|
||||||
|
}
|
||||||
|
cb();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertGetObjects(testParams, cb) {
|
||||||
|
console.log('assertGetObjecs');
|
||||||
|
const {
|
||||||
|
sourceBucket,
|
||||||
|
sourceLocation,
|
||||||
|
sourceKey,
|
||||||
|
sourceVersionId,
|
||||||
|
sourceVersioningState,
|
||||||
|
destBucket,
|
||||||
|
destLocation,
|
||||||
|
destKey,
|
||||||
|
destVersionId,
|
||||||
|
destVersioningState,
|
||||||
|
awsVersionId,
|
||||||
|
isEmptyObj,
|
||||||
|
directive,
|
||||||
|
} = testParams;
|
||||||
|
console.log('testParams in assertGetObjects..', testParams)
|
||||||
|
const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey,
|
||||||
|
VersionId: sourceVersionId };
|
||||||
|
const destGetParams = { Bucket: destBucket, Key: destKey,
|
||||||
|
VersionId: destVersionId };
|
||||||
|
const awsParams = { Bucket: awsBucket, Key: destKey,
|
||||||
|
VersionId: awsVersionId };
|
||||||
|
|
||||||
|
async.series([
|
||||||
|
cb => s3.getObject(sourceGetParams, cb),
|
||||||
|
cb => s3.getObject(destGetParams, cb),
|
||||||
|
cb => awsS3.getObject(awsParams, cb),
|
||||||
|
], (err, results) => {
|
||||||
|
assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`);
|
||||||
|
const [sourceRes, destRes, awsRes] = results;
|
||||||
|
console.log('******** sourceGetParams', sourceGetParams);
|
||||||
|
console.log('==== result of getting source object', sourceRes);
|
||||||
|
console.log('******** destGetParams', destGetParams);
|
||||||
|
console.log('==== result of getting dest object', destRes);
|
||||||
|
console.log('******** awsGetParams', awsParams);
|
||||||
|
console.log('==== result of getting aws object', awsRes)
|
||||||
|
// NOTE: assert version ids?
|
||||||
|
if (isEmptyObj) {
|
||||||
|
assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`);
|
||||||
|
assert.strictEqual(destRes.ETag, `"${emptyMD5}"`);
|
||||||
|
assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`);
|
||||||
|
assert.strictEqual(destRes.ETag, `"${correctMD5}"`);
|
||||||
|
assert.deepStrictEqual(sourceRes.Body, destRes.Body);
|
||||||
|
assert.strictEqual(awsRes.ETag, `"${correctMD5}"`);
|
||||||
|
assert.deepStrictEqual(sourceRes.Body, awsRes.Body);
|
||||||
|
}
|
||||||
|
if (directive === 'COPY') {
|
||||||
|
assert.deepStrictEqual(sourceRes.Metadata['test-header'],
|
||||||
|
destRes.Metadata['test-header']);
|
||||||
|
} else if (directive === 'REPLACE') {
|
||||||
|
assert.strictEqual(destRes.Metadata['test-header'],
|
||||||
|
undefined);
|
||||||
|
}
|
||||||
|
assert.strictEqual(awsRes.Metadata[locMetaHeader], destLocation);
|
||||||
|
if (directive === 'COPY') {
|
||||||
|
assert.deepStrictEqual(sourceRes.Metadata['test-header'],
|
||||||
|
awsRes.Metadata['test-header']);
|
||||||
|
} else if (directive === 'REPLACE') {
|
||||||
|
assert.strictEqual(awsRes.Metadata['test-header'],
|
||||||
|
undefined);
|
||||||
|
}
|
||||||
|
assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength);
|
||||||
|
assert.strictEqual(sourceRes.Metadata[locMetaHeader], sourceLocation);
|
||||||
|
assert.strictEqual(destRes.Metadata[locMetaHeader], destLocation);
|
||||||
|
cb();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
const testParams = {
|
||||||
|
sourceBucket: sourceBucketName,
|
||||||
|
sourceLocation: awsLocation,
|
||||||
|
sourceVersioningState: undefined,
|
||||||
|
destBucket: destBucketName,
|
||||||
|
destLocation: awsLocation,
|
||||||
|
destVersioningState: 'Enabled',
|
||||||
|
isEmptyObj: false,
|
||||||
|
directive: 'REPLACE',
|
||||||
|
};*/
|
||||||
|
|
||||||
|
// describeSkipIfNotMultiple
|
||||||
|
describe.only('AWS backend object copy with versioning',
|
||||||
|
function testSuite() {
|
||||||
|
this.timeout(250000);
|
||||||
|
withV4(sigCfg => {
|
||||||
|
beforeEach(() => {
|
||||||
|
bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
|
s3 = bucketUtil.s3;
|
||||||
|
process.stdout.write('Creating buckets\n');
|
||||||
|
/* if (process.env.ENABLE_KMS_ENCRYPTION === 'true') {
|
||||||
|
s3.createBucketAsync = createEncryptedBucketPromise;
|
||||||
|
} */
|
||||||
|
return s3.createBucketAsync({ Bucket: sourceBucketName })
|
||||||
|
.then(() => s3.createBucketAsync({ Bucket: destBucketName }))
|
||||||
|
.catch(err => {
|
||||||
|
process.stdout.write(`Error creating bucket: ${err}\n`);
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => bucketUtil.empty(sourceBucketName)
|
||||||
|
.then(() => bucketUtil.deleteOne(sourceBucketName))
|
||||||
|
.catch(err => {
|
||||||
|
process.stdout.write('Error deleting source bucket ' +
|
||||||
|
`in afterEach: ${err}\n`);
|
||||||
|
throw err;
|
||||||
|
})
|
||||||
|
.then(() => bucketUtil.empty(destBucketName))
|
||||||
|
.then(() => bucketUtil.deleteOne(destBucketName))
|
||||||
|
);
|
||||||
|
|
||||||
|
[{
|
||||||
|
directive: 'REPLACE',
|
||||||
|
isEmptyObj: true,
|
||||||
|
}, {
|
||||||
|
directive: 'REPLACE',
|
||||||
|
isEmptyObj: false,
|
||||||
|
}, {
|
||||||
|
directive: 'COPY',
|
||||||
|
isEmptyObj: false,
|
||||||
|
}].forEach(testParams => {
|
||||||
|
Object.assign(testParams, {
|
||||||
|
sourceBucket: sourceBucketName,
|
||||||
|
sourceLocation: awsLocation,
|
||||||
|
destBucket: destBucketName,
|
||||||
|
destLocation: awsLocation,
|
||||||
|
});
|
||||||
|
const { isEmptyObj, directive } = testParams;
|
||||||
|
it(`should copy a${isEmptyObj ? 'n empty' : ''} object from AWS ` +
|
||||||
|
'backend non-versioned bucket to AWS backend versioned bucket ' +
|
||||||
|
`with ${directive} directive`, done => {
|
||||||
|
Object.assign(testParams, {
|
||||||
|
sourceVersioningState: undefined,
|
||||||
|
destVersioningState: 'Enabled',
|
||||||
|
});
|
||||||
|
async.waterfall([
|
||||||
|
next => putSourceObj(testParams, next),
|
||||||
|
next => enableVersioning(s3, testParams.destBucket, next),
|
||||||
|
next => copyObject(testParams, next),
|
||||||
|
// put another version to test and make sure version id from
|
||||||
|
// copy was stored to get the right version
|
||||||
|
next => putToAwsBackend(s3, destBucketName,
|
||||||
|
testParams.destKey, wrongVersionBody, () => next()),
|
||||||
|
next => assertGetObjects(testParams, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it(`should copy ${isEmptyObj ? 'an empty' : ''}version from one ` +
|
||||||
|
`AWS backend versioned bucket to another on ${directive} directive`,
|
||||||
|
done => {
|
||||||
|
Object.assign(testParams, {
|
||||||
|
sourceVersioningState: 'Enabled',
|
||||||
|
destVersioningState: 'Enabled',
|
||||||
|
});
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, testParams.sourceBucket, next),
|
||||||
|
next => putSourceObj(testParams, next),
|
||||||
|
next => enableVersioning(s3, testParams.destBucket, next),
|
||||||
|
next => copyObject(testParams, next),
|
||||||
|
// put another version to test and make sure version id from
|
||||||
|
// copy was stored to get the right version
|
||||||
|
next => putToAwsBackend(s3, destBucketName,
|
||||||
|
testParams.destKey, wrongVersionBody, () => next()),
|
||||||
|
next => assertGetObjects(testParams, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it(`should copy ${isEmptyObj ? 'an empty' : ''} null version ` +
|
||||||
|
'from an AWS backend versioned bucket to a non-versioned one with '
|
||||||
|
+ `${directive} directive`, done => {
|
||||||
|
Object.assign(testParams, {
|
||||||
|
sourceVersioningState: 'Suspend',
|
||||||
|
destVersioningState: 'Suspended',
|
||||||
|
});
|
||||||
|
async.waterfall([
|
||||||
|
next => suspendVersioning(s3, testParams.sourceBucket,
|
||||||
|
next),
|
||||||
|
next => putSourceObj(testParams, next),
|
||||||
|
next => suspendVersioning(s3, testParams.destBucket, next),
|
||||||
|
next => copyObject(testParams, next),
|
||||||
|
next => enableVersioning(s3, testParams.destBucket, next),
|
||||||
|
// put another version to test and make sure version id from
|
||||||
|
// copy was stored to get the right version
|
||||||
|
next => putToAwsBackend(s3, destBucketName,
|
||||||
|
testParams.destKey, wrongVersionBody, () => next()),
|
||||||
|
next => assertGetObjects(testParams, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it(`should copy a ${isEmptyObj ? 'empty ' : ''}version from a ` +
|
||||||
|
'AWS backend versioned bucket to a non-versioned one with '
|
||||||
|
+ `${directive} directive`, done => {
|
||||||
|
Object.assign(testParams, {
|
||||||
|
sourceVersioningState: 'Enabled',
|
||||||
|
destVersioningState: 'Suspended',
|
||||||
|
});
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, testParams.sourceBucket, next),
|
||||||
|
next => putSourceObj(testParams, next),
|
||||||
|
next => suspendVersioning(s3, testParams.destBucket, next),
|
||||||
|
next => copyObject(testParams, next),
|
||||||
|
// put another version to test and make sure version id from
|
||||||
|
// copy was stored to get the right version
|
||||||
|
next => enableVersioning(s3, testParams.destBucket, next),
|
||||||
|
next => putToAwsBackend(s3, destBucketName,
|
||||||
|
testParams.destKey, wrongVersionBody, () => next()),
|
||||||
|
next => assertGetObjects(testParams, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
[{
|
||||||
|
sourceLocation: memLocation,
|
||||||
|
directive: 'REPLACE',
|
||||||
|
isEmptyObj: true,
|
||||||
|
}, {
|
||||||
|
sourceLocation: fileLocation,
|
||||||
|
directive: 'REPLACE',
|
||||||
|
isEmptyObj: true,
|
||||||
|
}, {
|
||||||
|
sourceLocation: memLocation,
|
||||||
|
directive: 'COPY',
|
||||||
|
isEmptyObj: false,
|
||||||
|
}, {
|
||||||
|
sourceLocation: fileLocation,
|
||||||
|
directive: 'COPY',
|
||||||
|
isEmptyObj: false,
|
||||||
|
}].forEach(testParams => {
|
||||||
|
Object.assign(testParams, {
|
||||||
|
sourceBucket: sourceBucketName,
|
||||||
|
sourceVersioningState: 'Enabled',
|
||||||
|
destBucket: sourceBucketName,
|
||||||
|
destLocation: awsLocation,
|
||||||
|
destVersioningState: 'Enabled',
|
||||||
|
});
|
||||||
|
const { sourceLocation, directive, isEmptyObj } = testParams;
|
||||||
|
|
||||||
|
it(`should copy ${isEmptyObj ? 'empty ' : ''}object from ` +
|
||||||
|
`${sourceLocation} to same bucket on AWS backend with ` +
|
||||||
|
`versioning with ${directive}`, done => {
|
||||||
|
async.waterfall([
|
||||||
|
next => putSourceObj(testParams, next),
|
||||||
|
next => enableVersioning(s3, testParams.sourceBucket, next),
|
||||||
|
next => copyObject(testParams, next),
|
||||||
|
next => assertGetObjects(testParams, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it(`should copy a ${isEmptyObj ? 'empty ' : ''}version from ` +
|
||||||
|
`${sourceLocation} to same bucket on AWS backend with ` +
|
||||||
|
`versioning with ${directive} directive`, done => {
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, testParams.sourceBucket, next),
|
||||||
|
// returns a version id which is added to testParams
|
||||||
|
// to be used in object copy
|
||||||
|
next => putSourceObj(testParams, next),
|
||||||
|
next => copyObject(testParams, next),
|
||||||
|
// put another version to test and make sure version id
|
||||||
|
// from copy was stored to get the right version
|
||||||
|
next => putToAwsBackend(s3, destBucketName,
|
||||||
|
testParams.destKey, wrongVersionBody, () => next()),
|
||||||
|
next => assertGetObjects(testParams, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -1,13 +1,10 @@
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const AWS = require('aws-sdk');
|
const withV4 = require('../../support/withV4');
|
||||||
const withV4 = require('../support/withV4');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const BucketUtility = require('../../lib/utility/bucket-util');
|
const { describeSkipIfNotMultiple, awsS3, getAzureClient, getAzureContainerName,
|
||||||
const { config } = require('../../../../../lib/Config');
|
convertMD5, memLocation, fileLocation, awsLocation, azureLocation } =
|
||||||
const { getRealAwsConfig } = require('../support/awsConfig');
|
require('../utils');
|
||||||
const { getAzureClient, getAzureContainerName, convertMD5,
|
|
||||||
memLocation, fileLocation, awsLocation, azureLocation } =
|
|
||||||
require('./utils');
|
|
||||||
|
|
||||||
const awsBucket = 'multitester555';
|
const awsBucket = 'multitester555';
|
||||||
const azureClient = getAzureClient();
|
const azureClient = getAzureClient();
|
||||||
|
@ -22,9 +19,6 @@ const cloudTimeout = 10000;
|
||||||
|
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
let s3;
|
let s3;
|
||||||
let awsS3;
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
const putParams = { Bucket: bucket, Body: body };
|
const putParams = { Bucket: bucket, Body: body };
|
||||||
const testBackends = [memLocation, fileLocation, awsLocation, azureLocation];
|
const testBackends = [memLocation, fileLocation, awsLocation, azureLocation];
|
||||||
|
@ -159,8 +153,6 @@ function testSuite() {
|
||||||
this.timeout(80000);
|
this.timeout(80000);
|
||||||
withV4(sigCfg => {
|
withV4(sigCfg => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
const awsConfig = getRealAwsConfig(awsLocation);
|
|
||||||
awsS3 = new AWS.S3(awsConfig);
|
|
||||||
bucketUtil = new BucketUtility('default', sigCfg);
|
bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
s3 = bucketUtil.s3;
|
s3 = bucketUtil.s3;
|
||||||
return s3.createBucketAsync({ Bucket: bucket })
|
return s3.createBucketAsync({ Bucket: bucket })
|
||||||
|
@ -274,7 +266,8 @@ function testSuite() {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should return error on putting tags to object deleted from AWS',
|
it('should not return error on putting tags to object ' +
|
||||||
|
'that has had a delete marker put directly on from AWS',
|
||||||
done => {
|
done => {
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
const params = Object.assign({ Key: key, Metadata:
|
const params = Object.assign({ Key: key, Metadata:
|
||||||
|
@ -289,7 +282,7 @@ function testSuite() {
|
||||||
Tagging: putTags };
|
Tagging: putTags };
|
||||||
process.stdout.write('Putting object tags\n');
|
process.stdout.write('Putting object tags\n');
|
||||||
s3.putObjectTagging(putTagParams, err => {
|
s3.putObjectTagging(putTagParams, err => {
|
||||||
assert.strictEqual(err.code, 'InternalError');
|
assert.strictEqual(err, null);
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -325,8 +318,8 @@ function testSuite() {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should not return error on getting tags from object deleted ' +
|
it('should not return error on getting tags from object that has ' +
|
||||||
'from AWS', done => {
|
'had a delete marker put directly on AWS', done => {
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
const params = Object.assign({ Key: key, Tagging: tagString,
|
const params = Object.assign({ Key: key, Tagging: tagString,
|
||||||
Metadata: { 'scal-location-constraint': awsLocation } },
|
Metadata: { 'scal-location-constraint': awsLocation } },
|
||||||
|
@ -367,8 +360,8 @@ function testSuite() {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should return error on deleting tags from object deleted ' +
|
it('should not return error on deleting tags from object that ' +
|
||||||
'from AWS', done => {
|
'has had delete markers put directly on AWS', done => {
|
||||||
const key = `somekey-${Date.now()}`;
|
const key = `somekey-${Date.now()}`;
|
||||||
const params = Object.assign({ Key: key, Tagging: tagString,
|
const params = Object.assign({ Key: key, Tagging: tagString,
|
||||||
Metadata: { 'scal-location-constraint': awsLocation } },
|
Metadata: { 'scal-location-constraint': awsLocation } },
|
||||||
|
@ -381,7 +374,7 @@ function testSuite() {
|
||||||
assert.equal(err, null);
|
assert.equal(err, null);
|
||||||
const tagParams = { Bucket: bucket, Key: key };
|
const tagParams = { Bucket: bucket, Key: key };
|
||||||
s3.deleteObjectTagging(tagParams, err => {
|
s3.deleteObjectTagging(tagParams, err => {
|
||||||
assert.strictEqual(err.code, 'InternalError');
|
assert.strictEqual(err, null);
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
});
|
});
|
|
@ -0,0 +1,223 @@
|
||||||
|
const async = require('async');
|
||||||
|
|
||||||
|
const withV4 = require('../../support/withV4');
|
||||||
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
|
const bucket = 'testawsbackendtaggingdeleteversioned';
|
||||||
|
|
||||||
|
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
|
||||||
|
const {
|
||||||
|
describeSkipIfNotMultiple,
|
||||||
|
awsS3,
|
||||||
|
awsBucket,
|
||||||
|
awsLocation,
|
||||||
|
enableVersioning,
|
||||||
|
putNullVersionsToAws,
|
||||||
|
putVersionsToAws,
|
||||||
|
awsGetLatestVerId,
|
||||||
|
tagging,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
|
const { putTaggingAndAssert, delTaggingAndAssert, awsGetAssertTags } = tagging;
|
||||||
|
const someBody = 'teststring';
|
||||||
|
|
||||||
|
describeSkipIfNotMultiple('AWS backend object tagging with versioning :: ' +
|
||||||
|
'delete', function testSuite() {
|
||||||
|
this.timeout(30000);
|
||||||
|
const tags = { key1: 'value1', key2: 'value2' };
|
||||||
|
|
||||||
|
withV4(sigCfg => {
|
||||||
|
const bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
|
const s3 = bucketUtil.s3;
|
||||||
|
beforeEach(done => s3.createBucket({
|
||||||
|
Bucket: bucket,
|
||||||
|
CreateBucketConfiguration: {
|
||||||
|
LocationConstraint: awsLocation,
|
||||||
|
},
|
||||||
|
}, done));
|
||||||
|
afterEach(done => {
|
||||||
|
removeAllVersions({ Bucket: bucket }, err => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
return s3.deleteBucket({ Bucket: bucket }, done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: should delete a tag set on the ' +
|
||||||
|
'latest version if no version is specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
expectedVersionId: false }, next),
|
||||||
|
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedVersionId: false }, next),
|
||||||
|
next => awsGetAssertTags({ key, expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: should delete a tag set on the ' +
|
||||||
|
'version if specified (null)', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: 'null', expectedVersionId: false }, next),
|
||||||
|
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: 'null', expectedVersionId: false }, next),
|
||||||
|
next => awsGetAssertTags({ key, expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should delete a tag set on the latest ' +
|
||||||
|
'version if no version is specified', done => {
|
||||||
|
const data = [undefined, 'test1', 'test2'];
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, data, next),
|
||||||
|
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, expectedVersionId: 'null' }, next),
|
||||||
|
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedVersionId: 'null' }, next),
|
||||||
|
next => awsGetAssertTags({ key, expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should delete a tag set on a specific ' +
|
||||||
|
'version (null)', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, [undefined],
|
||||||
|
next),
|
||||||
|
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, versionId: 'null', expectedVersionId: 'null' }, next),
|
||||||
|
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: 'null', expectedTags: tags,
|
||||||
|
expectedVersionId: 'null' }, next),
|
||||||
|
next => awsGetAssertTags({ key, expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled then suspended: should delete a tag set on ' +
|
||||||
|
'a specific (non-null) version if specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '',
|
||||||
|
(err, awsVid) => next(err, putData.VersionId, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key,
|
||||||
|
[someBody], () => next(null, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, versionId: s3Vid, expectedVersionId: s3Vid }, () =>
|
||||||
|
next(null, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: s3Vid, expectedVersionId: s3Vid },
|
||||||
|
() => next(null, awsVid)),
|
||||||
|
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
|
||||||
|
expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should delete a tag set on the latest ' +
|
||||||
|
'version if no version is specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
expectedVersionId: putData.VersionId }, next),
|
||||||
|
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedVersionId: versionId }, next),
|
||||||
|
next => awsGetAssertTags({ key, expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should delete a tag set on a specific version',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: putData.VersionId,
|
||||||
|
expectedVersionId: putData.VersionId }, next),
|
||||||
|
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId, expectedVersionId: versionId }, next),
|
||||||
|
next => awsGetAssertTags({ key, expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should delete a tag set on a specific ' +
|
||||||
|
'version that is not the latest version', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '',
|
||||||
|
(err, awsVid) => next(err, putData.VersionId, awsVid)),
|
||||||
|
// put another version
|
||||||
|
(s3Vid, awsVid, next) => s3.putObject({ Bucket: bucket,
|
||||||
|
Key: key, Body: someBody },
|
||||||
|
err => next(err, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, versionId: s3Vid, expectedVersionId: s3Vid }, err =>
|
||||||
|
next(err, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: s3Vid, expectedVersionId: s3Vid },
|
||||||
|
() => next(null, awsVid)),
|
||||||
|
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
|
||||||
|
expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended then enabled: should delete a tag set on ' +
|
||||||
|
'a specific version (null) if specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, [undefined],
|
||||||
|
() => next()),
|
||||||
|
next => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
() => next(null, awsVid)),
|
||||||
|
(awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: 'null', expectedVersionId: 'null' },
|
||||||
|
() => next(null, awsVid)),
|
||||||
|
(awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: 'null', expectedVersionId: 'null' },
|
||||||
|
() => next(null, awsVid)),
|
||||||
|
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
|
||||||
|
expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return an InternalError if trying to delete ' +
|
||||||
|
'tags from object that was deleted from AWS directly',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key, VersionId: awsVid }, next),
|
||||||
|
(delData, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedError: 'InternalError' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return an InternalError if trying to delete ' +
|
||||||
|
'tags from object that was deleted from AWS directly',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '',
|
||||||
|
(err, awsVid) => next(err, putData.VersionId, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
|
||||||
|
(s3Vid, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: s3Vid, expectedError: 'InternalError' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -0,0 +1,285 @@
|
||||||
|
const async = require('async');
|
||||||
|
|
||||||
|
const withV4 = require('../../support/withV4');
|
||||||
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
|
const bucket = 'testawsbackendtaggingversioned';
|
||||||
|
|
||||||
|
const { removeAllVersions } = require('../../../lib/utility/versioning-util');
|
||||||
|
const {
|
||||||
|
describeSkipIfNotMultiple,
|
||||||
|
awsS3,
|
||||||
|
awsBucket,
|
||||||
|
awsLocation,
|
||||||
|
enableVersioning,
|
||||||
|
putNullVersionsToAws,
|
||||||
|
putVersionsToAws,
|
||||||
|
awsGetLatestVerId,
|
||||||
|
tagging,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
|
const { putTaggingAndAssert, getTaggingAndAssert, delTaggingAndAssert,
|
||||||
|
awsGetAssertTags } = tagging;
|
||||||
|
const someBody = 'teststring';
|
||||||
|
|
||||||
|
describeSkipIfNotMultiple('AWS backend object tagging with versioning',
|
||||||
|
function testSuite() {
|
||||||
|
this.timeout(30000);
|
||||||
|
const tags = { key1: 'value1', key2: 'value2' };
|
||||||
|
|
||||||
|
withV4(sigCfg => {
|
||||||
|
const bucketUtil = new BucketUtility('default', sigCfg);
|
||||||
|
const s3 = bucketUtil.s3;
|
||||||
|
beforeEach(done => s3.createBucket({
|
||||||
|
Bucket: bucket,
|
||||||
|
CreateBucketConfiguration: {
|
||||||
|
LocationConstraint: awsLocation,
|
||||||
|
},
|
||||||
|
}, done));
|
||||||
|
afterEach(done => {
|
||||||
|
removeAllVersions({ Bucket: bucket }, err => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
return s3.deleteBucket({ Bucket: bucket }, done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: should put/get a tag set on the ' +
|
||||||
|
'latest version if no version is specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
expectedVersionId: false }, next),
|
||||||
|
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedTags: tags, expectedVersionId: false }, next),
|
||||||
|
(versionId, next) => awsGetAssertTags({ key,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning not configured: should put/get a tag set on a ' +
|
||||||
|
'specific version if specified (null)', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: 'null', expectedVersionId: false }, next),
|
||||||
|
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: 'null', expectedTags: tags,
|
||||||
|
expectedVersionId: false }, next),
|
||||||
|
(versionId, next) => awsGetAssertTags({ key,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should put/get a tag set on the latest ' +
|
||||||
|
'version if no version is specified', done => {
|
||||||
|
const data = [undefined, 'test1', 'test2'];
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, data, next),
|
||||||
|
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, expectedVersionId: 'null' }, next),
|
||||||
|
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedTags: tags, expectedVersionId: 'null' }, next),
|
||||||
|
(versionId, next) => awsGetAssertTags({ key,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning suspended: should put/get a tag set on a specific ' +
|
||||||
|
'version (null)', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, [undefined],
|
||||||
|
next),
|
||||||
|
(versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, versionId: 'null', expectedVersionId: 'null' }, next),
|
||||||
|
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: 'null', expectedTags: tags,
|
||||||
|
expectedVersionId: 'null' }, next),
|
||||||
|
(versionId, next) => awsGetAssertTags({ key,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled then suspended: should put/get a tag set on ' +
|
||||||
|
'a specific (non-null) version if specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '',
|
||||||
|
(err, awsVid) => next(err, putData.VersionId, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key,
|
||||||
|
[someBody], () => next(null, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, versionId: s3Vid, expectedVersionId: s3Vid }, () =>
|
||||||
|
next(null, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: s3Vid, expectedTags: tags,
|
||||||
|
expectedVersionId: s3Vid }, () => next(null, awsVid)),
|
||||||
|
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should put/get a tag set on the latest ' +
|
||||||
|
'version if no version is specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
expectedVersionId: putData.VersionId }, next),
|
||||||
|
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedTags: tags, expectedVersionId: versionId }, next),
|
||||||
|
(versionId, next) => awsGetAssertTags({ key,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should put/get a tag set on a specific version',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: putData.VersionId,
|
||||||
|
expectedVersionId: putData.VersionId }, next),
|
||||||
|
(versionId, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId, expectedTags: tags,
|
||||||
|
expectedVersionId: versionId }, next),
|
||||||
|
(versionId, next) => awsGetAssertTags({ key,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should put/get a tag set on a specific version',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: putData.VersionId,
|
||||||
|
expectedVersionId: putData.VersionId }, next),
|
||||||
|
(versionId, next) => delTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId, expectedVersionId: versionId }, next),
|
||||||
|
next => awsGetAssertTags({ key, expectedTags: {} }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('versioning enabled: should put/get a tag set on a specific ' +
|
||||||
|
'version that is not the latest version', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '',
|
||||||
|
(err, awsVid) => next(err, putData.VersionId, awsVid)),
|
||||||
|
// put another version
|
||||||
|
(s3Vid, awsVid, next) => s3.putObject({ Bucket: bucket,
|
||||||
|
Key: key, Body: someBody },
|
||||||
|
err => next(err, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, versionId: s3Vid, expectedVersionId: s3Vid }, err =>
|
||||||
|
next(err, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: s3Vid, expectedTags: tags,
|
||||||
|
expectedVersionId: s3Vid }, () => next(null, awsVid)),
|
||||||
|
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
it('versioning suspended then enabled: should put/get a tag set on ' +
|
||||||
|
'a specific version (null) if specified', done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => putNullVersionsToAws(s3, bucket, key, [undefined],
|
||||||
|
() => next()),
|
||||||
|
next => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody],
|
||||||
|
() => next(null, awsVid)),
|
||||||
|
(awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: 'null', expectedVersionId: 'null' },
|
||||||
|
() => next(null, awsVid)),
|
||||||
|
(awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: 'null', expectedTags: tags,
|
||||||
|
expectedVersionId: 'null' }, () => next(null, awsVid)),
|
||||||
|
(awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
|
||||||
|
expectedTags: tags }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get tags for an object even if it was deleted from ' +
|
||||||
|
'AWS directly (we rely on s3 metadata)',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
expectedVersionId: false }, () => next(null, awsVid)),
|
||||||
|
(awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key, VersionId: awsVid }, next),
|
||||||
|
(delData, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
expectedTags: tags, expectedVersionId: false,
|
||||||
|
getObject: false }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return an InternalError if trying to put ' +
|
||||||
|
'tags from object that was deleted from AWS directly',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '', next),
|
||||||
|
(awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key, VersionId: awsVid }, next),
|
||||||
|
(delData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
expectedError: 'InternalError' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get tags for an version even if it was deleted from ' +
|
||||||
|
'AWS directly (we rely on s3 metadata)',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => enableVersioning(s3, bucket, next),
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '',
|
||||||
|
(err, awsVid) => next(err, putData.VersionId, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
|
||||||
|
tags, versionId: s3Vid, expectedVersionId: s3Vid },
|
||||||
|
() => next(null, s3Vid, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
|
||||||
|
(s3Vid, next) => getTaggingAndAssert(s3, { bucket, key,
|
||||||
|
versionId: s3Vid, expectedTags: tags,
|
||||||
|
expectedVersionId: s3Vid, getObject: false }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return an InternalError if trying to put ' +
|
||||||
|
'tags on version that was deleted from AWS directly',
|
||||||
|
done => {
|
||||||
|
const key = `somekey-${Date.now()}`;
|
||||||
|
async.waterfall([
|
||||||
|
next => s3.putObject({ Bucket: bucket, Key: key }, next),
|
||||||
|
(putData, next) => awsGetLatestVerId(key, '',
|
||||||
|
(err, awsVid) => next(err, putData.VersionId, awsVid)),
|
||||||
|
(s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
|
||||||
|
Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
|
||||||
|
(s3Vid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
|
||||||
|
versionId: s3Vid, expectedError: 'InternalError' }, next),
|
||||||
|
], done);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -1,17 +1,15 @@
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const AWS = require('aws-sdk');
|
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
|
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { config } = require('../../../../../../lib/Config');
|
const { config } = require('../../../../../../lib/Config');
|
||||||
const { getRealAwsConfig } = require('../../support/awsConfig');
|
|
||||||
const { createEncryptedBucketPromise } =
|
const { createEncryptedBucketPromise } =
|
||||||
require('../../../lib/utility/createEncryptedBucket');
|
require('../../../lib/utility/createEncryptedBucket');
|
||||||
const { versioningEnabled } = require('../../../lib/utility/versioning-util');
|
const { versioningEnabled } = require('../../../lib/utility/versioning-util');
|
||||||
|
|
||||||
const { awsLocation, memLocation, fileLocation } = require('../utils');
|
const { describeSkipIfNotMultiple, awsS3, awsBucket, awsLocation,
|
||||||
const awsLocationEncryption = 'aws-test-encryption';
|
awsLocationEncryption, memLocation, fileLocation } = require('../utils');
|
||||||
const bucket = 'buckettestmultiplebackendput';
|
const bucket = 'buckettestmultiplebackendput';
|
||||||
const body = Buffer.from('I am a body', 'utf8');
|
const body = Buffer.from('I am a body', 'utf8');
|
||||||
const bigBody = Buffer.alloc(10485760);
|
const bigBody = Buffer.alloc(10485760);
|
||||||
|
@ -23,13 +21,9 @@ const bigAWSMD5 = 'a7d414b9133d6483d9a1c4e04e856e3b-2';
|
||||||
|
|
||||||
let bucketUtil;
|
let bucketUtil;
|
||||||
let s3;
|
let s3;
|
||||||
let awsS3;
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
const awsTimeout = 30000;
|
const awsTimeout = 30000;
|
||||||
const retryTimeout = 10000;
|
const retryTimeout = 10000;
|
||||||
let awsBucket;
|
|
||||||
|
|
||||||
function awsGetCheck(objectKey, s3MD5, awsMD5, location, cb) {
|
function awsGetCheck(objectKey, s3MD5, awsMD5, location, cb) {
|
||||||
process.stdout.write('Getting object\n');
|
process.stdout.write('Getting object\n');
|
||||||
|
@ -113,12 +107,6 @@ describe('MultipleBackend put object', function testSuite() {
|
||||||
if (!process.env.S3_END_TO_END) {
|
if (!process.env.S3_END_TO_END) {
|
||||||
this.retries(2);
|
this.retries(2);
|
||||||
}
|
}
|
||||||
before(() => {
|
|
||||||
awsBucket = config.locationConstraints[awsLocation].
|
|
||||||
details.bucketName;
|
|
||||||
const awsConfig = getRealAwsConfig(awsLocation);
|
|
||||||
awsS3 = new AWS.S3(awsConfig);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return an error to put request without a valid ' +
|
it('should return an error to put request without a valid ' +
|
||||||
'location constraint', done => {
|
'location constraint', done => {
|
||||||
|
|
|
@ -3,10 +3,17 @@ const async = require('async');
|
||||||
|
|
||||||
const withV4 = require('../../support/withV4');
|
const withV4 = require('../../support/withV4');
|
||||||
const BucketUtility = require('../../../lib/utility/bucket-util');
|
const BucketUtility = require('../../../lib/utility/bucket-util');
|
||||||
const { uniqName, getAzureClient, getAzureContainerName, getAzureKeys,
|
const {
|
||||||
convertMD5, fileLocation, azureLocation, azureLocationMismatch }
|
describeSkipIfNotMultiple,
|
||||||
= require('../utils');
|
uniqName,
|
||||||
const { config } = require('../../../../../../lib/Config');
|
getAzureClient,
|
||||||
|
getAzureContainerName,
|
||||||
|
getAzureKeys,
|
||||||
|
convertMD5,
|
||||||
|
fileLocation,
|
||||||
|
azureLocation,
|
||||||
|
azureLocationMismatch,
|
||||||
|
} = require('../utils');
|
||||||
|
|
||||||
const keyObject = 'putazure';
|
const keyObject = 'putazure';
|
||||||
const azureClient = getAzureClient();
|
const azureClient = getAzureClient();
|
||||||
|
@ -16,9 +23,6 @@ const { versioningEnabled } = require('../../../lib/utility/versioning-util');
|
||||||
const normalBody = Buffer.from('I am a body', 'utf8');
|
const normalBody = Buffer.from('I am a body', 'utf8');
|
||||||
const normalMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
|
const normalMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
|
||||||
|
|
||||||
const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
|
|
||||||
|| process.env.S3_END_TO_END) ? describe.skip : describe;
|
|
||||||
|
|
||||||
const keys = getAzureKeys();
|
const keys = getAzureKeys();
|
||||||
/* eslint-disable camelcase */
|
/* eslint-disable camelcase */
|
||||||
const azureMetadata = {
|
const azureMetadata = {
|
||||||
|
|
|
@ -1,9 +1,12 @@
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
|
const { errors } = require('arsenal');
|
||||||
|
const AWS = require('aws-sdk');
|
||||||
|
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const azure = require('azure-storage');
|
const azure = require('azure-storage');
|
||||||
|
|
||||||
|
const { getRealAwsConfig } = require('../support/awsConfig');
|
||||||
const { config } = require('../../../../../lib/Config');
|
const { config } = require('../../../../../lib/Config');
|
||||||
|
|
||||||
const memLocation = 'mem-test';
|
const memLocation = 'mem-test';
|
||||||
|
@ -14,10 +17,35 @@ const awsLocationMismatch = 'aws-test-mismatch';
|
||||||
const azureLocation = 'azuretest';
|
const azureLocation = 'azuretest';
|
||||||
const azureLocation2 = 'azuretest2';
|
const azureLocation2 = 'azuretest2';
|
||||||
const azureLocationMismatch = 'azuretestmismatch';
|
const azureLocationMismatch = 'azuretestmismatch';
|
||||||
|
const awsLocationEncryption = 'aws-test-encryption';
|
||||||
const versioningEnabled = { Status: 'Enabled' };
|
const versioningEnabled = { Status: 'Enabled' };
|
||||||
const versioningSuspended = { Status: 'Suspended' };
|
const versioningSuspended = { Status: 'Suspended' };
|
||||||
|
let describeSkipIfNotMultiple = describe.skip;
|
||||||
|
let awsS3;
|
||||||
|
let awsBucket;
|
||||||
|
|
||||||
|
if (config.backends.data === 'multiple' && !process.env.S3_END_TO_END) {
|
||||||
|
describeSkipIfNotMultiple = describe;
|
||||||
|
// can only get real aws config if not running end-to-end
|
||||||
|
const awsConfig = getRealAwsConfig(awsLocation);
|
||||||
|
awsS3 = new AWS.S3(awsConfig);
|
||||||
|
awsBucket = config.locationConstraints[awsLocation].details.bucketName;
|
||||||
|
}
|
||||||
|
|
||||||
|
function _assertErrorResult(err, expectedError, desc) {
|
||||||
|
if (!expectedError) {
|
||||||
|
assert.strictEqual(err, null, `got error for ${desc}: ${err}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
assert(err, `expected ${expectedError} but found no error`);
|
||||||
|
assert.strictEqual(err.code, expectedError);
|
||||||
|
assert.strictEqual(err.statusCode, errors[expectedError].code);
|
||||||
|
}
|
||||||
|
|
||||||
const utils = {
|
const utils = {
|
||||||
|
describeSkipIfNotMultiple,
|
||||||
|
awsS3,
|
||||||
|
awsBucket,
|
||||||
fileLocation,
|
fileLocation,
|
||||||
memLocation,
|
memLocation,
|
||||||
awsLocation,
|
awsLocation,
|
||||||
|
@ -26,6 +54,7 @@ const utils = {
|
||||||
azureLocation,
|
azureLocation,
|
||||||
azureLocation2,
|
azureLocation2,
|
||||||
azureLocationMismatch,
|
azureLocationMismatch,
|
||||||
|
awsLocationEncryption,
|
||||||
};
|
};
|
||||||
|
|
||||||
utils.uniqName = name => `${name}${new Date().getTime()}`;
|
utils.uniqName = name => `${name}${new Date().getTime()}`;
|
||||||
|
@ -119,6 +148,14 @@ utils.expectedETag = (body, getStringified = true) => {
|
||||||
return `"${eTagValue}"`;
|
return `"${eTagValue}"`;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
utils.putToAwsBackend = (s3, bucket, key, body, cb) => {
|
||||||
|
console.log('aaaain put to aws backend')
|
||||||
|
console.log('what is cb in putToAWsBackend?', cb)
|
||||||
|
s3.putObject({ Bucket: bucket, Key: key, Body: body,
|
||||||
|
Metadata: { 'scal-location-constraint': awsLocation } },
|
||||||
|
(err, result) => cb(err, result.VersionId));
|
||||||
|
};
|
||||||
|
|
||||||
utils.enableVersioning = (s3, bucket, cb) => {
|
utils.enableVersioning = (s3, bucket, cb) => {
|
||||||
s3.putBucketVersioning({ Bucket: bucket,
|
s3.putBucketVersioning({ Bucket: bucket,
|
||||||
VersioningConfiguration: versioningEnabled }, err => {
|
VersioningConfiguration: versioningEnabled }, err => {
|
||||||
|
@ -139,13 +176,11 @@ utils.suspendVersioning = (s3, bucket, cb) => {
|
||||||
|
|
||||||
utils.mapToAwsPuts = (s3, bucket, key, dataArray, cb) => {
|
utils.mapToAwsPuts = (s3, bucket, key, dataArray, cb) => {
|
||||||
async.mapSeries(dataArray, (data, next) => {
|
async.mapSeries(dataArray, (data, next) => {
|
||||||
s3.putObject({ Bucket: bucket, Key: key, Body: data,
|
utils.putToAwsBackend(s3, bucket, key, data, next);
|
||||||
Metadata: { 'scal-location-constraint': awsLocation } },
|
|
||||||
next);
|
|
||||||
}, (err, results) => {
|
}, (err, results) => {
|
||||||
assert.strictEqual(err, null, 'Expected success ' +
|
assert.strictEqual(err, null, 'Expected success ' +
|
||||||
`putting object, got error ${err}`);
|
`putting object, got error ${err}`);
|
||||||
cb(null, results.map(result => result.VersionId));
|
cb(null, results);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -161,4 +196,143 @@ utils.putNullVersionsToAws = (s3, bucket, key, versions, cb) => {
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
utils.getAndAssertResult = (s3, params, cb) => {
|
||||||
|
console.log('params', params);
|
||||||
|
const { bucket, key, body, versionId, expectedVersionId, expectedTagCount,
|
||||||
|
expectedError } = params;
|
||||||
|
s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
|
||||||
|
(err, data) => {
|
||||||
|
_assertErrorResult(err, expectedError, 'putting tags');
|
||||||
|
console.log('expected error...', expectedError)
|
||||||
|
if (expectedError) {
|
||||||
|
console.log('do we get here?', expectedError)
|
||||||
|
return cb();
|
||||||
|
}
|
||||||
|
console.log('we get to callback from get request to s3...')
|
||||||
|
assert.strictEqual(err, null, 'Expected success ' +
|
||||||
|
`getting object, got error ${err}`);
|
||||||
|
if (body) {
|
||||||
|
assert(data.Body, 'expected object body in response');
|
||||||
|
const expectedMD5 = utils.expectedETag(body, false);
|
||||||
|
const resultMD5 = utils.expectedETag(data.Body, false);
|
||||||
|
assert.strictEqual(resultMD5, expectedMD5);
|
||||||
|
}
|
||||||
|
if (!expectedVersionId) {
|
||||||
|
assert.strictEqual(data.VersionId, undefined);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(data.VersionId, expectedVersionId);
|
||||||
|
}
|
||||||
|
if (expectedTagCount && expectedTagCount === '0') {
|
||||||
|
assert.strictEqual(data.TagCount, undefined);
|
||||||
|
} else if (expectedTagCount) {
|
||||||
|
assert.strictEqual(data.TagCount, expectedTagCount);
|
||||||
|
}
|
||||||
|
return cb();
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
utils.awsGetLatestVerId = (key, body, cb) =>
|
||||||
|
awsS3.getObject({ Bucket: awsBucket, Key: key }, (err, result) => {
|
||||||
|
assert.strictEqual(err, null, 'Expected success ' +
|
||||||
|
`getting object from AWS, got error ${err}`);
|
||||||
|
const resultMD5 = utils.expectedETag(result.Body, false);
|
||||||
|
const expectedMD5 = utils.expectedETag(body, false);
|
||||||
|
assert.strictEqual(resultMD5, expectedMD5);
|
||||||
|
return cb(null, result.VersionId);
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
utils.tagging = {};
|
||||||
|
|
||||||
|
function _getTaggingConfig(tags) {
|
||||||
|
return {
|
||||||
|
// eslint-disable-next-line arrow-body-style
|
||||||
|
TagSet: Object.keys(tags).map(key => {
|
||||||
|
return {
|
||||||
|
Key: key,
|
||||||
|
Value: tags[key],
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
utils.tagging.putTaggingAndAssert = (s3, params, cb) => {
|
||||||
|
const { bucket, key, tags, versionId, expectedVersionId,
|
||||||
|
expectedError } = params;
|
||||||
|
const taggingConfig = _getTaggingConfig(tags);
|
||||||
|
return s3.putObjectTagging({ Bucket: bucket, Key: key, VersionId: versionId,
|
||||||
|
Tagging: taggingConfig }, (err, data) => {
|
||||||
|
_assertErrorResult(err, expectedError, 'putting tags');
|
||||||
|
if (expectedError) {
|
||||||
|
return cb();
|
||||||
|
}
|
||||||
|
assert.strictEqual(err, null, `got error for putting tags: ${err}`);
|
||||||
|
if (expectedVersionId) {
|
||||||
|
assert.strictEqual(data.VersionId, expectedVersionId);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(data.VersionId, undefined);
|
||||||
|
}
|
||||||
|
return cb(null, data.VersionId);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
utils.tagging.getTaggingAndAssert = (s3, params, cb) => {
|
||||||
|
const { bucket, key, expectedTags, versionId, expectedVersionId,
|
||||||
|
expectedError, getObject } = params;
|
||||||
|
s3.getObjectTagging({ Bucket: bucket, Key: key, VersionId: versionId },
|
||||||
|
(err, data) => {
|
||||||
|
_assertErrorResult(err, expectedError, 'putting tags');
|
||||||
|
if (expectedError) {
|
||||||
|
return cb();
|
||||||
|
}
|
||||||
|
const expectedTagResult = _getTaggingConfig(expectedTags);
|
||||||
|
const expectedTagCount = `${Object.keys(expectedTags).length}`;
|
||||||
|
assert.strictEqual(err, null, `got error for putting tags: ${err}`);
|
||||||
|
if (expectedVersionId) {
|
||||||
|
assert.strictEqual(data.VersionId, expectedVersionId);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(data.VersionId, undefined);
|
||||||
|
}
|
||||||
|
assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet);
|
||||||
|
if (getObject === false) {
|
||||||
|
return process.nextTick(cb, null, data.VersionId);
|
||||||
|
}
|
||||||
|
return utils.getAndAssertResult(s3, { bucket, key, versionId,
|
||||||
|
expectedVersionId, expectedTagCount },
|
||||||
|
() => cb(null, data.VersionId));
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
utils.tagging.delTaggingAndAssert = (s3, params, cb) => {
|
||||||
|
const { bucket, key, versionId, expectedVersionId, expectedError } = params;
|
||||||
|
return s3.deleteObjectTagging({ Bucket: bucket, Key: key,
|
||||||
|
VersionId: versionId }, (err, data) => {
|
||||||
|
_assertErrorResult(err, expectedError, 'putting tags');
|
||||||
|
if (expectedError) {
|
||||||
|
return cb();
|
||||||
|
}
|
||||||
|
assert.strictEqual(err, null, `got error for putting tags: ${err}`);
|
||||||
|
if (expectedVersionId) {
|
||||||
|
assert.strictEqual(data.VersionId, expectedVersionId);
|
||||||
|
} else {
|
||||||
|
assert.strictEqual(data.VersionId, undefined);
|
||||||
|
}
|
||||||
|
return utils.tagging.getTaggingAndAssert(s3, { bucket, key, versionId,
|
||||||
|
expectedVersionId, expectedTags: {} }, () => cb());
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
utils.tagging.awsGetAssertTags = (params, cb) => {
|
||||||
|
const { key, versionId, expectedTags } = params;
|
||||||
|
const expectedTagResult = _getTaggingConfig(expectedTags);
|
||||||
|
awsS3.getObjectTagging({ Bucket: awsBucket, Key: key,
|
||||||
|
VersionId: versionId }, (err, data) => {
|
||||||
|
assert.strictEqual(err, null, 'got unexpected error getting ' +
|
||||||
|
`tags directly from AWS: ${err}`);
|
||||||
|
assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet);
|
||||||
|
return cb();
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
module.exports = utils;
|
module.exports = utils;
|
||||||
|
|
|
@ -59,7 +59,7 @@ describe('Put object tagging with versioning', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should not create version puting object tags on a ' +
|
it('should not create version putting object tags on a ' +
|
||||||
' version-enabled bucket where no version id is specified ', done => {
|
' version-enabled bucket where no version id is specified ', done => {
|
||||||
async.waterfall([
|
async.waterfall([
|
||||||
next => s3.putBucketVersioning({ Bucket: bucketName,
|
next => s3.putBucketVersioning({ Bucket: bucketName,
|
||||||
|
|
Loading…
Reference in New Issue