finally + update test

lib/api/apiUtils/object/createAndStoreObject.js

@@ -61,6 +61,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
         log, callback) {
     const size = isDeleteMarker ? 0 : request.parsedContentLength;
+    // although the request method may actually be 'DELETE' if creating a
+    // delete marker, for our purposes we consider this to be a 'PUT'
+    // operation
+    const requestMethod = 'PUT';
     const websiteRedirectHeader =
         request.headers['x-amz-website-redirect-location'];
     if (!validateWebsiteHeader(websiteRedirectHeader)) {
@@ -70,8 +74,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         return callback(err);
     }
 
-    const metaHeaders = isDeleteMarker ? [] :
-        getMetaHeaders(request.headers);
+    const metaHeaders = isDeleteMarker ? [] : getMetaHeaders(request.headers);
     if (metaHeaders instanceof Error) {
         log.debug('user metadata validation failed', {
             error: metaHeaders,
@@ -79,6 +82,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         });
         return process.nextTick(() => callback(metaHeaders));
     }
+
     log.trace('meta headers', { metaHeaders, method: 'objectPut' });
     const objectKeyContext = {
         bucketName,
@@ -87,6 +91,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         objectKey,
         metaHeaders,
         tagging: request.headers['x-amz-tagging'],
+        isDeleteMarker,
     };
     // If the request was made with a pre-signed url, the x-amz-acl 'header'
     // might be in the query string rather than the actual headers so include
@@ -116,6 +121,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         metadataStoreParams.tagging = request.headers['x-amz-tagging'];
     }
 
+    // if creating new delete marker and there is an existing object, copy
+    // the object's location constraint metaheader to determine backend info
+    if (isDeleteMarker && objMD) {
+        // eslint-disable-next-line no-param-reassign
+        request.headers[constants.objectLocationConstraintHeader] =
+            objMD[constants.objectLocationConstraintHeader];
+    }
+
     const backendInfoObj =
         locationConstraintCheck(request, null, bucketMD, log);
     if (backendInfoObj.err) {
@@ -205,7 +218,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             metadataStoreParams.nullVersionId = options.nullVersionId;
             return _storeInMDandDeleteData(bucketName, infoArr,
                 cipherBundle, metadataStoreParams,
-                options.dataToDelete, requestLogger, request.method, next);
+                options.dataToDelete, requestLogger, requestMethod, next);
         },
     ], callback);
 }

lib/api/apiUtils/object/versioning.js

@@ -71,22 +71,29 @@ function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
     });
 }
 
-/** get location of data for deletion
+/** get location of null version data for deletion
 * @param {string} bucketName - name of bucket
 * @param {string} objKey - name of object key
 * @param {object} options - metadata options for getting object MD
 * @param {string} options.versionId - version to get from metadata
+* @param {object} mst - info about the master version
+* @param {string} mst.versionId - the master version's version id
 * @param {RequestLogger} log - logger instanceof
 * @param {function} cb - callback
 * @return {undefined} - and call callback with (err, dataToDelete)
 */
-function _getDeleteLocations(bucketName, objKey, options, log, cb) {
+function _getNullVersionsToDelete(bucketName, objKey, options, mst, log, cb) {
+    if (options.versionId === mst.versionId) {
+        // no need to get delete location, we already have the master's metadata
+        const dataToDelete = mst.objLocation;
+        return process.nextTick(cb, null, dataToDelete);
+    }
     return metadata.getObjectMD(bucketName, objKey, options, log,
         (err, versionMD) => {
             if (err) {
                 log.debug('err from metadata getting specified version', {
                     error: err,
-                    method: '_getDeleteLocations',
+                    method: '_getNullVersionsToDelete',
                 });
                 return cb(err);
             }
@@ -99,9 +106,8 @@ function _getDeleteLocations(bucketName, objKey, options, log, cb) {
         });
 }
 
-function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
-    // before deleting null version md, retrieve location of data to delete
-    return _getDeleteLocations(bucketName, objKey, options, log,
+function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
+    return _getNullVersionsToDelete(bucketName, objKey, options, mst, log,
         (err, nullDataToDelete) => {
             if (err) {
                 log.warn('could not find null version metadata', {
@@ -236,23 +242,24 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
             if (!delOptions) {
                 return process.nextTick(next, null, options);
             }
-            return _deleteNullVersionMD(bucketName, objectKey, delOptions, log,
-            (err, nullDataToDelete) => {
-                if (err) {
-                    log.warn('unexpected error deleting null version md', {
-                        error: err,
-                        method: 'versioningPreprocessing',
-                    });
-                    // it's possible there was a concurrent request to delete
-                    // the null version, so proceed with putting a new version
-                    if (err === errors.NoSuchKey) {
-                        return next(null, options);
+            return _deleteNullVersionMD(bucketName, objectKey, delOptions, mst,
+                log, (err, nullDataToDelete) => {
+                    if (err) {
+                        log.warn('unexpected error deleting null version md', {
+                            error: err,
+                            method: 'versioningPreprocessing',
+                        });
+                        // it's possible there was a concurrent request to
+                        // delete the null version, so proceed with putting a
+                        // new version
+                        if (err === errors.NoSuchKey) {
+                            return next(null, options);
+                        }
+                        return next(errors.InternalError);
                     }
-                    return next(errors.InternalError);
-                }
-                Object.assign(options, { dataToDelete: nullDataToDelete });
-                return next(null, options);
-            });
+                    Object.assign(options, { dataToDelete: nullDataToDelete });
+                    return next(null, options);
+                });
         },
     ], (err, options) => callback(err, options));
 }

lib/api/completeMultipartUpload.js

@@ -18,8 +18,7 @@ const { versioningPreprocessing, checkQueryVersionId }
 const metadata = require('../metadata/wrapper');
 const services = require('../services');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
-const { checkAzureBackendMatch, skipMpuPartProcessing } =
-    require('../data/external/utils');
+const { skipMpuPartProcessing } = require('../data/external/utils');
 
 const logger = require('../utilities/logger');
 
@@ -214,6 +213,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
                         key: completeObjData.key,
                         size: completeObjData.contentLength,
                         start: 0,
+                        dataStoreVersionId: completeObjData.dataStoreVersionId,
                         dataStoreName: storedMetadata.dataStoreName,
                         dataStoreETag: completeObjData.eTag,
                         dataStoreType: completeObjData.dataStoreType,
@@ -333,14 +333,15 @@ function completeMultipartUpload(authInfo, request, log, callback) {
                     // null version when versioning is suspended or versioning
                     // is not enabled, need to delete pre-existing data
                     // unless the preexisting object and the completed mpu
-                    // are on Azure data backend
+                    // are on external backends
                     if (dataToDelete) {
-                        if (!checkAzureBackendMatch(dataToDelete[0],
-                        completeObjData)) {
-                            data.batchDelete(dataToDelete, request.method, null,
-                                logger.newRequestLoggerFromSerializedUids(log
-                                .getSerializedUids()));
-                        }
+                        const newDataStoreName =
+                            Array.isArray(dataLocations) && dataLocations[0] ?
+                            dataLocations[0].dataStoreName : null;
+                        data.batchDelete(dataToDelete, request.method,
+                            newDataStoreName,
+                            logger.newRequestLoggerFromSerializedUids(log
+                            .getSerializedUids()));
                     }
                     return next(null, mpuBucket, keysToDelete, aggregateETag,
                         extraPartLocations, destinationBucket,

lib/api/initiateMultipartUpload.js

@@ -185,19 +185,17 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
                 if (err) {
                     return callback(err);
                 }
-              // NOTE: remove the following when we will support putting a
-              // versioned object to a location-constraint of type AWS or Azure.
                 if (locConstraint &&
                   config.locationConstraints[locConstraint] &&
                   config.locationConstraints[locConstraint].type &&
-                  constants.externalBackends[config
+                  constants.versioningNotImplBackends[config
                       .locationConstraints[locConstraint].type]
                 ) {
                     const vcfg = destinationBucket.getVersioningConfiguration();
                     const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
                     if (isVersionedObj) {
                         log.debug(externalVersioningErrorMessage,
-                            { method: 'multipleBackendGateway',
+                            { method: 'initiateMultipartUpload',
                                 error: errors.NotImplemented });
                         return callback(errors.NotImplemented
                         .customizeDescription(externalVersioningErrorMessage));

lib/api/objectCopy.js

@@ -23,6 +23,7 @@ const { config } = require('../Config');
 
 const versionIdUtils = versioning.VersionID;
 const locationHeader = constants.objectLocationConstraintHeader;
+const versioningNotImplBackends = constants.versioningNotImplBackends;
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
 'a versioned object to a location-constraint of type AWS or Azure.';
 
@@ -321,9 +322,7 @@ function objectCopy(authInfo, request, sourceBucket,
                 sourceLocationConstraintType = config.getLocationConstraintType(
                   storeMetadataParams.metaHeaders[locationHeader]);
             }
-            // NOTE: remove the following when we will support putting a
-            // versioned object to a location-constraint of type AWS or Azure.
-            if (constants.externalBackends[sourceLocationConstraintType]) {
+            if (versioningNotImplBackends[sourceLocationConstraintType]) {
                 const vcfg = destBucketMD.getVersioningConfiguration();
                 const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
                 if (isVersionedObj) {
@@ -337,6 +336,10 @@ function objectCopy(authInfo, request, sourceBucket,
             if (dataLocator.length === 0) {
                 if (!storeMetadataParams.locationMatch &&
                 constants.externalBackends[sourceLocationConstraintType]) {
+                    console.log('============================');
+                    console.log('detected zero byte object')
+                    console.log('location does not match and source ' +
+                    'is an external backend; about to use data.put for copy')
                     return data.put(null, null, storeMetadataParams.size,
                         dataStoreContext, backendInfo,
                         log, (error, objectRetrievalInfo) => {
@@ -359,7 +362,7 @@ function objectCopy(authInfo, request, sourceBucket,
                 return next(null, storeMetadataParams, dataLocator, destObjMD,
                     serverSideEncryption, destBucketMD);
             }
-
+            console.log('about to call data wrapper copy object')
             return data.copyObject(request, sourceLocationConstraintType,
               sourceLocationConstraintName, storeMetadataParams, dataLocator,
               dataStoreContext, backendInfo, serverSideEncryption, log,

lib/api/objectDelete.js

@@ -73,6 +73,13 @@ function objectDelete(authInfo, request, log, cb) {
                     // versioning has been configured
                     return next(null, bucketMD, objMD);
                 }
+                console.log('============== objMD.location? in objDelete', objMD.location)
+                if (reqVersionId && objMD.location &&
+                    Array.isArray(objMD.location) && objMD.location[0]) {
+                    // we need this information for data deletes to AWS
+                    // eslint-disable-next-line no-param-reassign
+                    objMD.location[0].deleteVersion = true;
+                }
                 if (objMD['content-length'] !== undefined) {
                     log.end().addDefaultFields({
                         bytesDeleted: objMD['content-length'],

lib/data/external/AwsClient.js

@@ -7,6 +7,11 @@ const { prepareStream } = require('../../api/apiUtils/object/prepareStream');
 const { logHelper, removeQuotes, trimXMetaPrefix } = require('./utils');
 const { config } = require('../../Config');
 
+const missingVerIdInternalError = errors.InternalError.customizeDescription(
+    'Invalid state. Please ensure versioning is enabled ' +
+    'in AWS for the location constraint and try again.'
+);
+
 class AwsClient {
     constructor(config) {
         this._s3Params = config.s3Params;
@@ -25,24 +30,15 @@ class AwsClient {
         return `${requestBucketName}/${requestObjectKey}`;
     }
     put(stream, size, keyContext, reqUids, callback) {
+        console.log('=========================')
+        console.log('got to AwsClient put')
         const awsKey = this._createAwsKey(keyContext.bucketName,
            keyContext.objectKey, this._bucketMatch);
         const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
         const log = createLogger(reqUids);
-        const uploadParams = {
-            Bucket: this._awsBucketName,
-            Key: awsKey,
-            Metadata: metaHeaders,
-            ContentLength: size,
-        };
-        if (this._serverSideEncryption) {
-            uploadParams.ServerSideEncryption = 'AES256';
-        }
-        if (keyContext.tagging) {
-            uploadParams.Tagging = keyContext.tagging;
-        }
 
         const putCb = (err, data) => {
+            console.log('result from aws put', data)
             if (err) {
                 logHelper(log, 'error', 'err from data backend',
                   err, this._dataStoreName);
@@ -52,21 +48,38 @@ class AwsClient {
                 );
             }
             if (!data.VersionId) {
-                const error = errors.InternalError.customizeDescription(
-                    'Invalid state. Please ensure versioning is enabled ' +
-                    'in AWS for the location constraint and try again.'
-                );
                 logHelper(log, 'error', 'missing version id for data ' +
-                'backend object', error, this._dataStoreName);
-                return callback(error);
+                    'backend object', missingVerIdInternalError,
+                    this._dataStoreName);
+                return callback(missingVerIdInternalError);
             }
             const dataStoreVersionId = data.VersionId;
             return callback(null, awsKey, dataStoreVersionId);
         };
 
+        const params = {
+            Bucket: this._awsBucketName,
+            Key: awsKey,
+        };
+        // we call data.put to create a delete marker, but it's actually a
+        // delete request in call to AWS
+        if (keyContext.isDeleteMarker) {
+            return this._client.deleteObject(params, putCb);
+        }
+        const uploadParams = params;
+        uploadParams.Metadata = metaHeaders;
+        uploadParams.ContentLength = size;
+        if (this._serverSideEncryption) {
+            uploadParams.ServerSideEncryption = 'AES256';
+        }
+        if (keyContext.tagging) {
+            uploadParams.Tagging = keyContext.tagging;
+        }
+
         if (!stream) {
             return this._client.putObject(uploadParams, putCb);
         }
+        console.log('uploadParams', uploadParams)
 
         uploadParams.Body = stream;
         return this._client.upload(uploadParams, putCb);
@@ -108,6 +121,14 @@ class AwsClient {
         }).on('success', response => {
             log.trace('AWS GET request response headers',
               { responseHeaders: response.httpResponse.headers });
+              console.log('====================')
+              console.log('params sent for aws get request', {
+                  Bucket: this._awsBucketName,
+                  Key: key,
+                  VersionId: dataStoreVersionId,
+                  Range: range,
+              });
+              console.log('AWS GET request response headers', response.httpResponse.headers)
         });
         const stream = request.createReadStream().on('error', err => {
             logHelper(log, 'error', 'error streaming data from AWS',
@@ -117,18 +138,29 @@ class AwsClient {
         return callback(null, stream);
     }
     delete(objectGetInfo, reqUids, callback) {
-        // for backwards compatibility
-        const key = typeof objectGetInfo === 'string' ? objectGetInfo :
-          objectGetInfo.key;
+        const { key, dataStoreVersionId, deleteVersion } = objectGetInfo;
+        const log = createLogger(reqUids);
         const params = {
             Bucket: this._awsBucketName,
             Key: key,
         };
-        return this._client.deleteObject(params, err => {
+        if (deleteVersion) {
+            params.VersionId = dataStoreVersionId;
+        }
+        console.log('=====================');
+        console.log('objectGetInfo', { key, dataStoreVersionId, deleteVersion })
+        console.log('params sending to AWS for delete', params)
+        return this._client.deleteObject(params, (err, data) => {
+            console.log('response from aws for delete', data)
             if (err) {
-                const log = createLogger(reqUids);
                 logHelper(log, 'error', 'error deleting object from ' +
                 'datastore', err, this._dataStoreName);
+                if (err.code === 'NoSuchVersion') {
+                    // data may have been deleted directly from the AWS backend
+                    // don't want to retry the delete and errors are not
+                    // sent back to client anyway, so no need to return err
+                    return callback();
+                }
                 return callback(errors.InternalError
                   .customizeDescription('Error returned from ' +
                   `AWS: ${err.message}`)
@@ -300,6 +332,9 @@ class AwsClient {
         const completeObjData = { key: awsKey };
         return this._client.completeMultipartUpload(mpuParams,
         (err, completeMpuRes) => {
+            console.log('==============================')
+            console.log('mpuParams', mpuParams);
+            console.log('completeMpuRes from AWS', completeMpuRes)
             if (err) {
                 if (mpuError[err.code]) {
                     logHelper(log, 'trace', 'err from data backend on ' +
@@ -313,6 +348,12 @@ class AwsClient {
                   `AWS: ${err.message}`)
                 );
             }
+            if (!completeMpuRes.VersionId) {
+                logHelper(log, 'error', 'missing version id for data ' +
+                'backend object', missingVerIdInternalError,
+                    this._dataStoreName);
+                return callback(missingVerIdInternalError);
+            }
             // need to get content length of new object to store
             // in our metadata
             return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
@@ -328,6 +369,7 @@ class AwsClient {
                 // remove quotes from eTag because they're added later
                 completeObjData.eTag = completeMpuRes.ETag
                     .substring(1, completeMpuRes.ETag.length - 1);
+                completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
                 completeObjData.contentLength = objHeaders.ContentLength;
                 return callback(null, completeObjData);
             });
@@ -357,10 +399,12 @@ class AwsClient {
     objectPutTagging(key, bucket, objectMD, log, callback) {
         const awsBucket = this._awsBucketName;
         const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
-        const tagParams = { Bucket: awsBucket, Key: awsKey };
-        if (objectMD.versionId) {
-            tagParams.VersionId = objectMD.versionId;
-        }
+        const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
+        const tagParams = {
+            Bucket: awsBucket,
+            Key: awsKey,
+            VersionId: dataStoreVersionId,
+        };
         const keyArray = Object.keys(objectMD.tags);
         tagParams.Tagging = {};
         tagParams.Tagging.TagSet = keyArray.map(key => {
@@ -383,10 +427,12 @@ class AwsClient {
     objectDeleteTagging(key, bucket, objectMD, log, callback) {
         const awsBucket = this._awsBucketName;
         const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
-        const tagParams = { Bucket: awsBucket, Key: awsKey };
-        if (objectMD.versionId) {
-            tagParams.VersionId = objectMD.versionId;
-        }
+        const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
+        const tagParams = {
+            Bucket: awsBucket,
+            Key: awsKey,
+            VersionId: dataStoreVersionId,
+        };
         return this._client.deleteObjectTagging(tagParams, err => {
             if (err) {
                 logHelper(log, 'error', 'error from data backend on ' +
@@ -417,7 +463,7 @@ class AwsClient {
             CopySource: `${sourceAwsBucketName}/${sourceKey}`,
             Metadata: metaHeaders,
             MetadataDirective: metadataDirective,
-        }, err => {
+        }, (err, copyResult) => {
             if (err) {
                 if (err.code === 'AccessDenied') {
                     logHelper(log, 'error', 'Unable to access ' +
@@ -435,7 +481,15 @@ class AwsClient {
                   `AWS: ${err.message}`)
                 );
             }
-            return callback(null, destAwsKey);
+            if (!copyResult.VersionId) {
+                logHelper(log, 'error', 'missing version id for data ' +
+                'backend object', missingVerIdInternalError,
+                    this._dataStoreName);
+                return callback(missingVerIdInternalError);
+            }
+            console.log('destAwsKey in AwsClient', destAwsKey);
+            console.log('callback??', callback)
+            return callback(null, destAwsKey, copyResult.VersionId);
         });
     }
     uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,

lib/data/multipleBackendGateway.js

@@ -247,11 +247,13 @@ const multipleBackendGateway = {
         const client = clients[location];
         if (client.copyObject) {
             return client.copyObject(request, externalSourceKey,
-            sourceLocationConstraintName, log, (err, key) => {
+            sourceLocationConstraintName, log, (err, key,
+                dataStoreVersionId) => {
                 const dataRetrievalInfo = {
                     key,
                     dataStoreName: location,
                     dataStoreType: client.clientType,
+                    dataStoreVersionId,
                 };
                 cb(err, dataRetrievalInfo);
             });

lib/data/wrapper.js

@@ -56,6 +56,33 @@ if (config.backends.data === 'mem') {
  */
 const MAX_RETRY = 2;
 
+// This check is done because on a put, complete mpu or copy request to AWS,
+// if the object already exists on AWS, the existing object should not be
+// deleted, which is the functionality for all other backends
+function _shouldSkipDelete(locations, requestMethod, newObjDataStoreName) {
+    const skipBackend = externalBackends;
+    const skipMethods = { PUT: true, POST: true };
+    if (!Array.isArray(locations) || !locations[0] ||
+        !locations[0].dataStoreType) {
+        return false;
+    }
+    const isSkipBackend = skipBackend[locations[0].dataStoreType];
+    const isMatchingBackends =
+        locations[0].dataStoreName === newObjDataStoreName;
+    const isSkipMethod = skipMethods[requestMethod];
+    console.log('============================')
+    console.log('requestMethod', requestMethod)
+    console.log('isSkipMethod', isSkipMethod)
+    console.log('isSkipBackend', isSkipBackend);
+    console.log('isMatchingBackends', isMatchingBackends)
+    console.log('locations[0].dataStoreName', locations[0].dataStoreName);
+    console.log('newObjDataStoreName', newObjDataStoreName)
+    if (!isSkipBackend || !isMatchingBackends || !isSkipMethod) {
+        return false;
+    }
+    return true;
+}
+
 function _retryDelete(objectGetInfo, log, count, cb) {
     if (count > MAX_RETRY) {
         return cb(errors.InternalError);
@@ -216,7 +243,6 @@ const data = {
             return callback(err);
         });
     },
-
     // It would be preferable to have an sproxyd batch delete route to
     // replace this
     batchDelete: (locations, requestMethod, newObjDataStoreName, log) => {
@@ -224,20 +250,12 @@ const data = {
         // be finalized; refer Issue #312 for the discussion. In the
         // meantime, we at least log the location of the data we are
         // about to delete before attempting its deletion.
-        /* eslint-disable camelcase */
-        const skipBackend = externalBackends;
-        /* eslint-enable camelcase */
-        const isSkipBackend = (locations[0] && locations[0].dataStoreType) ?
-            skipBackend[locations[0].dataStoreType] : false;
-        const isMatchingBackends = locations[0] ?
-            locations[0].dataStoreName === newObjDataStoreName : false;
-        // This check is done because on a PUT request to AWS, if the object
-        // already exists on AWS, the existing object should not be deleted,
-        // which is the functionality for all other backends
-        // TODO: update for mpu and object copy
-        if (requestMethod === 'PUT' && isSkipBackend && isMatchingBackends) {
+        console.log('got to batchDelete... requestMethod', requestMethod)
+        console.log('should skip method?', _shouldSkipDelete(locations, requestMethod, newObjDataStoreName))
+        if (_shouldSkipDelete(locations, requestMethod, newObjDataStoreName)) {
             return;
         }
+        console.log('are we continuing any way? )!(@)(!)')
         log.trace('initiating batch delete', {
             keys: locations,
             implName,
@@ -326,6 +344,7 @@ const data = {
     copyObject: (request, sourceLocationConstraintType,
       sourceLocationConstraintName, storeMetadataParams, dataLocator,
       dataStoreContext, destBackendInfo, serverSideEncryption, log, cb) => {
+          console.log('============ data wrapper copy object')
         // NOTE: using copyObject only if copying object from one external
         // backend to the same external backend
         // and for Azure if it is the same account since Azure copy outside
@@ -338,22 +357,26 @@ const data = {
         (sourceLocationConstraintType === 'aws_s3' ||
         (sourceLocationConstraintType === 'azure' && config.isSameAzureAccount(
         sourceLocationConstraintName, request.headers[locationHeader])))) {
+            console.log('detected location match and external backend')
             const location = storeMetadataParams
             .metaHeaders[locationHeader];
             const objectGetInfo = dataLocator[0];
             const externalSourceKey = objectGetInfo.key;
+            console.log('about to use client.copyObject')
             return client.copyObject(request, location,
             externalSourceKey, sourceLocationConstraintName, log,
             (error, objectRetrievalInfo) => {
                 if (error) {
                     return cb(error);
                 }
+                console.log('objectRetrievalInfo - awsClient', objectRetrievalInfo)
                 const putResult = {
                     key: objectRetrievalInfo.key,
                     dataStoreName: objectRetrievalInfo.
                         dataStoreName,
                     dataStoreType: objectRetrievalInfo.
                         dataStoreType,
+                    dataStoreVersionId: objectRetrievalInfo.dataStoreVersionId,
                     size: storeMetadataParams.size,
                     dataStoreETag: objectGetInfo.dataStoreETag,
                     start: objectGetInfo.start,
@@ -362,7 +385,6 @@ const data = {
                 return cb(null, putResultArr);
             });
         }
-
         // dataLocator is an array.  need to get and put all parts
         // For now, copy 1 part at a time. Could increase the second
         // argument here to increase the number of parts
@@ -438,6 +460,7 @@ const data = {
                     }
                     // Copied object is not encrypted so just put it
                     // without a cipherBundle
+                    console.log('about to call data.put?');
                     return data.put(null, stream, part.size,
                     dataStoreContext, destBackendInfo,
                     log, (error, partRetrievalInfo) => {
@@ -451,6 +474,8 @@ const data = {
                             dataStoreName: partRetrievalInfo.
                                 dataStoreName,
                             dataStoreETag: part.dataStoreETag,
+                            dataStoreVersionId: partRetrievalInfo.
+                                dataStoreVersionId,
                             start: part.start,
                             size: part.size,
                         };

tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js

@@ -2,9 +2,13 @@ const assert = require('assert');
 
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { config } = require('../../../../../../lib/Config');
-const { memLocation, fileLocation, awsLocation, awsLocationMismatch }
-    = require('../utils');
+const {
+    describeSkipIfNotMultiple,
+    memLocation,
+    fileLocation,
+    awsLocation,
+    awsLocationMismatch,
+} = require('../utils');
 
 const bucket = 'buckettestmultiplebackenddelete';
 const memObject = `memObject-${Date.now()}`;
@@ -16,9 +20,6 @@ const mismatchObject = `mismatchOjbect-${Date.now()}`;
 const body = Buffer.from('I am a body', 'utf8');
 const bigBody = Buffer.alloc(10485760);
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
 describeSkipIfNotMultiple('Multiple backend delete', () => {
     withV4(sigCfg => {
         let bucketUtil;

tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js

@@ -0,0 +1,560 @@
+const assert = require('assert');
+const async = require('async');
+const { errors } = require('arsenal');
+
+const withV4 = require('../../support/withV4');
+const BucketUtility = require('../../../lib/utility/bucket-util');
+
+const {
+    describeSkipIfNotMultiple,
+    awsS3,
+    awsLocation,
+    awsBucket,
+    putToAwsBackend,
+    enableVersioning,
+    suspendVersioning,
+    mapToAwsPuts,
+    putNullVersionsToAws,
+    putVersionsToAws,
+    getAndAssertResult,
+    awsGetLatestVerId,
+} = require('../utils');
+
+const someBody = 'testbody';
+const bucket = 'buckettestmultiplebackenddeleteversioning';
+
+// order of items by index:
+// 0 - whether to expect a version id
+// 1 - whether version id should match request version id (undef if n/a)
+// 2 - whether x-amz-delete-marker response header should be true
+const _deleteResultSchema = {
+    nonVersionedDelete: [false, undefined, false],
+    newDeleteMarker: [true, false, true],
+    deleteVersion: [true, true, false],
+    deleteDeleteMarker: [true, true, true],
+};
+
+const [nonVersionedDelete, newDeleteMarker, deleteVersion, deleteDeleteMarker]
+    = Object.keys(_deleteResultSchema);
+
+function _assertDeleteResult(result, resultType, requestVersionId) {
+    if (!_deleteResultSchema[resultType]) {
+        throw new Error(`undefined result type "${resultType}"`);
+    }
+    const [expectVersionId, matchReqVersionId, expectDeleteMarker] =
+        _deleteResultSchema[resultType];
+    if (expectVersionId && matchReqVersionId) {
+        assert.strictEqual(result.VersionId, requestVersionId);
+    } else if (expectVersionId) {
+        assert(result.VersionId, 'expected version id in result');
+    } else {
+        assert.strictEqual(result.VersionId, undefined,
+            `did not expect version id in result, got "${result.VersionId}"`);
+    }
+    if (expectDeleteMarker) {
+        assert.strictEqual(result.DeleteMarker, 'true');
+    } else {
+        assert.strictEqual(result.DeleteMarker, undefined);
+    }
+}
+
+function delAndAssertResult(s3, params, cb) {
+    const { bucket, key, versionId, resultType, resultError } = params;
+    return s3.deleteObject({ Bucket: bucket, Key: key, VersionId:
+        versionId }, (err, result) => {
+        if (resultError) {
+            assert(err, `expected ${resultError} but found no error`);
+            assert.strictEqual(err.code, resultError);
+            assert.strictEqual(err.statusCode, errors[resultError].code);
+            return cb(null);
+        }
+        assert.strictEqual(err, null, 'Expected success ' +
+            `deleting object, got error ${err}`);
+        _assertDeleteResult(result, resultType, versionId);
+        return cb(null, result.VersionId);
+    });
+}
+
+function _createDeleteMarkers(s3, bucket, key, count, cb) {
+    return async.timesSeries(count,
+        (i, next) => delAndAssertResult(s3, { bucket, key,
+            resultType: newDeleteMarker }, next),
+        cb);
+}
+
+function _deleteDeleteMarkers(s3, bucket, key, deleteMarkerVids, cb) {
+    return async.mapSeries(deleteMarkerVids, (versionId, next) => {
+        delAndAssertResult(s3, { bucket, key, versionId,
+            resultType: deleteDeleteMarker }, next);
+    }, () => cb());
+}
+
+function _getAssertDeleted(s3, params, cb) {
+    const { key, versionId, errorCode } = params;
+    return s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
+        err => {
+            assert.strictEqual(err.code, errorCode);
+            assert.strictEqual(err.statusCode, 404);
+            return cb();
+        });
+}
+
+function _awsGetAssertDeleted(params, cb) {
+    const { key, versionId, errorCode } = params;
+    return awsS3.getObject({ Bucket: awsBucket, Key: key, VersionId:
+        versionId }, err => {
+        assert.strictEqual(err.code, errorCode);
+        assert.strictEqual(err.statusCode, 404);
+        return cb();
+    });
+}
+
+describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' +
+    'using object location constraint', function testSuite() {
+    this.timeout(30000);
+    withV4(sigCfg => {
+        let bucketUtil;
+        let s3;
+        beforeEach(() => {
+            process.stdout.write('Creating bucket\n');
+            bucketUtil = new BucketUtility('default', sigCfg);
+            s3 = bucketUtil.s3;
+            return s3.createBucketAsync({ Bucket: bucket })
+            .catch(err => {
+                process.stdout.write(`Error creating bucket: ${err}\n`);
+                throw err;
+            });
+        });
+
+        afterEach(() => {
+            process.stdout.write('Emptying bucket\n');
+            return bucketUtil.empty(bucket)
+            .then(() => {
+                process.stdout.write('Deleting bucket\n');
+                return bucketUtil.deleteOne(bucket);
+            })
+            .catch(err => {
+                process.stdout.write('Error emptying/deleting bucket: ' +
+                `${err}\n`);
+                throw err;
+            });
+        });
+
+        it('versioning not configured: if specifying "null" version, should ' +
+        'delete specific version in AWS backend', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putToAwsBackend(s3, bucket, key, someBody,
+                    err => next(err)),
+                next => awsGetLatestVerId(key, someBody, next),
+                (awsVerId, next) => delAndAssertResult(s3, { bucket,
+                    key, versionId: 'null', resultType: deleteVersion },
+                    err => next(err, awsVerId)),
+                (awsVerId, next) => _awsGetAssertDeleted({ key,
+                    versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
+            ], done);
+        });
+
+        it('versioning not configured: specifying any version id other ' +
+        'than null should not result in its deletion in AWS backend', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putToAwsBackend(s3, bucket, key, someBody,
+                    err => next(err)),
+                next => awsGetLatestVerId(key, someBody, next),
+                (awsVerId, next) => delAndAssertResult(s3, { bucket,
+                    key, versionId: 'awsVerId', resultError:
+                    'InvalidArgument' }, err => next(err, awsVerId)),
+                (awsVerId, next) => awsGetLatestVerId(key, someBody,
+                    (err, resultVid) => {
+                        assert.strictEqual(resultVid, awsVerId);
+                        next();
+                    }),
+            ], done);
+        });
+
+        it('versioning suspended: should delete a specific version in AWS ' +
+        'backend successfully', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, [someBody],
+                    err => next(err)),
+                next => awsGetLatestVerId(key, someBody, next),
+                (awsVerId, next) => delAndAssertResult(s3, { bucket,
+                    key, versionId: 'null', resultType: deleteVersion },
+                    err => next(err, awsVerId)),
+                (awsVerId, next) => _awsGetAssertDeleted({ key,
+                    versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should delete a specific version in AWS ' +
+        'backend successfully', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putVersionsToAws(s3, bucket, key, [someBody],
+                    (err, versionIds) => next(err, versionIds[0])),
+                (s3vid, next) => awsGetLatestVerId(key, someBody,
+                    (err, awsVid) => next(err, s3vid, awsVid)),
+                (s3VerId, awsVerId, next) => delAndAssertResult(s3, { bucket,
+                    key, versionId: s3VerId, resultType: deleteVersion },
+                    err => next(err, awsVerId)),
+                (awsVerId, next) => _awsGetAssertDeleted({ key,
+                    versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
+            ], done);
+        });
+
+        it('versioning not configured: deleting existing object should ' +
+        'not return version id or x-amz-delete-marker: true but should ' +
+        'create a delete marker in aws ', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putToAwsBackend(s3, bucket, key, someBody,
+                    err => next(err)),
+                next => delAndAssertResult(s3, { bucket, key,
+                    resultType: nonVersionedDelete }, err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+
+        it('versioning suspended: should create a delete marker in s3 ' +
+        'and aws successfully when deleting existing object', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, [someBody],
+                    err => next(err)),
+                next => delAndAssertResult(s3, { bucket, key, resultType:
+                    newDeleteMarker }, err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+
+        // NOTE: Normal deletes when versioning is suspended create a
+        // delete marker with the version id "null", which overwrites an
+        // existing null version in s3 metadata.
+        it('versioning suspended: creating a delete marker will overwrite an ' +
+        'existing null version that is the latest version in s3 metadata,' +
+        ' but the data of the first null version will remain in AWS',
+        function itF(done) {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, [someBody],
+                    err => next(err)),
+                next => awsGetLatestVerId(key, someBody, next),
+                (awsNullVid, next) => {
+                    this.test.awsNullVid = awsNullVid;
+                    next();
+                },
+                // following call should generate a delete marker
+                next => delAndAssertResult(s3, { bucket, key, resultType:
+                    newDeleteMarker }, next),
+                // delete delete marker
+                (dmVid, next) => delAndAssertResult(s3, { bucket, key,
+                    versionId: dmVid, resultType: deleteDeleteMarker },
+                    err => next(err)),
+                // should get no such object even after deleting del marker
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                // get directly to aws however will give us first null version
+                next => awsGetLatestVerId(key, someBody, next),
+                (awsLatestVid, next) => {
+                    assert.strictEqual(awsLatestVid, this.test.awsNullVid);
+                    next();
+                },
+            ], done);
+        });
+
+        // NOTE: Normal deletes when versioning is suspended create a
+        // delete marker with the version id "null" which is supposed to
+        // overwrite any existing null version.
+        it('versioning suspended: creating a delete marker will overwrite an ' +
+        'existing null version that is not the latest version in s3 metadata,' +
+        ' but the data of the first null version will remain in AWS',
+        function itF(done) {
+            const key = `somekey-${Date.now()}`;
+            const data = [undefined, 'data1'];
+            async.waterfall([
+                // put null version
+                next => putToAwsBackend(s3, bucket, key, data[0],
+                    err => next(err)),
+                next => awsGetLatestVerId(key, '', next),
+                (awsNullVid, next) => {
+                    this.test.awsNullVid = awsNullVid;
+                    next();
+                },
+                // enable versioning and put another version
+                next => putVersionsToAws(s3, bucket, key, [data[1]], next),
+                (versions, next) => {
+                    this.test.s3vid = versions[0];
+                    next();
+                },
+                next => suspendVersioning(s3, bucket, next),
+                // overwrites null version in s3 metadata but does not send
+                // additional delete to AWS to clean up previous "null" version
+                // -- see note above
+                next => delAndAssertResult(s3, { bucket, key,
+                    resultType: newDeleteMarker }, next),
+                (s3dmVid, next) => {
+                    this.test.s3DeleteMarkerId = s3dmVid;
+                    next();
+                },
+                // delete delete marker
+                next => delAndAssertResult(s3, { bucket, key,
+                    versionId: this.test.s3DeleteMarkerId,
+                    resultType: deleteDeleteMarker }, err => next(err)),
+                // deleting latest version after del marker
+                next => delAndAssertResult(s3, { bucket, key,
+                    versionId: this.test.s3vid, resultType: deleteVersion },
+                    err => next(err)),
+                // should get no such object instead of null version
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                // we get the null version that should have been "overwritten"
+                // when getting the latest version in AWS now
+                next => awsGetLatestVerId(key, '', next),
+                (awsLatestVid, next) => {
+                    assert.strictEqual(awsLatestVid, this.test.awsNullVid);
+                    next();
+                },
+            ], done);
+        });
+
+        it('versioning enabled: should create a delete marker in s3 and ' +
+        'aws successfully when deleting existing object', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putVersionsToAws(s3, bucket, key, [someBody],
+                    err => next(err)),
+                next => delAndAssertResult(s3, { bucket, key, resultType:
+                    newDeleteMarker }, err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+
+        it('versioning enabled: should delete a delete marker in s3 and ' +
+        'aws successfully', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putVersionsToAws(s3, bucket, key, [someBody],
+                    (err, versionIds) => next(err, versionIds[0])),
+                // create a delete marker
+                (s3vid, next) => delAndAssertResult(s3, { bucket, key,
+                    resultType: newDeleteMarker }, (err, delMarkerVid) =>
+                    next(err, s3vid, delMarkerVid)),
+                // delete delete marker
+                (s3vid, dmVid, next) => delAndAssertResult(s3, { bucket, key,
+                    versionId: dmVid, resultType: deleteDeleteMarker },
+                    err => next(err, s3vid)),
+                // should be able to get object originally put from s3
+                (s3vid, next) => getAndAssertResult(s3, { bucket, key,
+                    body: someBody, expectedVersionId: s3vid }, next),
+                // latest version in aws should now be object originally put
+                next => awsGetLatestVerId(key, someBody, next),
+            ], done);
+        });
+
+        it('multiple delete markers: should be able to get pre-existing ' +
+        'versions after creating and deleting several delete markers', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putVersionsToAws(s3, bucket, key, [someBody],
+                    (err, versionIds) => next(err, versionIds[0])),
+                (s3vid, next) => _createDeleteMarkers(s3, bucket, key, 3,
+                    (err, dmVids) => next(err, s3vid, dmVids)),
+                (s3vid, dmVids, next) => _deleteDeleteMarkers(s3, bucket, key,
+                    dmVids, () => next(null, s3vid)),
+                // should be able to get object originally put from s3
+                (s3vid, next) => getAndAssertResult(s3, { bucket, key,
+                    body: someBody, expectedVersionId: s3vid }, next),
+                // latest version in aws should now be object originally put
+                next => awsGetLatestVerId(key, someBody, next),
+            ], done);
+        });
+
+        it('multiple delete markers: should get NoSuchObject if only ' +
+        'one of the delete markers is deleted', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putVersionsToAws(s3, bucket, key, [someBody],
+                    err => next(err)),
+                next => _createDeleteMarkers(s3, bucket, key, 3,
+                    (err, dmVids) => next(err, dmVids[2])),
+                (lastDmVid, next) => delAndAssertResult(s3, { bucket,
+                    key, versionId: lastDmVid, resultType: deleteDeleteMarker },
+                    err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+
+        it('should get the new latest version after deleting the latest' +
+        'specific version', done => {
+            const key = `somekey-${Date.now()}`;
+            const data = [...Array(4).keys()].map(i => i.toString());
+            async.waterfall([
+                // put 3 null versions
+                next => mapToAwsPuts(s3, bucket, key, data.slice(0, 3),
+                    err => next(err)),
+                // put one version
+                next => putVersionsToAws(s3, bucket, key, [data[3]],
+                    (err, versionIds) => next(err, versionIds[0])),
+                // delete the latest version
+                (versionId, next) => delAndAssertResult(s3, { bucket,
+                    key, versionId, resultType: deleteVersion },
+                    err => next(err)),
+                // should get the last null version
+                next => getAndAssertResult(s3, { bucket, key,
+                    body: data[2], expectedVersionId: 'null' }, next),
+                next => awsGetLatestVerId(key, data[2],
+                    err => next(err)),
+                // delete the null version
+                next => delAndAssertResult(s3, { bucket,
+                    key, versionId: 'null', resultType: deleteVersion },
+                    err => next(err)),
+                // s3 metadata should report no existing versions for keyname
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                // NOTE: latest version in aws will be the second null version
+                next => awsGetLatestVerId(key, data[1],
+                    err => next(err)),
+            ], done);
+        });
+
+        it('should delete the correct version even if other versions or ' +
+        'delete markers put directly on aws', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putVersionsToAws(s3, bucket, key, [someBody],
+                    (err, versionIds) => next(err, versionIds[0])),
+                (s3vid, next) => awsGetLatestVerId(key, someBody,
+                    (err, awsVid) => next(err, s3vid, awsVid)),
+                // put an object in AWS
+                (s3vid, awsVid, next) => awsS3.putObject({ Bucket: awsBucket,
+                    Key: key }, err => next(err, s3vid, awsVid)),
+                // create a delete marker in AWS
+                (s3vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key }, err => next(err, s3vid, awsVid)),
+                // delete original version in s3
+                (s3vid, awsVid, next) => delAndAssertResult(s3, { bucket, key,
+                    versionId: s3vid, resultType: deleteVersion },
+                    err => next(err, awsVid)),
+                (awsVid, next) => _getAssertDeleted(s3, { key,
+                    errorCode: 'NoSuchKey' }, () => next(null, awsVid)),
+                (awsVerId, next) => _awsGetAssertDeleted({ key,
+                    versionId: awsVerId, errorCode: 'NoSuchVersion' }, next),
+            ], done);
+        });
+
+        it('should not return an error deleting a version that was already ' +
+        'deleted directly from AWS backend', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putVersionsToAws(s3, bucket, key, [someBody],
+                    (err, versionIds) => next(err, versionIds[0])),
+                (s3vid, next) => awsGetLatestVerId(key, someBody,
+                    (err, awsVid) => next(err, s3vid, awsVid)),
+                // delete the object in AWS
+                (s3vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key, VersionId: awsVid }, err => next(err, s3vid)),
+                // then try to delete in S3
+                (s3vid, next) => delAndAssertResult(s3, { bucket, key,
+                    versionId: s3vid, resultType: deleteVersion },
+                    err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+    });
+});
+
+describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' +
+    'using bucket location constraint', function testSuite() {
+    this.timeout(30000);
+    const createBucketParams = {
+        Bucket: bucket,
+        CreateBucketConfiguration: {
+            LocationConstraint: awsLocation,
+        },
+    };
+    withV4(sigCfg => {
+        let bucketUtil;
+        let s3;
+        beforeEach(() => {
+            process.stdout.write('Creating bucket\n');
+            bucketUtil = new BucketUtility('default', sigCfg);
+            s3 = bucketUtil.s3;
+            return s3.createBucketAsync(createBucketParams)
+            .catch(err => {
+                process.stdout.write(`Error creating bucket: ${err}\n`);
+                throw err;
+            });
+        });
+
+        afterEach(() => {
+            process.stdout.write('Emptying bucket\n');
+            return bucketUtil.empty(bucket)
+            .then(() => {
+                process.stdout.write('Deleting bucket\n');
+                return bucketUtil.deleteOne(bucket);
+            })
+            .catch(err => {
+                process.stdout.write('Error emptying/deleting bucket: ' +
+                `${err}\n`);
+                throw err;
+            });
+        });
+
+        it('versioning not configured: deleting non-existing object should ' +
+        'not return version id or x-amz-delete-marker: true nor create a ' +
+        'delete marker in aws ', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => delAndAssertResult(s3, { bucket, key,
+                    resultType: nonVersionedDelete }, err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+
+        it('versioning suspended: should create a delete marker in s3 ' +
+        'and aws successfully when deleting non-existing object', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => suspendVersioning(s3, bucket, next),
+                next => delAndAssertResult(s3, { bucket, key, resultType:
+                    newDeleteMarker }, err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+
+        it('versioning enabled: should create a delete marker in s3 and ' +
+        'aws successfully when deleting non-existing object', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => delAndAssertResult(s3, { bucket, key, resultType:
+                    newDeleteMarker }, err => next(err)),
+                next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' },
+                    next),
+                next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' },
+                    next),
+            ], done);
+        });
+    });
+});

tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js

@@ -3,10 +3,15 @@ const async = require('async');
 
 const BucketUtility = require('../../../lib/utility/bucket-util');
 const withV4 = require('../../support/withV4');
-const { config } = require('../../../../../../lib/Config');
-const { uniqName, getAzureClient, getAzureContainerName, getAzureKeys,
-    azureLocation, azureLocationMismatch } =
-  require('../utils');
+const {
+    describeSkipIfNotMultiple,
+    uniqName,
+    getAzureClient,
+    getAzureContainerName,
+    getAzureKeys,
+    azureLocation,
+    azureLocationMismatch,
+} = require('../utils');
 
 const keyObject = 'deleteazure';
 const azureContainerName = getAzureContainerName();
@@ -20,10 +25,6 @@ const nonExistingId = process.env.AWS_ON_AIR ?
     'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' :
     '3939393939393939393936493939393939393939756e6437';
 
-
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
 describeSkipIfNotMultiple('Multiple backend delete object from Azure',
 function testSuite() {
     this.timeout(250000);

tests/functional/aws-node-sdk/test/multipleBackend/get/get.js

@@ -2,9 +2,13 @@ const assert = require('assert');
 const async = require('async');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { config } = require('../../../../../../lib/Config');
-const { memLocation, fileLocation, awsLocation, awsLocationMismatch }
-    = require('../utils');
+const {
+    describeSkipIfNotMultiple,
+    memLocation,
+    fileLocation,
+    awsLocation,
+    awsLocationMismatch,
+} = require('../utils');
 
 const bucket = 'buckettestmultiplebackendget';
 const memObject = `memobject-${Date.now()}`;
@@ -20,9 +24,6 @@ const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
 const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
 const bigMD5 = 'f1c9645dbc14efddc7d8a322685f26eb';
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
 describe('Multiple backend get object', function testSuite() {
     this.timeout(30000);
     withV4(sigCfg => {

tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js

@@ -1,46 +1,23 @@
 const assert = require('assert');
-const AWS = require('aws-sdk');
 const async = require('async');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { config } = require('../../../../../../lib/Config');
-const { getRealAwsConfig } = require('../../support/awsConfig');
 const {
+    awsS3,
     awsLocation,
+    awsBucket,
     enableVersioning,
     suspendVersioning,
     mapToAwsPuts,
     putNullVersionsToAws,
     putVersionsToAws,
-    expectedETag,
+    getAndAssertResult,
+    describeSkipIfNotMultiple,
 } = require('../utils');
 
 const someBody = 'testbody';
 const bucket = 'buckettestmultiplebackendgetawsversioning';
 
-let awsS3;
-
-function getAndAssertResult(s3, params, cb) {
-    const { bucket, key, body, versionId, expectedVersionId } = params;
-    s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
-        (err, data) => {
-            assert.strictEqual(err, null, 'Expected success ' +
-                `getting object, got error ${err}`);
-            if (body) {
-                assert(data.Body, 'expected object body in response');
-                const expectedMD5 = expectedETag(body, false);
-                const resultMD5 = expectedETag(data.Body, false);
-                assert.strictEqual(resultMD5, expectedMD5);
-            }
-            if (!expectedVersionId) {
-                assert.strictEqual(data.VersionId, undefined);
-            } else {
-                assert.strictEqual(data.VersionId, expectedVersionId);
-            }
-            cb();
-        });
-}
-
 function getAndAssertVersions(s3, bucket, key, versionIds, expectedData,
     cb) {
     async.mapSeries(versionIds, (versionId, next) => {
@@ -58,16 +35,7 @@ function getAndAssertVersions(s3, bucket, key, versionIds, expectedData,
     });
 }
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
-if (describeSkipIfNotMultiple !== describe.skip) {
-    // can only get real aws config if not running end-to-end
-    const awsConfig = getRealAwsConfig(awsLocation);
-    awsS3 = new AWS.S3(awsConfig);
-}
-
-describeSkipIfNotMultiple('Multiple backend get object with versioning',
+describeSkipIfNotMultiple('AWS backend get object with versioning',
 function testSuite() {
     this.timeout(30000);
     withV4(sigCfg => {
@@ -308,8 +276,6 @@ function testSuite() {
         it('should return the correct data getting versioned object ' +
         'even if object was deleted from AWS (creating a delete marker)',
         done => {
-            const awsBucket = config.locationConstraints[awsLocation].
-                details.bucketName;
             const key = `somekey-${Date.now()}`;
             async.waterfall([
                 next => enableVersioning(s3, bucket, next),
@@ -327,15 +293,13 @@ function testSuite() {
         it('should return the correct data getting versioned object ' +
         'even if object is put directly to AWS (creating new version)',
         done => {
-            const awsBucket = config.locationConstraints[awsLocation].
-                details.bucketName;
             const key = `somekey-${Date.now()}`;
             async.waterfall([
                 next => enableVersioning(s3, bucket, next),
                 next => s3.putObject({ Bucket: bucket, Key: key, Body: someBody,
                     Metadata: { 'scal-location-constraint': awsLocation } },
                     (err, res) => next(err, res.VersionId)),
-                // create a delete marker in AWS
+                // put an object in AWS
                 (versionId, next) => awsS3.putObject({ Bucket: awsBucket,
                     Key: key }, err => next(err, versionId)),
                 (versionId, next) => getAndAssertResult(s3, { bucket, key,
@@ -346,8 +310,6 @@ function testSuite() {
         it('should return a InternalError if trying to get an object ' +
         'that was deleted in AWS but exists in s3 metadata',
         done => {
-            const awsBucket = config.locationConstraints[awsLocation].
-                details.bucketName;
             const key = `somekey-${Date.now()}`;
             async.waterfall([
                 next => enableVersioning(s3, bucket, next),
@@ -372,8 +334,6 @@ function testSuite() {
         it('should return a InternalError if trying to get a version ' +
         'that was deleted in AWS but exists in s3 metadata',
         done => {
-            const awsBucket = config.locationConstraints[awsLocation].
-                details.bucketName;
             const key = `somekey-${Date.now()}`;
             async.waterfall([
                 next => enableVersioning(s3, bucket, next),

tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js

@@ -3,21 +3,22 @@ const assert = require('assert');
 const BucketUtility = require('../../../lib/utility/bucket-util');
 const withV4 = require('../../support/withV4');
 
-const { uniqName, getAzureClient, getAzureContainerName,
-  getAzureKeys, azureLocation } = require('../utils');
+const {
+    describeSkipIfNotMultiple,
+    uniqName,
+    getAzureClient,
+    getAzureContainerName,
+    getAzureKeys,
+    azureLocation,
+} = require('../utils');
 
 const azureClient = getAzureClient();
 const azureContainerName = getAzureContainerName();
 const keys = getAzureKeys();
 const keyObject = 'getazure';
 
-const { config } = require('../../../../../../lib/Config');
-
 const normalBody = Buffer.from('I am a body', 'utf8');
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
 const azureTimeout = 10000;
 
 describeSkipIfNotMultiple('Multiple backend get object from Azure',

tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js

@@ -1,22 +1,10 @@
 const async = require('async');
 const assert = require('assert');
 
-const { config } = require('../../../../../../lib/Config');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { azureLocation } = require('../utils');
-
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
-let azureContainerName;
-
-if (config.locationConstraints[azureLocation] &&
-config.locationConstraints[azureLocation].details &&
-config.locationConstraints[azureLocation].details.azureContainerName) {
-    azureContainerName =
-      config.locationConstraints[azureLocation].details.azureContainerName;
-}
+const { describeSkipIfNotMultiple, azureLocation, azureContainerName }
+    = require('../utils');
 
 const keyName = `somekey-${Date.now()}`;
 

tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js

@@ -1,24 +1,14 @@
 const assert = require('assert');
 
-const { config } = require('../../../../../../lib/Config');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { azureLocation } = require('../utils');
+const { describeSkipIfNotMultiple, azureLocation, getAzureContainerName }
+    = require('../utils');
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
-let azureContainerName;
+const azureContainerName = getAzureContainerName();
 const bodyFirstPart = Buffer.alloc(10);
 const bodySecondPart = Buffer.alloc(104857610);
 
-if (config.locationConstraints[azureLocation] &&
-config.locationConstraints[azureLocation].details &&
-config.locationConstraints[azureLocation].details.azureContainerName) {
-    azureContainerName =
-      config.locationConstraints[azureLocation].details.azureContainerName;
-}
-
 let bucketUtil;
 let s3;
 

tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js

@@ -4,9 +4,8 @@ const async = require('async');
 const { s3middleware } = require('arsenal');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { uniqName, getAzureClient, getAzureContainerName, convertMD5,
-    azureLocation } = require('../utils');
-const { config } = require('../../../../../../lib/Config');
+const { describeSkipIfNotMultiple, uniqName, getAzureClient,
+    getAzureContainerName, convertMD5, azureLocation } = require('../utils');
 const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
 const maxSubPartSize = azureMpuUtils.maxSubPartSize;
 
@@ -15,9 +14,6 @@ const azureClient = getAzureClient();
 const azureContainerName = getAzureContainerName();
 const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
 let bucketUtil;
 let s3;
 

tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js

@@ -1,21 +1,23 @@
 const async = require('async');
 const assert = require('assert');
-const AWS = require('aws-sdk');
 
 const { s3middleware } = require('arsenal');
-const { config } = require('../../../../../../lib/Config');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { fileLocation, awsLocation, azureLocation, azureLocationMismatch,
-    getAzureClient, getAzureContainerName } = require('../utils');
-const { getRealAwsConfig } =
-    require('../../support/awsConfig');
+const {
+    describeSkipIfNotMultiple,
+    fileLocation,
+    awsS3,
+    awsLocation,
+    awsBucket,
+    azureLocation,
+    azureLocationMismatch,
+    getAzureClient,
+    getAzureContainerName,
+} = require('../utils');
 
 const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
 
-const awsBucket = 'multitester555';
 const azureContainerName = getAzureContainerName();
 const azureClient = getAzureClient();
 const azureTimeout = 20000;
@@ -106,8 +108,7 @@ function testSuite() {
             this.currentTest.key = `somekey-${Date.now()}`;
             bucketUtil = new BucketUtility('default', sigCfg);
             s3 = bucketUtil.s3;
-            const awsConfig = getRealAwsConfig(awsLocation);
-            this.currentTest.awsClient = new AWS.S3(awsConfig);
+            this.currentTest.awsClient = awsS3;
             return s3.createBucketAsync({ Bucket: azureContainerName })
             .catch(err => {
                 process.stdout.write(`Error creating bucket: ${err}\n`);

tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js

@@ -0,0 +1,176 @@
+const assert = require('assert');
+const async = require('async');
+const withV4 = require('../../support/withV4');
+const BucketUtility = require('../../../lib/utility/bucket-util');
+const { minimumAllowedPartSize } = require('../../../../../../constants');
+const { removeAllVersions } = require('../../../lib/utility/versioning-util');
+const {
+    awsLocation,
+    enableVersioning,
+    suspendVersioning,
+    putToAwsBackend,
+    awsGetLatestVerId,
+    getAndAssertResult,
+    describeSkipIfNotMultiple,
+} = require('../utils');
+
+const data = ['a', 'b'].map(char => Buffer.alloc(minimumAllowedPartSize, char));
+const concattedData = Buffer.concat(data);
+
+const bucket = 'buckettestmultiplebackendmpuawsversioning';
+
+function mpuSetup(s3, key, location, cb) {
+    const partArray = [];
+    async.waterfall([
+        next => {
+            const params = {
+                Bucket: bucket,
+                Key: key,
+                Metadata: { 'scal-location-constraint': location },
+            };
+            s3.createMultipartUpload(params, (err, res) => {
+                assert.strictEqual(err, null, `err creating mpu: ${err}`);
+                const uploadId = res.UploadId;
+                assert(uploadId);
+                assert.strictEqual(res.Bucket, bucket);
+                assert.strictEqual(res.Key, key);
+                next(err, uploadId);
+            });
+        },
+        (uploadId, next) => {
+            const partParams = {
+                Bucket: bucket,
+                Key: key,
+                PartNumber: 1,
+                UploadId: uploadId,
+                Body: data[0],
+            };
+            s3.uploadPart(partParams, (err, res) => {
+                assert.strictEqual(err, null, `err uploading part 1: ${err}`);
+                partArray.push({ ETag: res.ETag, PartNumber: 1 });
+                next(err, uploadId);
+            });
+        },
+        (uploadId, next) => {
+            const partParams = {
+                Bucket: bucket,
+                Key: key,
+                PartNumber: 2,
+                UploadId: uploadId,
+                Body: data[1],
+            };
+            s3.uploadPart(partParams, (err, res) => {
+                assert.strictEqual(err, null, `err uploading part 2: ${err}`);
+                partArray.push({ ETag: res.ETag, PartNumber: 2 });
+                next(err, uploadId);
+            });
+        },
+    ], (err, uploadId) => {
+        process.stdout.write('Created MPU and put two parts\n');
+        cb(err, uploadId, partArray);
+    });
+}
+
+function completeAndAssertMpu(s3, params, cb) {
+    const { bucket, key, uploadId, partArray, expectVersionId,
+        expectedGetVersionId } = params;
+    s3.completeMultipartUpload({
+        Bucket: bucket,
+        Key: key,
+        UploadId: uploadId,
+        MultipartUpload: { Parts: partArray },
+    }, (err, data) => {
+        assert.strictEqual(err, null, `Err completing MPU: ${err}`);
+        if (expectVersionId) {
+            assert.notEqual(data.VersionId, undefined);
+        } else {
+            assert.strictEqual(data.VersionId, undefined);
+        }
+        const expectedVersionId = expectedGetVersionId || data.VersionId;
+        getAndAssertResult(s3, { bucket, key, body: concattedData,
+            expectedVersionId }, cb);
+    });
+}
+
+describeSkipIfNotMultiple('AWS backend complete mpu with versioning',
+function testSuite() {
+    this.timeout(30000);
+    withV4(sigCfg => {
+        const bucketUtil = new BucketUtility('default', sigCfg);
+        const s3 = bucketUtil.s3;
+        beforeEach(done => s3.createBucket({
+            Bucket: bucket,
+            CreateBucketConfiguration: {
+                LocationConstraint: awsLocation,
+            },
+        }, done));
+        afterEach(done => {
+            removeAllVersions({ Bucket: bucket }, err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3.deleteBucket({ Bucket: bucket }, done);
+            });
+        });
+
+        it('versioning not configured: should not return version id ' +
+        'completing mpu', done => {
+            const key = `somekey-${Date.now()}`;
+            mpuSetup(s3, key, awsLocation, (err, uploadId, partArray) => {
+                completeAndAssertMpu(s3, { bucket, key, uploadId, partArray,
+                    expectVersionId: false }, done);
+            });
+        });
+
+        it('versioning not configured: if complete mpu on already-existing ' +
+        'object, metadata should be overwritten but data of previous version' +
+        'in AWS should not be deleted', function itF(done) {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putToAwsBackend(s3, bucket, key, '', err => next(err)),
+                next => awsGetLatestVerId(key, '', next),
+                (awsVerId, next) => {
+                    this.test.awsVerId = awsVerId;
+                    next();
+                },
+                next => mpuSetup(s3, key, awsLocation, next),
+                (uploadId, partArray, next) => completeAndAssertMpu(s3,
+                    { bucket, key, uploadId, partArray, expectVersionId:
+                        false }, next),
+                next => s3.deleteObject({ Bucket: bucket, Key: key, VersionId:
+                    'null' }, next),
+                (delData, next) => getAndAssertResult(s3, { bucket, key,
+                    expectedError: 'NoSuchKey' }, next),
+                next => awsGetLatestVerId(key, '', next),
+                (awsVerId, next) => {
+                    assert.strictEqual(awsVerId, this.test.awsVerId);
+                    next();
+                },
+            ], done);
+        });
+
+        it('versioning suspended: should not return version id completing mpu',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => suspendVersioning(s3, bucket, next),
+                next => mpuSetup(s3, key, awsLocation, next),
+                (uploadId, partArray, next) => completeAndAssertMpu(s3,
+                    { bucket, key, uploadId, partArray, expectVersionId: false,
+                    expectedGetVersionId: 'null' }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should return version id completing mpu',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => mpuSetup(s3, key, awsLocation, next),
+                (uploadId, partArray, next) => completeAndAssertMpu(s3,
+                    { bucket, key, uploadId, partArray, expectVersionId: true },
+                    next),
+            ], done);
+        });
+    });
+});

tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js

@@ -4,9 +4,9 @@ const async = require('async');
 const { s3middleware } = require('arsenal');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { expectedETag, uniqName, getAzureClient, getAzureContainerName,
-    convertMD5, azureLocation } = require('../utils');
-const { config } = require('../../../../../../lib/Config');
+const { describeSkipIfNotMultiple, expectedETag, uniqName, getAzureClient,
+    getAzureContainerName, convertMD5, azureLocation, azureLocationMismatch }
+    = require('../utils');
 const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
 const maxSubPartSize = azureMpuUtils.maxSubPartSize;
 const getBlockId = azureMpuUtils.getBlockId;
@@ -16,10 +16,6 @@ const azureClient = getAzureClient();
 const azureContainerName = getAzureContainerName();
 const expectedMD5 = 'a63c90cc3684ad8b0a2176a6a8fe9005';
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-const azureLocationMismatch = 'azuretestmismatch';
-
 let bucketUtil;
 let s3;
 

tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js

@@ -4,10 +4,17 @@ const async = require('async');
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
 const constants = require('../../../../../../constants');
-const { config } = require('../../../../../../lib/Config');
-const { getAzureClient, getAzureContainerName, convertMD5, memLocation,
-    awsLocation, azureLocation, azureLocation2, azureLocationMismatch } =
-    require('../utils');
+const {
+    describeSkipIfNotMultiple,
+    getAzureClient,
+    getAzureContainerName,
+    convertMD5,
+    memLocation,
+    awsLocation,
+    azureLocation,
+    azureLocation2,
+    azureLocationMismatch,
+} = require('../utils');
 const { createEncryptedBucketPromise } =
     require('../../../lib/utility/createEncryptedBucket');
 
@@ -26,8 +33,6 @@ const azureTimeout = 40000;
 
 let bucketUtil;
 let s3;
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
 
 function putSourceObj(key, location, objSize, cb) {
     const sourceParams = { Bucket: bucket, Key: key,

tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js

@@ -6,10 +6,12 @@ const BucketUtility = require('../../../lib/utility/bucket-util');
 const constants = require('../../../../../../constants');
 const { config } = require('../../../../../../lib/Config');
 const { getRealAwsConfig } = require('../../support/awsConfig');
+const { removeAllVersions } = require('../../../lib/utility/versioning-util');
 const { createEncryptedBucketPromise } =
     require('../../../lib/utility/createEncryptedBucket');
-const { memLocation, awsLocation, awsLocation2, awsLocationMismatch } =
-    require('../utils');
+const { describeSkipIfNotMultiple, awsS3, awsBucket, memLocation, awsLocation,
+    awsLocation2, awsLocationMismatch, awsLocationEncryption }
+    = require('../utils');
 
 const bucket = 'buckettestmultiplebackendobjectcopy';
 const body = Buffer.from('I am a body', 'utf8');
@@ -18,13 +20,8 @@ const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
 const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
 const { versioningEnabled } = require('../../../lib/utility/versioning-util');
 
-const awsLocationEncryption = 'aws-test-encryption';
-
 let bucketUtil;
 let s3;
-let awsS3;
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
 
 function putSourceObj(location, isEmptyObj, cb) {
     const key = `somekey-${Date.now()}`;
@@ -52,8 +49,6 @@ function putSourceObj(location, isEmptyObj, cb) {
 function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey,
 destBucket, destLoc, awsKey, mdDirective, isEmptyObj, awsS3, awsLocation,
 callback) {
-    const awsBucket =
-        config.locationConstraints[awsLocation].details.bucketName;
     const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey };
     const destGetParams = { Bucket: destBucket, Key: destKey };
     const awsParams = { Bucket: awsBucket, Key: awsKey };
@@ -109,32 +104,20 @@ describeSkipIfNotMultiple('MultipleBackend object copy',
 function testSuite() {
     this.timeout(250000);
     withV4(sigCfg => {
-        beforeEach(() => {
-            bucketUtil = new BucketUtility('default', sigCfg);
-            s3 = bucketUtil.s3;
-            const awsConfig = getRealAwsConfig(awsLocation);
-            awsS3 = new AWS.S3(awsConfig);
-            process.stdout.write('Creating bucket\n');
-            if (process.env.ENABLE_KMS_ENCRYPTION === 'true') {
-                s3.createBucketAsync = createEncryptedBucketPromise;
-            }
-            return s3.createBucketAsync({ Bucket: bucket })
-            .catch(err => {
-                process.stdout.write(`Error creating bucket: ${err}\n`);
-                throw err;
-            });
-        });
-
-        afterEach(() => {
-            process.stdout.write('Emptying bucket\n');
-            return bucketUtil.empty(bucket)
-            .then(() => {
-                process.stdout.write('Deleting bucket\n');
-                return bucketUtil.deleteOne(bucket);
-            })
-            .catch(err => {
-                process.stdout.write(`Error in afterEach: ${err}\n`);
-                throw err;
+        const bucketUtil = new BucketUtility('default', sigCfg);
+        const s3 = bucketUtil.s3;
+        beforeEach(done => s3.createBucket({
+            Bucket: bucket,
+            CreateBucketConfiguration: {
+                LocationConstraint: awsLocation,
+            },
+        }, done));
+        afterEach(done => {
+            removeAllVersions({ Bucket: bucket }, err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3.deleteBucket({ Bucket: bucket }, done);
             });
         });
 

tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js

@@ -0,0 +1,384 @@
+const assert = require('assert');
+const async = require('async');
+const withV4 = require('../../support/withV4');
+const BucketUtility = require('../../../lib/utility/bucket-util');
+const constants = require('../../../../../../constants');
+const {
+    describeSkipIfNotMultiple,
+    awsS3,
+    awsBucket,
+    memLocation,
+    fileLocation,
+    awsLocation,
+    enableVersioning,
+    suspendVersioning,
+    putToAwsBackend,
+    awsGetLatestVerId,
+} = require('../utils');
+
+const sourceBucketName = 'buckettestobjectcopyawsversioning-source';
+const destBucketName = 'buckettestobjectcopyawsversioning-dest';
+
+const someBody = Buffer.from('I am a body', 'utf8');
+const wrongVersionBody = 'this is not the content you wanted';
+const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
+const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
+const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
+
+let bucketUtil;
+let s3;
+
+function _getTestMetadata(location) {
+    return {
+        'scal-location-constraint': location,
+        'test-header': 'copyme',
+    };
+}
+
+function putSourceObj(testParams, cb) {
+    const { sourceBucket, sourceLocation, isEmptyObj } = testParams;
+    const sourceKey = `sourcekey-${Date.now()}`;
+    const sourceParams = {
+        Bucket: sourceBucket,
+        Key: sourceKey,
+        Metadata: _getTestMetadata(sourceLocation),
+    };
+    if (!isEmptyObj) {
+        sourceParams.Body = someBody;
+    }
+    s3.putObject(sourceParams, (err, result) => {
+        assert.strictEqual(err, null,
+            `Error putting source object: ${err}`);
+        if (isEmptyObj) {
+            assert.strictEqual(result.ETag, `"${emptyMD5}"`);
+        } else {
+            assert.strictEqual(result.ETag, `"${correctMD5}"`);
+        }
+        Object.assign(testParams, {
+            sourceKey,
+            sourceVersionId: result.VersionId,
+        });
+        cb();
+    });
+}
+
+function copyObject(testParams, cb) {
+    const { sourceBucket, sourceKey, sourceVersionId, sourceVersioningState,
+        destBucket, destLocation, directive, destVersioningState, isEmptyObj }
+        = testParams;
+    const destKey = `destkey-${Date.now()}`;
+    const copyParams = {
+        Bucket: destBucket,
+        Key: destKey,
+        CopySource: `/${sourceBucket}/${sourceKey}`,
+        MetadataDirective: directive,
+        Metadata: {
+            'scal-location-constraint': destLocation,
+        },
+    };
+    if (sourceVersionId) {
+        copyParams.CopySource =
+            `${copyParams.CopySource}?versionId=${sourceVersionId}`;
+    } else if (sourceVersioningState === 'Suspended') {
+        copyParams.CopySource =
+            `${copyParams.CopySource}?versionId=null`;
+    }
+    console.log('===================');
+    console.log('params sent to object copy', copyParams)
+    s3.copyObject(copyParams, (err, data) => {
+        assert.strictEqual(err, null,
+            `Error copying object to destination: ${err}`);
+            console.log('copy object result', data)
+        if (destVersioningState === 'Enabled') {
+            console.log('got version id for dest object', data.VersionId)
+            assert.notEqual(data.VersionId, undefined);
+        } else {
+            assert.strictEqual(data.VersionId, undefined);
+        }
+        const expectedBody = isEmptyObj ? '' : someBody;
+        return awsGetLatestVerId(destKey, expectedBody, (err, awsVersionId) => {
+            Object.assign(testParams, {
+                destKey,
+                destVersionId: data.VersionId,
+                awsVersionId,
+            });
+            if (!data.VersionId && destVersioningState === 'Suspended') {
+                // eslint-disable-next-line no-param-reassign
+                testParams.destVersionId = 'null';
+            }
+            cb();
+        });
+    });
+}
+
+function assertGetObjects(testParams, cb) {
+    console.log('assertGetObjecs');
+    const {
+        sourceBucket,
+        sourceLocation,
+        sourceKey,
+        sourceVersionId,
+        sourceVersioningState,
+        destBucket,
+        destLocation,
+        destKey,
+        destVersionId,
+        destVersioningState,
+        awsVersionId,
+        isEmptyObj,
+        directive,
+    } = testParams;
+    console.log('testParams in assertGetObjects..', testParams)
+    const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey,
+        VersionId: sourceVersionId };
+    const destGetParams = { Bucket: destBucket, Key: destKey,
+        VersionId: destVersionId };
+    const awsParams = { Bucket: awsBucket, Key: destKey,
+        VersionId: awsVersionId };
+
+    async.series([
+        cb => s3.getObject(sourceGetParams, cb),
+        cb => s3.getObject(destGetParams, cb),
+        cb => awsS3.getObject(awsParams, cb),
+    ], (err, results) => {
+        assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`);
+        const [sourceRes, destRes, awsRes] = results;
+        console.log('******** sourceGetParams', sourceGetParams);
+        console.log('==== result of getting source object', sourceRes);
+        console.log('******** destGetParams', destGetParams);
+        console.log('==== result of getting dest object', destRes);
+        console.log('******** awsGetParams', awsParams);
+        console.log('==== result of getting aws object', awsRes)
+        // NOTE: assert version ids?
+        if (isEmptyObj) {
+            assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`);
+            assert.strictEqual(destRes.ETag, `"${emptyMD5}"`);
+            assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`);
+        } else {
+            assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`);
+            assert.strictEqual(destRes.ETag, `"${correctMD5}"`);
+            assert.deepStrictEqual(sourceRes.Body, destRes.Body);
+            assert.strictEqual(awsRes.ETag, `"${correctMD5}"`);
+            assert.deepStrictEqual(sourceRes.Body, awsRes.Body);
+        }
+        if (directive === 'COPY') {
+            assert.deepStrictEqual(sourceRes.Metadata['test-header'],
+                destRes.Metadata['test-header']);
+        } else if (directive === 'REPLACE') {
+            assert.strictEqual(destRes.Metadata['test-header'],
+              undefined);
+        }
+        assert.strictEqual(awsRes.Metadata[locMetaHeader], destLocation);
+        if (directive === 'COPY') {
+            assert.deepStrictEqual(sourceRes.Metadata['test-header'],
+                awsRes.Metadata['test-header']);
+        } else if (directive === 'REPLACE') {
+            assert.strictEqual(awsRes.Metadata['test-header'],
+              undefined);
+        }
+        assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength);
+        assert.strictEqual(sourceRes.Metadata[locMetaHeader], sourceLocation);
+        assert.strictEqual(destRes.Metadata[locMetaHeader], destLocation);
+        cb();
+    });
+}
+
+/*
+const testParams = {
+    sourceBucket: sourceBucketName,
+    sourceLocation: awsLocation,
+    sourceVersioningState: undefined,
+    destBucket: destBucketName,
+    destLocation: awsLocation,
+    destVersioningState: 'Enabled',
+    isEmptyObj: false,
+    directive: 'REPLACE',
+};*/
+
+// describeSkipIfNotMultiple
+describe.only('AWS backend object copy with versioning',
+function testSuite() {
+    this.timeout(250000);
+    withV4(sigCfg => {
+        beforeEach(() => {
+            bucketUtil = new BucketUtility('default', sigCfg);
+            s3 = bucketUtil.s3;
+            process.stdout.write('Creating buckets\n');
+            /* if (process.env.ENABLE_KMS_ENCRYPTION === 'true') {
+                s3.createBucketAsync = createEncryptedBucketPromise;
+            } */
+            return s3.createBucketAsync({ Bucket: sourceBucketName })
+            .then(() => s3.createBucketAsync({ Bucket: destBucketName }))
+            .catch(err => {
+                process.stdout.write(`Error creating bucket: ${err}\n`);
+                throw err;
+            });
+        });
+
+        afterEach(() => bucketUtil.empty(sourceBucketName)
+            .then(() => bucketUtil.deleteOne(sourceBucketName))
+            .catch(err => {
+                process.stdout.write('Error deleting source bucket ' +
+                `in afterEach: ${err}\n`);
+                throw err;
+            })
+            .then(() => bucketUtil.empty(destBucketName))
+            .then(() => bucketUtil.deleteOne(destBucketName))
+        );
+
+        [{
+            directive: 'REPLACE',
+            isEmptyObj: true,
+        }, {
+            directive: 'REPLACE',
+            isEmptyObj: false,
+        }, {
+            directive: 'COPY',
+            isEmptyObj: false,
+        }].forEach(testParams => {
+            Object.assign(testParams, {
+                sourceBucket: sourceBucketName,
+                sourceLocation: awsLocation,
+                destBucket: destBucketName,
+                destLocation: awsLocation,
+            });
+            const { isEmptyObj, directive } = testParams;
+            it(`should copy a${isEmptyObj ? 'n empty' : ''} object from AWS ` +
+            'backend non-versioned bucket to AWS backend versioned bucket ' +
+            `with ${directive} directive`, done => {
+                Object.assign(testParams, {
+                    sourceVersioningState: undefined,
+                    destVersioningState: 'Enabled',
+                });
+                async.waterfall([
+                    next => putSourceObj(testParams, next),
+                    next => enableVersioning(s3, testParams.destBucket, next),
+                    next => copyObject(testParams, next),
+                    // put another version to test and make sure version id from
+                    // copy was stored to get the right version
+                    next => putToAwsBackend(s3, destBucketName,
+                        testParams.destKey, wrongVersionBody, () => next()),
+                    next => assertGetObjects(testParams, next),
+                ], done);
+            });
+
+            it(`should copy ${isEmptyObj ? 'an empty' : ''}version from one ` +
+            `AWS backend versioned bucket to another on ${directive} directive`,
+            done => {
+                Object.assign(testParams, {
+                    sourceVersioningState: 'Enabled',
+                    destVersioningState: 'Enabled',
+                });
+                async.waterfall([
+                    next => enableVersioning(s3, testParams.sourceBucket, next),
+                    next => putSourceObj(testParams, next),
+                    next => enableVersioning(s3, testParams.destBucket, next),
+                    next => copyObject(testParams, next),
+                    // put another version to test and make sure version id from
+                    // copy was stored to get the right version
+                    next => putToAwsBackend(s3, destBucketName,
+                        testParams.destKey, wrongVersionBody, () => next()),
+                    next => assertGetObjects(testParams, next),
+                ], done);
+            });
+
+            it(`should copy ${isEmptyObj ? 'an empty' : ''} null version ` +
+            'from an AWS backend versioned bucket to a non-versioned one with '
+            + `${directive} directive`, done => {
+                Object.assign(testParams, {
+                    sourceVersioningState: 'Suspend',
+                    destVersioningState: 'Suspended',
+                });
+                async.waterfall([
+                    next => suspendVersioning(s3, testParams.sourceBucket,
+                        next),
+                    next => putSourceObj(testParams, next),
+                    next => suspendVersioning(s3, testParams.destBucket, next),
+                    next => copyObject(testParams, next),
+                    next => enableVersioning(s3, testParams.destBucket, next),
+                    // put another version to test and make sure version id from
+                    // copy was stored to get the right version
+                    next => putToAwsBackend(s3, destBucketName,
+                        testParams.destKey, wrongVersionBody, () => next()),
+                    next => assertGetObjects(testParams, next),
+                ], done);
+            });
+
+            it(`should copy a ${isEmptyObj ? 'empty ' : ''}version from a ` +
+            'AWS backend versioned bucket to a non-versioned one with '
+            + `${directive} directive`, done => {
+                Object.assign(testParams, {
+                    sourceVersioningState: 'Enabled',
+                    destVersioningState: 'Suspended',
+                });
+                async.waterfall([
+                    next => enableVersioning(s3, testParams.sourceBucket, next),
+                    next => putSourceObj(testParams, next),
+                    next => suspendVersioning(s3, testParams.destBucket, next),
+                    next => copyObject(testParams, next),
+                    // put another version to test and make sure version id from
+                    // copy was stored to get the right version
+                    next => enableVersioning(s3, testParams.destBucket, next),
+                    next => putToAwsBackend(s3, destBucketName,
+                        testParams.destKey, wrongVersionBody, () => next()),
+                    next => assertGetObjects(testParams, next),
+                ], done);
+            });
+        });
+
+        [{
+            sourceLocation: memLocation,
+            directive: 'REPLACE',
+            isEmptyObj: true,
+        }, {
+            sourceLocation: fileLocation,
+            directive: 'REPLACE',
+            isEmptyObj: true,
+        }, {
+            sourceLocation: memLocation,
+            directive: 'COPY',
+            isEmptyObj: false,
+        }, {
+            sourceLocation: fileLocation,
+            directive: 'COPY',
+            isEmptyObj: false,
+        }].forEach(testParams => {
+            Object.assign(testParams, {
+                sourceBucket: sourceBucketName,
+                sourceVersioningState: 'Enabled',
+                destBucket: sourceBucketName,
+                destLocation: awsLocation,
+                destVersioningState: 'Enabled',
+            });
+            const { sourceLocation, directive, isEmptyObj } = testParams;
+
+            it(`should copy ${isEmptyObj ? 'empty ' : ''}object from ` +
+            `${sourceLocation} to same bucket on AWS backend with ` +
+            `versioning with ${directive}`, done => {
+                async.waterfall([
+                    next => putSourceObj(testParams, next),
+                    next => enableVersioning(s3, testParams.sourceBucket, next),
+                    next => copyObject(testParams, next),
+                    next => assertGetObjects(testParams, next),
+                ], done);
+            });
+
+            it(`should copy a ${isEmptyObj ? 'empty ' : ''}version from ` +
+            `${sourceLocation} to same bucket on AWS backend with ` +
+            `versioning with ${directive} directive`, done => {
+                async.waterfall([
+                    next => enableVersioning(s3, testParams.sourceBucket, next),
+                    // returns a version id which is added to testParams
+                    // to be used in object copy
+                    next => putSourceObj(testParams, next),
+                    next => copyObject(testParams, next),
+                    // put another version to test and make sure version id
+                    // from copy was stored to get the right version
+                    next => putToAwsBackend(s3, destBucketName,
+                        testParams.destKey, wrongVersionBody, () => next()),
+                    next => assertGetObjects(testParams, next),
+                ], done);
+            });
+        });
+    });
+});

tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js

@@ -1,13 +1,10 @@
 const assert = require('assert');
 const async = require('async');
-const AWS = require('aws-sdk');
-const withV4 = require('../support/withV4');
-const BucketUtility = require('../../lib/utility/bucket-util');
-const { config } = require('../../../../../lib/Config');
-const { getRealAwsConfig } = require('../support/awsConfig');
-const { getAzureClient, getAzureContainerName, convertMD5,
-    memLocation, fileLocation, awsLocation, azureLocation } =
-    require('./utils');
+const withV4 = require('../../support/withV4');
+const BucketUtility = require('../../../lib/utility/bucket-util');
+const { describeSkipIfNotMultiple, awsS3, getAzureClient, getAzureContainerName,
+    convertMD5, memLocation, fileLocation, awsLocation, azureLocation } =
+    require('../utils');
 
 const awsBucket = 'multitester555';
 const azureClient = getAzureClient();
@@ -22,9 +19,6 @@ const cloudTimeout = 10000;
 
 let bucketUtil;
 let s3;
-let awsS3;
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
 
 const putParams = { Bucket: bucket, Body: body };
 const testBackends = [memLocation, fileLocation, awsLocation, azureLocation];
@@ -159,8 +153,6 @@ function testSuite() {
     this.timeout(80000);
     withV4(sigCfg => {
         beforeEach(() => {
-            const awsConfig = getRealAwsConfig(awsLocation);
-            awsS3 = new AWS.S3(awsConfig);
             bucketUtil = new BucketUtility('default', sigCfg);
             s3 = bucketUtil.s3;
             return s3.createBucketAsync({ Bucket: bucket })
@@ -274,7 +266,8 @@ function testSuite() {
                 });
             });
 
-            it('should return error on putting tags to object deleted from AWS',
+            it('should not return error on putting tags to object ' +
+            'that has had a delete marker put directly on from AWS',
             done => {
                 const key = `somekey-${Date.now()}`;
                 const params = Object.assign({ Key: key, Metadata:
@@ -289,7 +282,7 @@ function testSuite() {
                             Tagging: putTags };
                         process.stdout.write('Putting object tags\n');
                         s3.putObjectTagging(putTagParams, err => {
-                            assert.strictEqual(err.code, 'InternalError');
+                            assert.strictEqual(err, null);
                             done();
                         });
                     });
@@ -325,8 +318,8 @@ function testSuite() {
                 });
             });
 
-            it('should not return error on getting tags from object deleted ' +
-            'from AWS', done => {
+            it('should not return error on getting tags from object that has ' +
+            'had a delete marker put directly on AWS', done => {
                 const key = `somekey-${Date.now()}`;
                 const params = Object.assign({ Key: key, Tagging: tagString,
                     Metadata: { 'scal-location-constraint': awsLocation } },
@@ -367,8 +360,8 @@ function testSuite() {
                 });
             });
 
-            it('should return error on deleting tags from object deleted ' +
-            'from AWS', done => {
+            it('should not return error on deleting tags from object that ' +
+            'has had delete markers put directly on AWS', done => {
                 const key = `somekey-${Date.now()}`;
                 const params = Object.assign({ Key: key, Tagging: tagString,
                     Metadata: { 'scal-location-constraint': awsLocation } },
@@ -381,7 +374,7 @@ function testSuite() {
                         assert.equal(err, null);
                         const tagParams = { Bucket: bucket, Key: key };
                         s3.deleteObjectTagging(tagParams, err => {
-                            assert.strictEqual(err.code, 'InternalError');
+                            assert.strictEqual(err, null);
                             done();
                         });
                     });

tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js

@@ -0,0 +1,223 @@
+const async = require('async');
+
+const withV4 = require('../../support/withV4');
+const BucketUtility = require('../../../lib/utility/bucket-util');
+const bucket = 'testawsbackendtaggingdeleteversioned';
+
+const { removeAllVersions } = require('../../../lib/utility/versioning-util');
+const {
+    describeSkipIfNotMultiple,
+    awsS3,
+    awsBucket,
+    awsLocation,
+    enableVersioning,
+    putNullVersionsToAws,
+    putVersionsToAws,
+    awsGetLatestVerId,
+    tagging,
+} = require('../utils');
+
+const { putTaggingAndAssert, delTaggingAndAssert, awsGetAssertTags } = tagging;
+const someBody = 'teststring';
+
+describeSkipIfNotMultiple('AWS backend object tagging with versioning :: ' +
+'delete', function testSuite() {
+    this.timeout(30000);
+    const tags = { key1: 'value1', key2: 'value2' };
+
+    withV4(sigCfg => {
+        const bucketUtil = new BucketUtility('default', sigCfg);
+        const s3 = bucketUtil.s3;
+        beforeEach(done => s3.createBucket({
+            Bucket: bucket,
+            CreateBucketConfiguration: {
+                LocationConstraint: awsLocation,
+            },
+        }, done));
+        afterEach(done => {
+            removeAllVersions({ Bucket: bucket }, err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3.deleteBucket({ Bucket: bucket }, done);
+            });
+        });
+
+        it('versioning not configured: should delete a tag set on the ' +
+        'latest version if no version is specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    expectedVersionId: false }, next),
+                (versionId, next) => delTaggingAndAssert(s3, { bucket, key,
+                    expectedVersionId: false }, next),
+                next => awsGetAssertTags({ key, expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning not configured: should delete a tag set on the ' +
+        'version if specified (null)', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: 'null', expectedVersionId: false }, next),
+                (versionId, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId: 'null', expectedVersionId: false }, next),
+                next => awsGetAssertTags({ key, expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning suspended: should delete a tag set on the latest ' +
+        'version if no version is specified', done => {
+            const data = [undefined, 'test1', 'test2'];
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, data, next),
+                (versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, expectedVersionId: 'null' }, next),
+                (versionId, next) => delTaggingAndAssert(s3, { bucket, key,
+                    expectedVersionId: 'null' }, next),
+                next => awsGetAssertTags({ key, expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning suspended: should delete a tag set on a specific ' +
+        'version (null)', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, [undefined],
+                    next),
+                (versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, versionId: 'null', expectedVersionId: 'null' }, next),
+                (versionId, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId: 'null', expectedTags: tags,
+                    expectedVersionId: 'null' }, next),
+                next => awsGetAssertTags({ key, expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning enabled then suspended: should delete a tag set on ' +
+        'a specific (non-null) version if specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '',
+                    (err, awsVid) => next(err, putData.VersionId, awsVid)),
+                (s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key,
+                    [someBody], () => next(null, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, versionId: s3Vid, expectedVersionId: s3Vid }, () =>
+                    next(null, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId: s3Vid, expectedVersionId: s3Vid },
+                    () => next(null, awsVid)),
+                (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
+                    expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should delete a tag set on the latest ' +
+        'version if no version is specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    expectedVersionId: putData.VersionId }, next),
+                (versionId, next) => delTaggingAndAssert(s3, { bucket, key,
+                    expectedVersionId: versionId }, next),
+                next => awsGetAssertTags({ key, expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should delete a tag set on a specific version',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: putData.VersionId,
+                    expectedVersionId: putData.VersionId }, next),
+                (versionId, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId, expectedVersionId: versionId }, next),
+                next => awsGetAssertTags({ key, expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should delete a tag set on a specific ' +
+        'version that is not the latest version', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '',
+                    (err, awsVid) => next(err, putData.VersionId, awsVid)),
+                // put another version
+                (s3Vid, awsVid, next) => s3.putObject({ Bucket: bucket,
+                    Key: key, Body: someBody },
+                    err => next(err, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, versionId: s3Vid, expectedVersionId: s3Vid }, err =>
+                    next(err, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId: s3Vid, expectedVersionId: s3Vid },
+                    () => next(null, awsVid)),
+                (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
+                    expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning suspended then enabled: should delete a tag set on ' +
+        'a specific version (null) if specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, [undefined],
+                    () => next()),
+                next => awsGetLatestVerId(key, '', next),
+                (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody],
+                    () => next(null, awsVid)),
+                (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: 'null', expectedVersionId: 'null' },
+                    () => next(null, awsVid)),
+                (awsVid, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId: 'null', expectedVersionId: 'null' },
+                    () => next(null, awsVid)),
+                (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
+                    expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('should return an InternalError if trying to delete ' +
+        'tags from object that was deleted from AWS directly',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '', next),
+                (awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key, VersionId: awsVid }, next),
+                (delData, next) => delTaggingAndAssert(s3, { bucket, key,
+                    expectedError: 'InternalError' }, next),
+            ], done);
+        });
+
+        it('should return an InternalError if trying to delete ' +
+        'tags from object that was deleted from AWS directly',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '',
+                    (err, awsVid) => next(err, putData.VersionId, awsVid)),
+                (s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
+                (s3Vid, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId: s3Vid, expectedError: 'InternalError' }, next),
+            ], done);
+        });
+    });
+});

tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js

@@ -0,0 +1,285 @@
+const async = require('async');
+
+const withV4 = require('../../support/withV4');
+const BucketUtility = require('../../../lib/utility/bucket-util');
+const bucket = 'testawsbackendtaggingversioned';
+
+const { removeAllVersions } = require('../../../lib/utility/versioning-util');
+const {
+    describeSkipIfNotMultiple,
+    awsS3,
+    awsBucket,
+    awsLocation,
+    enableVersioning,
+    putNullVersionsToAws,
+    putVersionsToAws,
+    awsGetLatestVerId,
+    tagging,
+} = require('../utils');
+
+const { putTaggingAndAssert, getTaggingAndAssert, delTaggingAndAssert,
+    awsGetAssertTags } = tagging;
+const someBody = 'teststring';
+
+describeSkipIfNotMultiple('AWS backend object tagging with versioning',
+function testSuite() {
+    this.timeout(30000);
+    const tags = { key1: 'value1', key2: 'value2' };
+
+    withV4(sigCfg => {
+        const bucketUtil = new BucketUtility('default', sigCfg);
+        const s3 = bucketUtil.s3;
+        beforeEach(done => s3.createBucket({
+            Bucket: bucket,
+            CreateBucketConfiguration: {
+                LocationConstraint: awsLocation,
+            },
+        }, done));
+        afterEach(done => {
+            removeAllVersions({ Bucket: bucket }, err => {
+                if (err) {
+                    return done(err);
+                }
+                return s3.deleteBucket({ Bucket: bucket }, done);
+            });
+        });
+
+        it('versioning not configured: should put/get a tag set on the ' +
+        'latest version if no version is specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    expectedVersionId: false }, next),
+                (versionId, next) => getTaggingAndAssert(s3, { bucket, key,
+                    expectedTags: tags, expectedVersionId: false }, next),
+                (versionId, next) => awsGetAssertTags({ key,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('versioning not configured: should put/get a tag set on a ' +
+        'specific version if specified (null)', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: 'null', expectedVersionId: false }, next),
+                (versionId, next) => getTaggingAndAssert(s3, { bucket, key,
+                    versionId: 'null', expectedTags: tags,
+                    expectedVersionId: false }, next),
+                (versionId, next) => awsGetAssertTags({ key,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('versioning suspended: should put/get a tag set on the latest ' +
+        'version if no version is specified', done => {
+            const data = [undefined, 'test1', 'test2'];
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, data, next),
+                (versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, expectedVersionId: 'null' }, next),
+                (versionId, next) => getTaggingAndAssert(s3, { bucket, key,
+                    expectedTags: tags, expectedVersionId: 'null' }, next),
+                (versionId, next) => awsGetAssertTags({ key,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('versioning suspended: should put/get a tag set on a specific ' +
+        'version (null)', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, [undefined],
+                    next),
+                (versionIds, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, versionId: 'null', expectedVersionId: 'null' }, next),
+                (versionId, next) => getTaggingAndAssert(s3, { bucket, key,
+                    versionId: 'null', expectedTags: tags,
+                    expectedVersionId: 'null' }, next),
+                (versionId, next) => awsGetAssertTags({ key,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('versioning enabled then suspended: should put/get a tag set on ' +
+        'a specific (non-null) version if specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '',
+                    (err, awsVid) => next(err, putData.VersionId, awsVid)),
+                (s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key,
+                    [someBody], () => next(null, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, versionId: s3Vid, expectedVersionId: s3Vid }, () =>
+                    next(null, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
+                    versionId: s3Vid, expectedTags: tags,
+                    expectedVersionId: s3Vid }, () => next(null, awsVid)),
+                (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should put/get a tag set on the latest ' +
+        'version if no version is specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    expectedVersionId: putData.VersionId }, next),
+                (versionId, next) => getTaggingAndAssert(s3, { bucket, key,
+                    expectedTags: tags, expectedVersionId: versionId }, next),
+                (versionId, next) => awsGetAssertTags({ key,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should put/get a tag set on a specific version',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: putData.VersionId,
+                    expectedVersionId: putData.VersionId }, next),
+                (versionId, next) => getTaggingAndAssert(s3, { bucket, key,
+                    versionId, expectedTags: tags,
+                    expectedVersionId: versionId }, next),
+                (versionId, next) => awsGetAssertTags({ key,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should put/get a tag set on a specific version',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: putData.VersionId,
+                    expectedVersionId: putData.VersionId }, next),
+                (versionId, next) => delTaggingAndAssert(s3, { bucket, key,
+                    versionId, expectedVersionId: versionId }, next),
+                next => awsGetAssertTags({ key, expectedTags: {} }, next),
+            ], done);
+        });
+
+        it('versioning enabled: should put/get a tag set on a specific ' +
+        'version that is not the latest version', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '',
+                    (err, awsVid) => next(err, putData.VersionId, awsVid)),
+                // put another version
+                (s3Vid, awsVid, next) => s3.putObject({ Bucket: bucket,
+                    Key: key, Body: someBody },
+                    err => next(err, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, versionId: s3Vid, expectedVersionId: s3Vid }, err =>
+                    next(err, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
+                    versionId: s3Vid, expectedTags: tags,
+                    expectedVersionId: s3Vid }, () => next(null, awsVid)),
+                (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+
+        it('versioning suspended then enabled: should put/get a tag set on ' +
+        'a specific version (null) if specified', done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => putNullVersionsToAws(s3, bucket, key, [undefined],
+                    () => next()),
+                next => awsGetLatestVerId(key, '', next),
+                (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody],
+                    () => next(null, awsVid)),
+                (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: 'null', expectedVersionId: 'null' },
+                    () => next(null, awsVid)),
+                (awsVid, next) => getTaggingAndAssert(s3, { bucket, key,
+                    versionId: 'null', expectedTags: tags,
+                    expectedVersionId: 'null' }, () => next(null, awsVid)),
+                (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid,
+                    expectedTags: tags }, next),
+            ], done);
+        });
+
+        it('should get tags for an object even if it was deleted from ' +
+        'AWS directly (we rely on s3 metadata)',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '', next),
+                (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    expectedVersionId: false }, () => next(null, awsVid)),
+                (awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key, VersionId: awsVid }, next),
+                (delData, next) => getTaggingAndAssert(s3, { bucket, key,
+                    expectedTags: tags, expectedVersionId: false,
+                    getObject: false }, next),
+            ], done);
+        });
+
+        it('should return an InternalError if trying to put ' +
+        'tags from object that was deleted from AWS directly',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '', next),
+                (awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key, VersionId: awsVid }, next),
+                (delData, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    expectedError: 'InternalError' }, next),
+            ], done);
+        });
+
+        it('should get tags for an version even if it was deleted from ' +
+        'AWS directly (we rely on s3 metadata)',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => enableVersioning(s3, bucket, next),
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '',
+                    (err, awsVid) => next(err, putData.VersionId, awsVid)),
+                (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key,
+                    tags, versionId: s3Vid, expectedVersionId: s3Vid },
+                    () => next(null, s3Vid, awsVid)),
+                (s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
+                (s3Vid, next) => getTaggingAndAssert(s3, { bucket, key,
+                    versionId: s3Vid, expectedTags: tags,
+                    expectedVersionId: s3Vid, getObject: false }, next),
+            ], done);
+        });
+
+        it('should return an InternalError if trying to put ' +
+        'tags on version that was deleted from AWS directly',
+        done => {
+            const key = `somekey-${Date.now()}`;
+            async.waterfall([
+                next => s3.putObject({ Bucket: bucket, Key: key }, next),
+                (putData, next) => awsGetLatestVerId(key, '',
+                    (err, awsVid) => next(err, putData.VersionId, awsVid)),
+                (s3Vid, awsVid, next) => awsS3.deleteObject({ Bucket: awsBucket,
+                    Key: key, VersionId: awsVid }, err => next(err, s3Vid)),
+                (s3Vid, next) => putTaggingAndAssert(s3, { bucket, key, tags,
+                    versionId: s3Vid, expectedError: 'InternalError' }, next),
+            ], done);
+        });
+    });
+});

tests/functional/aws-node-sdk/test/multipleBackend/put/put.js

@@ -1,17 +1,15 @@
 const assert = require('assert');
-const AWS = require('aws-sdk');
 const async = require('async');
 
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
 const { config } = require('../../../../../../lib/Config');
-const { getRealAwsConfig } = require('../../support/awsConfig');
 const { createEncryptedBucketPromise } =
     require('../../../lib/utility/createEncryptedBucket');
 const { versioningEnabled } = require('../../../lib/utility/versioning-util');
 
-const { awsLocation, memLocation, fileLocation } = require('../utils');
-const awsLocationEncryption = 'aws-test-encryption';
+const { describeSkipIfNotMultiple, awsS3, awsBucket, awsLocation,
+    awsLocationEncryption, memLocation, fileLocation } = require('../utils');
 const bucket = 'buckettestmultiplebackendput';
 const body = Buffer.from('I am a body', 'utf8');
 const bigBody = Buffer.alloc(10485760);
@@ -23,13 +21,9 @@ const bigAWSMD5 = 'a7d414b9133d6483d9a1c4e04e856e3b-2';
 
 let bucketUtil;
 let s3;
-let awsS3;
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
 
 const awsTimeout = 30000;
 const retryTimeout = 10000;
-let awsBucket;
 
 function awsGetCheck(objectKey, s3MD5, awsMD5, location, cb) {
     process.stdout.write('Getting object\n');
@@ -113,12 +107,6 @@ describe('MultipleBackend put object', function testSuite() {
             if (!process.env.S3_END_TO_END) {
                 this.retries(2);
             }
-            before(() => {
-                awsBucket = config.locationConstraints[awsLocation].
-                  details.bucketName;
-                const awsConfig = getRealAwsConfig(awsLocation);
-                awsS3 = new AWS.S3(awsConfig);
-            });
 
             it('should return an error to put request without a valid ' +
                 'location constraint', done => {

tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js

@@ -3,10 +3,17 @@ const async = require('async');
 
 const withV4 = require('../../support/withV4');
 const BucketUtility = require('../../../lib/utility/bucket-util');
-const { uniqName, getAzureClient, getAzureContainerName, getAzureKeys,
-    convertMD5, fileLocation, azureLocation, azureLocationMismatch }
-    = require('../utils');
-const { config } = require('../../../../../../lib/Config');
+const {
+    describeSkipIfNotMultiple,
+    uniqName,
+    getAzureClient,
+    getAzureContainerName,
+    getAzureKeys,
+    convertMD5,
+    fileLocation,
+    azureLocation,
+    azureLocationMismatch,
+} = require('../utils');
 
 const keyObject = 'putazure';
 const azureClient = getAzureClient();
@@ -16,9 +23,6 @@ const { versioningEnabled } = require('../../../lib/utility/versioning-util');
 const normalBody = Buffer.from('I am a body', 'utf8');
 const normalMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
 
-const describeSkipIfNotMultiple = (config.backends.data !== 'multiple'
-    || process.env.S3_END_TO_END) ? describe.skip : describe;
-
 const keys = getAzureKeys();
 /* eslint-disable camelcase */
 const azureMetadata = {

tests/functional/aws-node-sdk/test/multipleBackend/utils.js

@@ -1,9 +1,12 @@
 const assert = require('assert');
 const crypto = require('crypto');
+const { errors } = require('arsenal');
+const AWS = require('aws-sdk');
 
 const async = require('async');
 const azure = require('azure-storage');
 
+const { getRealAwsConfig } = require('../support/awsConfig');
 const { config } = require('../../../../../lib/Config');
 
 const memLocation = 'mem-test';
@@ -14,10 +17,35 @@ const awsLocationMismatch = 'aws-test-mismatch';
 const azureLocation = 'azuretest';
 const azureLocation2 = 'azuretest2';
 const azureLocationMismatch = 'azuretestmismatch';
+const awsLocationEncryption = 'aws-test-encryption';
 const versioningEnabled = { Status: 'Enabled' };
 const versioningSuspended = { Status: 'Suspended' };
+let describeSkipIfNotMultiple = describe.skip;
+let awsS3;
+let awsBucket;
+
+if (config.backends.data === 'multiple' && !process.env.S3_END_TO_END) {
+    describeSkipIfNotMultiple = describe;
+    // can only get real aws config if not running end-to-end
+    const awsConfig = getRealAwsConfig(awsLocation);
+    awsS3 = new AWS.S3(awsConfig);
+    awsBucket = config.locationConstraints[awsLocation].details.bucketName;
+}
+
+function _assertErrorResult(err, expectedError, desc) {
+    if (!expectedError) {
+        assert.strictEqual(err, null, `got error for ${desc}: ${err}`);
+        return;
+    }
+    assert(err, `expected ${expectedError} but found no error`);
+    assert.strictEqual(err.code, expectedError);
+    assert.strictEqual(err.statusCode, errors[expectedError].code);
+}
 
 const utils = {
+    describeSkipIfNotMultiple,
+    awsS3,
+    awsBucket,
     fileLocation,
     memLocation,
     awsLocation,
@@ -26,6 +54,7 @@ const utils = {
     azureLocation,
     azureLocation2,
     azureLocationMismatch,
+    awsLocationEncryption,
 };
 
 utils.uniqName = name => `${name}${new Date().getTime()}`;
@@ -119,6 +148,14 @@ utils.expectedETag = (body, getStringified = true) => {
     return `"${eTagValue}"`;
 };
 
+utils.putToAwsBackend = (s3, bucket, key, body, cb) => {
+    console.log('aaaain put to aws backend')
+    console.log('what is cb in putToAWsBackend?', cb)
+    s3.putObject({ Bucket: bucket, Key: key, Body: body,
+    Metadata: { 'scal-location-constraint': awsLocation } },
+        (err, result) => cb(err, result.VersionId));
+};
+
 utils.enableVersioning = (s3, bucket, cb) => {
     s3.putBucketVersioning({ Bucket: bucket,
         VersioningConfiguration: versioningEnabled }, err => {
@@ -139,13 +176,11 @@ utils.suspendVersioning = (s3, bucket, cb) => {
 
 utils.mapToAwsPuts = (s3, bucket, key, dataArray, cb) => {
     async.mapSeries(dataArray, (data, next) => {
-        s3.putObject({ Bucket: bucket, Key: key, Body: data,
-        Metadata: { 'scal-location-constraint': awsLocation } },
-            next);
+        utils.putToAwsBackend(s3, bucket, key, data, next);
     }, (err, results) => {
         assert.strictEqual(err, null, 'Expected success ' +
             `putting object, got error ${err}`);
-        cb(null, results.map(result => result.VersionId));
+        cb(null, results);
     });
 };
 
@@ -161,4 +196,143 @@ utils.putNullVersionsToAws = (s3, bucket, key, versions, cb) => {
     });
 };
 
+utils.getAndAssertResult = (s3, params, cb) => {
+    console.log('params', params);
+    const { bucket, key, body, versionId, expectedVersionId, expectedTagCount,
+    expectedError } = params;
+    s3.getObject({ Bucket: bucket, Key: key, VersionId: versionId },
+        (err, data) => {
+            _assertErrorResult(err, expectedError, 'putting tags');
+            console.log('expected error...', expectedError)
+            if (expectedError) {
+                console.log('do we get here?', expectedError)
+                return cb();
+            }
+            console.log('we get to callback from get request to s3...')
+            assert.strictEqual(err, null, 'Expected success ' +
+                `getting object, got error ${err}`);
+            if (body) {
+                assert(data.Body, 'expected object body in response');
+                const expectedMD5 = utils.expectedETag(body, false);
+                const resultMD5 = utils.expectedETag(data.Body, false);
+                assert.strictEqual(resultMD5, expectedMD5);
+            }
+            if (!expectedVersionId) {
+                assert.strictEqual(data.VersionId, undefined);
+            } else {
+                assert.strictEqual(data.VersionId, expectedVersionId);
+            }
+            if (expectedTagCount && expectedTagCount === '0') {
+                assert.strictEqual(data.TagCount, undefined);
+            } else if (expectedTagCount) {
+                assert.strictEqual(data.TagCount, expectedTagCount);
+            }
+            return cb();
+        });
+};
+
+utils.awsGetLatestVerId = (key, body, cb) =>
+    awsS3.getObject({ Bucket: awsBucket, Key: key }, (err, result) => {
+        assert.strictEqual(err, null, 'Expected success ' +
+            `getting object from AWS, got error ${err}`);
+        const resultMD5 = utils.expectedETag(result.Body, false);
+        const expectedMD5 = utils.expectedETag(body, false);
+        assert.strictEqual(resultMD5, expectedMD5);
+        return cb(null, result.VersionId);
+    });
+
+
+utils.tagging = {};
+
+function _getTaggingConfig(tags) {
+    return {
+        // eslint-disable-next-line arrow-body-style
+        TagSet: Object.keys(tags).map(key => {
+            return {
+                Key: key,
+                Value: tags[key],
+            };
+        }),
+    };
+}
+
+utils.tagging.putTaggingAndAssert = (s3, params, cb) => {
+    const { bucket, key, tags, versionId, expectedVersionId,
+        expectedError } = params;
+    const taggingConfig = _getTaggingConfig(tags);
+    return s3.putObjectTagging({ Bucket: bucket, Key: key, VersionId: versionId,
+        Tagging: taggingConfig }, (err, data) => {
+        _assertErrorResult(err, expectedError, 'putting tags');
+        if (expectedError) {
+            return cb();
+        }
+        assert.strictEqual(err, null, `got error for putting tags: ${err}`);
+        if (expectedVersionId) {
+            assert.strictEqual(data.VersionId, expectedVersionId);
+        } else {
+            assert.strictEqual(data.VersionId, undefined);
+        }
+        return cb(null, data.VersionId);
+    });
+};
+
+utils.tagging.getTaggingAndAssert = (s3, params, cb) => {
+    const { bucket, key, expectedTags, versionId, expectedVersionId,
+        expectedError, getObject } = params;
+    s3.getObjectTagging({ Bucket: bucket, Key: key, VersionId: versionId },
+        (err, data) => {
+            _assertErrorResult(err, expectedError, 'putting tags');
+            if (expectedError) {
+                return cb();
+            }
+            const expectedTagResult = _getTaggingConfig(expectedTags);
+            const expectedTagCount = `${Object.keys(expectedTags).length}`;
+            assert.strictEqual(err, null, `got error for putting tags: ${err}`);
+            if (expectedVersionId) {
+                assert.strictEqual(data.VersionId, expectedVersionId);
+            } else {
+                assert.strictEqual(data.VersionId, undefined);
+            }
+            assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet);
+            if (getObject === false) {
+                return process.nextTick(cb, null, data.VersionId);
+            }
+            return utils.getAndAssertResult(s3, { bucket, key, versionId,
+                expectedVersionId, expectedTagCount },
+                () => cb(null, data.VersionId));
+        });
+};
+
+utils.tagging.delTaggingAndAssert = (s3, params, cb) => {
+    const { bucket, key, versionId, expectedVersionId, expectedError } = params;
+    return s3.deleteObjectTagging({ Bucket: bucket, Key: key,
+        VersionId: versionId }, (err, data) => {
+        _assertErrorResult(err, expectedError, 'putting tags');
+        if (expectedError) {
+            return cb();
+        }
+        assert.strictEqual(err, null, `got error for putting tags: ${err}`);
+        if (expectedVersionId) {
+            assert.strictEqual(data.VersionId, expectedVersionId);
+        } else {
+            assert.strictEqual(data.VersionId, undefined);
+        }
+        return utils.tagging.getTaggingAndAssert(s3, { bucket, key, versionId,
+            expectedVersionId, expectedTags: {} }, () => cb());
+    });
+};
+
+utils.tagging.awsGetAssertTags = (params, cb) => {
+    const { key, versionId, expectedTags } = params;
+    const expectedTagResult = _getTaggingConfig(expectedTags);
+    awsS3.getObjectTagging({ Bucket: awsBucket, Key: key,
+        VersionId: versionId }, (err, data) => {
+        assert.strictEqual(err, null, 'got unexpected error getting ' +
+            `tags directly from AWS: ${err}`);
+        assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet);
+        return cb();
+    });
+};
+
+
 module.exports = utils;

tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js

@@ -59,7 +59,7 @@ describe('Put object tagging with versioning', () => {
             });
         });
 
-        it('should not create version puting object tags on a ' +
+        it('should not create version putting object tags on a ' +
         ' version-enabled bucket where no version id is specified ', done => {
             async.waterfall([
                 next => s3.putBucketVersioning({ Bucket: bucketName,