Merge branch 'bugfix/ARSN-143/include-json-files-in-tsc-include' into tmp/octopus/w/8.1/bugfix/ARSN-143/include-json-files-in-tsc-include

When the last version of an object is deleted
a placeholder master key is created and is kept
for 15 seconds before it gets repaired.

To avoid listing the PHD keys we added a transform
stream that replaces PHD master keys with the last
version of that object

Added tests for PHD cases + fixed unit tests

.github/workflows/tests.yaml

@@ -39,10 +39,14 @@ jobs:
       run: yarn --silent lint -- --max-warnings 0
     - name: lint markdown
       run: yarn --silent lint_md
-    - name: run unit tests
-      run: yarn test
+    - name: add hostname
+      run: |
+        sudo sh -c "echo '127.0.0.1 testrequestbucket.localhost' >> /etc/hosts"
+    - name: test and coverage
+      run: yarn --silent coverage
     - name: run functional tests
       run: yarn ft_test
+    - uses: codecov/codecov-action@v2
     - name: run executables tests
       run: yarn install && yarn test
       working-directory: 'lib/executables/pensieveCreds/'

README.md

@@ -1,5 +1,7 @@
 # Arsenal
 
+[![codecov](https://codecov.io/gh/scality/Arsenal/branch/development/8.1/graph/badge.svg?token=X0esXhJSwb)](https://codecov.io/gh/scality/Arsenal)
+
 Common utilities for the S3 project components
 
 Within this repository, you will be able to find the shared libraries for the

documentation/BucketInfoModelVersion.md

@@ -85,6 +85,66 @@ Used to store the bucket lifecycle configuration info
 
 ### Properties Added
 
+```javascript
+this._uid = uid || uuid();
+```
+
+### Usage
+
+Used to set a unique identifier on a bucket
+
+## Model version 8
+
+### Properties Added
+
+```javascript
+this._readLocationConstraint = readLocationConstraint || null;
+```
+
+### Usage
+
+Used to store default read location of the bucket
+
+## Model version 9
+
+### Properties Added
+
+```javascript
+this._isNFS = isNFS || null;
+```
+
+### Usage
+
+Used to determine whether the bucket may be accessed through NFS
+
+## Model version 10
+
+### Properties Added
+
+```javascript
+this._ingestion = ingestionConfig || null;
+```
+
+### Usage
+
+Used to store the ingestion status of a bucket
+
+## Model version 11
+
+### Properties Added
+
+```javascript
+this._azureInfo = azureInfo || null;
+```
+
+### Usage
+
+Used to store Azure storage account specific information
+
+## Model version 12
+
+### Properties Added
+
 ```javascript
 this._objectLockEnabled = objectLockEnabled || false;
 this._objectLockConfiguration = objectLockConfiguration || null;
@@ -95,7 +155,7 @@ this._objectLockConfiguration = objectLockConfiguration || null;
 Used to determine whether object lock capabilities are enabled on a bucket and
 to store the object lock configuration of the bucket
 
-## Model version 8
+## Model version 13
 
 ### Properties Added
 
@@ -107,7 +167,7 @@ this._notificationConfiguration = notificationConfiguration || null;
 
 Used to store the bucket notification configuration info
 
-## Model version 9
+## Model version 14
 
 ### Properties Added
 
@@ -118,15 +178,3 @@ this._serverSideEncryption.configuredMasterKeyId = configuredMasterKeyId || unde
 ### Usage
 
 Used to store the users configured KMS key id
-
-## Model version 10
-
-### Properties Added
-
-```javascript
-this._uid = uid || uuid();
-```
-
-### Usage
-
-Used to set a unique identifier on a bucket

errors/arsenalErrors.json

@@ -26,7 +26,7 @@
   },
   "BucketAlreadyOwnedByYou": {
     "code": 409,
-    "description": "Your previous request to create the named bucket succeeded and you already own it. You get this error in all AWS regions except US Standard, us-east-1. In us-east-1 region, you will get 200 OK, but it is no-op (if bucket exists S3 will not do anything)."
+    "description": "A bucket with this name exists and is already owned by you"
   },
   "BucketNotEmpty": {
     "code": 409,
@@ -403,6 +403,10 @@
     "code": 409,
     "description": "The request was rejected because it attempted to create a resource that already exists."
   },
+  "KeyAlreadyExists": {
+    "code": 409,
+    "description": "The request was rejected because it attempted to create a resource that already exists."
+  },
   "ServiceFailure": {
     "code": 500,
     "description": "Server error: the request processing has failed because of an unknown error, exception or failure."
@@ -760,5 +764,10 @@
   "ReadOnly": {
     "description": "trying to write to read only back-end",
     "code": 403
+  },
+  "_comment": "----------------------- authbackend -----------------------",
+  "AuthMethodNotImplemented": {
+    "description": "AuthMethodNotImplemented",
+    "code": 501
   }
 }

greenkeeper.json

@@ -0,0 +1,28 @@
+{
+  "groups": {
+    "default": {
+      "packages": [
+        "lib/executables/pensieveCreds/package.json",
+        "package.json"
+      ]
+    }
+  },
+  "branchPrefix": "improvement/greenkeeper.io/",
+  "commitMessages": {
+    "initialBadge": "docs(readme): add Greenkeeper badge",
+    "initialDependencies": "chore(package): update dependencies",
+    "initialBranches": "chore(bert-e): whitelist greenkeeper branches",
+    "dependencyUpdate": "fix(package): update ${dependency} to version ${version}",
+    "devDependencyUpdate": "chore(package): update ${dependency} to version ${version}",
+    "dependencyPin": "fix: pin ${dependency} to ${oldVersionResolved}",
+    "devDependencyPin": "chore: pin ${dependency} to ${oldVersionResolved}",
+    "closes": "\n\nCloses #${number}"
+  },
+  "ignore": [
+    "ajv",
+    "eslint",
+    "eslint-plugin-react",
+    "eslint-config-airbnb",
+    "eslint-config-scality"
+  ]
+}

index.js

@@ -0,0 +1,202 @@
+module.exports = {
+    auth: require('./lib/auth/auth'),
+    constants: require('./lib/constants'),
+    db: require('./lib/db'),
+    errors: require('./lib/errors.js'),
+    errorUtils: require('./lib/errorUtils'),
+    shuffle: require('./lib/shuffle'),
+    stringHash: require('./lib/stringHash'),
+    ipCheck: require('./lib/ipCheck'),
+    jsutil: require('./lib/jsutil'),
+    https: {
+        ciphers: require('./lib/https/ciphers.js'),
+        dhparam: require('./lib/https/dh2048.js'),
+    },
+    algorithms: {
+        list: require('./lib/algos/list/exportAlgos'),
+        listTools: {
+            DelimiterTools: require('./lib/algos/list/tools'),
+            Skip: require('./lib/algos/list/skip'),
+        },
+        cache: {
+            LRUCache: require('./lib/algos/cache/LRUCache'),
+        },
+        stream: {
+            MergeStream: require('./lib/algos/stream/MergeStream'),
+        },
+        SortedSet: require('./lib/algos/set/SortedSet'),
+    },
+    policies: {
+        evaluators: require('./lib/policyEvaluator/evaluator.js'),
+        validateUserPolicy: require('./lib/policy/policyValidator')
+            .validateUserPolicy,
+        evaluatePrincipal: require('./lib/policyEvaluator/principal'),
+        RequestContext: require('./lib/policyEvaluator/RequestContext.js'),
+        requestUtils: require('./lib/policyEvaluator/requestUtils'),
+        actionMaps: require('./lib/policyEvaluator/utils/actionMaps'),
+    },
+    Clustering: require('./lib/Clustering'),
+    testing: {
+        matrix: require('./lib/testing/matrix.js'),
+    },
+    versioning: {
+        VersioningConstants: require('./lib/versioning/constants.js')
+            .VersioningConstants,
+        Version: require('./lib/versioning/Version.js').Version,
+        VersionID: require('./lib/versioning/VersionID.js'),
+        WriteGatheringManager: require('./lib/versioning/WriteGatheringManager.js'),
+        WriteCache: require('./lib/versioning/WriteCache.js'),
+        VersioningRequestProcessor: require('./lib/versioning/VersioningRequestProcessor.js'),
+    },
+    network: {
+        http: {
+            server: require('./lib/network/http/server'),
+            utils: require('./lib/network/http/utils'),
+        },
+        rpc: require('./lib/network/rpc/rpc'),
+        level: require('./lib/network/rpc/level-net'),
+        rest: {
+            RESTServer: require('./lib/network/rest/RESTServer'),
+            RESTClient: require('./lib/network/rest/RESTClient'),
+        },
+        RoundRobin: require('./lib/network/RoundRobin'),
+        probe: {
+            ProbeServer: require('./lib/network/probe/ProbeServer'),
+            HealthProbeServer:
+                require('./lib/network/probe/HealthProbeServer.js'),
+            Utils: require('./lib/network/probe/Utils.js'),
+        },
+        kmip: require('./lib/network/kmip'),
+        kmipClient: require('./lib/network/kmip/Client'),
+    },
+    s3routes: {
+        routes: require('./lib/s3routes/routes'),
+        routesUtils: require('./lib/s3routes/routesUtils'),
+    },
+    s3middleware: {
+        userMetadata: require('./lib/s3middleware/userMetadata'),
+        convertToXml: require('./lib/s3middleware/convertToXml'),
+        escapeForXml: require('./lib/s3middleware/escapeForXml'),
+        objectLegalHold: require('./lib/s3middleware/objectLegalHold'),
+        tagging: require('./lib/s3middleware/tagging'),
+        checkDateModifiedHeaders:
+            require('./lib/s3middleware/validateConditionalHeaders')
+                .checkDateModifiedHeaders,
+        validateConditionalHeaders:
+            require('./lib/s3middleware/validateConditionalHeaders')
+                .validateConditionalHeaders,
+        MD5Sum: require('./lib/s3middleware/MD5Sum'),
+        NullStream: require('./lib/s3middleware/nullStream'),
+        objectUtils: require('./lib/s3middleware/objectUtils'),
+        azureHelper: {
+            mpuUtils:
+                require('./lib/s3middleware/azureHelpers/mpuUtils'),
+            ResultsCollector:
+                require('./lib/s3middleware/azureHelpers/ResultsCollector'),
+            SubStreamInterface:
+                require('./lib/s3middleware/azureHelpers/SubStreamInterface'),
+        },
+        prepareStream: require('./lib/s3middleware/prepareStream'),
+        processMpuParts: require('./lib/s3middleware/processMpuParts'),
+        retention: require('./lib/s3middleware/objectRetention'),
+        lifecycleHelpers: require('./lib/s3middleware/lifecycleHelpers'),
+    },
+    storage: {
+        metadata: {
+            MetadataWrapper: require('./lib/storage/metadata/MetadataWrapper'),
+            bucketclient: {
+                BucketClientInterface:
+                require('./lib/storage/metadata/bucketclient/' +
+                    'BucketClientInterface'),
+                LogConsumer:
+                require('./lib/storage/metadata/bucketclient/LogConsumer'),
+            },
+            file: {
+                BucketFileInterface:
+                require('./lib/storage/metadata/file/BucketFileInterface'),
+                MetadataFileServer:
+                require('./lib/storage/metadata/file/MetadataFileServer'),
+                MetadataFileClient:
+                require('./lib/storage/metadata/file/MetadataFileClient'),
+            },
+            inMemory: {
+                metastore:
+                require('./lib/storage/metadata/in_memory/metastore'),
+                metadata: require('./lib/storage/metadata/in_memory/metadata'),
+                bucketUtilities:
+                require('./lib/storage/metadata/in_memory/bucket_utilities'),
+            },
+            mongoclient: {
+                MongoClientInterface:
+                require('./lib/storage/metadata/mongoclient/' +
+                    'MongoClientInterface'),
+                LogConsumer:
+                require('./lib/storage/metadata/mongoclient/LogConsumer'),
+            },
+            proxy: {
+                Server: require('./lib/storage/metadata/proxy/Server'),
+            },
+        },
+        data: {
+            DataWrapper: require('./lib/storage/data/DataWrapper'),
+            MultipleBackendGateway:
+            require('./lib/storage/data/MultipleBackendGateway'),
+            parseLC: require('./lib/storage/data/LocationConstraintParser'),
+            file: {
+                DataFileStore:
+                require('./lib/storage/data/file/DataFileStore'),
+                DataFileInterface:
+                require('./lib/storage/data/file/DataFileInterface'),
+            },
+            external: {
+                AwsClient: require('./lib/storage/data/external/AwsClient'),
+                AzureClient: require('./lib/storage/data/external/AzureClient'),
+                GcpClient: require('./lib/storage/data/external/GcpClient'),
+                GCP: require('./lib/storage/data/external/GCP/GcpService'),
+                GcpUtils: require('./lib/storage/data/external/GCP/GcpUtils'),
+                GcpSigner: require('./lib/storage/data/external/GCP/GcpSigner'),
+                PfsClient: require('./lib/storage/data/external/PfsClient'),
+                backendUtils: require('./lib/storage/data/external/utils'),
+            },
+            inMemory: {
+                datastore: require('./lib/storage/data/in_memory/datastore'),
+            },
+        },
+        utils: require('./lib/storage/utils'),
+    },
+    models: {
+        BackendInfo: require('./lib/models/BackendInfo'),
+        BucketInfo: require('./lib/models/BucketInfo'),
+        BucketAzureInfo: require('./lib/models/BucketAzureInfo'),
+        ObjectMD: require('./lib/models/ObjectMD'),
+        ObjectMDLocation: require('./lib/models/ObjectMDLocation'),
+        ObjectMDAzureInfo: require('./lib/models/ObjectMDAzureInfo'),
+        ARN: require('./lib/models/ARN'),
+        WebsiteConfiguration: require('./lib/models/WebsiteConfiguration'),
+        ReplicationConfiguration:
+          require('./lib/models/ReplicationConfiguration'),
+        LifecycleConfiguration:
+            require('./lib/models/LifecycleConfiguration'),
+        LifecycleRule: require('./lib/models/LifecycleRule'),
+        BucketPolicy: require('./lib/models/BucketPolicy'),
+        ObjectLockConfiguration:
+            require('./lib/models/ObjectLockConfiguration'),
+        NotificationConfiguration:
+            require('./lib/models/NotificationConfiguration'),
+    },
+    metrics: {
+        StatsClient: require('./lib/metrics/StatsClient'),
+        StatsModel: require('./lib/metrics/StatsModel'),
+        RedisClient: require('./lib/metrics/RedisClient'),
+        ZenkoMetrics: require('./lib/metrics/ZenkoMetrics'),
+    },
+    pensieve: {
+        credentialUtils: require('./lib/executables/pensieveCreds/utils'),
+    },
+    stream: {
+        readJSONStreamObject: require('./lib/stream/readJSONStreamObject'),
+    },
+    patches: {
+        locationConstraints: require('./lib/patches/locationConstraints'),
+    },
+};

index.ts

@@ -2,6 +2,7 @@ export const auth = require('./lib/auth/auth');
 export const constants = require('./lib/constants');
 export const db = require('./lib/db');
 export const errors = require('./lib/errors.js');
+export const errorUtils = require('./lib/errorUtils');
 export const shuffle = require('./lib/shuffle');
 export const stringHash = require('./lib/stringHash');
 export const ipCheck = require('./lib/ipCheck');
@@ -14,15 +15,10 @@ export const https = {
 };
 
 export const algorithms = {
-    list: {
-        Basic: require('./lib/algos/list/basic').List,
-        Delimiter: require('./lib/algos/list/delimiter').Delimiter,
-        DelimiterVersions: require('./lib/algos/list/delimiterVersions').DelimiterVersions,
-        DelimiterMaster: require('./lib/algos/list/delimiterMaster').DelimiterMaster,
-        MPU: require('./lib/algos/list/MPU').MultipartUploads,
-    },
+    list: require('./lib/algos/list/exportAlgos'),
     listTools: {
         DelimiterTools: require('./lib/algos/list/tools'),
+        Skip: require('./lib/algos/list/skip'),
     },
     cache: {
         LRUCache: require('./lib/algos/cache/LRUCache'),
@@ -58,6 +54,7 @@ export const versioning = {
 export const network = {
     http: {
         server: require('./lib/network/http/server'),
+        utils: require('./lib/network/http/utils'),
     },
     rpc: require('./lib/network/rpc/rpc'),
     level: require('./lib/network/rpc/level-net'),
@@ -65,10 +62,13 @@ export const network = {
         RESTServer: require('./lib/network/rest/RESTServer'),
         RESTClient: require('./lib/network/rest/RESTClient'),
     },
+    RoundRobin: require('./lib/network/RoundRobin'),
     probe: {
         ProbeServer: require('./lib/network/probe/ProbeServer'),
+        HealthProbeServer:
+            require('./lib/network/probe/HealthProbeServer.js'),
+        Utils: require('./lib/network/probe/Utils.js'),
     },
-    RoundRobin: require('./lib/network/RoundRobin'),
     kmip: require('./lib/network/kmip'),
     kmipClient: require('./lib/network/kmip/Client'),
 };
@@ -84,16 +84,24 @@ export const s3middleware = {
     escapeForXml: require('./lib/s3middleware/escapeForXml'),
     objectLegalHold: require('./lib/s3middleware/objectLegalHold'),
     tagging: require('./lib/s3middleware/tagging'),
+    checkDateModifiedHeaders:
+        require('./lib/s3middleware/validateConditionalHeaders')
+            .checkDateModifiedHeaders,
     validateConditionalHeaders:
-        require('./lib/s3middleware/validateConditionalHeaders').validateConditionalHeaders,
+        require('./lib/s3middleware/validateConditionalHeaders')
+            .validateConditionalHeaders,
     MD5Sum: require('./lib/s3middleware/MD5Sum'),
     NullStream: require('./lib/s3middleware/nullStream'),
     objectUtils: require('./lib/s3middleware/objectUtils'),
     azureHelper: {
-        mpuUtils: require('./lib/s3middleware/azureHelpers/mpuUtils'),
-        ResultsCollector: require('./lib/s3middleware/azureHelpers/ResultsCollector'),
-        SubStreamInterface: require('./lib/s3middleware/azureHelpers/SubStreamInterface'),
+        mpuUtils:
+            require('./lib/s3middleware/azureHelpers/mpuUtils'),
+        ResultsCollector:
+            require('./lib/s3middleware/azureHelpers/ResultsCollector'),
+        SubStreamInterface:
+            require('./lib/s3middleware/azureHelpers/SubStreamInterface'),
     },
+    prepareStream: require('./lib/s3middleware/prepareStream'),
     processMpuParts: require('./lib/s3middleware/processMpuParts'),
     retention: require('./lib/s3middleware/objectRetention'),
     lifecycleHelpers: require('./lib/s3middleware/lifecycleHelpers'),
@@ -164,17 +172,24 @@ export const storage = {
 };
 
 export const models = {
+    BackendInfo: require('./lib/models/BackendInfo'),
     BucketInfo: require('./lib/models/BucketInfo'),
+    BucketAzureInfo: require('./lib/models/BucketAzureInfo'),
     ObjectMD: require('./lib/models/ObjectMD'),
     ObjectMDLocation: require('./lib/models/ObjectMDLocation'),
+    ObjectMDAzureInfo: require('./lib/models/ObjectMDAzureInfo'),
     ARN: require('./lib/models/ARN'),
     WebsiteConfiguration: require('./lib/models/WebsiteConfiguration'),
-    ReplicationConfiguration: require('./lib/models/ReplicationConfiguration'),
-    LifecycleConfiguration: require('./lib/models/LifecycleConfiguration'),
+    ReplicationConfiguration:
+      require('./lib/models/ReplicationConfiguration'),
+    LifecycleConfiguration:
+        require('./lib/models/LifecycleConfiguration'),
     LifecycleRule: require('./lib/models/LifecycleRule'),
     BucketPolicy: require('./lib/models/BucketPolicy'),
-    ObjectLockConfiguration: require('./lib/models/ObjectLockConfiguration'),
-    NotificationConfiguration: require('./lib/models/NotificationConfiguration'),
+    ObjectLockConfiguration:
+        require('./lib/models/ObjectLockConfiguration'),
+    NotificationConfiguration:
+        require('./lib/models/NotificationConfiguration'),
 };
 
 export const metrics = {
@@ -191,3 +206,7 @@ export const pensieve = {
 export const stream = {
     readJSONStreamObject: require('./lib/stream/readJSONStreamObject'),
 };
+
+export const patches = {
+    locationConstraints: require('./lib/patches/locationConstraints'),
+};

lib/auth/Vault.js

@@ -91,7 +91,7 @@ class Vault {
                 requestContext: serializedRCsArr,
             },
             (err, userInfo) => vaultSignatureCb(err, userInfo,
-                                                params.log, callback)
+                params.log, callback),
         );
     }
 
@@ -146,7 +146,7 @@ class Vault {
                 requestContext: serializedRCs,
             },
             (err, userInfo) => vaultSignatureCb(err, userInfo,
-                params.log, callback, streamingV4Params)
+                params.log, callback, streamingV4Params),
         );
     }
 
@@ -232,28 +232,28 @@ class Vault {
     */
     getAccountIds(canonicalIDs, log, callback) {
         log.trace('getting accountIds from Vault based on canonicalIDs',
-        { canonicalIDs });
+            { canonicalIDs });
         this.client.getAccountIds(canonicalIDs,
-        { reqUid: log.getSerializedUids() },
-        (err, info) => {
-            if (err) {
-                log.debug('received error message from vault',
-                    { errorMessage: err });
-                return callback(err);
-            }
-            const infoFromVault = info.message.body;
-            log.trace('info received from vault', { infoFromVault });
-            const result = {};
-            /* If the accountId was not found in Vault, do not
-            send the canonicalID back to the API */
-            Object.keys(infoFromVault).forEach(key => {
-                if (infoFromVault[key] !== 'NotFound' &&
-                infoFromVault[key] !== 'WrongFormat') {
-                    result[key] = infoFromVault[key];
+            { reqUid: log.getSerializedUids() },
+            (err, info) => {
+                if (err) {
+                    log.debug('received error message from vault',
+                        { errorMessage: err });
+                    return callback(err);
                 }
+                const infoFromVault = info.message.body;
+                log.trace('info received from vault', { infoFromVault });
+                const result = {};
+                /* If the accountId was not found in Vault, do not
+            send the canonicalID back to the API */
+                Object.keys(infoFromVault).forEach(key => {
+                    if (infoFromVault[key] !== 'NotFound' &&
+                infoFromVault[key] !== 'WrongFormat') {
+                        result[key] = infoFromVault[key];
+                    }
+                });
+                return callback(null, result);
             });
-            return callback(null, result);
-        });
     }
 
     /** checkPolicies -- call Vault to evaluate policies

lib/auth/auth.js

@@ -10,11 +10,13 @@ const constants = require('../constants');
 const constructStringToSignV2 = require('./v2/constructStringToSign');
 const constructStringToSignV4 = require('./v4/constructStringToSign');
 const convertUTCtoISO8601 = require('./v4/timeUtils').convertUTCtoISO8601;
-const vaultUtilities = require('./in_memory/vaultUtilities');
-const backend = require('./in_memory/Backend');
-const validateAuthConfig = require('./in_memory/validateAuthConfig');
-const AuthLoader = require('./in_memory/AuthLoader');
+const vaultUtilities = require('./backends/in_memory/vaultUtilities');
+const inMemoryBackend = require('./backends/in_memory/Backend');
+const validateAuthConfig = require('./backends/in_memory/validateAuthConfig');
+const AuthLoader = require('./backends/in_memory/AuthLoader');
 const Vault = require('./Vault');
+const baseBackend = require('./backends/base');
+const chainBackend = require('./backends/ChainBackend');
 
 let vault = null;
 const auth = {};
@@ -72,7 +74,7 @@ function extractParams(request, log, awsService, data) {
             version = 'v4';
         } else {
             log.trace('invalid authorization security header',
-                      { header: authHeader });
+                { header: authHeader });
             return { err: errors.AccessDenied };
         }
     } else if (data.Signature) {
@@ -87,7 +89,7 @@ function extractParams(request, log, awsService, data) {
     if (version !== null && method !== null) {
         if (!checkFunctions[version] || !checkFunctions[version][method]) {
             log.trace('invalid auth version or method',
-                      { version, authMethod: method });
+                { version, authMethod: method });
             return { err: errors.NotImplemented };
         }
         log.trace('identified auth method', { version, authMethod: method });
@@ -159,7 +161,7 @@ function doAuth(request, log, cb, awsService, requestContexts) {
  * @return {undefined}
  */
 function generateV4Headers(request, data, accessKey, secretKeyValue,
-                           awsService, proxyPath, sessionToken) {
+    awsService, proxyPath, sessionToken) {
     Object.assign(request, { headers: {} });
     const amzDate = convertUTCtoISO8601(Date.now());
     // get date without time
@@ -192,16 +194,16 @@ function generateV4Headers(request, data, accessKey, secretKeyValue,
         .filter(headerName =>
             headerName.startsWith('x-amz-')
             || headerName.startsWith('x-scal-')
-            || headerName === 'host'
+            || headerName === 'host',
         ).sort().join(';');
     const params = { request, signedHeaders, payloadChecksum,
         credentialScope, timestamp, query: data,
         awsService: service, proxyPath };
     const stringToSign = constructStringToSignV4(params);
     const signingKey = vaultUtilities.calculateSigningKey(secretKeyValue,
-                                                          region,
-                                                          scopeDate,
-                                                          service);
+        region,
+        scopeDate,
+        service);
     const signature = crypto.createHmac('sha256', signingKey)
         .update(stringToSign, 'binary').digest('hex');
     const authorizationHeader = `${algorithm} Credential=${accessKey}` +
@@ -222,10 +224,14 @@ module.exports = {
         constructStringToSignV2,
     },
     inMemory: {
-        backend,
+        backend: inMemoryBackend,
         validateAuthConfig,
         AuthLoader,
     },
+    backends: {
+        baseBackend,
+        chainBackend,
+    },
     AuthInfo,
     Vault,
 };

lib/auth/backends/ChainBackend.js

@@ -0,0 +1,189 @@
+'use strict'; // eslint-disable-line strict
+
+const assert = require('assert');
+const async = require('async');
+
+const errors = require('../../errors');
+const BaseBackend = require('./base');
+
+/**
+ * Class that provides an authentication backend that will verify signatures
+ * and retrieve emails and canonical ids associated with an account using a
+ * given list of authentication backends and vault clients.
+ *
+ * @class ChainBackend
+ */
+class ChainBackend extends BaseBackend {
+    /**
+     * @constructor
+     * @param {string} service - service id
+     * @param {object[]} clients - list of authentication backends or vault clients
+     */
+    constructor(service, clients) {
+        super(service);
+
+        assert(Array.isArray(clients) && clients.length > 0, 'invalid client list');
+        assert(clients.every(client =>
+            typeof client.verifySignatureV4 === 'function' &&
+            typeof client.verifySignatureV2 === 'function' &&
+            typeof client.getCanonicalIds === 'function' &&
+            typeof client.getEmailAddresses === 'function' &&
+            typeof client.checkPolicies === 'function' &&
+            typeof client.healthcheck === 'function',
+        ), 'invalid client: missing required auth backend methods');
+        this._clients = clients;
+    }
+
+
+    /*
+     * try task against each client for one to be successful
+     */
+    _tryEachClient(task, cb) {
+        async.tryEach(this._clients.map(client => done => task(client, done)), cb);
+    }
+
+    /*
+     * apply task to all clients
+     */
+    _forEachClient(task, cb) {
+        async.map(this._clients, task, cb);
+    }
+
+    verifySignatureV2(stringToSign, signatureFromRequest, accessKey, options, callback) {
+        this._tryEachClient((client, done) => client.verifySignatureV2(
+            stringToSign,
+            signatureFromRequest,
+            accessKey,
+            options,
+            done,
+        ), callback);
+    }
+
+    verifySignatureV4(stringToSign, signatureFromRequest, accessKey, region, scopeDate, options, callback) {
+        this._tryEachClient((client, done) => client.verifySignatureV4(
+            stringToSign,
+            signatureFromRequest,
+            accessKey,
+            region,
+            scopeDate,
+            options,
+            done,
+        ), callback);
+    }
+
+    static _mergeObjects(objectResponses) {
+        return objectResponses.reduce(
+            (retObj, resObj) => Object.assign(retObj, resObj.message.body),
+            {});
+    }
+
+    getCanonicalIds(emailAddresses, options, callback) {
+        this._forEachClient(
+            (client, done) => client.getCanonicalIds(emailAddresses, options, done),
+            (err, res) => {
+                if (err) {
+                    return callback(err);
+                }
+                // TODO: atm naive merge, better handling of conflicting email results
+                return callback(null, {
+                    message: {
+                        body: ChainBackend._mergeObjects(res),
+                    },
+                });
+            });
+    }
+
+    getEmailAddresses(canonicalIDs, options, callback) {
+        this._forEachClient(
+            (client, done) => client.getEmailAddresses(canonicalIDs, options, done),
+            (err, res) => {
+                if (err) {
+                    return callback(err);
+                }
+                return callback(null, {
+                    message: {
+                        body: ChainBackend._mergeObjects(res),
+                    },
+                });
+            });
+    }
+
+    /*
+     * merge policy responses into a single message
+     */
+    static _mergePolicies(policyResponses) {
+        const policyMap = {};
+
+        policyResponses.forEach(resp => {
+            if (!resp.message || !Array.isArray(resp.message.body)) {
+                return;
+            }
+
+            resp.message.body.forEach(policy => {
+                const key = (policy.arn || '') + (policy.versionId || '');
+                if (!policyMap[key] || !policyMap[key].isAllowed) {
+                    policyMap[key] = policy;
+                }
+                // else is duplicate policy
+            });
+        });
+
+        return Object.keys(policyMap).map(key => {
+            const policyRes = { isAllowed: policyMap[key].isAllowed };
+            if (policyMap[key].arn !== '') {
+                policyRes.arn = policyMap[key].arn;
+            }
+            if (policyMap[key].versionId) {
+                policyRes.versionId = policyMap[key].versionId;
+            }
+            return policyRes;
+        });
+    }
+
+    /*
+        response format:
+            {  message: {
+                body: [{}],
+                code: number,
+                message: string,
+            } }
+     */
+    checkPolicies(requestContextParams, userArn, options, callback) {
+        this._forEachClient((client, done) => client.checkPolicies(
+            requestContextParams,
+            userArn,
+            options,
+            done,
+        ), (err, res) => {
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, {
+                message: {
+                    body: ChainBackend._mergePolicies(res),
+                },
+            });
+        });
+    }
+
+    healthcheck(reqUid, callback) {
+        this._forEachClient((client, done) =>
+            client.healthcheck(reqUid, (err, res) => done(null, {
+                error: !!err ? err : null,
+                status: res,
+            }),
+            ), (err, res) => {
+            if (err) {
+                return callback(err);
+            }
+
+            const isError = res.some(results => !!results.error);
+            if (isError) {
+                return callback(errors.InternalError, res);
+            }
+            return callback(null, res);
+        });
+    }
+}
+
+module.exports = ChainBackend;

lib/auth/backends/base.js

@@ -0,0 +1,86 @@
+'use strict'; // eslint-disable-line strict
+
+const errors = require('../../errors');
+
+/**
+ * Base backend class
+ *
+ * @class BaseBackend
+ */
+class BaseBackend {
+    /**
+     * @constructor
+     * @param {string} service - service identifer for construction arn
+     */
+    constructor(service) {
+        this.service = service;
+    }
+
+    /** verifySignatureV2
+    * @param {string} stringToSign - string to sign built per AWS rules
+    * @param {string} signatureFromRequest - signature sent with request
+    * @param {string} accessKey - account accessKey
+    * @param {object} options - contains algorithm (SHA1 or SHA256)
+    * @param {function} callback - callback with either error or user info
+    * @return {function} calls callback
+    */
+    verifySignatureV2(stringToSign, signatureFromRequest,
+        accessKey, options, callback) {
+        return callback(errors.AuthMethodNotImplemented);
+    }
+
+
+    /** verifySignatureV4
+     * @param {string} stringToSign - string to sign built per AWS rules
+     * @param {string} signatureFromRequest - signature sent with request
+     * @param {string} accessKey - account accessKey
+     * @param {string} region - region specified in request credential
+     * @param {string} scopeDate - date specified in request credential
+     * @param {object} options - options to send to Vault
+     * (just contains reqUid for logging in Vault)
+     * @param {function} callback - callback with either error or user info
+     * @return {function} calls callback
+     */
+    verifySignatureV4(stringToSign, signatureFromRequest, accessKey,
+        region, scopeDate, options, callback) {
+        return callback(errors.AuthMethodNotImplemented);
+    }
+
+    /**
+     * Gets canonical ID's for a list of accounts
+     * based on email associated with account
+     * @param {array} emails - list of email addresses
+     * @param {object} options - to send log id to vault
+     * @param {function} callback - callback to calling function
+     * @returns {function} callback with either error or
+     * object with email addresses as keys and canonical IDs
+     * as values
+     */
+    getCanonicalIds(emails, options, callback) {
+        return callback(errors.AuthMethodNotImplemented);
+    }
+
+    /**
+     * Gets email addresses (referred to as diplay names for getACL's)
+     * for a list of accounts based on canonical IDs associated with account
+     * @param {array} canonicalIDs - list of canonicalIDs
+     * @param {object} options - to send log id to vault
+     * @param {function} callback - callback to calling function
+     * @returns {function} callback with either error or
+     * an object from Vault containing account canonicalID
+     * as each object key and an email address as the value (or "NotFound")
+     */
+    getEmailAddresses(canonicalIDs, options, callback) {
+        return callback(errors.AuthMethodNotImplemented);
+    }
+
+    checkPolicies(requestContextParams, userArn, options, callback) {
+        return callback(null, { message: { body: [] } });
+    }
+
+    healthcheck(reqUid, callback) {
+        return callback(null, { code: 200, message: 'OK' });
+    }
+}
+
+module.exports = BaseBackend;

lib/auth/backends/in_memory/AuthLoader.js

@@ -3,7 +3,7 @@ const glob = require('simple-glob');
 const joi = require('@hapi/joi');
 const werelogs = require('werelogs');
 
-const ARN = require('../../models/ARN');
+const ARN = require('../../../models/ARN');
 
 /**
  * Load authentication information from files or pre-loaded account
@@ -26,20 +26,20 @@ class AuthLoader {
             .required();
 
         const accountsJoi = joi.array()
-                .items({
-                    name: joi.string().required(),
-                    email: joi.string().email().required(),
-                    arn: joi.string().required(),
-                    canonicalID: joi.string().required(),
-                    shortid: joi.string().regex(/^[0-9]{12}$/).required(),
-                    keys: this._joiKeysValidator,
-                    // backward-compat
-                    users: joi.array(),
-                })
-                .required()
-                .unique('arn')
-                .unique('email')
-                .unique('canonicalID');
+            .items({
+                name: joi.string().required(),
+                email: joi.string().email().required(),
+                arn: joi.string().required(),
+                canonicalID: joi.string().required(),
+                shortid: joi.string().regex(/^[0-9]{12}$/).required(),
+                keys: this._joiKeysValidator,
+                // backward-compat
+                users: joi.array(),
+            })
+            .required()
+            .unique('arn')
+            .unique('email')
+            .unique('canonicalID');
         this._joiValidator = joi.object({ accounts: accountsJoi });
     }
 
@@ -136,7 +136,7 @@ class AuthLoader {
 
     _validateData(authData, filePath) {
         const res = joi.validate(authData, this._joiValidator,
-                                 { abortEarly: false });
+            { abortEarly: false });
         if (res.error) {
             this._dumpJoiErrors(res.error.details, filePath);
             return false;
@@ -156,7 +156,7 @@ class AuthLoader {
                         'master/conf/authdata.json). Also note that support ' +
                         'for account users has been dropped.',
                     { accountName: account.name, accountArn: account.arn,
-                      filePath });
+                        filePath });
                 arnError = true;
                 return;
             }
@@ -167,7 +167,7 @@ class AuthLoader {
                         'https://github.com/scality/S3/blob/master/conf/' +
                         'authdata.json)',
                     { accountName: account.name, accountArn: account.arn,
-                      filePath });
+                        filePath });
                 arnError = true;
                 return;
             }
@@ -176,8 +176,8 @@ class AuthLoader {
                 this._log.error(
                     'authentication config validation error',
                     { reason: arnObj.error.description,
-                      accountName: account.name, accountArn: account.arn,
-                      filePath });
+                        accountName: account.name, accountArn: account.arn,
+                        filePath });
                 arnError = true;
                 return;
             }
@@ -185,8 +185,8 @@ class AuthLoader {
                 this._log.error(
                     'authentication config validation error',
                     { reason: 'not an IAM account ARN',
-                      accountName: account.name, accountArn: account.arn,
-                      filePath });
+                        accountName: account.name, accountArn: account.arn,
+                        filePath });
                 arnError = true;
                 return;
             }
@@ -215,7 +215,7 @@ class AuthLoader {
                 logInfo.context = err.context;
             }
             this._log.error('authentication config validation error',
-                            logInfo);
+                logInfo);
         });
     }
 }

lib/auth/backends/in_memory/Backend.js

@@ -2,10 +2,11 @@
 
 const crypto = require('crypto');
 
-const errors = require('../../errors');
+const errors = require('../../../errors');
 const calculateSigningKey = require('./vaultUtilities').calculateSigningKey;
 const hashSignature = require('./vaultUtilities').hashSignature;
 const Indexer = require('./Indexer');
+const BaseBackend = require('../base');
 
 function _formatResponse(userInfoToSend) {
     return {
@@ -19,9 +20,9 @@ function _formatResponse(userInfoToSend) {
  * Class that provides a memory backend for verifying signatures and getting
  * emails and canonical ids associated with an account.
  *
- * @class Backend
+ * @class InMemoryBackend
  */
-class Backend {
+class InMemoryBackend extends BaseBackend {
     /**
      * @constructor
      * @param {string} service - service identifer for construction arn
@@ -30,19 +31,11 @@ class Backend {
      * back and returns it in an object
      */
     constructor(service, indexer, formatter) {
-        this.service = service;
+        super(service);
         this.indexer = indexer;
         this.formatResponse = formatter;
     }
 
-    /** verifySignatureV2
-    * @param {string} stringToSign - string to sign built per AWS rules
-    * @param {string} signatureFromRequest - signature sent with request
-    * @param {string} accessKey - account accessKey
-    * @param {object} options - contains algorithm (SHA1 or SHA256)
-    * @param {function} callback - callback with either error or user info
-    * @return {function} calls callback
-    */
     verifySignatureV2(stringToSign, signatureFromRequest,
         accessKey, options, callback) {
         const entity = this.indexer.getEntityByKey(accessKey);
@@ -65,18 +58,6 @@ class Backend {
         return callback(null, vaultReturnObject);
     }
 
-
-    /** verifySignatureV4
-     * @param {string} stringToSign - string to sign built per AWS rules
-     * @param {string} signatureFromRequest - signature sent with request
-     * @param {string} accessKey - account accessKey
-     * @param {string} region - region specified in request credential
-     * @param {string} scopeDate - date specified in request credential
-     * @param {object} options - options to send to Vault
-     * (just contains reqUid for logging in Vault)
-     * @param {function} callback - callback with either error or user info
-     * @return {function} calls callback
-     */
     verifySignatureV4(stringToSign, signatureFromRequest, accessKey,
         region, scopeDate, options, callback) {
         const entity = this.indexer.getEntityByKey(accessKey);
@@ -100,16 +81,6 @@ class Backend {
         return callback(null, vaultReturnObject);
     }
 
-    /**
-     * Gets canonical ID's for a list of accounts
-     * based on email associated with account
-     * @param {array} emails - list of email addresses
-     * @param {object} log - log object
-     * @param {function} cb - callback to calling function
-     * @returns {function} callback with either error or
-     * object with email addresses as keys and canonical IDs
-     * as values
-     */
     getCanonicalIds(emails, log, cb) {
         const results = {};
         emails.forEach(email => {
@@ -130,16 +101,6 @@ class Backend {
         return cb(null, vaultReturnObject);
     }
 
-    /**
-     * Gets email addresses (referred to as diplay names for getACL's)
-     * for a list of accounts based on canonical IDs associated with account
-     * @param {array} canonicalIDs - list of canonicalIDs
-     * @param {object} options - to send log id to vault
-     * @param {function} cb - callback to calling function
-     * @returns {function} callback with either error or
-     * an object from Vault containing account canonicalID
-     * as each object key and an email address as the value (or "NotFound")
-     */
     getEmailAddresses(canonicalIDs, options, cb) {
         const results = {};
         canonicalIDs.forEach(canonicalId => {
@@ -188,7 +149,7 @@ class Backend {
 }
 
 
-class S3AuthBackend extends Backend {
+class S3AuthBackend extends InMemoryBackend {
     /**
      * @constructor
      * @param {object} authdata - the authentication config file's data

lib/auth/v2/getCanonicalizedAmzHeaders.js

@@ -41,7 +41,7 @@ function getCanonicalizedAmzHeaders(headers, clientType) {
     // Build headerString
     return amzHeaders.reduce((headerStr, current) =>
         `${headerStr}${current[0]}:${current[1]}\n`,
-        '');
+    '');
 }
 
 module.exports = getCanonicalizedAmzHeaders;

lib/auth/v2/headerAuthCheck.js

@@ -22,9 +22,9 @@ function check(request, log, data) {
     timestamp = Date.parse(timestamp);
     if (!timestamp) {
         log.debug('missing or invalid date header',
-        { method: 'auth/v2/headerAuthCheck.check' });
+            { method: 'auth/v2/headerAuthCheck.check' });
         return { err: errors.AccessDenied.
-          customizeDescription('Authentication requires a valid Date or ' +
+            customizeDescription('Authentication requires a valid Date or ' +
           'x-amz-date header') };
     }
 

lib/auth/v2/queryAuthCheck.js

@@ -42,12 +42,12 @@ function check(request, log, data) {
 
     if (expirationTime > currentTime + preSignedURLExpiry) {
         log.debug('expires parameter too far in future',
-        { expires: request.query.Expires });
+            { expires: request.query.Expires });
         return { err: errors.AccessDenied };
     }
     if (currentTime > expirationTime) {
         log.debug('current time exceeds expires time',
-        { expires: request.query.Expires });
+            { expires: request.query.Expires });
         return { err: errors.RequestTimeTooSkewed };
     }
     const accessKey = data.AWSAccessKeyId;

lib/auth/v4/awsURIencode.js

@@ -43,7 +43,7 @@ function awsURIencode(input, encodeSlash, noEncodeStar) {
         return encoded;
     }
     for (let i = 0; i < input.length; i++) {
-        const ch = input.charAt(i);
+        let ch = input.charAt(i);
         if ((ch >= 'A' && ch <= 'Z') ||
             (ch >= 'a' && ch <= 'z') ||
             (ch >= '0' && ch <= '9') ||
@@ -57,6 +57,20 @@ function awsURIencode(input, encodeSlash, noEncodeStar) {
         } else if (ch === '*') {
             encoded = encoded.concat(noEncodeStar ? '*' : '%2A');
         } else {
+            if (ch >= '\uD800' && ch <= '\uDBFF') {
+                // If this character is a high surrogate peek the next character
+                // and join it with this one if the next character is a low
+                // surrogate.
+                // Otherwise the encoded URI will contain the two surrogates as
+                // two distinct UTF-8 sequences which is not valid UTF-8.
+                if (i + 1 < input.length) {
+                    const ch2 = input.charAt(i + 1);
+                    if (ch2 >= '\uDC00' && ch2 <= '\uDFFF') {
+                        i++;
+                        ch += ch2;
+                    }
+                }
+            }
             encoded = encoded.concat(_toHexUTF8(ch));
         }
     }

lib/auth/v4/headerAuthCheck.js

@@ -88,14 +88,14 @@ function check(request, log, data, awsService) {
     }
     if (!timestamp) {
         log.debug('missing or invalid date header',
-          { method: 'auth/v4/headerAuthCheck.check' });
+            { method: 'auth/v4/headerAuthCheck.check' });
         return { err: errors.AccessDenied.
-          customizeDescription('Authentication requires a valid Date or ' +
+            customizeDescription('Authentication requires a valid Date or ' +
           'x-amz-date header') };
     }
 
     const validationResult = validateCredentials(credentialsArr, timestamp,
-      log);
+        log);
     if (validationResult instanceof Error) {
         log.debug('credentials in improper format', { credentialsArr,
             timestamp, validationResult });
@@ -127,6 +127,17 @@ function check(request, log, data, awsService) {
         return { err: errors.RequestTimeTooSkewed };
     }
 
+    let proxyPath = null;
+    if (request.headers.proxy_path) {
+        try {
+            proxyPath = decodeURIComponent(request.headers.proxy_path);
+        } catch (err) {
+            log.debug('invalid proxy_path header', { proxyPath, err });
+            return { err: errors.InvalidArgument.customizeDescription(
+                'invalid proxy_path header') };
+        }
+    }
+
     const stringToSign = constructStringToSign({
         log,
         request,
@@ -136,6 +147,7 @@ function check(request, log, data, awsService) {
         timestamp,
         payloadChecksum,
         awsService: service,
+        proxyPath,
     });
     log.trace('constructed stringToSign', { stringToSign });
     if (stringToSign instanceof Error) {

lib/auth/v4/queryAuthCheck.js

@@ -45,7 +45,7 @@ function check(request, log, data) {
     }
 
     const validationResult = validateCredentials(credential, timestamp,
-      log);
+        log);
     if (validationResult instanceof Error) {
         log.debug('credentials in improper format', { credential,
             timestamp, validationResult });
@@ -62,6 +62,17 @@ function check(request, log, data) {
         return { err: errors.RequestTimeTooSkewed };
     }
 
+    let proxyPath = null;
+    if (request.headers.proxy_path) {
+        try {
+            proxyPath = decodeURIComponent(request.headers.proxy_path);
+        } catch (err) {
+            log.debug('invalid proxy_path header', { proxyPath });
+            return { err: errors.InvalidArgument.customizeDescription(
+                'invalid proxy_path header') };
+        }
+    }
+
     // In query v4 auth, the canonical request needs
     // to include the query params OTHER THAN
     // the signature so create a
@@ -87,6 +98,7 @@ function check(request, log, data) {
         credentialScope:
             `${scopeDate}/${region}/${service}/${requestType}`,
         awsService: service,
+        proxyPath,
     });
     if (stringToSign instanceof Error) {
         return { err: stringToSign };

lib/auth/v4/validateInputs.js

@@ -25,20 +25,20 @@ function validateCredentials(credentials, timestamp, log) {
         log.warn('accessKey provided is wrong format', { accessKey });
         return errors.InvalidArgument;
     }
-     // The scope date (format YYYYMMDD) must be same date as the timestamp
-     // on the request from the x-amz-date param (if queryAuthCheck)
-     // or from the x-amz-date header or date header (if headerAuthCheck)
-     // Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
-     // http://docs.aws.amazon.com/AmazonS3/latest/API/
-     // sigv4-query-string-auth.html
-     // http://docs.aws.amazon.com/general/latest/gr/
-     // sigv4-date-handling.html
+    // The scope date (format YYYYMMDD) must be same date as the timestamp
+    // on the request from the x-amz-date param (if queryAuthCheck)
+    // or from the x-amz-date header or date header (if headerAuthCheck)
+    // Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
+    // http://docs.aws.amazon.com/AmazonS3/latest/API/
+    // sigv4-query-string-auth.html
+    // http://docs.aws.amazon.com/general/latest/gr/
+    // sigv4-date-handling.html
 
-     // convert timestamp to format of scopeDate YYYYMMDD
+    // convert timestamp to format of scopeDate YYYYMMDD
     const timestampDate = timestamp.split('T')[0];
     if (scopeDate.length !== 8 || scopeDate !== timestampDate) {
         log.warn('scope date must be the same date as the timestamp date',
-           { scopeDate, timestampDate });
+            { scopeDate, timestampDate });
         return errors.RequestTimeTooSkewed;
     }
     if (service !== 's3' && service !== 'iam' && service !== 'ring' &&
@@ -50,7 +50,7 @@ function validateCredentials(credentials, timestamp, log) {
     }
     if (requestType !== 'aws4_request') {
         log.warn('requestType contained in params is not aws4_request',
-           { requestType });
+            { requestType });
         return errors.InvalidArgument;
     }
     return {};
@@ -68,7 +68,7 @@ function extractQueryParams(queryObj, log) {
     // Do not need the algorithm sent back
     if (queryObj['X-Amz-Algorithm'] !== 'AWS4-HMAC-SHA256') {
         log.warn('algorithm param incorrect',
-        { algo: queryObj['X-Amz-Algorithm'] });
+            { algo: queryObj['X-Amz-Algorithm'] });
         return authParams;
     }
 

lib/constants.js

@@ -1,20 +1,21 @@
 'use strict'; // eslint-disable-line strict
+const crypto = require('crypto');
 
 // The min value here is to manage further backward compat if we
 // need it
-const iamSecurityTokenSizeMin = 128;
-const iamSecurityTokenSizeMax = 128;
-// Security token is an hex string (no real format from amazon)
-const iamSecurityTokenPattern =
-    new RegExp(`^[a-f0-9]{${iamSecurityTokenSizeMin},` +
-        `${iamSecurityTokenSizeMax}}$`);
+// Default value
+const vaultGeneratedIamSecurityTokenSizeMin = 128;
+// Safe to assume that a typical token size is less than 8192 bytes
+const vaultGeneratedIamSecurityTokenSizeMax = 8192;
+// Base-64
+const vaultGeneratedIamSecurityTokenPattern = /^[A-Za-z0-9/+=]*$/;
 
 module.exports = {
     // info about the iam security token
     iamSecurityToken: {
-        min: iamSecurityTokenSizeMin,
-        max: iamSecurityTokenSizeMax,
-        pattern: iamSecurityTokenPattern,
+        min: vaultGeneratedIamSecurityTokenSizeMin,
+        max: vaultGeneratedIamSecurityTokenSizeMax,
+        pattern: vaultGeneratedIamSecurityTokenPattern,
     },
     // PublicId is used as the canonicalID for a request that contains
     // no authentication information.  Requestor can access
@@ -23,6 +24,7 @@ module.exports = {
     zenkoServiceAccount: 'http://acs.zenko.io/accounts/service',
     metadataFileNamespace: '/MDFile',
     dataFileURL: '/DataFile',
+    passthroughFileURL: '/PassthroughFile',
     // AWS states max size for user-defined metadata
     // (x-amz-meta- headers) is 2 KB:
     // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
@@ -32,7 +34,10 @@ module.exports = {
     emptyFileMd5: 'd41d8cd98f00b204e9800998ecf8427e',
     // Version 2 changes the format of the data location property
     // Version 3 adds the dataStoreName attribute
-    mdModelVersion: 3,
+    // Version 4 add the Creation-Time and Content-Language attributes,
+    //     and add support for x-ms-meta-* headers in UserMetadata
+    // Version 5 adds the azureInfo structure
+    mdModelVersion: 5,
     /*
      * Splitter is used to build the object name for the overview of a
      * multipart upload and to build the object names for each part of a
@@ -72,9 +77,44 @@ module.exports = {
     permittedCapitalizedBuckets: {
         METADATA: true,
     },
+    // Setting a lower object key limit to account for:
+    // - Mongo key limit of 1012 bytes
+    // - Version ID in Mongo Key if versioned of 33
+    // - Max bucket name length if bucket match false of 63
+    // - Extra prefix slash for bucket prefix if bucket match of 1
+    objectKeyByteLimit: 915,
+    /* delimiter for location-constraint. The location constraint will be able
+     * to include the ingestion flag
+     */
+    zenkoSeparator: ':',
     /* eslint-disable camelcase */
     externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true },
-    /* eslint-enable camelcase */
+    replicationBackends: { aws_s3: true, azure: true, gcp: true },
+
+    // hex digest of sha256 hash of empty string:
+    emptyStringHash: crypto.createHash('sha256')
+        .update('', 'binary').digest('hex'),
+    mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
+    // AWS sets a minimum size limit for parts except for the last part.
+    // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
+    minimumAllowedPartSize: 5242880,
+    gcpMaximumAllowedPartCount: 1024,
+    // GCP Object Tagging Prefix
+    gcpTaggingPrefix: 'aws-tag-',
+    productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
+    legacyLocations: ['sproxyd', 'legacy'],
+    // healthcheck default call from nginx is every 2 seconds
+    // for external backends, don't call unless at least 1 minute
+    // (60,000 milliseconds) since last call
+    externalBackendHealthCheckInterval: 60000,
+    // some of the available data backends  (if called directly rather
+    // than through the multiple backend gateway) need a key provided
+    // as a string as first parameter of the get/delete methods.
+    clientsRequireStringKey: { sproxyd: true, cdmi: true },
+    hasCopyPartBackends: { aws_s3: true, gcp: true },
+    versioningNotImplBackends: { azure: true, gcp: true },
+    // user metadata applied on zenko-created objects
+    zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
     // Default expiration value of the S3 pre-signed URL duration
     // 604800 seconds (seven days).
     defaultPreSignedURLExpiry: 7 * 24 * 60 * 60,
@@ -91,10 +131,6 @@ module.exports = {
         's3:ObjectRemoved:DeleteMarkerCreated',
     ]),
     notificationArnPrefix: 'arn:scality:bucketnotif',
-    // some of the available data backends  (if called directly rather
-    // than through the multiple backend gateway) need a key provided
-    // as a string as first parameter of the get/delete methods.
-    clientsRequireStringKey: { sproxyd: true, cdmi: true },
     // HTTP server keep-alive timeout is set to a higher value than
     // client's free sockets timeout to avoid the risk of triggering
     // ECONNRESET errors if the server closes the connection at the

lib/db.js

@@ -64,12 +64,12 @@ class IndexTransaction {
     push(op) {
         if (this.closed) {
             throw propError('pushOnCommittedTransaction',
-                            'can not add ops to already committed transaction');
+                'can not add ops to already committed transaction');
         }
 
         if (op.type !== 'put' && op.type !== 'del') {
             throw propError('invalidTransactionVerb',
-                            `unknown action type: ${op.type}`);
+                `unknown action type: ${op.type}`);
         }
 
         if (op.key === undefined) {
@@ -137,7 +137,7 @@ class IndexTransaction {
     addCondition(condition) {
         if (this.closed) {
             throw propError('pushOnCommittedTransaction',
-                            'can not add conditions to already committed transaction');
+                'can not add conditions to already committed transaction');
         }
         if (condition === undefined || Object.keys(condition).length === 0) {
             throw propError('missingCondition', 'missing condition for conditional put');
@@ -159,12 +159,12 @@ class IndexTransaction {
     commit(cb) {
         if (this.closed) {
             return cb(propError('alreadyCommitted',
-                                'transaction was already committed'));
+                'transaction was already committed'));
         }
 
         if (this.operations.length === 0) {
             return cb(propError('emptyTransaction',
-                                'tried to commit an empty transaction'));
+                'tried to commit an empty transaction'));
         }
 
         this.closed = true;

lib/errors.js

@@ -76,11 +76,11 @@ function errorsGen() {
     const errorsObj = require('../errors/arsenalErrors.json');
 
     Object.keys(errorsObj)
-          .filter(index => index !== '_comment')
-          .forEach(index => {
-              errors[index] = new ArsenalError(index, errorsObj[index].code,
-                                               errorsObj[index].description);
-          });
+        .filter(index => index !== '_comment')
+        .forEach(index => {
+            errors[index] = new ArsenalError(index, errorsObj[index].code,
+                errorsObj[index].description);
+        });
     return errors;
 }
 

lib/executables/pensieveCreds/package.json

@@ -7,8 +7,8 @@
     "test": "mocha --recursive --timeout 5500 tests/unit"
   },
   "dependencies": {
-    "mocha": "2.5.3",
-    "async": "^2.6.0",
+    "mocha": "5.2.0",
+    "async": "~2.6.1",
     "node-forge": "^0.7.1"
   }
 }

lib/executables/pensieveCreds/tests/unit/utilsSpec.js

@@ -17,9 +17,9 @@ describe('decyrptSecret', () => {
 describe('parseServiceCredentials', () => {
     const conf = {
         users: [{ accessKey,
-        accountType: 'service-clueso',
-        secretKey,
-        userName: 'Search Service Account' }],
+            accountType: 'service-clueso',
+            secretKey,
+            userName: 'Search Service Account' }],
     };
     const auth = JSON.stringify({ privateKey });
 

lib/jsutil.js

@@ -25,7 +25,7 @@ module.exports.once = function once(func) {
             state.res = func.apply(func, args);
         } else {
             debug('function already called:', func,
-                  'returning cached result:', state.res);
+                'returning cached result:', state.res);
         }
         return state.res;
     };

lib/metrics/RedisClient.js

@@ -17,11 +17,33 @@ class RedisClient {
                 method: 'RedisClient.constructor',
                 redisHost: config.host,
                 redisPort: config.port,
-            })
+            }),
         );
         return this;
     }
 
+    /**
+    * scan a pattern and return matching keys
+    * @param {string} pattern - string pattern to match with all existing keys
+    * @param {number} [count=10] - scan count
+    * @param {callback} cb - callback (error, result)
+    * @return {undefined}
+    */
+    scan(pattern, count = 10, cb) {
+        const params = { match: pattern, count };
+        const keys = [];
+
+        const stream = this._client.scanStream(params);
+        stream.on('data', resultKeys => {
+            for (let i = 0; i < resultKeys.length; i++) {
+                keys.push(resultKeys[i]);
+            }
+        });
+        stream.on('end', () => {
+            cb(null, keys);
+        });
+    }
+
     /**
     * increment value of a key by 1 and set a ttl
     * @param {string} key - key holding the value
@@ -35,6 +57,17 @@ class RedisClient {
             .exec(cb);
     }
 
+    /**
+     * increment value of a key by a given amount
+     * @param {string} key - key holding the value
+     * @param {number} amount - amount to increase by
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    incrby(key, amount, cb) {
+        return this._client.incrby(key, amount, cb);
+    }
+
     /**
      * increment value of a key by a given amount and set a ttl
      * @param {string} key - key holding the value
@@ -50,13 +83,24 @@ class RedisClient {
     }
 
     /**
-    * execute a batch of commands
-    * @param {string[]} cmds - list of commands
-    * @param {callback} cb - callback
-    * @return {undefined}
-    */
-    batch(cmds, cb) {
-        return this._client.pipeline(cmds).exec(cb);
+     * decrement value of a key by a given amount
+     * @param {string} key - key holding the value
+     * @param {number} amount - amount to increase by
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    decrby(key, amount, cb) {
+        return this._client.decrby(key, amount, cb);
+    }
+
+    /**
+     * get value stored at key
+     * @param {string} key - key holding the value
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    get(key, cb) {
+        return this._client.get(key, cb);
     }
 
     /**
@@ -71,6 +115,16 @@ class RedisClient {
         return this._client.exists(key, cb);
     }
 
+    /**
+    * execute a batch of commands
+    * @param {string[]} cmds - list of commands
+    * @param {callback} cb - callback
+    * @return {undefined}
+    */
+    batch(cmds, cb) {
+        return this._client.pipeline(cmds).exec(cb);
+    }
+
     /**
      * Add a value and its score to a sorted set. If no sorted set exists, this
      * will create a new one for the given key.
@@ -150,12 +204,26 @@ class RedisClient {
         return this._client.zrangebyscore(key, min, max, cb);
     }
 
+    /**
+     * get TTL or expiration in seconds
+     * @param {string} key - name of key
+     * @param {function} cb - callback
+     * @return {undefined}
+     */
+    ttl(key, cb) {
+        return this._client.ttl(key, cb);
+    }
+
     clear(cb) {
         return this._client.flushdb(cb);
     }
 
-    disconnect() {
-        this._client.disconnect();
+    disconnect(cb) {
+        return this._client.quit(cb);
+    }
+
+    listClients(cb) {
+        return this._client.client('list', cb);
     }
 }
 

lib/metrics/StatsClient.js

@@ -41,11 +41,11 @@ class StatsClient {
     /**
     * build redis key to get total number of occurrences on the server
     * @param {string} name - key name identifier
-    * @param {object} d - Date instance
+    * @param {Date} date - Date instance
     * @return {string} key - key for redis
     */
-    _buildKey(name, d) {
-        return `${name}:${this._normalizeTimestamp(d)}`;
+    buildKey(name, date) {
+        return `${name}:${this._normalizeTimestamp(date)}`;
     }
 
     /**
@@ -85,11 +85,35 @@ class StatsClient {
             amount = (typeof incr === 'number') ? incr : 1;
         }
 
-        const key = this._buildKey(`${id}:requests`, new Date());
+        const key = this.buildKey(`${id}:requests`, new Date());
 
         return this._redis.incrbyEx(key, amount, this._expiry, callback);
     }
 
+    /**
+     * Increment the given key by the given value.
+     * @param {String} key - The Redis key to increment
+     * @param {Number} incr - The value to increment by
+     * @param {function} [cb] - callback
+     * @return {undefined}
+     */
+    incrementKey(key, incr, cb) {
+        const callback = cb || this._noop;
+        return this._redis.incrby(key, incr, callback);
+    }
+
+    /**
+     * Decrement the given key by the given value.
+     * @param {String} key - The Redis key to decrement
+     * @param {Number} decr - The value to decrement by
+     * @param {function} [cb] - callback
+     * @return {undefined}
+     */
+    decrementKey(key, decr, cb) {
+        const callback = cb || this._noop;
+        return this._redis.decrby(key, decr, callback);
+    }
+
     /**
     * report/record a request that ended up being a 500 on the server
     * @param {string} id - service identifier
@@ -101,10 +125,54 @@ class StatsClient {
             return undefined;
         }
         const callback = cb || this._noop;
-        const key = this._buildKey(`${id}:500s`, new Date());
+        const key = this.buildKey(`${id}:500s`, new Date());
         return this._redis.incrEx(key, this._expiry, callback);
     }
 
+    /**
+    * wrapper on `getStats` that handles a list of keys
+    * @param {object} log - Werelogs request logger
+    * @param {array} ids - service identifiers
+    * @param {callback} cb - callback to call with the err/result
+    * @return {undefined}
+    */
+    getAllStats(log, ids, cb) {
+        if (!this._redis) {
+            return cb(null, {});
+        }
+
+        const statsRes = {
+            'requests': 0,
+            '500s': 0,
+            'sampleDuration': this._expiry,
+        };
+        let requests = 0;
+        let errors = 0;
+
+        // for now set concurrency to default of 10
+        return async.eachLimit(ids, 10, (id, done) => {
+            this.getStats(log, id, (err, res) => {
+                if (err) {
+                    return done(err);
+                }
+                requests += res.requests;
+                errors += res['500s'];
+                return done();
+            });
+        }, error => {
+            if (error) {
+                log.error('error getting stats', {
+                    error,
+                    method: 'StatsClient.getAllStats',
+                });
+                return cb(null, statsRes);
+            }
+            statsRes.requests = requests;
+            statsRes['500s'] = errors;
+            return cb(null, statsRes);
+        });
+    }
+
     /**
     * get stats for the last x seconds, x being the sampling duration
     * @param {object} log - Werelogs request logger
@@ -121,8 +189,8 @@ class StatsClient {
         const reqsKeys = [];
         const req500sKeys = [];
         for (let i = 0; i < totalKeys; i++) {
-            reqsKeys.push(['get', this._buildKey(`${id}:requests`, d)]);
-            req500sKeys.push(['get', this._buildKey(`${id}:500s`, d)]);
+            reqsKeys.push(['get', this.buildKey(`${id}:requests`, d)]);
+            req500sKeys.push(['get', this.buildKey(`${id}:500s`, d)]);
             this._setPrevInterval(d);
         }
         return async.parallel([

lib/metrics/StatsModel.js

@@ -1,11 +1,148 @@
+const async = require('async');
+
 const StatsClient = require('./StatsClient');
- /**
+
+/**
  * @class StatsModel
  *
  * @classdesc Extend and overwrite how timestamps are normalized by minutes
  * rather than by seconds
  */
 class StatsModel extends StatsClient {
+    /**
+    * Utility method to convert 2d array rows to columns, and vice versa
+    * See also: https://docs.ruby-lang.org/en/2.0.0/Array.html#method-i-zip
+    * @param {array} arrays - 2d array of integers
+    * @return {array} converted array
+    */
+    _zip(arrays) {
+        if (arrays.length > 0 && arrays.every(a => Array.isArray(a))) {
+            return arrays[0].map((_, i) => arrays.map(a => a[i]));
+        }
+        return [];
+    }
+
+    /**
+    * normalize to the nearest interval
+    * @param {object} d - Date instance
+    * @return {number} timestamp - normalized to the nearest interval
+    */
+    _normalizeTimestamp(d) {
+        const m = d.getMinutes();
+        return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
+    }
+
+    /**
+    * override the method to get the count as an array of integers separated
+    * by each interval
+    * typical input looks like [[null, '1'], [null, '2'], [null, null]...]
+    * @param {array} arr - each index contains the result of each batch command
+    *   where index 0 signifies the error and index 1 contains the result
+    * @return {array} array of integers, ordered from most recent interval to
+    *   oldest interval with length of (expiry / interval)
+    */
+    _getCount(arr) {
+        const size = Math.floor(this._expiry / this._interval);
+        const array = arr.reduce((store, i) => {
+            let num = parseInt(i[1], 10);
+            num = Number.isNaN(num) ? 0 : num;
+            store.push(num);
+            return store;
+        }, []);
+
+        if (array.length < size) {
+            array.push(...Array(size - array.length).fill(0));
+        }
+        return array;
+    }
+
+    /**
+    * wrapper on `getStats` that handles a list of keys
+    * override the method to reduce the returned 2d array from `_getCount`
+    * @param {object} log - Werelogs request logger
+    * @param {array} ids - service identifiers
+    * @param {callback} cb - callback to call with the err/result
+    * @return {undefined}
+    */
+    getAllStats(log, ids, cb) {
+        if (!this._redis) {
+            return cb(null, {});
+        }
+
+        const size = Math.floor(this._expiry / this._interval);
+        const statsRes = {
+            'requests': Array(size).fill(0),
+            '500s': Array(size).fill(0),
+            'sampleDuration': this._expiry,
+        };
+        const requests = [];
+        const errors = [];
+
+        if (ids.length === 0) {
+            return cb(null, statsRes);
+        }
+
+        // for now set concurrency to default of 10
+        return async.eachLimit(ids, 10, (id, done) => {
+            this.getStats(log, id, (err, res) => {
+                if (err) {
+                    return done(err);
+                }
+                requests.push(res.requests);
+                errors.push(res['500s']);
+                return done();
+            });
+        }, error => {
+            if (error) {
+                log.error('error getting stats', {
+                    error,
+                    method: 'StatsModel.getAllStats',
+                });
+                return cb(null, statsRes);
+            }
+
+            statsRes.requests = this._zip(requests).map(arr =>
+                arr.reduce((acc, i) => acc + i), 0);
+            statsRes['500s'] = this._zip(errors).map(arr =>
+                arr.reduce((acc, i) => acc + i), 0);
+
+            return cb(null, statsRes);
+        });
+    }
+
+    /**
+     * Handles getting a list of global keys.
+     * @param {array} ids - Service identifiers
+     * @param {object} log - Werelogs request logger
+     * @param {function} cb - Callback
+     * @return {undefined}
+     */
+    getAllGlobalStats(ids, log, cb) {
+        const reqsKeys = ids.map(key => (['get', key]));
+        return this._redis.batch(reqsKeys, (err, res) => {
+            const statsRes = { requests: 0 };
+            if (err) {
+                log.error('error getting metrics', {
+                    error: err,
+                    method: 'StatsClient.getAllGlobalStats',
+                });
+                return cb(null, statsRes);
+            }
+            statsRes.requests = res.reduce((sum, curr) => {
+                const [cmdErr, val] = curr;
+                if (cmdErr) {
+                    // Log any individual request errors from the batch request.
+                    log.error('error getting metrics', {
+                        error: cmdErr,
+                        method: 'StatsClient.getAllGlobalStats',
+                    });
+                }
+                return sum + (Number.parseInt(val, 10) || 0);
+            }, 0);
+            return cb(null, statsRes);
+        });
+    }
+
     /**
      * normalize date timestamp to the nearest hour
      * @param {Date} d - Date instance
@@ -24,34 +161,6 @@ class StatsModel extends StatsClient {
         return d.setHours(d.getHours() - 1);
     }
 
-    /**
-     * normalize to the nearest interval
-     * @param {object} d - Date instance
-     * @return {number} timestamp - normalized to the nearest interval
-     */
-    _normalizeTimestamp(d) {
-        const m = d.getMinutes();
-        return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
-    }
-
-    /**
-     * override the method to get the result as an array of integers separated
-     * by each interval
-     * typical input looks like [[null, '1'], [null, '2'], [null, null]...]
-     * @param {array} arr - each index contains the result of each batch command
-     *   where index 0 signifies the error and index 1 contains the result
-     * @return {array} array of integers, ordered from most recent interval to
-     *   oldest interval
-     */
-    _getCount(arr) {
-        return arr.reduce((store, i) => {
-            let num = parseInt(i[1], 10);
-            num = Number.isNaN(num) ? 0 : num;
-            store.push(num);
-            return store;
-        }, []);
-    }
-
     /**
      * get list of sorted set key timestamps
      * @param {number} epoch - epoch time

lib/metrics/ZenkoMetrics.js

@@ -2,8 +2,8 @@ const promClient = require('prom-client');
 
 const collectDefaultMetricsIntervalMs =
       process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS !== undefined ?
-      Number.parseInt(process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS, 10) :
-      10000;
+          Number.parseInt(process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS, 10) :
+          10000;
 
 promClient.collectDefaultMetrics({ timeout: collectDefaultMetricsIntervalMs });
 

lib/models/ARN.js

@@ -27,7 +27,7 @@ class ARN {
 
     static createFromString(arnStr) {
         const [arn, partition, service, region, accountId,
-               resourceType, resource] = arnStr.split(':');
+            resourceType, resource] = arnStr.split(':');
 
         if (arn !== 'arn') {
             return { error: errors.InvalidArgument.customizeDescription(
@@ -58,7 +58,7 @@ class ARN {
                     'must be a 12-digit number or "*"') };
         }
         const fullResource = (resource !== undefined ?
-                              `${resourceType}:${resource}` : resourceType);
+            `${resourceType}:${resource}` : resourceType);
         return new ARN(partition, service, region, accountId, fullResource);
     }
 
@@ -98,7 +98,7 @@ class ARN {
 
     toString() {
         return ['arn', this.getPartition(), this.getService(),
-                this.getRegion(), this.getAccountId(), this.getResource()]
+            this.getRegion(), this.getAccountId(), this.getResource()]
             .join(':');
     }
 }

lib/models/BucketAzureInfo.js

@@ -0,0 +1,237 @@
+/**
+ * Helper class to ease access to the Azure specific information for
+ * storage accounts mapped to buckets.
+ */
+class BucketAzureInfo {
+    /**
+     * @constructor
+     * @param {object} obj - Raw structure for the Azure info on storage account
+     * @param {string} obj.sku - SKU name of this storage account
+     * @param {string} obj.accessTier - Access Tier name of this storage account
+     * @param {string} obj.kind - Kind name of this storage account
+     * @param {string[]} obj.systemKeys - pair of shared keys for the system
+     * @param {string[]} obj.tenantKeys - pair of shared keys for the tenant
+     * @param {string} obj.subscriptionId - subscription ID the storage account
+     *   belongs to
+     * @param {string} obj.resourceGroup - Resource group name the storage
+     *   account belongs to
+     * @param {object} obj.deleteRetentionPolicy - Delete retention policy
+     * @param {boolean} obj.deleteRetentionPolicy.enabled -
+     * @param {number} obj.deleteRetentionPolicy.days -
+     * @param {object[]} obj.managementPolicies - Management policies for this
+     *   storage account
+     * @param {boolean} obj.httpsOnly - Server the content of this storage
+     *   account through HTTPS only
+     * @param {object} obj.tags - Set of tags applied on this storage account
+     * @param {object[]} obj.networkACL - Network ACL of this storage account
+     * @param {string} obj.cname - CNAME of this storage account
+     * @param {boolean} obj.azureFilesAADIntegration - whether or not Azure
+     *   Files AAD Integration is enabled for this storage account
+     * @param {boolean} obj.hnsEnabled - whether or not a hierarchical namespace
+     *   is enabled for this storage account
+     * @param {object} obj.logging - service properties: logging
+     * @param {object} obj.hourMetrics - service properties: hourMetrics
+     * @param {object} obj.minuteMetrics - service properties: minuteMetrics
+     * @param {string} obj.serviceVersion - service properties: serviceVersion
+     */
+    constructor(obj) {
+        this._data = {
+            sku: obj.sku,
+            accessTier: obj.accessTier,
+            kind: obj.kind,
+            systemKeys: obj.systemKeys,
+            tenantKeys: obj.tenantKeys,
+            subscriptionId: obj.subscriptionId,
+            resourceGroup: obj.resourceGroup,
+            deleteRetentionPolicy: obj.deleteRetentionPolicy,
+            managementPolicies: obj.managementPolicies,
+            httpsOnly: obj.httpsOnly,
+            tags: obj.tags,
+            networkACL: obj.networkACL,
+            cname: obj.cname,
+            azureFilesAADIntegration: obj.azureFilesAADIntegration,
+            hnsEnabled: obj.hnsEnabled,
+            logging: obj.logging,
+            hourMetrics: obj.hourMetrics,
+            minuteMetrics: obj.minuteMetrics,
+            serviceVersion: obj.serviceVersion,
+        };
+    }
+
+    getSku() {
+        return this._data.sku;
+    }
+
+    setSku(sku) {
+        this._data.sku = sku;
+        return this;
+    }
+
+    getAccessTier() {
+        return this._data.accessTier;
+    }
+
+    setAccessTier(accessTier) {
+        this._data.accessTier = accessTier;
+        return this;
+    }
+
+    getKind() {
+        return this._data.kind;
+    }
+
+    setKind(kind) {
+        this._data.kind = kind;
+        return this;
+    }
+
+    getSystemKeys() {
+        return this._data.systemKeys;
+    }
+
+    setSystemKeys(systemKeys) {
+        this._data.systemKeys = systemKeys;
+        return this;
+    }
+
+    getTenantKeys() {
+        return this._data.tenantKeys;
+    }
+
+    setTenantKeys(tenantKeys) {
+        this._data.tenantKeys = tenantKeys;
+        return this;
+    }
+
+    getSubscriptionId() {
+        return this._data.subscriptionId;
+    }
+
+    setSubscriptionId(subscriptionId) {
+        this._data.subscriptionId = subscriptionId;
+        return this;
+    }
+
+    getResourceGroup() {
+        return this._data.resourceGroup;
+    }
+
+    setResourceGroup(resourceGroup) {
+        this._data.resourceGroup = resourceGroup;
+        return this;
+    }
+
+    getDeleteRetentionPolicy() {
+        return this._data.deleteRetentionPolicy;
+    }
+
+    setDeleteRetentionPolicy(deleteRetentionPolicy) {
+        this._data.deleteRetentionPolicy = deleteRetentionPolicy;
+        return this;
+    }
+
+    getManagementPolicies() {
+        return this._data.managementPolicies;
+    }
+
+    setManagementPolicies(managementPolicies) {
+        this._data.managementPolicies = managementPolicies;
+        return this;
+    }
+
+    getHttpsOnly() {
+        return this._data.httpsOnly;
+    }
+
+    setHttpsOnly(httpsOnly) {
+        this._data.httpsOnly = httpsOnly;
+        return this;
+    }
+
+    getTags() {
+        return this._data.tags;
+    }
+
+    setTags(tags) {
+        this._data.tags = tags;
+        return this;
+    }
+
+    getNetworkACL() {
+        return this._data.networkACL;
+    }
+
+    setNetworkACL(networkACL) {
+        this._data.networkACL = networkACL;
+        return this;
+    }
+
+    getCname() {
+        return this._data.cname;
+    }
+
+    setCname(cname) {
+        this._data.cname = cname;
+        return this;
+    }
+
+    getAzureFilesAADIntegration() {
+        return this._data.azureFilesAADIntegration;
+    }
+
+    setAzureFilesAADIntegration(azureFilesAADIntegration) {
+        this._data.azureFilesAADIntegration = azureFilesAADIntegration;
+        return this;
+    }
+
+    getHnsEnabled() {
+        return this._data.hnsEnabled;
+    }
+
+    setHnsEnabled(hnsEnabled) {
+        this._data.hnsEnabled = hnsEnabled;
+        return this;
+    }
+
+    getLogging() {
+        return this._data.logging;
+    }
+
+    setLogging(logging) {
+        this._data.logging = logging;
+        return this;
+    }
+
+    getHourMetrics() {
+        return this._data.hourMetrics;
+    }
+
+    setHourMetrics(hourMetrics) {
+        this._data.hourMetrics = hourMetrics;
+        return this;
+    }
+
+    getMinuteMetrics() {
+        return this._data.minuteMetrics;
+    }
+
+    setMinuteMetrics(minuteMetrics) {
+        this._data.minuteMetrics = minuteMetrics;
+        return this;
+    }
+
+    getServiceVersion() {
+        return this._data.serviceVersion;
+    }
+
+    setServiceVersion(serviceVersion) {
+        this._data.serviceVersion = serviceVersion;
+        return this;
+    }
+
+    getValue() {
+        return this._data;
+    }
+}
+
+module.exports = BucketAzureInfo;

lib/models/BucketInfo.js

@@ -9,8 +9,9 @@ const BucketPolicy = require('./BucketPolicy');
 const NotificationConfiguration = require('./NotificationConfiguration');
 
 // WHEN UPDATING THIS NUMBER, UPDATE BucketInfoModelVersion.md CHANGELOG
-// BucketInfoModelVersion.md can be found in the root of this repository
-const modelVersion = 10;
+// BucketInfoModelVersion.md can be found in documentation/ at the root
+// of this repository
+const modelVersion = 14;
 
 class BucketInfo {
     /**
@@ -41,7 +42,8 @@ class BucketInfo {
     * @param {object} versioningConfiguration - versioning configuration
     * @param {string} versioningConfiguration.Status - versioning status
     * @param {object} versioningConfiguration.MfaDelete - versioning mfa delete
-    * @param {string} locationConstraint - locationConstraint for bucket
+    * @param {string} locationConstraint - locationConstraint for bucket that
+    * also includes the ingestion flag
     * @param {WebsiteConfiguration} [websiteConfiguration] - website
     * configuration
     * @param {object[]} [cors] - collection of CORS rules to apply
@@ -57,17 +59,23 @@ class BucketInfo {
     * @param {object} [lifecycleConfiguration] - lifecycle configuration
     * @param {object} [bucketPolicy] - bucket policy
     * @param {string} [uid] - unique identifier for the bucket, necessary
+    * @param {string} readLocationConstraint - readLocationConstraint for bucket
+    * addition for use with lifecycle operations
+    * @param {boolean} [isNFS] - whether the bucket is on NFS
+    * @param {object} [ingestionConfig] - object for ingestion status: en/dis
+    * @param {object} [azureInfo] - Azure storage account specific info
     * @param {boolean} [objectLockEnabled] - true when object lock enabled
     * @param {object} [objectLockConfiguration] - object lock configuration
     * @param {object} [notificationConfiguration] - bucket notification configuration
     */
     constructor(name, owner, ownerDisplayName, creationDate,
-                mdBucketModelVersion, acl, transient, deleted,
-                serverSideEncryption, versioningConfiguration,
-                locationConstraint, websiteConfiguration, cors,
-                replicationConfiguration, lifecycleConfiguration,
-                bucketPolicy, uid, objectLockEnabled, objectLockConfiguration,
-                notificationConfiguration) {
+        mdBucketModelVersion, acl, transient, deleted,
+        serverSideEncryption, versioningConfiguration,
+        locationConstraint, websiteConfiguration, cors,
+        replicationConfiguration, lifecycleConfiguration,
+        bucketPolicy, uid, readLocationConstraint, isNFS,
+        ingestionConfig, azureInfo, objectLockEnabled,
+        objectLockConfiguration, notificationConfiguration) {
         assert.strictEqual(typeof name, 'string');
         assert.strictEqual(typeof owner, 'string');
         assert.strictEqual(typeof ownerDisplayName, 'string');
@@ -86,7 +94,7 @@ class BucketInfo {
         if (serverSideEncryption) {
             assert.strictEqual(typeof serverSideEncryption, 'object');
             const { cryptoScheme, algorithm, masterKeyId,
-                    configuredMasterKeyId, mandatory } = serverSideEncryption;
+                configuredMasterKeyId, mandatory } = serverSideEncryption;
             assert.strictEqual(typeof cryptoScheme, 'number');
             assert.strictEqual(typeof algorithm, 'string');
             assert.strictEqual(typeof masterKeyId, 'string');
@@ -108,6 +116,15 @@ class BucketInfo {
         if (locationConstraint) {
             assert.strictEqual(typeof locationConstraint, 'string');
         }
+        if (ingestionConfig) {
+            assert.strictEqual(typeof ingestionConfig, 'object');
+        }
+        if (azureInfo) {
+            assert.strictEqual(typeof azureInfo, 'object');
+        }
+        if (readLocationConstraint) {
+            assert.strictEqual(typeof readLocationConstraint, 'string');
+        }
         if (websiteConfiguration) {
             assert(websiteConfiguration instanceof WebsiteConfiguration);
             const { indexDocument, errorDocument, redirectAllRequestsTo,
@@ -164,12 +181,16 @@ class BucketInfo {
         this._serverSideEncryption = serverSideEncryption || null;
         this._versioningConfiguration = versioningConfiguration || null;
         this._locationConstraint = locationConstraint || null;
+        this._readLocationConstraint = readLocationConstraint || null;
         this._websiteConfiguration = websiteConfiguration || null;
         this._replicationConfiguration = replicationConfiguration || null;
         this._cors = cors || null;
         this._lifecycleConfiguration = lifecycleConfiguration || null;
         this._bucketPolicy = bucketPolicy || null;
         this._uid = uid || uuid();
+        this._isNFS = isNFS || null;
+        this._ingestion = ingestionConfig || null;
+        this._azureInfo = azureInfo || null;
         this._objectLockEnabled = objectLockEnabled || false;
         this._objectLockConfiguration = objectLockConfiguration || null;
         this._notificationConfiguration = notificationConfiguration || null;
@@ -192,12 +213,16 @@ class BucketInfo {
             serverSideEncryption: this._serverSideEncryption,
             versioningConfiguration: this._versioningConfiguration,
             locationConstraint: this._locationConstraint,
+            readLocationConstraint: this._readLocationConstraint,
             websiteConfiguration: undefined,
             cors: this._cors,
             replicationConfiguration: this._replicationConfiguration,
             lifecycleConfiguration: this._lifecycleConfiguration,
             bucketPolicy: this._bucketPolicy,
             uid: this._uid,
+            isNFS: this._isNFS,
+            ingestion: this._ingestion,
+            azureInfo: this._azureInfo,
             objectLockEnabled: this._objectLockEnabled,
             objectLockConfiguration: this._objectLockConfiguration,
             notificationConfiguration: this._notificationConfiguration,
@@ -222,7 +247,8 @@ class BucketInfo {
             obj.transient, obj.deleted, obj.serverSideEncryption,
             obj.versioningConfiguration, obj.locationConstraint, websiteConfig,
             obj.cors, obj.replicationConfiguration, obj.lifecycleConfiguration,
-            obj.bucketPolicy, obj.uid, obj.objectLockEnabled,
+            obj.bucketPolicy, obj.uid, obj.readLocationConstraint, obj.isNFS,
+            obj.ingestion, obj.azureInfo, obj.objectLockEnabled,
             obj.objectLockConfiguration, obj.notificationConfiguration);
     }
 
@@ -247,8 +273,10 @@ class BucketInfo {
             data._versioningConfiguration, data._locationConstraint,
             data._websiteConfiguration, data._cors,
             data._replicationConfiguration, data._lifecycleConfiguration,
-            data._bucketPolicy, data._uid, data._objectLockEnabled,
-            data._objectLockConfiguration, data._notificationConfiguration);
+            data._bucketPolicy, data._uid, data._readLocationConstraint,
+            data._isNFS, data._ingestion, data._azureInfo,
+            data._objectLockEnabled, data._objectLockConfiguration,
+            data._notificationConfiguration);
     }
 
     /**
@@ -545,6 +573,17 @@ class BucketInfo {
         return this._locationConstraint;
     }
 
+    /**
+    * Get read location constraint.
+    * @return {string} - bucket read location constraint
+    */
+    getReadLocationConstraint() {
+        if (this._readLocationConstraint) {
+            return this._readLocationConstraint;
+        }
+        return this._locationConstraint;
+    }
+
     /**
      * Set Bucket model version
      *
@@ -633,6 +672,85 @@ class BucketInfo {
         this._uid = uid;
         return this;
     }
+    /**
+     * Check if the bucket is an NFS bucket.
+     * @return {boolean} - Wether the bucket is NFS or not
+     */
+    isNFS() {
+        return this._isNFS;
+    }
+    /**
+     * Set whether the bucket is an NFS bucket.
+     * @param {boolean} isNFS - Wether the bucket is NFS or not
+     * @return {BucketInfo} - bucket info instance
+     */
+    setIsNFS(isNFS) {
+        this._isNFS = isNFS;
+        return this;
+    }
+    /**
+     * enable ingestion, set 'this._ingestion' to { status: 'enabled' }
+     * @return {BucketInfo} - bucket info instance
+     */
+    enableIngestion() {
+        this._ingestion = { status: 'enabled' };
+        return this;
+    }
+    /**
+     * disable ingestion, set 'this._ingestion' to { status: 'disabled' }
+     * @return {BucketInfo} - bucket info instance
+     */
+    disableIngestion() {
+        this._ingestion = { status: 'disabled' };
+        return this;
+    }
+    /**
+     * Get ingestion configuration
+     * @return {object} - bucket ingestion configuration: Enabled or Disabled
+     */
+    getIngestion() {
+        return this._ingestion;
+    }
+
+    /**
+     ** Check if bucket is an ingestion bucket
+     * @return {boolean} - 'true' if bucket is ingestion bucket, 'false' if
+     * otherwise
+     */
+    isIngestionBucket() {
+        const ingestionConfig = this.getIngestion();
+        if (ingestionConfig) {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Check if ingestion is enabled
+     * @return {boolean} - 'true' if ingestion is enabled, otherwise 'false'
+     */
+    isIngestionEnabled() {
+        const ingestionConfig = this.getIngestion();
+        return ingestionConfig ? ingestionConfig.status === 'enabled' : false;
+    }
+
+    /**
+     * Return the Azure specific storage account information for this bucket
+     * @return {object} - a structure suitable for {@link BucketAzureIno}
+     *   constructor
+     */
+    getAzureInfo() {
+        return this._azureInfo;
+    }
+    /**
+     * Set the Azure specific storage account information for this bucket
+     * @param {object} azureInfo - a structure suitable for
+     *   {@link BucketAzureInfo} construction
+     * @return {BucketInfo} - bucket info instance
+     */
+    setAzureInfo(azureInfo) {
+        this._azureInfo = azureInfo;
+        return this;
+    }
     /**
     * Check if object lock is enabled.
     * @return {boolean} - depending on whether object lock is enabled

lib/models/LifecycleConfiguration.js

@@ -5,6 +5,8 @@ const errors = require('../errors');
 const LifecycleRule = require('./LifecycleRule');
 const escapeForXml = require('../s3middleware/escapeForXml');
 
+const MAX_DAYS = 2147483647; // Max 32-bit signed binary integer.
+
 /**
  * Format of xml request:
 
@@ -85,10 +87,13 @@ class LifecycleConfiguration {
     /**
      * Create a Lifecycle Configuration instance
      * @param {string} xml - the parsed xml
+     * @param {object} config - the CloudServer config
      * @return {object} - LifecycleConfiguration instance
      */
-    constructor(xml) {
+    constructor(xml, config) {
         this._parsedXML = xml;
+        this._storageClasses =
+            config.replicationEndpoints.map(endpoint => endpoint.site);
         this._ruleIDs = [];
         this._tagKeys = [];
         this._config = {};
@@ -211,9 +216,10 @@ class LifecycleConfiguration {
      */
     _parseRule(rule) {
         const ruleObj = {};
-        if (rule.Transition || rule.NoncurrentVersionTransition) {
+        if (rule.NoncurrentVersionTransition) {
             ruleObj.error = errors.NotImplemented.customizeDescription(
-                'Transition lifecycle action not yet implemented');
+                'NoncurrentVersionTransition lifecycle action not yet ' +
+                'implemented');
             return ruleObj;
         }
         // Either Prefix or Filter must be included, but can be empty string
@@ -375,7 +381,7 @@ class LifecycleConfiguration {
             if (!tags[i].Key || !tags[i].Value) {
                 tagObj.error =
                     errors.MissingRequiredParameter.customizeDescription(
-                    'Tag XML does not contain both Key and Value');
+                        'Tag XML does not contain both Key and Value');
                 break;
             }
 
@@ -468,6 +474,315 @@ class LifecycleConfiguration {
         return statusObj;
     }
 
+    /**
+     * Finds the prefix and/or tags of the given rule and gets the error message
+     * @param {object} rule - The rule to find the prefix in
+     * @return {string} - The prefix of filter information
+     */
+    _getRuleFilterDesc(rule) {
+        if (rule.Prefix) {
+            return `prefix '${rule.Prefix[0]}'`;
+        }
+        // There must be a filter if no top-level prefix is provided. First
+        // check if there are multiple filters (i.e. `Filter.And`).
+        if (rule.Filter[0] === undefined || rule.Filter[0].And === undefined) {
+            const { Prefix, Tag } = rule.Filter[0] || {};
+            if (Prefix) {
+                return `filter '(prefix=${Prefix[0]})'`;
+            }
+            if (Tag) {
+                const { Key, Value } = Tag[0];
+                return `filter '(tag: key=${Key[0]}, value=${Value[0]})'`;
+            }
+            return 'filter (all)';
+        }
+        const filters = [];
+        const { Prefix, Tag } = rule.Filter[0].And[0];
+        if (Prefix) {
+            filters.push(`prefix=${Prefix[0]}`);
+        }
+        Tag.forEach(tag => {
+            const { Key, Value } = tag;
+            filters.push(`tag: key=${Key[0]}, value=${Value[0]}`);
+        });
+        const joinedFilters = filters.join(' and ');
+        return `filter '(${joinedFilters})'`;
+    }
+
+    /**
+     * Checks the validity of the given field
+     * @param {object} params - Given function parameters
+     * @param {string} params.days - The value of the field to check
+     * @param {string} params.field - The field name with the value
+     * @param {string} params.ancestor - The immediate ancestor field
+     * @return {object|null} Returns an error object or `null`
+     */
+    _checkDays(params) {
+        const { days, field, ancestor } = params;
+        if (days < 0) {
+            const msg = `'${field}' in ${ancestor} action must be nonnegative`;
+            return errors.InvalidArgument.customizeDescription(msg);
+        }
+        if (days > MAX_DAYS) {
+            return errors.MalformedXML.customizeDescription(
+                `'${field}' in ${ancestor} action must not exceed ${MAX_DAYS}`);
+        }
+        return null;
+    }
+
+    /**
+     * Checks the validity of the given storage class
+     * @param {object} params - Given function parameters
+     * @param {array} params.usedStorageClasses - Storage classes used in other
+     * rules
+     * @param {string} params.storageClass - The storage class of the current
+     * rule
+     * @param {string} params.ancestor - The immediate ancestor field
+     * @param {string} params.prefix - The prefix of the rule
+     * @return {object|null} Returns an error object or `null`
+     */
+    _checkStorageClasses(params) {
+        const { usedStorageClasses, storageClass, ancestor, rule } = params;
+        if (!this._storageClasses.includes(storageClass)) {
+            // This differs from the AWS message. This will help the user since
+            // the StorageClass does not conform to AWS specs.
+            const list = `'${this._storageClasses.join("', '")}'`;
+            const msg = `'StorageClass' must be one of ${list}`;
+            return errors.MalformedXML.customizeDescription(msg);
+        }
+        if (usedStorageClasses.includes(storageClass)) {
+            const msg = `'StorageClass' must be different for '${ancestor}' ` +
+                `actions in same 'Rule' with ${this._getRuleFilterDesc(rule)}`;
+            return errors.InvalidRequest.customizeDescription(msg);
+        }
+        return null;
+    }
+
+    /**
+     * Ensure that transition rules are at least a day apart from each other.
+     * @param {object} params - Given function parameters
+     * @param {string} [params.days] - The days of the current transition
+     * @param {string} [params.date] - The date of the current transition
+     * @param {string} params.storageClass - The storage class of the current
+     * rule
+     * @param {string} params.rule - The current rule
+     * @return {undefined}
+     */
+    _checkTimeGap(params) {
+        const { days, date, storageClass, rule } = params;
+        const invalidTransition = rule.Transition.find(transition => {
+            if (storageClass === transition.StorageClass[0]) {
+                return false;
+            }
+            if (days !== undefined) {
+                return Number.parseInt(transition.Days[0], 10) === days;
+            }
+            if (date !== undefined) {
+                const timestamp = new Date(date).getTime();
+                const compareTimestamp = new Date(transition.Date[0]).getTime();
+                const oneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
+                return Math.abs(timestamp - compareTimestamp) < oneDay;
+            }
+            return false;
+        });
+        if (invalidTransition) {
+            const timeType = days !== undefined ? 'Days' : 'Date';
+            const filterMsg = this._getRuleFilterDesc(rule);
+            const compareStorageClass = invalidTransition.StorageClass[0];
+            const msg = `'${timeType}' in the 'Transition' action for ` +
+                `StorageClass '${storageClass}' for ${filterMsg} must be at ` +
+                `least one day apart from ${filterMsg} in the 'Transition' ` +
+                `action for StorageClass '${compareStorageClass}'`;
+            return errors.InvalidArgument.customizeDescription(msg);
+        }
+        return undefined;
+    }
+
+    /**
+     * Checks transition time type (i.e. 'Date' or 'Days') only occurs once
+     * across transitions and across transitions and expiration policies
+     * @param {object} params - Given function parameters
+     * @param {string} params.usedTimeType - The time type that has been used by
+     * another rule
+     * @param {string} params.currentTimeType - the time type used by the
+     * current rule
+     * @param {string} params.rule - The current rule
+     * @return {object|null} Returns an error object or `null`
+     */
+    _checkTimeType(params) {
+        const { usedTimeType, currentTimeType, rule } = params;
+        if (usedTimeType && usedTimeType !== currentTimeType) {
+            const msg = "Found mixed 'Date' and 'Days' based Transition " +
+                'actions in lifecycle rule for ' +
+                `${this._getRuleFilterDesc(rule)}`;
+            return errors.InvalidRequest.customizeDescription(msg);
+        }
+        // Transition time type cannot differ from the expiration, if provided.
+        if (rule.Expiration &&
+            rule.Expiration[0][currentTimeType] === undefined) {
+            const msg = "Found mixed 'Date' and 'Days' based Expiration and " +
+                'Transition actions in lifecycle rule for ' +
+                `${this._getRuleFilterDesc(rule)}`;
+            return errors.InvalidRequest.customizeDescription(msg);
+        }
+        return null;
+    }
+
+    /**
+     * Checks the validity of the given date
+     * @param {string} date - The date the check
+     * @return {object|null} Returns an error object or `null`
+     */
+    _checkDate(date) {
+        const isoRegex = new RegExp('^(-?(?:[1-9][0-9]*)?[0-9]{4})-' +
+            '(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9])' +
+            ':([0-5][0-9]):([0-5][0-9])(.[0-9]+)?(Z)?$');
+        if (!isoRegex.test(date)) {
+            const msg = 'Date must be in ISO 8601 format';
+            return errors.InvalidArgument.customizeDescription(msg);
+        }
+        return null;
+    }
+
+    /**
+     * Parses the NonCurrentVersionTransition value
+     * @param {object} rule - Rule object from Rule array from this._parsedXml
+     * @return {object} - Contains error if parsing failed, otherwise contains
+     * the parsed nonCurrentVersionTransition array
+     *
+     * Format of result:
+     * result = {
+     *      error: <error>,
+     *      nonCurrentVersionTransition: [
+     *          {
+     *              noncurrentDays: <non-current-days>,
+     *              storageClass: <storage-class>,
+     *          },
+     *          ...
+     *      ]
+     * }
+     */
+    _parseNoncurrentVersionTransition(rule) {
+        const nonCurrentVersionTransition = [];
+        const usedStorageClasses = [];
+        for (let i = 0; i < rule.NoncurrentVersionTransition.length; i++) {
+            const t = rule.NoncurrentVersionTransition[i]; // Transition object
+            const noncurrentDays =
+                t.NoncurrentDays && Number.parseInt(t.NoncurrentDays[0], 10);
+            const storageClass = t.StorageClass && t.StorageClass[0];
+            if (noncurrentDays === undefined || storageClass === undefined) {
+                return { error: errors.MalformedXML };
+            }
+            let error = this._checkDays({
+                days: noncurrentDays,
+                field: 'NoncurrentDays',
+                ancestor: 'NoncurrentVersionTransition',
+            });
+            if (error) {
+                return { error };
+            }
+            error = this._checkStorageClasses({
+                storageClass,
+                usedStorageClasses,
+                ancestor: 'NoncurrentVersionTransition',
+                rule,
+            });
+            if (error) {
+                return { error };
+            }
+            nonCurrentVersionTransition.push({ noncurrentDays, storageClass });
+            usedStorageClasses.push(storageClass);
+        }
+        return { nonCurrentVersionTransition };
+    }
+
+    /**
+     * Parses the Transition value
+     * @param {object} rule - Rule object from Rule array from this._parsedXml
+     * @return {object} - Contains error if parsing failed, otherwise contains
+     * the parsed transition array
+     *
+     * Format of result:
+     * result = {
+     *      error: <error>,
+     *      transition: [
+     *          {
+     *              days: <days>,
+     *              date: <date>,
+     *              storageClass: <storage-class>,
+     *          },
+     *          ...
+     *      ]
+     * }
+     */
+    _parseTransition(rule) {
+        const transition = [];
+        const usedStorageClasses = [];
+        let usedTimeType = null;
+        for (let i = 0; i < rule.Transition.length; i++) {
+            const t = rule.Transition[i]; // Transition object
+            const days = t.Days && Number.parseInt(t.Days[0], 10);
+            const date = t.Date && t.Date[0];
+            const storageClass = t.StorageClass && t.StorageClass[0];
+            if ((days === undefined && date === undefined) ||
+                (days !== undefined && date !== undefined) ||
+                (storageClass === undefined)) {
+                return { error: errors.MalformedXML };
+            }
+            let error = this._checkStorageClasses({
+                storageClass,
+                usedStorageClasses,
+                ancestor: 'Transition',
+                rule,
+            });
+            if (error) {
+                return { error };
+            }
+            usedStorageClasses.push(storageClass);
+            if (days !== undefined) {
+                error = this._checkTimeType({
+                    usedTimeType,
+                    currentTimeType: 'Days',
+                    rule,
+                });
+                if (error) {
+                    return { error };
+                }
+                usedTimeType = 'Days';
+                error = this._checkDays({
+                    days,
+                    field: 'Days',
+                    ancestor: 'Transition',
+                });
+                if (error) {
+                    return { error };
+                }
+                transition.push({ days, storageClass });
+            }
+            if (date !== undefined) {
+                error = this._checkTimeType({
+                    usedTimeType,
+                    currentTimeType: 'Date',
+                    rule,
+                });
+                if (error) {
+                    return { error };
+                }
+                usedTimeType = 'Date';
+                error = this._checkDate(date);
+                if (error) {
+                    return { error };
+                }
+                transition.push({ date, storageClass });
+            }
+            error = this._checkTimeGap({ days, date, storageClass, rule });
+            if (error) {
+                return { error };
+            }
+        }
+        return { transition };
+    }
+
     /**
      * Check that action component of rule is valid
      * @param {object} rule - a rule object from Rule array from this._parsedXml
@@ -492,8 +807,13 @@ class LifecycleConfiguration {
         const actionsObj = {};
         actionsObj.propName = 'actions';
         actionsObj.actions = [];
-        const validActions = ['AbortIncompleteMultipartUpload',
-            'Expiration', 'NoncurrentVersionExpiration'];
+        const validActions = [
+            'AbortIncompleteMultipartUpload',
+            'Expiration',
+            'NoncurrentVersionExpiration',
+            'NoncurrentVersionTransition',
+            'Transition',
+        ];
         validActions.forEach(a => {
             if (rule[a]) {
                 actionsObj.actions.push({ actionName: `${a}` });
@@ -510,7 +830,8 @@ class LifecycleConfiguration {
             if (action.error) {
                 actionsObj.error = action.error;
             } else {
-                const actionTimes = ['days', 'date', 'deleteMarker'];
+                const actionTimes = ['days', 'date', 'deleteMarker',
+                    'transition', 'nonCurrentVersionTransition'];
                 actionTimes.forEach(t => {
                     if (action[t]) {
                         // eslint-disable-next-line no-param-reassign
@@ -597,12 +918,9 @@ class LifecycleConfiguration {
             return expObj;
         }
         if (subExp.Date) {
-            const isoRegex = new RegExp('^(-?(?:[1-9][0-9]*)?[0-9]{4})-' +
-                '(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9])' +
-                ':([0-5][0-9]):([0-5][0-9])(.[0-9]+)?(Z)?$');
-            if (!isoRegex.test(subExp.Date[0])) {
-                expObj.error = errors.InvalidArgument.customizeDescription(
-                    'Date must be in ISO 8601 format');
+            const error = this._checkDate(subExp.Date[0]);
+            if (error) {
+                expObj.error = error;
             } else {
                 expObj.date = subExp.Date[0];
             }
@@ -611,7 +929,7 @@ class LifecycleConfiguration {
             const daysInt = parseInt(subExp.Days[0], 10);
             if (daysInt < 1) {
                 expObj.error = errors.InvalidArgument.customizeDescription(
-                   'Expiration days is not a positive integer');
+                    'Expiration days is not a positive integer');
             } else {
                 expObj.days = daysInt;
             }
@@ -714,6 +1032,26 @@ class LifecycleConfiguration {
                 if (a.deleteMarker) {
                     assert.strictEqual(typeof a.deleteMarker, 'string');
                 }
+                if (a.nonCurrentVersionTransition) {
+                    assert.strictEqual(
+                        typeof a.nonCurrentVersionTransition, 'object');
+                    a.nonCurrentVersionTransition.forEach(t => {
+                        assert.strictEqual(typeof t.noncurrentDays, 'number');
+                        assert.strictEqual(typeof t.storageClass, 'string');
+                    });
+                }
+                if (a.transition) {
+                    assert.strictEqual(typeof a.transition, 'object');
+                    a.transition.forEach(t => {
+                        if (t.days || t.days === 0) {
+                            assert.strictEqual(typeof t.days, 'number');
+                        }
+                        if (t.date !== undefined) {
+                            assert.strictEqual(typeof t.date, 'string');
+                        }
+                        assert.strictEqual(typeof t.storageClass, 'string');
+                    });
+                }
             });
         });
     }
@@ -763,7 +1101,8 @@ class LifecycleConfiguration {
             }
 
             const Actions = actions.map(action => {
-                const { actionName, days, date, deleteMarker } = action;
+                const { actionName, days, date, deleteMarker,
+                    nonCurrentVersionTransition, transition } = action;
                 let Action;
                 if (actionName === 'AbortIncompleteMultipartUpload') {
                     Action = `<${actionName}><DaysAfterInitiation>${days}` +
@@ -780,6 +1119,40 @@ class LifecycleConfiguration {
                     Action = `<${actionName}>${Days}${Date}${DelMarker}` +
                         `</${actionName}>`;
                 }
+                if (actionName === 'NoncurrentVersionTransition') {
+                    const xml = [];
+                    nonCurrentVersionTransition.forEach(transition => {
+                        const { noncurrentDays, storageClass } = transition;
+                        xml.push(
+                            `<${actionName}>`,
+                            `<NoncurrentDays>${noncurrentDays}` +
+                                '</NoncurrentDays>',
+                            `<StorageClass>${storageClass}</StorageClass>`,
+                            `</${actionName}>`,
+                        );
+                    });
+                    Action = xml.join('');
+                }
+                if (actionName === 'Transition') {
+                    const xml = [];
+                    transition.forEach(transition => {
+                        const { days, date, storageClass } = transition;
+                        let element;
+                        if (days !== undefined) {
+                            element = `<Days>${days}</Days>`;
+                        }
+                        if (date !== undefined) {
+                            element = `<Date>${date}</Date>`;
+                        }
+                        xml.push(
+                            `<${actionName}>`,
+                            element,
+                            `<StorageClass>${storageClass}</StorageClass>`,
+                            `</${actionName}>`,
+                        );
+                    });
+                    Action = xml.join('');
+                }
                 return Action;
             }).join('');
             return `<Rule>${ID}${Status}${Filter}${Actions}</Rule>`;

lib/models/NotificationConfiguration.js

@@ -27,7 +27,7 @@ const errors = require('../errors');
  * </NotificationConfiguration>
  */
 
- /**
+/**
   * Format of config:
   *
   * config = {

lib/models/ObjectLockConfiguration.js

@@ -17,7 +17,7 @@ const errors = require('../errors');
  * </ObjectLockConfiguration>
  */
 
- /**
+/**
   * Format of config:
   *
   * config = {

lib/models/ObjectMD.js

@@ -1,3 +1,5 @@
+const crypto = require('crypto');
+
 const constants = require('../constants');
 const VersionIDUtils = require('../versioning/VersionID');
 
@@ -8,7 +10,6 @@ const ObjectMDLocation = require('./ObjectMDLocation');
  * mpuPart metadata for example)
  */
 class ObjectMD {
-
     /**
      * Create a new instance of ObjectMD. Parameter <tt>objMd</tt> is
      * reserved for internal use, users should call
@@ -28,9 +29,14 @@ class ObjectMD {
             } else {
                 this._updateFromParsedJSON(objMd);
             }
+            if (!this._data['creation-time']) {
+                this.setCreationTime(this.getLastModified());
+            }
         } else {
             // set newly-created object md modified time to current time
-            this._data['last-modified'] = new Date().toJSON();
+            const dt = new Date().toJSON();
+            this.setLastModified(dt);
+            this.setCreationTime(dt);
         }
         // set latest md model version now that we ensured
         // backward-compat conversion
@@ -85,6 +91,8 @@ class ObjectMD {
             'content-length': 0,
             'content-type': '',
             'content-md5': '',
+            'content-language': '',
+            'creation-time': undefined,
             // simple/no version. will expand once object versioning is
             // introduced
             'x-amz-version-id': 'null',
@@ -106,6 +114,7 @@ class ObjectMD {
             },
             'key': '',
             'location': null,
+            'azureInfo': undefined,
             // versionId, isNull, nullVersionId and isDeleteMarker
             // should be undefined when not set explicitly
             'isNull': undefined,
@@ -124,6 +133,7 @@ class ObjectMD {
                 role: '',
                 storageType: '',
                 dataStoreVersionId: '',
+                isNFS: null,
             },
             'dataStoreName': '',
             'originOp': '',
@@ -138,7 +148,7 @@ class ObjectMD {
 
         Object.assign(this._data, objMd._data);
         Object.assign(this._data.replicationInfo,
-                      objMd._data.replicationInfo);
+            objMd._data.replicationInfo);
     }
 
     _updateFromParsedJSON(objMd) {
@@ -356,6 +366,50 @@ class ObjectMD {
         return this._data['content-md5'];
     }
 
+    /**
+     * Set content-language
+     *
+     * @param {string} contentLanguage - content-language
+     * @return {ObjectMD} itself
+     */
+    setContentLanguage(contentLanguage) {
+        this._data['content-language'] = contentLanguage;
+        return this;
+    }
+
+    /**
+     * Returns content-language
+     *
+     * @return {string} content-language
+     */
+    getContentLanguage() {
+        return this._data['content-language'];
+    }
+
+    /**
+     * Set Creation Date
+     *
+     * @param {string} creationTime - Creation Date
+     * @return {ObjectMD} itself
+     */
+    setCreationTime(creationTime) {
+        this._data['creation-time'] = creationTime;
+        return this;
+    }
+
+    /**
+     * Returns Creation Date
+     *
+     * @return {string} Creation Date
+     */
+    getCreationTime() {
+        // If creation-time is not set fallback to LastModified
+        if (!this._data['creation-time']) {
+            return this.getLastModified();
+        }
+        return this._data['creation-time'];
+    }
+
     /**
      * Set version id
      *
@@ -599,6 +653,29 @@ class ObjectMD {
         return reducedLocations;
     }
 
+    /**
+     * Set the Azure specific information
+     * @param {object} azureInfo - a plain JS structure representing the
+     *   Azure specific information for a Blob or a Container (see constructor
+     *   of {@link ObjectMDAzureInfo} for a description of the fields of this
+     *   structure
+     * @return {ObjectMD} itself
+     */
+    setAzureInfo(azureInfo) {
+        this._data.azureInfo = azureInfo;
+        return this;
+    }
+
+    /**
+     * Get the Azure specific information
+     * @return {object} a plain JS structure representing the Azure specific
+     *   information for a Blob or a Container an suitable for the constructor
+     *   of {@link ObjectMDAzureInfo}.
+     */
+    getAzureInfo() {
+        return this._data.azureInfo;
+    }
+
     /**
      * Set metadata isNull value
      *
@@ -680,6 +757,19 @@ class ObjectMD {
         return this._data.isDeleteMarker || false;
     }
 
+    /**
+     * Get if the object is a multipart upload (MPU)
+     *
+     * The function checks the "content-md5" field: if it contains a
+     * dash ('-') it is a MPU, as the content-md5 string ends with
+     * "-[nbparts]" for MPUs.
+     *
+     * @return {boolean} Whether object is a multipart upload
+     */
+    isMultipartUpload() {
+        return this.getContentMd5().includes('-');
+    }
+
     /**
      * Set metadata versionId value
      *
@@ -707,7 +797,10 @@ class ObjectMD {
      * @return {string|undefined} The encoded object versionId
      */
     getEncodedVersionId() {
-        return VersionIDUtils.encode(this.getVersionId());
+        if (this.getVersionId()) {
+            return VersionIDUtils.encode(this.getVersionId());
+        }
+        return undefined;
     }
 
     /**
@@ -750,6 +843,20 @@ class ObjectMD {
         return this._data.tags;
     }
 
+    getUserMetadata() {
+        const metaHeaders = {};
+        const data = this.getValue();
+        Object.keys(data).forEach(key => {
+            if (key.startsWith('x-amz-meta-')) {
+                metaHeaders[key] = data[key];
+            }
+        });
+        if (Object.keys(metaHeaders).length > 0) {
+            return JSON.stringify(metaHeaders);
+        }
+        return undefined;
+    }
+
     /**
      * Set replication information
      *
@@ -758,7 +865,7 @@ class ObjectMD {
      */
     setReplicationInfo(replicationInfo) {
         const { status, backends, content, destination, storageClass, role,
-            storageType, dataStoreVersionId } = replicationInfo;
+            storageType, dataStoreVersionId, isNFS } = replicationInfo;
         this._data.replicationInfo = {
             status,
             backends,
@@ -768,6 +875,7 @@ class ObjectMD {
             role,
             storageType: storageType || '',
             dataStoreVersionId: dataStoreVersionId || '',
+            isNFS: isNFS || null,
         };
         return this;
     }
@@ -786,6 +894,24 @@ class ObjectMD {
         return this;
     }
 
+    /**
+     * Set whether the replication is occurring from an NFS bucket.
+     * @param {Boolean} isNFS - Whether replication from an NFS bucket
+     * @return {ObjectMD} itself
+     */
+    setReplicationIsNFS(isNFS) {
+        this._data.replicationInfo.isNFS = isNFS;
+        return this;
+    }
+
+    /**
+     * Get whether the replication is occurring from an NFS bucket.
+     * @return {Boolean} Whether replication from an NFS bucket
+     */
+    getReplicationIsNFS() {
+        return this._data.replicationInfo.isNFS;
+    }
+
     setReplicationSiteStatus(site, status) {
         const backend = this._data.replicationInfo.backends
             .find(o => o.site === site);
@@ -832,6 +958,11 @@ class ObjectMD {
         return this;
     }
 
+    setReplicationStorageType(storageType) {
+        this._data.replicationInfo.storageType = storageType;
+        return this;
+    }
+
     setReplicationStorageClass(storageClass) {
         this._data.replicationInfo.storageClass = storageClass;
         return this;
@@ -913,6 +1044,9 @@ class ObjectMD {
         Object.keys(metaHeaders).forEach(key => {
             if (key.startsWith('x-amz-meta-')) {
                 this._data[key] = metaHeaders[key];
+            } else if (key.startsWith('x-ms-meta-')) {
+                const _key = key.replace('x-ms-meta-', 'x-amz-meta-');
+                this._data[_key] = metaHeaders[key];
             }
         });
         // If a multipart object and the acl is already parsed, we update it
@@ -922,6 +1056,20 @@ class ObjectMD {
         return this;
     }
 
+    /**
+     * Clear all existing meta headers (used for Azure)
+     *
+     * @return {ObjectMD} itself
+     */
+    clearMetadataValues() {
+        Object.keys(this._data).forEach(key => {
+            if (key.startsWith('x-amz-meta')) {
+                delete this._data[key];
+            }
+        });
+        return this;
+    }
+
     /**
      * overrideMetadataValues (used for complete MPU and object copy)
      *
@@ -933,6 +1081,39 @@ class ObjectMD {
         return this;
     }
 
+    /**
+     * Create or update the microVersionId field
+     *
+     * This field can be used to force an update in MongoDB. This can
+     * be needed in the following cases:
+     *
+     * - in case no other metadata field changes
+     *
+     * - to detect a change when fields change but object version does
+     *   not change e.g. when ingesting a putObjectTagging coming from
+     *   S3C to Zenko
+     *
+     * - to manage conflicts during concurrent updates, using
+     *   conditions on the microVersionId field.
+     *
+     * It's a field of 16 hexadecimal characters randomly generated
+     *
+     * @return {ObjectMD} itself
+     */
+    updateMicroVersionId() {
+        this._data.microVersionId = crypto.randomBytes(8).toString('hex');
+    }
+
+    /**
+     * Get the microVersionId field, or null if not set
+     *
+     * @return {string|null} the microVersionId field if exists, or
+     * {null} if it does not exist
+     */
+    getMicroVersionId() {
+        return this._data.microVersionId || null;
+    }
+
     /**
      * Set object legal hold status
      * @param {boolean} legalHold - true if legal hold is 'ON' false if 'OFF'

lib/models/ObjectMDAzureInfo.js

@@ -0,0 +1,162 @@
+/**
+ * Helper class to ease access to the Azure specific information for
+ * Blob and Container objects.
+ */
+class ObjectMDAzureInfo {
+    /**
+     * @constructor
+     * @param {object} obj - Raw structure for the Azure info on Blob/Container
+     * @param {string} obj.containerPublicAccess - Public access authorization
+     *   type
+     * @param {object[]} obj.containerStoredAccessPolicies - Access policies
+     *   for Shared Access Signature bearer
+     * @param {object} obj.containerImmutabilityPolicy - data immutability
+     *   policy for this container
+     * @param {boolean} obj.containerLegalHoldStatus - legal hold status for
+     *   this container
+     * @param {boolean} obj.containerDeletionInProgress - deletion in progress
+     *   indicator for this container
+     * @param {string} obj.blobType - defines the type of blob for this object
+     * @param {string} obj.blobContentMD5 - whole object MD5 sum set by the
+     *   client through the Azure API
+     * @param {string} obj.blobIssuedETag - backup of the issued ETag on MD only
+     *   operations like Set Blob Properties and Set Blob Metadata
+     * @param {object} obj.blobCopyInfo - information pertaining to past and
+     *   pending copy operation targeting this object
+     * @param {number} obj.blobSequenceNumber - sequence number for a PageBlob
+     * @param {Date} obj.blobAccessTierChangeTime - date of change of tier
+     * @param {boolean} obj.blobUncommitted - A block has been put for a
+     *   nonexistent blob which is about to be created
+     */
+    constructor(obj) {
+        this._data = {
+            containerPublicAccess: obj.containerPublicAccess,
+            containerStoredAccessPolicies: obj.containerStoredAccessPolicies,
+            containerImmutabilityPolicy: obj.containerImmutabilityPolicy,
+            containerLegalHoldStatus: obj.containerLegalHoldStatus,
+            containerDeletionInProgress: obj.containerDeletionInProgress,
+            blobType: obj.blobType,
+            blobContentMD5: obj.blobContentMD5,
+            blobIssuedETag: obj.blobIssuedETag,
+            blobCopyInfo: obj.blobCopyInfo,
+            blobSequenceNumber: obj.blobSequenceNumber,
+            blobAccessTierChangeTime: obj.blobAccessTierChangeTime,
+            blobUncommitted: obj.blobUncommitted,
+        };
+    }
+
+    getContainerPublicAccess() {
+        return this._data.containerPublicAccess;
+    }
+
+    setContainerPublicAccess(containerPublicAccess) {
+        this._data.containerPublicAccess = containerPublicAccess;
+        return this;
+    }
+
+    getContainerStoredAccessPolicies() {
+        return this._data.containerStoredAccessPolicies;
+    }
+
+    setContainerStoredAccessPolicies(containerStoredAccessPolicies) {
+        this._data.containerStoredAccessPolicies =
+            containerStoredAccessPolicies;
+        return this;
+    }
+
+    getContainerImmutabilityPolicy() {
+        return this._data.containerImmutabilityPolicy;
+    }
+
+    setContainerImmutabilityPolicy(containerImmutabilityPolicy) {
+        this._data.containerImmutabilityPolicy = containerImmutabilityPolicy;
+        return this;
+    }
+
+    getContainerLegalHoldStatus() {
+        return this._data.containerLegalHoldStatus;
+    }
+
+    setContainerLegalHoldStatus(containerLegalHoldStatus) {
+        this._data.containerLegalHoldStatus = containerLegalHoldStatus;
+        return this;
+    }
+
+    getContainerDeletionInProgress() {
+        return this._data.containerDeletionInProgress;
+    }
+
+    setContainerDeletionInProgress(containerDeletionInProgress) {
+        this._data.containerDeletionInProgress = containerDeletionInProgress;
+        return this;
+    }
+
+    getBlobType() {
+        return this._data.blobType;
+    }
+
+    setBlobType(blobType) {
+        this._data.blobType = blobType;
+        return this;
+    }
+
+    getBlobContentMD5() {
+        return this._data.blobContentMD5;
+    }
+
+    setBlobContentMD5(blobContentMD5) {
+        this._data.blobContentMD5 = blobContentMD5;
+        return this;
+    }
+
+    getBlobIssuedETag() {
+        return this._data.blobIssuedETag;
+    }
+
+    setBlobIssuedETag(blobIssuedETag) {
+        this._data.blobIssuedETag = blobIssuedETag;
+        return this;
+    }
+
+    getBlobCopyInfo() {
+        return this._data.blobCopyInfo;
+    }
+
+    setBlobCopyInfo(blobCopyInfo) {
+        this._data.blobCopyInfo = blobCopyInfo;
+        return this;
+    }
+
+    getBlobSequenceNumber() {
+        return this._data.blobSequenceNumber;
+    }
+
+    setBlobSequenceNumber(blobSequenceNumber) {
+        this._data.blobSequenceNumber = blobSequenceNumber;
+        return this;
+    }
+
+    getBlobAccessTierChangeTime() {
+        return this._data.blobAccessTierChangeTime;
+    }
+
+    setBlobAccessTierChangeTime(blobAccessTierChangeTime) {
+        this._data.blobAccessTierChangeTime = blobAccessTierChangeTime;
+        return this;
+    }
+
+    getBlobUncommitted() {
+        return this._data.blobUncommitted;
+    }
+
+    setBlobUncommitted(blobUncommitted) {
+        this._data.blobUncommitted = blobUncommitted;
+        return this;
+    }
+
+    getValue() {
+        return this._data;
+    }
+}
+
+module.exports = ObjectMDAzureInfo;

lib/models/ObjectMDLocation.js

@@ -3,7 +3,6 @@
  * 'location' array
  */
 class ObjectMDLocation {
-
     /**
      * @constructor
      * @param {object} locationObj - single data location info
@@ -14,10 +13,14 @@ class ObjectMDLocation {
      * @param {string} locationObj.dataStoreName - type of data store
      * @param {string} locationObj.dataStoreETag - internal ETag of
      *   data part
+     * @param {string} [locationObj.dataStoreVersionId] - versionId,
+     *   needed for cloud backends
      * @param {number} [location.cryptoScheme] - if location data is
      * encrypted: the encryption scheme version
      * @param {string} [location.cipheredDataKey] - if location data
      * is encrypted: the base64-encoded ciphered data key
+     * @param {string} [locationObj.blockId] - blockId of the part,
+     *   set by the Azure Blob Service REST API frontend
      */
     constructor(locationObj) {
         this._data = {
@@ -26,6 +29,8 @@ class ObjectMDLocation {
             size: locationObj.size,
             dataStoreName: locationObj.dataStoreName,
             dataStoreETag: locationObj.dataStoreETag,
+            dataStoreVersionId: locationObj.dataStoreVersionId,
+            blockId: locationObj.blockId,
         };
         if (locationObj.cryptoScheme) {
             this._data.cryptoScheme = locationObj.cryptoScheme;
@@ -47,6 +52,7 @@ class ObjectMDLocation {
      * @param {object} location - single data location info
      * @param {string} location.key - data backend key
      * @param {string} location.dataStoreName - type of data store
+     * @param {string} [location.dataStoreVersionId] - data backend version ID
      * @param {number} [location.cryptoScheme] - if location data is
      * encrypted: the encryption scheme version
      * @param {string} [location.cipheredDataKey] - if location data
@@ -57,6 +63,7 @@ class ObjectMDLocation {
         [
             'key',
             'dataStoreName',
+            'dataStoreVersionId',
             'cryptoScheme',
             'cipheredDataKey',
         ].forEach(attrName => {
@@ -73,6 +80,10 @@ class ObjectMDLocation {
         return this._data.dataStoreETag;
     }
 
+    getDataStoreVersionId() {
+        return this._data.dataStoreVersionId;
+    }
+
     getPartNumber() {
         return Number.parseInt(this._data.dataStoreETag.split(':')[0], 10);
     }
@@ -107,6 +118,15 @@ class ObjectMDLocation {
         return this._data.cipheredDataKey;
     }
 
+    getBlockId() {
+        return this._data.blockId;
+    }
+
+    setBlockId(blockId) {
+        this._data.blockId = blockId;
+        return this;
+    }
+
     getValue() {
         return this._data;
     }

lib/models/ReplicationConfiguration.js

@@ -59,6 +59,7 @@ class ReplicationConfiguration {
         this._rules = null;
         this._prevStorageClass = null;
         this._hasScalityDestination = null;
+        this._preferredReadLocation = null;
     }
 
     /**
@@ -85,6 +86,18 @@ class ReplicationConfiguration {
         return this._rules;
     }
 
+    /**
+     * The preferred read location
+     * @return {string|null} - The preferred read location if defined,
+     * otherwise null
+     *
+     * FIXME ideally we should be able to specify one preferred read
+     * location for each rule
+     */
+    getPreferredReadLocation() {
+        return this._preferredReadLocation;
+    }
+
     /**
      * Get the replication configuration
      * @return {object} - The replication configuration
@@ -94,6 +107,7 @@ class ReplicationConfiguration {
             role: this.getRole(),
             destination: this.getDestination(),
             rules: this.getRules(),
+            preferredReadLocation: this.getPreferredReadLocation(),
         };
     }
 
@@ -292,6 +306,14 @@ class ReplicationConfiguration {
             return undefined;
         }
         const storageClasses = destination.StorageClass[0].split(',');
+        const prefReadIndex = storageClasses.findIndex(storageClass =>
+            storageClass.endsWith(':preferred_read'));
+        if (prefReadIndex !== -1) {
+            const prefRead = storageClasses[prefReadIndex].split(':')[0];
+            // remove :preferred_read tag from storage class name
+            storageClasses[prefReadIndex] = prefRead;
+            this._preferredReadLocation = prefRead;
+        }
         const isValidStorageClass = storageClasses.every(storageClass => {
             if (validStorageClasses.includes(storageClass)) {
                 this._hasScalityDestination =

lib/network/RoundRobin.js

@@ -111,7 +111,7 @@ class RoundRobin {
     pickHost() {
         if (this.logger) {
             this.logger.debug('pick host',
-                              { host: this.getCurrentHost() });
+                { host: this.getCurrentHost() });
         }
         const curHost = this.getCurrentHost();
         ++this.pickCount;
@@ -163,7 +163,7 @@ class RoundRobin {
         }
         if (this.logger) {
             this.logger.debug('round robin host',
-                              { newHost: this.getCurrentHost() });
+                { newHost: this.getCurrentHost() });
         }
     }
 }

lib/network/http/server.js

@@ -10,7 +10,6 @@ const { checkSupportIPv6 } = require('./utils');
 
 
 class Server {
-
     /**
      * @constructor
      *
@@ -369,6 +368,8 @@ class Server {
             error: err.stack || err,
             address: sock.address(),
         });
+        // socket is not systematically destroyed
+        sock.destroy();
     }
 
     /**
@@ -429,16 +430,16 @@ class Server {
                 // Setting no delay of the socket to the value configured
                 sock.setNoDelay(this.isNoDelay());
                 sock.on('error', err => this._logger.info(
-                 'socket error - request rejected', { error: err }));
+                    'socket error - request rejected', { error: err }));
             });
             this._server.on('tlsClientError', (err, sock) =>
-                    this._onClientError(err, sock));
+                this._onClientError(err, sock));
             this._server.on('clientError', (err, sock) =>
-                    this._onClientError(err, sock));
+                this._onClientError(err, sock));
             this._server.on('checkContinue', (req, res) =>
-                    this._onCheckContinue(req, res));
+                this._onCheckContinue(req, res));
             this._server.on('checkExpectation', (req, res) =>
-                    this._onCheckExpectation(req, res));
+                this._onCheckExpectation(req, res));
             this._server.on('listening', () => this._onListening());
         }
         this._server.listen(this._port, this._address);

lib/network/http/utils.js

@@ -72,8 +72,8 @@ function getByteRangeFromSpec(rangeSpec, objectSize) {
     if (rangeSpec.start < objectSize) {
         // test is false if end is undefined
         return { range: [rangeSpec.start,
-                         (rangeSpec.end < objectSize ?
-                          rangeSpec.end : objectSize - 1)] };
+            (rangeSpec.end < objectSize ?
+                rangeSpec.end : objectSize - 1)] };
     }
     return { error: errors.InvalidRange };
 }

lib/network/kmip/Client.js

@@ -95,8 +95,8 @@ function _negotiateProtocolVersion(client, logger, cb) {
         if (err) {
             const error = _arsenalError(err);
             logger.error('KMIP::negotiateProtocolVersion',
-                         { error,
-                           vendorIdentification: client.vendorIdentification });
+                { error,
+                    vendorIdentification: client.vendorIdentification });
             return cb(error);
         }
         const majorVersions =
@@ -107,8 +107,8 @@ function _negotiateProtocolVersion(client, logger, cb) {
             majorVersions.length !== minorVersions.length) {
             const error = _arsenalError('No suitable protocol version');
             logger.error('KMIP::negotiateProtocolVersion',
-                         { error,
-                           vendorIdentification: client.vendorIdentification });
+                { error,
+                    vendorIdentification: client.vendorIdentification });
             return cb(error);
         }
         client.kmip.changeProtocolVersion(majorVersions[0], minorVersions[0]);
@@ -131,8 +131,8 @@ function _mapExtensions(client, logger, cb) {
         if (err) {
             const error = _arsenalError(err);
             logger.error('KMIP::mapExtensions',
-                         { error,
-                           vendorIdentification: client.vendorIdentification });
+                { error,
+                    vendorIdentification: client.vendorIdentification });
             return cb(error);
         }
         const extensionNames = response.lookup(searchFilter.extensionName);
@@ -140,8 +140,8 @@ function _mapExtensions(client, logger, cb) {
         if (extensionNames.length !== extensionTags.length) {
             const error = _arsenalError('Inconsistent extension list');
             logger.error('KMIP::mapExtensions',
-                         { error,
-                           vendorIdentification: client.vendorIdentification });
+                { error,
+                    vendorIdentification: client.vendorIdentification });
             return cb(error);
         }
         extensionNames.forEach((extensionName, idx) => {
@@ -165,7 +165,7 @@ function _queryServerInformation(client, logger, cb) {
         if (err) {
             const error = _arsenalError(err);
             logger.warn('KMIP::queryServerInformation',
-                        { error });
+                { error });
             /* no error returned, caller can keep going */
             return cb();
         }
@@ -175,9 +175,9 @@ function _queryServerInformation(client, logger, cb) {
             JSON.stringify(response.lookup(searchFilter.serverInformation)[0]));
 
         logger.info('KMIP Server identified',
-                    { vendorIdentification: client.vendorIdentification,
-                      serverInformation: client.serverInformation,
-                      negotiatedProtocolVersion: client.kmip.protocolVersion });
+            { vendorIdentification: client.vendorIdentification,
+                serverInformation: client.serverInformation,
+                negotiatedProtocolVersion: client.kmip.protocolVersion });
         return cb();
     });
 }
@@ -201,8 +201,8 @@ function _queryOperationsAndObjects(client, logger, cb) {
         if (err) {
             const error = _arsenalError(err);
             logger.error('KMIP::queryOperationsAndObjects',
-                         { error,
-                           vendorIdentification: client.vendorIdentification });
+                { error,
+                    vendorIdentification: client.vendorIdentification });
             return cb(error);
         }
         const supportedOperations = response.lookup(searchFilter.operation);
@@ -227,15 +227,15 @@ function _queryOperationsAndObjects(client, logger, cb) {
             logger.warn('KMIP::queryOperationsAndObjects: ' +
                         'The KMIP Server announces that it ' +
                         'does not support all of the required features',
-                        { vendorIdentification: client.vendorIdentification,
-                          serverInformation: client.serverInformation,
-                          supportsEncrypt, supportsDecrypt,
-                          supportsActivate, supportsRevoke,
-                          supportsCreate, supportsDestroy,
-                          supportsQuery, supportsSymmetricKeys });
+            { vendorIdentification: client.vendorIdentification,
+                serverInformation: client.serverInformation,
+                supportsEncrypt, supportsDecrypt,
+                supportsActivate, supportsRevoke,
+                supportsCreate, supportsDestroy,
+                supportsQuery, supportsSymmetricKeys });
         } else {
             logger.info('KMIP Server provides the necessary feature set',
-                        { vendorIdentification: client.vendorIdentification });
+                { vendorIdentification: client.vendorIdentification });
         }
         return cb();
     });
@@ -269,8 +269,8 @@ class Client {
         this.vendorIdentification = '';
         this.serverInformation = [];
         this.kmip = new KMIP(CodecClass || TTLVCodec,
-                             TransportClass || TlsTransport,
-                             options);
+            TransportClass || TlsTransport,
+            options);
         this.kmip.registerHandshakeFunction((logger, cb) => {
             this._kmipHandshake(logger, cb);
         });
@@ -327,8 +327,8 @@ class Client {
             if (err) {
                 const error = _arsenalError(err);
                 logger.error('KMIP::_activateBucketKey',
-                             { error,
-                               serverInformation: this.serverInformation });
+                    { error,
+                        serverInformation: this.serverInformation });
                 return cb(error);
             }
             const uniqueIdentifier =
@@ -337,7 +337,7 @@ class Client {
                 const error = _arsenalError(
                     'Server did not return the expected identifier');
                 logger.error('KMIP::cipherDataKey',
-                             { error, uniqueIdentifier });
+                    { error, uniqueIdentifier });
                 return cb(error);
             }
             return cb(null, keyIdentifier);
@@ -356,20 +356,20 @@ class Client {
         const attributes = [];
         if (!!this.options.bucketNameAttributeName) {
             attributes.push(KMIP.Attribute('TextString',
-                                           this.options.bucketNameAttributeName,
-                                           bucketName));
+                this.options.bucketNameAttributeName,
+                bucketName));
         }
         attributes.push(...[
             KMIP.Attribute('Enumeration', 'Cryptographic Algorithm',
-                           CRYPTOGRAPHIC_ALGORITHM),
+                CRYPTOGRAPHIC_ALGORITHM),
             KMIP.Attribute('Integer', 'Cryptographic Length',
-                           CRYPTOGRAPHIC_LENGTH),
+                CRYPTOGRAPHIC_LENGTH),
             KMIP.Attribute('Integer', 'Cryptographic Usage Mask',
-                           this.kmip.encodeMask('Cryptographic Usage Mask',
-                                                CRYPTOGRAPHIC_USAGE_MASK))]);
+                this.kmip.encodeMask('Cryptographic Usage Mask',
+                    CRYPTOGRAPHIC_USAGE_MASK))]);
         if (this.options.compoundCreateActivate) {
             attributes.push(KMIP.Attribute('Date-Time', 'Activation Date',
-                                           new Date(Date.UTC())));
+                new Date(Date.UTC())));
         }
 
         return this.kmip.request(logger, 'Create', [
@@ -379,8 +379,8 @@ class Client {
             if (err) {
                 const error = _arsenalError(err);
                 logger.error('KMIP::createBucketKey',
-                             { error,
-                               serverInformation: this.serverInformation });
+                    { error,
+                        serverInformation: this.serverInformation });
                 return cb(error);
             }
             const createdObjectType =
@@ -391,7 +391,7 @@ class Client {
                 const error = _arsenalError(
                     'Server created an object of wrong type');
                 logger.error('KMIP::createBucketKey',
-                             { error, createdObjectType });
+                    { error, createdObjectType });
                 return cb(error);
             }
             if (!this.options.compoundCreateActivate) {
@@ -416,16 +416,16 @@ class Client {
             KMIP.TextString('Unique Identifier', bucketKeyId),
             KMIP.Structure('Revocation Reason', [
                 KMIP.Enumeration('Revocation Reason Code',
-                                 'Cessation of Operation'),
+                    'Cessation of Operation'),
                 KMIP.TextString('Revocation Message',
-                                'About to be deleted'),
+                    'About to be deleted'),
             ]),
         ], (err, response) => {
             if (err) {
                 const error = _arsenalError(err);
                 logger.error('KMIP::_revokeBucketKey',
-                             { error,
-                               serverInformation: this.serverInformation });
+                    { error,
+                        serverInformation: this.serverInformation });
                 return cb(error);
             }
             const uniqueIdentifier =
@@ -434,7 +434,7 @@ class Client {
                 const error = _arsenalError(
                     'Server did not return the expected identifier');
                 logger.error('KMIP::_revokeBucketKey',
-                             { error, uniqueIdentifier });
+                    { error, uniqueIdentifier });
                 return cb(error);
             }
             return cb();
@@ -453,8 +453,8 @@ class Client {
             if (err) {
                 const error = _arsenalError(err);
                 logger.error('KMIP::destroyBucketKey: revocation failed',
-                             { error,
-                               serverInformation: this.serverInformation });
+                    { error,
+                        serverInformation: this.serverInformation });
                 return cb(error);
             }
             return this.kmip.request(logger, 'Destroy', [
@@ -463,8 +463,8 @@ class Client {
                 if (err) {
                     const error = _arsenalError(err);
                     logger.error('KMIP::destroyBucketKey',
-                                 { error,
-                                   serverInformation: this.serverInformation });
+                        { error,
+                            serverInformation: this.serverInformation });
                     return cb(error);
                 }
                 const uniqueIdentifier =
@@ -473,7 +473,7 @@ class Client {
                     const error = _arsenalError(
                         'Server did not return the expected identifier');
                     logger.error('KMIP::destroyBucketKey',
-                                 { error, uniqueIdentifier });
+                        { error, uniqueIdentifier });
                     return cb(error);
                 }
                 return cb();
@@ -492,19 +492,19 @@ class Client {
      * @callback called with (err, cipheredDataKey: Buffer)
      */
     cipherDataKey(cryptoScheme,
-                  masterKeyId,
-                  plainTextDataKey,
-                  logger,
-                  cb) {
+        masterKeyId,
+        plainTextDataKey,
+        logger,
+        cb) {
         return this.kmip.request(logger, 'Encrypt', [
             KMIP.TextString('Unique Identifier', masterKeyId),
             KMIP.Structure('Cryptographic Parameters', [
                 KMIP.Enumeration('Block Cipher Mode',
-                                CRYPTOGRAPHIC_CIPHER_MODE),
+                    CRYPTOGRAPHIC_CIPHER_MODE),
                 KMIP.Enumeration('Padding Method',
-                                CRYPTOGRAPHIC_PADDING_METHOD),
+                    CRYPTOGRAPHIC_PADDING_METHOD),
                 KMIP.Enumeration('Cryptographic Algorithm',
-                               CRYPTOGRAPHIC_ALGORITHM),
+                    CRYPTOGRAPHIC_ALGORITHM),
             ]),
             KMIP.ByteString('Data', plainTextDataKey),
             KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
@@ -512,8 +512,8 @@ class Client {
             if (err) {
                 const error = _arsenalError(err);
                 logger.error('KMIP::cipherDataKey',
-                             { error,
-                               serverInformation: this.serverInformation });
+                    { error,
+                        serverInformation: this.serverInformation });
                 return cb(error);
             }
             const uniqueIdentifier =
@@ -523,7 +523,7 @@ class Client {
                 const error = _arsenalError(
                     'Server did not return the expected identifier');
                 logger.error('KMIP::cipherDataKey',
-                             { error, uniqueIdentifier });
+                    { error, uniqueIdentifier });
                 return cb(error);
             }
             return cb(null, data);
@@ -541,19 +541,19 @@ class Client {
      * @callback called with (err, plainTextDataKey: Buffer)
      */
     decipherDataKey(cryptoScheme,
-                    masterKeyId,
-                    cipheredDataKey,
-                    logger,
-                    cb) {
+        masterKeyId,
+        cipheredDataKey,
+        logger,
+        cb) {
         return this.kmip.request(logger, 'Decrypt', [
             KMIP.TextString('Unique Identifier', masterKeyId),
             KMIP.Structure('Cryptographic Parameters', [
                 KMIP.Enumeration('Block Cipher Mode',
-                                CRYPTOGRAPHIC_CIPHER_MODE),
+                    CRYPTOGRAPHIC_CIPHER_MODE),
                 KMIP.Enumeration('Padding Method',
-                                 CRYPTOGRAPHIC_PADDING_METHOD),
+                    CRYPTOGRAPHIC_PADDING_METHOD),
                 KMIP.Enumeration('Cryptographic Algorithm',
-                                 CRYPTOGRAPHIC_ALGORITHM),
+                    CRYPTOGRAPHIC_ALGORITHM),
             ]),
             KMIP.ByteString('Data', cipheredDataKey),
             KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
@@ -561,8 +561,8 @@ class Client {
             if (err) {
                 const error = _arsenalError(err);
                 logger.error('KMIP::decipherDataKey',
-                             { error,
-                               serverInformation: this.serverInformation });
+                    { error,
+                        serverInformation: this.serverInformation });
                 return cb(error);
             }
             const uniqueIdentifier =
@@ -572,7 +572,7 @@ class Client {
                 const error = _arsenalError(
                     'Server did not return the right identifier');
                 logger.error('KMIP::decipherDataKey',
-                             { error, uniqueIdentifier });
+                    { error, uniqueIdentifier });
                 return cb(error);
             }
             return cb(null, data);

lib/network/kmip/codec/ttlv.js

@@ -55,15 +55,15 @@ function TTLVCodec() {
                     const property = {};
                     if (!TypeDecoder[elementType]) {
                         _throwError(logger,
-                                    'Unknown element type',
-                                    { funcName, elementTag, elementType });
+                            'Unknown element type',
+                            { funcName, elementTag, elementType });
                     }
                     const elementValue = value.slice(i + 8,
-                                                   i + 8 + elementLength);
+                        i + 8 + elementLength);
                     if (elementValue.length !== elementLength) {
                         _throwError(logger, 'BUG: Wrong buffer size',
-                                    { funcName, elementLength,
-                                      bufferLength: elementValue.length });
+                            { funcName, elementLength,
+                                bufferLength: elementValue.length });
                     }
                     property.type = TypeDecoder[elementType].name;
                     property.value = TypeDecoder[elementType]
@@ -75,7 +75,7 @@ function TTLVCodec() {
                     const tagInfo = TagDecoder[elementTag];
                     if (!tagInfo) {
                         logger.debug('Unknown element tag',
-                                     { funcName, elementTag });
+                            { funcName, elementTag });
                         property.tag = elementTag;
                         element['Unknown Tag'] = property;
                     } else {
@@ -83,8 +83,8 @@ function TTLVCodec() {
                         if (tagInfo.name === 'Attribute Name') {
                             if (property.type !== 'TextString') {
                                 _throwError(logger,
-                                            'Invalide type',
-                                            { funcName, type: property.type });
+                                    'Invalide type',
+                                    { funcName, type: property.type });
                             }
                             diversion = property.value;
                         }
@@ -114,8 +114,8 @@ function TTLVCodec() {
                         }
                         const itemResult =
                               TypeEncoder[itemType].encode(itemTagName,
-                                                           itemValue,
-                                                           itemDiversion);
+                                  itemValue,
+                                  itemDiversion);
                         encodedValue = encodedValue
                             .concat(_ttlvPadVector(itemResult));
                     });
@@ -133,9 +133,9 @@ function TTLVCodec() {
                 const fixedLength = 4;
                 if (fixedLength !== value.length) {
                     _throwError(logger,
-                                'Length mismatch',
-                                { funcName, fixedLength,
-                                  bufferLength: value.length });
+                        'Length mismatch',
+                        { funcName, fixedLength,
+                            bufferLength: value.length });
                 }
                 return value.readUInt32BE(0);
             },
@@ -156,16 +156,16 @@ function TTLVCodec() {
                 const fixedLength = 8;
                 if (fixedLength !== value.length) {
                     _throwError(logger,
-                                'Length mismatch',
-                                { funcName, fixedLength,
-                                  bufferLength: value.length });
+                        'Length mismatch',
+                        { funcName, fixedLength,
+                            bufferLength: value.length });
                 }
                 const longUInt = UINT32_MAX * value.readUInt32BE(0) +
                       value.readUInt32BE(4);
                 if (longUInt > Number.MAX_SAFE_INTEGER) {
                     _throwError(logger,
-                                '53-bit overflow',
-                                { funcName, longUInt });
+                        '53-bit overflow',
+                        { funcName, longUInt });
                 }
                 return longUInt;
             },
@@ -200,9 +200,9 @@ function TTLVCodec() {
                 const fixedLength = 4;
                 if (fixedLength !== value.length) {
                     _throwError(logger,
-                                'Length mismatch',
-                                { funcName, fixedLength,
-                                  bufferLength: value.length });
+                        'Length mismatch',
+                        { funcName, fixedLength,
+                            bufferLength: value.length });
                 }
                 const enumValue = value.toString('hex');
                 const actualTag = diversion ? TagEncoder[diversion].value : tag;
@@ -211,10 +211,10 @@ function TTLVCodec() {
                     !enumInfo.enumeration ||
                     !enumInfo.enumeration[enumValue]) {
                     return { tag,
-                             value: enumValue,
-                             message: 'Unknown enumeration value',
-                             diversion,
-                           };
+                        value: enumValue,
+                        message: 'Unknown enumeration value',
+                        diversion,
+                    };
                 }
                 return enumInfo.enumeration[enumValue];
             },
@@ -227,7 +227,7 @@ function TTLVCodec() {
                 const actualTag = diversion || tagName;
                 const encodedValue =
                       Buffer.from(TagEncoder[actualTag].enumeration[value],
-                                  'hex');
+                          'hex');
                 return _ttlvPadVector([tag, type, length, encodedValue]);
             },
         },
@@ -238,9 +238,9 @@ function TTLVCodec() {
                 const fixedLength = 8;
                 if (fixedLength !== value.length) {
                     _throwError(logger,
-                                'Length mismatch',
-                                { funcName, fixedLength,
-                                  bufferLength: value.length });
+                        'Length mismatch',
+                        { funcName, fixedLength,
+                            bufferLength: value.length });
                 }
                 const msUInt = value.readUInt32BE(0);
                 const lsUInt = value.readUInt32BE(4);
@@ -267,7 +267,7 @@ function TTLVCodec() {
                 const length = Buffer.alloc(4);
                 length.writeUInt32BE(value.length);
                 return _ttlvPadVector([tag, type, length,
-                               Buffer.from(value, 'utf8')]);
+                    Buffer.from(value, 'utf8')]);
             },
         },
         '08': {
@@ -289,17 +289,17 @@ function TTLVCodec() {
                 const fixedLength = 8;
                 if (fixedLength !== value.length) {
                     _throwError(logger,
-                                'Length mismatch',
-                                { funcName, fixedLength,
-                                  bufferLength: value.length });
+                        'Length mismatch',
+                        { funcName, fixedLength,
+                            bufferLength: value.length });
                 }
                 const d = new Date(0);
                 const utcSeconds = UINT32_MAX * value.readUInt32BE(0) +
                       value.readUInt32BE(4);
                 if (utcSeconds > Number.MAX_SAFE_INTEGER) {
                     _throwError(logger,
-                                '53-bit overflow',
-                                { funcName, utcSeconds });
+                        '53-bit overflow',
+                        { funcName, utcSeconds });
                 }
                 d.setUTCSeconds(utcSeconds);
                 return d;
@@ -323,9 +323,9 @@ function TTLVCodec() {
                 const fixedLength = 4;
                 if (fixedLength !== value.length) {
                     _throwError(logger,
-                                'Length mismatch',
-                                { funcName, fixedLength,
-                                  bufferLength: value.length });
+                        'Length mismatch',
+                        { funcName, fixedLength,
+                            bufferLength: value.length });
                 }
                 return value.readInt32BE(0);
             },
@@ -415,8 +415,8 @@ function TTLVCodec() {
                     throw Error(`Unknown Type '${type}'`);
                 }
                 const itemValue = TypeEncoder[type].encode(key,
-                                                        item[key].value,
-                                                        item[key].diversion);
+                    item[key].value,
+                    item[key].diversion);
                 result = result.concat(_ttlvPadVector(itemValue));
             });
         });

lib/network/kmip/index.js

@@ -275,11 +275,11 @@ class KMIP {
                 KMIP.Structure('Request Header', [
                     KMIP.Structure('Protocol Version', [
                         KMIP.Integer('Protocol Version Major',
-                                     this.protocolVersion.major),
+                            this.protocolVersion.major),
                         KMIP.Integer('Protocol Version Minor',
-                                     this.protocolVersion.minor)]),
+                            this.protocolVersion.minor)]),
                     KMIP.Integer('Maximum Response Size',
-                                 this.maximumResponseSize),
+                        this.maximumResponseSize),
                     KMIP.Integer('Batch Count', 1)]),
                 KMIP.Structure('Batch Item', [
                     KMIP.Enumeration('Operation', operation),
@@ -292,7 +292,7 @@ class KMIP {
             (err, conversation, rawResponse) => {
                 if (err) {
                     logger.error('KMIP::request: Failed to send message',
-                                 { error: err });
+                        { error: err });
                     return cb(err);
                 }
                 const response = this._decodeMessage(logger, rawResponse);
@@ -311,16 +311,16 @@ class KMIP {
                     this.transport.abortPipeline(conversation);
                     const error = Error('Invalid batch item ID returned');
                     logger.error('KMIP::request: failed',
-                                 { resultUniqueBatchItemID, uuid, error });
+                        { resultUniqueBatchItemID, uuid, error });
                     return cb(error);
                 }
                 if (performedOperation !== operation) {
                     this.transport.abortPipeline(conversation);
                     const error = Error('Operation mismatch',
-                                        { got: performedOperation,
-                                         expected: operation });
+                        { got: performedOperation,
+                            expected: operation });
                     logger.error('KMIP::request: Operation mismatch',
-                                 { error });
+                        { error });
                     return cb(error);
                 }
                 if (resultStatus !== 'Success') {
@@ -331,19 +331,17 @@ class KMIP {
                           response.lookup(
                               'Response Message/Batch Item/Result Message')[0];
                     const error = Error('KMIP request failure',
-                                        { resultStatus,
-                                          resultReason,
-                                          resultMessage });
+                        { resultStatus,
+                            resultReason,
+                            resultMessage });
                     logger.error('KMIP::request: request failed',
-                                 { error, resultStatus,
-                                   resultReason, resultMessage });
+                        { error, resultStatus,
+                            resultReason, resultMessage });
                     return cb(error);
                 }
                 return cb(null, response);
             });
     }
-
-
 }
 
 

lib/network/kmip/transport/TransportTemplate.js

@@ -86,8 +86,8 @@ class TransportTemplate {
                     const deferedRequest = this.deferedRequests.shift();
                     process.nextTick(() => {
                         this.send(logger,
-                                  deferedRequest.encodedMessage,
-                                  deferedRequest.cb);
+                            deferedRequest.encodedMessage,
+                            deferedRequest.cb);
                     });
                 } else if (this.callbackPipeline.length === 0 &&
                            this.deferedRequests.length === 0 &&

lib/network/probe/HealthProbeServer.js

@@ -0,0 +1,76 @@
+const httpServer = require('../http/server');
+const werelogs = require('werelogs');
+const errors = require('../../errors');
+const ZenkoMetrics = require('../../metrics/ZenkoMetrics');
+const { sendSuccess, sendError } = require('./Utils');
+
+function checkStub(log) { // eslint-disable-line
+    return true;
+}
+
+class HealthProbeServer extends httpServer {
+    constructor(params) {
+        const logging = new werelogs.Logger('HealthProbeServer');
+        super(params.port, logging);
+        this.logging = logging;
+        this.setBindAddress(params.bindAddress || 'localhost');
+        // hooking our request processing function by calling the
+        // parent's method for that
+        this.onRequest(this._onRequest);
+        this._reqHandlers = {
+            '/_/health/liveness': this._onLiveness.bind(this),
+            '/_/health/readiness': this._onReadiness.bind(this),
+            '/_/monitoring/metrics': this._onMetrics.bind(this),
+        };
+        this._livenessCheck = params.livenessCheck || checkStub;
+        this._readinessCheck = params.readinessCheck || checkStub;
+    }
+
+    onLiveCheck(f) {
+        this._livenessCheck = f;
+    }
+
+    onReadyCheck(f) {
+        this._readinessCheck = f;
+    }
+
+    _onRequest(req, res) {
+        const log = this.logging.newRequestLogger();
+        log.debug('request received', { method: req.method,
+            url: req.url });
+
+        if (req.method !== 'GET') {
+            sendError(res, log, errors.MethodNotAllowed);
+        } else if (req.url in this._reqHandlers) {
+            this._reqHandlers[req.url](req, res, log);
+        } else {
+            sendError(res, log, errors.InvalidURI);
+        }
+    }
+
+    _onLiveness(req, res, log) {
+        if (this._livenessCheck(log)) {
+            sendSuccess(res, log);
+        } else {
+            sendError(res, log, errors.ServiceUnavailable);
+        }
+    }
+
+    _onReadiness(req, res, log) {
+        if (this._readinessCheck(log)) {
+            sendSuccess(res, log);
+        } else {
+            sendError(res, log, errors.ServiceUnavailable);
+        }
+    }
+
+    // expose metrics to Prometheus
+    _onMetrics(req, res) {
+        res.writeHead(200, {
+            'Content-Type': ZenkoMetrics.asPrometheusContentType(),
+        });
+        res.end(ZenkoMetrics.asPrometheus());
+    }
+}
+
+module.exports = HealthProbeServer;

lib/network/probe/ProbeServer.js

@@ -3,19 +3,17 @@ const werelogs = require('werelogs');
 const errors = require('../../errors');
 
 const DEFAULT_LIVE_ROUTE = '/_/live';
-const DEFAULT_READY_ROUTE = '/_/live';
-const DEFAULT_METRICS_ROUTE = '/_/metrics';
+const DEFAULT_READY_ROUTE = '/_/ready';
+const DEFAULT_METRICS_ROUTE = '/metrics';
 
 /**
- * ProbeDelegate is used to determine if a probe is successful or
- * if any errors are present.
- * If everything is working as intended, it is a no-op.
- * Otherwise, return a string representing what is failing.
+ * ProbeDelegate is used to handle probe checks.
+ * You can sendSuccess and sendError from Utils to handle success
+ * and failure conditions.
  * @callback ProbeDelegate
  * @param { import('http').ServerResponse } res - HTTP response for writing
  * @param {werelogs.Logger} log - Werelogs instance for logging if you choose to
- * @return {(string|undefined)} String representing issues to report. An empty
- * string or undefined is used to represent no issues.
+ * @return {undefined}
  */
 
 /**
@@ -91,13 +89,7 @@ class ProbeServer extends httpServer {
             return;
         }
 
-        const probeResponse = this._handlers.get(req.url)(res, log);
-        if (probeResponse !== undefined && probeResponse !== '') {
-            // Return an internal error with the response
-            errors.InternalError
-                .customizeDescription(probeResponse)
-                .writeResponse(res);
-        }
+        this._handlers.get(req.url)(res, log);
     }
 }
 

lib/network/probe/Utils.js

@@ -0,0 +1,41 @@
+/**
+ * Send a successful HTTP response of 200 OK
+ * @param {http.ServerResponse} res - HTTP response for writing
+ * @param {werelogs.Logger} log - Werelogs instance for logging if you choose to
+ * @param {string} [message] - Message to send as response, defaults to OK
+ * @returns {undefined}
+ */
+function sendSuccess(res, log, message = 'OK') {
+    log.debug('replying with success');
+    res.writeHead(200);
+    res.end(message);
+}
+
+/**
+ * Send an Arsenal Error response
+ * @param {http.ServerResponse} res - HTTP response for writing
+ * @param {werelogs.Logger} log - Werelogs instance for logging if you choose to
+ * @param {ArsenalError} error - Error to send back to the user
+ * @param {string} [optMessage] - Message to use instead of the errors message
+ * @returns {undefined}
+ */
+function sendError(res, log, error, optMessage) {
+    const message = optMessage || error.description || '';
+    log.debug('sending back error response',
+        {
+            httpCode: error.code,
+            errorType: error.message,
+            error: message,
+        },
+    );
+    res.writeHead(error.code);
+    res.end(JSON.stringify({
+        errorType: error.message,
+        errorMessage: message,
+    }));
+}
+
+module.exports = {
+    sendSuccess,
+    sendError,
+};

lib/network/rest/RESTClient.js

@@ -81,6 +81,7 @@ class RESTClient {
 
         this.host = params.host;
         this.port = params.port;
+        this.isPassthrough = params.isPassthrough || false;
         this.setupLogging(params.logApi);
         this.httpAgent = new HttpAgent({
             keepAlive: true,
@@ -119,11 +120,13 @@ class RESTClient {
     doRequest(method, headers, key, log, responseCb) {
         const reqHeaders = headers || {};
         const urlKey = key || '';
+        const prefix = this.isPassthrough ?
+            constants.passthroughFileURL : constants.dataFileURL;
         const reqParams = {
             hostname: this.host,
             port: this.port,
             method,
-            path: `${constants.dataFileURL}/${urlKey}`,
+            path: encodeURI(`${prefix}/${urlKey}`),
             headers: reqHeaders,
             agent: this.httpAgent,
         };

lib/network/rest/RESTServer.js

@@ -7,7 +7,7 @@ const werelogs = require('werelogs');
 
 const httpServer = require('../http/server');
 const constants = require('../../constants');
-const utils = require('./utils');
+const { parseURL } = require('./utils');
 const httpUtils = require('../http/utils');
 const errors = require('../../errors');
 
@@ -19,7 +19,7 @@ function setContentRange(response, byteRange, objectSize) {
     const [start, end] = byteRange;
     assert(start !== undefined && end !== undefined);
     response.setHeader('Content-Range',
-                       `bytes ${start}-${end}/${objectSize}`);
+        `bytes ${start}-${end}/${objectSize}`);
 }
 
 function sendError(res, log, error, optMessage) {
@@ -37,42 +37,6 @@ function sendError(res, log, error, optMessage) {
         errorMessage: message })}\n`);
 }
 
-/**
- * Parse the given url and return a pathInfo object. Sanity checks are
- * performed.
- *
- * @param {String} urlStr - URL to parse
- * @param {Boolean} expectKey - whether the command expects to see a
- *   key in the URL
- * @return {Object} a pathInfo object with URL items containing the
- * following attributes:
- *   - pathInfo.service {String} - The name of REST service ("DataFile")
- *   - pathInfo.key {String} - The requested key
- */
-function parseURL(urlStr, expectKey) {
-    const urlObj = url.parse(urlStr);
-    const pathInfo = utils.explodePath(urlObj.path);
-    if (pathInfo.service !== constants.dataFileURL) {
-        throw errors.InvalidAction.customizeDescription(
-            `unsupported service '${pathInfo.service}'`);
-    }
-    if (expectKey && pathInfo.key === undefined) {
-        throw errors.MissingParameter.customizeDescription(
-            'URL is missing key');
-    }
-    if (!expectKey && pathInfo.key !== undefined) {
-        // note: we may implement rewrite functionality by allowing a
-        // key in the URL, though we may still provide the new key in
-        // the Location header to keep immutability property and
-        // atomicity of the update (we would just remove the old
-        // object when the new one has been written entirely in this
-        // case, saving a request over an equivalent PUT + DELETE).
-        throw errors.InvalidURI.customizeDescription(
-            'PUT url cannot contain a key');
-    }
-    return pathInfo;
-}
-
 /**
  * @class
  * @classdesc REST Server interface
@@ -81,7 +45,6 @@ function parseURL(urlStr, expectKey) {
  * start() to start listening to the configured port.
  */
 class RESTServer extends httpServer {
-
     /**
      * @constructor
      * @param {Object} params - constructor params
@@ -263,7 +226,7 @@ class RESTServer extends httpServer {
                     return sendError(res, log, err);
                 }
                 log.debug('sending back 200/206 response with contents',
-                          { key: pathInfo.key });
+                    { key: pathInfo.key });
                 setContentLength(res, contentLength);
                 res.setHeader('Accept-Ranges', 'bytes');
                 if (byteRange) {
@@ -301,7 +264,7 @@ class RESTServer extends httpServer {
                 return sendError(res, log, err);
             }
             log.debug('sending back 204 response to DELETE',
-                      { key: pathInfo.key });
+                { key: pathInfo.key });
             res.writeHead(204);
             return res.end(() => {
                 log.debug('DELETE response sent', { key: pathInfo.key });

lib/network/rest/utils.js

@@ -1,15 +1,68 @@
 'use strict'; // eslint-disable-line
 
 const errors = require('../../errors');
+const constants = require('../../constants');
+const url = require('url');
 
-module.exports.explodePath = function explodePath(path) {
+const passthroughPrefixLength = constants.passthroughFileURL.length;
+
+function explodePath(path) {
+    if (path.startsWith(constants.passthroughFileURL)) {
+        const key = path.slice(passthroughPrefixLength + 1);
+        return {
+            service: constants.passthroughFileURL,
+            key: key.length > 0 ? key : undefined,
+        };
+    }
     const pathMatch = /^(\/[a-zA-Z0-9]+)(\/([0-9a-f]*))?$/.exec(path);
     if (pathMatch) {
         return {
             service: pathMatch[1],
             key: (pathMatch[3] !== undefined && pathMatch[3].length > 0 ?
-                  pathMatch[3] : undefined),
+                pathMatch[3] : undefined),
         };
     }
     throw errors.InvalidURI.customizeDescription('malformed URI');
+}
+
+/**
+ * Parse the given url and return a pathInfo object. Sanity checks are
+ * performed.
+ *
+ * @param {String} urlStr - URL to parse
+ * @param {Boolean} expectKey - whether the command expects to see a
+ *   key in the URL
+ * @return {Object} a pathInfo object with URL items containing the
+ * following attributes:
+ *   - pathInfo.service {String} - The name of REST service ("DataFile")
+ *   - pathInfo.key {String} - The requested key
+ */
+function parseURL(urlStr, expectKey) {
+    const urlObj = url.parse(urlStr);
+    const pathInfo = explodePath(decodeURI(urlObj.path));
+    if ((pathInfo.service !== constants.dataFileURL)
+        && (pathInfo.service !== constants.passthroughFileURL)) {
+        throw errors.InvalidAction.customizeDescription(
+            `unsupported service '${pathInfo.service}'`);
+    }
+    if (expectKey && pathInfo.key === undefined) {
+        throw errors.MissingParameter.customizeDescription(
+            'URL is missing key');
+    }
+    if (!expectKey && pathInfo.key !== undefined) {
+        // note: we may implement rewrite functionality by allowing a
+        // key in the URL, though we may still provide the new key in
+        // the Location header to keep immutability property and
+        // atomicity of the update (we would just remove the old
+        // object when the new one has been written entirely in this
+        // case, saving a request over an equivalent PUT + DELETE).
+        throw errors.InvalidURI.customizeDescription(
+            'PUT url cannot contain a key');
+    }
+    return pathInfo;
+}
+
+module.exports = {
+    explodePath,
+    parseURL,
 };

lib/network/rpc/level-net.js

@@ -17,7 +17,6 @@ const rpc = require('./rpc.js');
  * RPC client object accessing the sub-level transparently.
  */
 class LevelDbClient extends rpc.BaseClient {
-
     /**
      * @constructor
      *
@@ -78,7 +77,6 @@ class LevelDbClient extends rpc.BaseClient {
  * env.subDb (env is passed as first parameter of received RPC calls).
  */
 class LevelDbService extends rpc.BaseService {
-
     /**
      * @constructor
      *

lib/network/rpc/rpc.js

@@ -37,7 +37,6 @@ let streamRPCJSONObj;
  *   an error occurred).
  */
 class BaseClient extends EventEmitter {
-
     /**
      * @constructor
      *
@@ -54,7 +53,7 @@ class BaseClient extends EventEmitter {
      */
     constructor(params) {
         const { url, logger, callTimeoutMs,
-                streamMaxPendingAck, streamAckTimeoutMs } = params;
+            streamMaxPendingAck, streamAckTimeoutMs } = params;
         assert(url);
         assert(logger);
 
@@ -82,11 +81,11 @@ class BaseClient extends EventEmitter {
     _call(remoteCall, args, cb) {
         const wrapCb = (err, data) => {
             cb(reconstructError(err),
-               this.socketStreams.decodeStreams(data));
+                this.socketStreams.decodeStreams(data));
         };
         this.logger.debug('remote call', { remoteCall, args });
         this.socket.emit('call', remoteCall,
-                         this.socketStreams.encodeStreams(args), wrapCb);
+            this.socketStreams.encodeStreams(args), wrapCb);
         return undefined;
     }
 
@@ -113,8 +112,8 @@ class BaseClient extends EventEmitter {
             throw new Error(`argument cb=${cb} is not a callback`);
         }
         async.timeout(this._call.bind(this), timeoutMs,
-                      `operation ${remoteCall} timed out`)(remoteCall,
-                                                           args, cb);
+            `operation ${remoteCall} timed out`)(remoteCall,
+            args, cb);
         return undefined;
     }
 
@@ -142,7 +141,7 @@ class BaseClient extends EventEmitter {
         const url = this.url;
         this.socket.on('error', err => {
             this.logger.warn('connectivity error to the RPC service',
-                             { url, error: err });
+                { url, error: err });
         });
         this.socket.on('connect', () => {
             this.emit('connect');
@@ -156,7 +155,7 @@ class BaseClient extends EventEmitter {
         this.getManifest((err, manifest) => {
             if (err) {
                 this.logger.error('Error fetching manifest from RPC server',
-                                  { error: err });
+                    { error: err });
             } else {
                 manifest.api.forEach(apiItem => {
                     this.createCall(apiItem.name);
@@ -251,7 +250,6 @@ class BaseClient extends EventEmitter {
  *
  */
 class BaseService {
-
     /**
      * @constructor
      *
@@ -497,7 +495,7 @@ function RPCServer(params) {
 
                 conn.on('error', err => {
                     log.error('error on socket.io connection',
-                              { namespace: service.namespace, error: err });
+                        { namespace: service.namespace, error: err });
                 });
                 conn.on('call', (remoteCall, args, cb) => {
                     const decodedArgs = streamsSocket.decodeStreams(args);
@@ -647,8 +645,8 @@ streamRPCJSONObj = function _streamRPCJSONObj(obj, wstream, cb) {
     // primitive types
     if (obj === undefined) {
         wstream.write('null'); // if undefined elements are present in
-                               // arrays, convert them to JSON null
-                               // objects
+        // arrays, convert them to JSON null
+        // objects
     } else {
         wstream.write(JSON.stringify(obj));
     }

lib/network/rpc/sio-stream.js

@@ -16,7 +16,7 @@ class SIOOutputStream extends stream.Writable {
     constructor(socket, streamId, maxPendingAck, ackTimeoutMs) {
         super({ objectMode: true });
         this._initOutputStream(socket, streamId, maxPendingAck,
-                               ackTimeoutMs);
+            ackTimeoutMs);
     }
 
     _initOutputStream(socket, streamId, maxPendingAck, ackTimeoutMs) {
@@ -194,7 +194,7 @@ class SIOStreamSocket {
         this.socket.on('stream-data', (payload, cb) => {
             const { streamId, data } = payload;
             log.debug('received \'stream-data\' event',
-                      { streamId, size: data.length });
+                { streamId, size: data.length });
             const stream = this.remoteStreams[streamId];
             if (!stream) {
                 log.debug('no such remote stream registered', { streamId });
@@ -280,15 +280,15 @@ class SIOStreamSocket {
             let transportStream;
             if (isReadStream) {
                 transportStream = new SIOOutputStream(this, streamId,
-                                                      this.maxPendingAck,
-                                                      this.ackTimeoutMs);
+                    this.maxPendingAck,
+                    this.ackTimeoutMs);
             } else {
                 transportStream = new SIOInputStream(this, streamId);
             }
             this.localStreams[streamId] = arg;
             arg.once('close', () => {
                 log.debug('stream closed, removing from local streams',
-                          { streamId });
+                    { streamId });
                 delete this.localStreams[streamId];
             });
             arg.on('error', error => {
@@ -350,8 +350,8 @@ class SIOStreamSocket {
                 stream = new SIOInputStream(this, streamId);
             } else if (arg.writable) {
                 stream = new SIOOutputStream(this, streamId,
-                                             this.maxPendingAck,
-                                             this.ackTimeoutMs);
+                    this.maxPendingAck,
+                    this.ackTimeoutMs);
             } else {
                 throw new Error('can\'t decode stream neither readable ' +
                                 'nor writable');
@@ -360,14 +360,14 @@ class SIOStreamSocket {
             if (arg.readable) {
                 stream.once('close', () => {
                     log.debug('stream closed, removing from remote streams',
-                              { streamId });
+                        { streamId });
                     delete this.remoteStreams[streamId];
                 });
             }
             if (arg.writable) {
                 stream.once('finish', () => {
                     log.debug('stream finished, removing from remote streams',
-                              { streamId });
+                        { streamId });
                     delete this.remoteStreams[streamId];
                 });
             }
@@ -399,7 +399,7 @@ class SIOStreamSocket {
 
     _write(streamId, data, cb) {
         this.logger.debug('emit \'stream-data\' event',
-                          { streamId, size: data.length });
+            { streamId, size: data.length });
         this.socket.emit('stream-data', { streamId, data }, cb);
     }
 

lib/patches/locationConstraints.js

@@ -0,0 +1,159 @@
+'use strict'; // eslint-disable-line strict
+
+const { URL } = require('url');
+const { decryptSecret } = require('../executables/pensieveCreds/utils');
+
+function patchLocations(overlayLocations, creds, log) {
+    if (!overlayLocations) {
+        return {};
+    }
+
+    const locations = {};
+    Object.keys(overlayLocations).forEach(k => {
+        const l = overlayLocations[k];
+        const location = {
+            name: k,
+            objectId: l.objectId,
+            details: l.details || {},
+            locationType: l.locationType,
+        };
+        let supportsVersioning = false;
+        let pathStyle = process.env.CI_CEPH !== undefined;
+
+        switch (l.locationType) {
+        case 'location-mem-v1':
+            location.type = 'mem';
+            location.details = { supportsVersioning: true };
+            break;
+        case 'location-file-v1':
+            location.type = 'file';
+            location.details = { supportsVersioning: true };
+            break;
+        case 'location-azure-v1':
+            location.type = 'azure';
+            if (l.details.secretKey && l.details.secretKey.length > 0) {
+                location.details = {
+                    bucketMatch: l.details.bucketMatch,
+                    azureStorageEndpoint: l.details.endpoint,
+                    azureStorageAccountName: l.details.accessKey,
+                    azureStorageAccessKey: decryptSecret(creds,
+                        l.details.secretKey),
+                    azureContainerName: l.details.bucketName,
+                };
+            }
+            break;
+        case 'location-ceph-radosgw-s3-v1':
+        case 'location-scality-ring-s3-v1':
+            pathStyle = true; // fallthrough
+        case 'location-aws-s3-v1':
+        case 'location-wasabi-v1':
+            supportsVersioning = true; // fallthrough
+        case 'location-do-spaces-v1':
+            location.type = 'aws_s3';
+            if (l.details.secretKey && l.details.secretKey.length > 0) {
+                let https = true;
+                let awsEndpoint = l.details.endpoint ||
+                    's3.amazonaws.com';
+                if (awsEndpoint.includes('://')) {
+                    const url = new URL(awsEndpoint);
+                    awsEndpoint = url.host;
+                    https = url.protocol.includes('https');
+                }
+
+                location.details = {
+                    credentials: {
+                        accessKey: l.details.accessKey,
+                        secretKey: decryptSecret(creds,
+                            l.details.secretKey),
+                    },
+                    bucketName: l.details.bucketName,
+                    bucketMatch: l.details.bucketMatch,
+                    serverSideEncryption:
+                        Boolean(l.details.serverSideEncryption),
+                    region: l.details.region,
+                    awsEndpoint,
+                    supportsVersioning,
+                    pathStyle,
+                    https,
+                };
+            }
+            break;
+        case 'location-gcp-v1':
+            location.type = 'gcp';
+            if (l.details.secretKey && l.details.secretKey.length > 0) {
+                location.details = {
+                    credentials: {
+                        accessKey: l.details.accessKey,
+                        secretKey: decryptSecret(creds,
+                            l.details.secretKey),
+                    },
+                    bucketName: l.details.bucketName,
+                    mpuBucketName: l.details.mpuBucketName,
+                    bucketMatch: l.details.bucketMatch,
+                    gcpEndpoint: l.details.endpoint ||
+                        'storage.googleapis.com',
+                    https: true,
+                };
+            }
+            break;
+        case 'location-scality-sproxyd-v1':
+            location.type = 'scality';
+            if (l.details && l.details.bootstrapList &&
+                l.details.proxyPath) {
+                location.details = {
+                    supportsVersioning: true,
+                    connector: {
+                        sproxyd: {
+                            chordCos: l.details.chordCos || null,
+                            bootstrap: l.details.bootstrapList,
+                            path: l.details.proxyPath,
+                        },
+                    },
+                };
+            }
+            break;
+        case 'location-nfs-mount-v1':
+            location.type = 'pfs';
+            if (l.details) {
+                location.details = {
+                    supportsVersioning: true,
+                    bucketMatch: true,
+                    pfsDaemonEndpoint: {
+                        host: `${l.name}-cosmos-pfsd`,
+                        port: 80,
+                    },
+                };
+            }
+            break;
+        case 'location-scality-hdclient-v2':
+            location.type = 'scality';
+            if (l.details && l.details.bootstrapList) {
+                location.details = {
+                    supportsVersioning: true,
+                    connector: {
+                        hdclient: {
+                            bootstrap: l.details.bootstrapList,
+                        },
+                    },
+                };
+            }
+            break;
+        default:
+            log.info(
+                'unknown location type',
+                { locationType: l.locationType },
+            );
+            return;
+        }
+        location.sizeLimitGB = l.sizeLimitGB || null;
+        location.isTransient = Boolean(l.isTransient);
+        location.legacyAwsBehavior = Boolean(l.legacyAwsBehavior);
+        locations[location.name] = location;
+        return;
+    });
+    return locations;
+}
+
+module.exports = {
+    patchLocations,
+};

lib/policy/userPolicySchema.json

@@ -38,6 +38,10 @@
             "type": "string",
             "pattern": "^arn:aws:iam::[0-9]{12}:saml-provider/[\\w._-]{1,128}$"
         },
+        "principalFederatedOidcIdp": {
+            "type": "string",
+            "pattern": "^(?:http(s)?:\/\/)?[\\w.-]+(?:\\.[\\w\\.-]+)+[\\w\\-\\._~:/?#[\\]@!\\$&'\\(\\)\\*\\+,;=.]+$"
+        },
         "principalAWSItem": {
             "type": "object",
             "properties": {
@@ -98,6 +102,9 @@
                     "oneOf": [
                         {
                             "$ref": "#/definitions/principalFederatedSamlIdp"
+                        },
+                        {
+                            "$ref": "#/definitions/principalFederatedOidcIdp"
                         }
                     ]
                 }

lib/policyEvaluator/evaluator.js

@@ -50,7 +50,7 @@ evaluators.isResourceApplicable = (requestContext, statementResource, log) => {
                 requestResourceArr, true);
         if (arnSegmentsMatch) {
             log.trace('policy resource is applicable to request',
-            { requestResource: resource, policyResource });
+                { requestResource: resource, policyResource });
             return true;
         }
         continue;
@@ -224,21 +224,21 @@ evaluators.evaluatePolicy = (requestContext, policy, log) => {
         // in policy, move on to next statement
         if (currentStatement.NotResource &&
             evaluators.isResourceApplicable(requestContext,
-            currentStatement.NotResource, log)) {
+                currentStatement.NotResource, log)) {
             continue;
         }
         // If affirmative action is in policy and request action is not
         // applicable, move on to next statement
         if (currentStatement.Action &&
             !evaluators.isActionApplicable(requestContext.getAction(),
-            currentStatement.Action, log)) {
+                currentStatement.Action, log)) {
             continue;
         }
         // If NotAction is in policy and action matches NotAction in policy,
         // move on to next statement
         if (currentStatement.NotAction &&
             evaluators.isActionApplicable(requestContext.getAction(),
-            currentStatement.NotAction, log)) {
+                currentStatement.NotAction, log)) {
             continue;
         }
         const conditionEval = currentStatement.Condition ?

lib/policyEvaluator/utils/actionMaps.js

@@ -30,6 +30,7 @@ const sharedActionMap = {
     bypassGovernanceRetention: 's3:BypassGovernanceRetention',
     listMultipartUploads: 's3:ListBucketMultipartUploads',
     listParts: 's3:ListMultipartUploadParts',
+    metadataSearch: 's3:MetadataSearch',
     multipartDelete: 's3:AbortMultipartUpload',
     objectDelete: 's3:DeleteObject',
     objectDeleteTagging: 's3:DeleteObjectTagging',
@@ -116,6 +117,7 @@ const actionMonitoringMapS3 = {
     initiateMultipartUpload: 'CreateMultipartUpload',
     listMultipartUploads: 'ListMultipartUploads',
     listParts: 'ListParts',
+    metadataSearch: 'MetadataSearch',
     multiObjectDelete: 'DeleteObjects',
     multipartDelete: 'AbortMultipartUpload',
     objectCopy: 'CopyObject',
@@ -159,6 +161,7 @@ const actionMapIAM = {
     getPolicyVersion: 'iam:GetPolicyVersion',
     getUser: 'iam:GetUser',
     listAccessKeys: 'iam:ListAccessKeys',
+    listEntitiesForPolicy: 'iam:ListEntitiesForPolicy',
     listGroupPolicies: 'iam:ListGroupPolicies',
     listGroups: 'iam:ListGroups',
     listGroupsForUser: 'iam:ListGroupsForUser',

lib/policyEvaluator/utils/conditions.js

@@ -39,11 +39,11 @@ conditions.findConditionKey = (key, requestContext) => {
     // (see Boolean Condition Operators).
     // Note: This key is only present if MFA was used. So, the following
     // will not work:
-        //     "Condition" :
-        //     { "Bool" : { "aws:MultiFactorAuthPresent" : false } }
+    //     "Condition" :
+    //     { "Bool" : { "aws:MultiFactorAuthPresent" : false } }
     // Instead use:
-        //     "Condition" :
-        //     { "Null" : { "aws:MultiFactorAuthPresent" : true } }
+    //     "Condition" :
+    //     { "Null" : { "aws:MultiFactorAuthPresent" : true } }
     map.set('aws:MultiFactorAuthPresent',
         requestContext.getMultiFactorAuthPresent());
     // aws:MultiFactorAuthAge – Used to check how many seconds since
@@ -146,6 +146,8 @@ conditions.findConditionKey = (key, requestContext) => {
     map.set('s3:ObjLocationConstraint',
         headers['x-amz-meta-scal-location-constraint']);
     map.set('sts:ExternalId', requestContext.getRequesterExternalId());
+    map.set('keycloak:groups', requesterInfo.keycloakGroup);
+    map.set('keycloak:roles', requesterInfo.keycloakRole);
     map.set('iam:PolicyArn', requestContext.getPolicyArn());
     // s3:ExistingObjectTag - Used to check that existing object tag has
     // specific tag key and value. Extraction of correct tag key is done in CloudServer.
@@ -164,8 +166,8 @@ conditions.findConditionKey = (key, requestContext) => {
     // so evaluation should be skipped
     map.set('s3:RequestObjectTagKeys',
         requestContext.getNeedTagEval() && requestContext.getRequestObjTags()
-        ? getTagKeys(requestContext.getRequestObjTags())
-        : undefined);
+            ? getTagKeys(requestContext.getRequestObjTags())
+            : undefined);
     return map.get(key);
 };
 
@@ -189,7 +191,7 @@ function convertSpecialChars(string) {
         return map[char];
     }
     return string.replace(/(\$\{\*\})|(\$\{\?\})|(\$\{\$\})/g,
-    characterMap);
+        characterMap);
 }
 
 /**
@@ -423,10 +425,10 @@ conditions.convertConditionOperator = operator => {
             return !operatorMap.ArnLike(key, value);
         },
         Null: function nullOperator(key, value) {
-         // Null is used to check if a condition key is present.
-         // The policy statement value should be either true (the key doesn't
-         // exist — it is null) or false (the key exists and its value is
-         // not null).
+            // Null is used to check if a condition key is present.
+            // The policy statement value should be either true (the key doesn't
+            // exist — it is null) or false (the key exists and its value is
+            // not null).
             if ((key === undefined || key === null)
                 && value[0] === 'true' ||
                 (key !== undefined && key !== null)

lib/policyEvaluator/utils/wildcards.js

@@ -51,10 +51,10 @@ wildcards.handleWildcardInResource = arn => {
 // Wildcards can be part of the resource ARN.
 // Wildcards do NOT span segments of the ARN (separated by ":")
 
-// Example: all elements in specific bucket:
-// "Resource": "arn:aws:s3:::my_corporate_bucket/*"
-// ARN format:
-// arn:partition:service:region:namespace:relative-id
+    // Example: all elements in specific bucket:
+    // "Resource": "arn:aws:s3:::my_corporate_bucket/*"
+    // ARN format:
+    // arn:partition:service:region:namespace:relative-id
     const arnArr = arn.split(':');
     return arnArr.map(portion => wildcards.handleWildcards(portion));
 };

lib/s3middleware/MD5Sum.js

@@ -6,7 +6,6 @@ const crypto = require('crypto');
  * data through a stream
  */
 class MD5Sum extends Transform {
-
     /**
      * @constructor
      */
@@ -40,7 +39,6 @@ class MD5Sum extends Transform {
         this.emit('hashed');
         callback(null);
     }
-
 }
 
 module.exports = MD5Sum;

lib/s3middleware/azureHelpers/ResultsCollector.js

@@ -73,7 +73,7 @@ class ResultsCollector extends EventEmitter {
  * @property {Error} [results[].error] - error returned by Azure putting subpart
  * @property {number} results[].subPartIndex - index of the subpart
  */
- /**
+/**
   * "error" event
   * @event ResultCollector#error
   * @type {(Error|undefined)} error - error returned by Azure last subpart

lib/s3middleware/azureHelpers/mpuUtils.js

@@ -94,7 +94,7 @@ azureMpuUtils.getSubPartIds = (part, uploadId) =>
         azureMpuUtils.getBlockId(uploadId, part.partNumber, subPartIndex));
 
 azureMpuUtils.putSinglePart = (errorWrapperFn, request, params, dataStoreName,
-log, cb) => {
+    log, cb) => {
     const { bucketName, partNumber, size, objectKey, contentMD5, uploadId }
         = params;
     const blockId = azureMpuUtils.getBlockId(uploadId, partNumber, 0);
@@ -107,31 +107,31 @@ log, cb) => {
     request.pipe(passThrough);
     return errorWrapperFn('uploadPart', 'createBlockFromStream',
         [blockId, bucketName, objectKey, passThrough, size, options,
-        (err, result) => {
-            if (err) {
-                log.error('Error from Azure data backend uploadPart',
-                    { error: err.message, dataStoreName });
-                if (err.code === 'ContainerNotFound') {
-                    return cb(errors.NoSuchBucket);
+            (err, result) => {
+                if (err) {
+                    log.error('Error from Azure data backend uploadPart',
+                        { error: err.message, dataStoreName });
+                    if (err.code === 'ContainerNotFound') {
+                        return cb(errors.NoSuchBucket);
+                    }
+                    if (err.code === 'InvalidMd5') {
+                        return cb(errors.InvalidDigest);
+                    }
+                    if (err.code === 'Md5Mismatch') {
+                        return cb(errors.BadDigest);
+                    }
+                    return cb(errors.InternalError.customizeDescription(
+                        `Error returned from Azure: ${err.message}`),
+                    );
                 }
-                if (err.code === 'InvalidMd5') {
-                    return cb(errors.InvalidDigest);
-                }
-                if (err.code === 'Md5Mismatch') {
-                    return cb(errors.BadDigest);
-                }
-                return cb(errors.InternalError.customizeDescription(
-                    `Error returned from Azure: ${err.message}`)
-                );
-            }
-            const md5 = result.headers['content-md5'] || '';
-            const eTag = objectUtils.getHexMD5(md5);
-            return cb(null, eTag, size);
-        }], log, cb);
+                const md5 = result.headers['content-md5'] || '';
+                const eTag = objectUtils.getHexMD5(md5);
+                return cb(null, eTag, size);
+            }], log, cb);
 };
 
 azureMpuUtils.putNextSubPart = (errorWrapperFn, partParams, subPartInfo,
-subPartStream, subPartIndex, resultsCollector, log, cb) => {
+    subPartStream, subPartIndex, resultsCollector, log, cb) => {
     const { uploadId, partNumber, bucketName, objectKey } = partParams;
     const subPartSize = azureMpuUtils.getSubPartSize(
         subPartInfo, subPartIndex);
@@ -140,11 +140,11 @@ subPartStream, subPartIndex, resultsCollector, log, cb) => {
     resultsCollector.pushOp();
     errorWrapperFn('uploadPart', 'createBlockFromStream',
         [subPartId, bucketName, objectKey, subPartStream, subPartSize,
-        {}, err => resultsCollector.pushResult(err, subPartIndex)], log, cb);
+            {}, err => resultsCollector.pushResult(err, subPartIndex)], log, cb);
 };
 
 azureMpuUtils.putSubParts = (errorWrapperFn, request, params,
-dataStoreName, log, cb) => {
+    dataStoreName, log, cb) => {
     const subPartInfo = azureMpuUtils.getSubPartInfo(params.size);
     const resultsCollector = new ResultsCollector();
     const hashedStream = new MD5Sum();

lib/s3middleware/convertToXml.js

@@ -31,9 +31,9 @@ convertMethods.listMultipartUploads = xmlParams => {
     const l = xmlParams.list;
 
     xml.push('<?xml version="1.0" encoding="UTF-8"?>',
-             '<ListMultipartUploadsResult ' +
+        '<ListMultipartUploadsResult ' +
                 'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">',
-             `<Bucket>${escapeForXml(xmlParams.bucketName)}</Bucket>`
+        `<Bucket>${escapeForXml(xmlParams.bucketName)}</Bucket>`,
     );
 
     // For certain XML elements, if it is `undefined`, AWS returns either an
@@ -58,7 +58,7 @@ convertMethods.listMultipartUploads = xmlParams => {
     });
 
     xml.push(`<MaxUploads>${escapeForXml(l.MaxKeys)}</MaxUploads>`,
-             `<IsTruncated>${escapeForXml(l.IsTruncated)}</IsTruncated>`
+        `<IsTruncated>${escapeForXml(l.IsTruncated)}</IsTruncated>`,
     );
 
     l.Uploads.forEach(upload => {
@@ -69,29 +69,29 @@ convertMethods.listMultipartUploads = xmlParams => {
         }
 
         xml.push('<Upload>',
-                 `<Key>${escapeForXml(key)}</Key>`,
-                 `<UploadId>${escapeForXml(val.UploadId)}</UploadId>`,
-                 '<Initiator>',
-                 `<ID>${escapeForXml(val.Initiator.ID)}</ID>`,
-                 `<DisplayName>${escapeForXml(val.Initiator.DisplayName)}` +
+            `<Key>${escapeForXml(key)}</Key>`,
+            `<UploadId>${escapeForXml(val.UploadId)}</UploadId>`,
+            '<Initiator>',
+            `<ID>${escapeForXml(val.Initiator.ID)}</ID>`,
+            `<DisplayName>${escapeForXml(val.Initiator.DisplayName)}` +
                     '</DisplayName>',
-                 '</Initiator>',
-                 '<Owner>',
-                 `<ID>${escapeForXml(val.Owner.ID)}</ID>`,
-                 `<DisplayName>${escapeForXml(val.Owner.DisplayName)}` +
+            '</Initiator>',
+            '<Owner>',
+            `<ID>${escapeForXml(val.Owner.ID)}</ID>`,
+            `<DisplayName>${escapeForXml(val.Owner.DisplayName)}` +
                     '</DisplayName>',
-                 '</Owner>',
-                 `<StorageClass>${escapeForXml(val.StorageClass)}` +
+            '</Owner>',
+            `<StorageClass>${escapeForXml(val.StorageClass)}` +
                     '</StorageClass>',
-                 `<Initiated>${escapeForXml(val.Initiated)}</Initiated>`,
-                 '</Upload>'
+            `<Initiated>${escapeForXml(val.Initiated)}</Initiated>`,
+            '</Upload>',
         );
     });
 
     l.CommonPrefixes.forEach(prefix => {
         xml.push('<CommonPrefixes>',
-                 `<Prefix>${escapeForXml(prefix)}</Prefix>`,
-                 '</CommonPrefixes>'
+            `<Prefix>${escapeForXml(prefix)}</Prefix>`,
+            '</CommonPrefixes>',
         );
     });
 

lib/s3middleware/nullStream.js

@@ -5,7 +5,6 @@ const Readable = require('stream').Readable;
  * This class is used to produce zeros filled buffers for a reader consumption
  */
 class NullStream extends Readable {
-
     /**
      * Construct a new zeros filled buffers producer that will
      * produce as much bytes as specified by the range parameter, or the size
@@ -32,8 +31,8 @@ class NullStream extends Readable {
     _read(size) {
         const toRead = Math.min(size, this.bytesToRead);
         const buffer = toRead > 0
-                  ? Buffer.alloc(toRead, 0)
-                  : null;
+            ? Buffer.alloc(toRead, 0)
+            : null;
         this.bytesToRead -= toRead;
         this.push(buffer);
     }

lib/s3middleware/tagging.js

@@ -4,11 +4,11 @@ const errors = require('../errors');
 const escapeForXml = require('./escapeForXml');
 
 const errorInvalidArgument = errors.InvalidArgument
-  .customizeDescription('The header \'x-amz-tagging\' shall be ' +
+    .customizeDescription('The header \'x-amz-tagging\' shall be ' +
   'encoded as UTF-8 then URLEncoded URL query parameters without ' +
   'tag name duplicates.');
 const errorBadRequestLimit50 = errors.BadRequest
-  .customizeDescription('Object tags cannot be greater than 50');
+    .customizeDescription('Object tags cannot be greater than 50');
 
 /*
     Format of xml request:
@@ -38,7 +38,7 @@ const _validator = {
         result.Tagging.TagSet &&
         result.Tagging.TagSet.length === 1 &&
         (
-          result.Tagging.TagSet[0] === '' ||
+            result.Tagging.TagSet[0] === '' ||
           result.Tagging.TagSet[0] &&
           Object.keys(result.Tagging.TagSet[0]).length === 1 &&
           result.Tagging.TagSet[0].Tag &&
@@ -155,7 +155,7 @@ function parseTagXml(xml, log, cb) {
 function convertToXml(objectTags) {
     const xml = [];
     xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
-    '<Tagging> <TagSet>');
+        '<Tagging> <TagSet>');
     if (objectTags && Object.keys(objectTags).length > 0) {
         Object.keys(objectTags).forEach(key => {
             xml.push(`<Tag><Key>${escapeForXml(key)}</Key>` +

lib/s3middleware/validateConditionalHeaders.js

@@ -68,6 +68,31 @@ function _checkUnmodifiedSince(ifUnmodifiedSinceTime, lastModified) {
     return res;
 }
 
+/**
+ * checks 'if-modified-since' and 'if-unmodified-since' headers if included in
+ * request against last-modified date of object
+ * @param {object} headers - headers from request object
+ * @param {string} lastModified - last modified date of object
+ * @return {object} contains modifiedSince and unmodifiedSince res objects
+ */
+function checkDateModifiedHeaders(headers, lastModified) {
+    let lastModifiedDate = new Date(lastModified);
+    lastModifiedDate.setMilliseconds(0);
+    lastModifiedDate = lastModifiedDate.getTime();
+
+    const ifModifiedSinceHeader = headers['if-modified-since'] ||
+        headers['x-amz-copy-source-if-modified-since'];
+    const ifUnmodifiedSinceHeader = headers['if-unmodified-since'] ||
+        headers['x-amz-copy-source-if-unmodified-since'];
+
+    const modifiedSinceRes = _checkModifiedSince(ifModifiedSinceHeader,
+        lastModifiedDate);
+    const unmodifiedSinceRes = _checkUnmodifiedSince(ifUnmodifiedSinceHeader,
+        lastModifiedDate);
+
+    return { modifiedSinceRes, unmodifiedSinceRes };
+}
+
 /**
  * validateConditionalHeaders - validates 'if-modified-since',
  * 'if-unmodified-since', 'if-match' or 'if-none-match' headers if included in
@@ -79,23 +104,14 @@ function _checkUnmodifiedSince(ifUnmodifiedSinceTime, lastModified) {
  * empty object if no error
  */
 function validateConditionalHeaders(headers, lastModified, contentMD5) {
-    let lastModifiedDate = new Date(lastModified);
-    lastModifiedDate.setMilliseconds(0);
-    lastModifiedDate = lastModifiedDate.getTime();
     const ifMatchHeader = headers['if-match'] ||
         headers['x-amz-copy-source-if-match'];
     const ifNoneMatchHeader = headers['if-none-match'] ||
         headers['x-amz-copy-source-if-none-match'];
-    const ifModifiedSinceHeader = headers['if-modified-since'] ||
-        headers['x-amz-copy-source-if-modified-since'];
-    const ifUnmodifiedSinceHeader = headers['if-unmodified-since'] ||
-        headers['x-amz-copy-source-if-unmodified-since'];
     const etagMatchRes = _checkEtagMatch(ifMatchHeader, contentMD5);
     const etagNoneMatchRes = _checkEtagNoneMatch(ifNoneMatchHeader, contentMD5);
-    const modifiedSinceRes = _checkModifiedSince(ifModifiedSinceHeader,
-        lastModifiedDate);
-    const unmodifiedSinceRes = _checkUnmodifiedSince(ifUnmodifiedSinceHeader,
-        lastModifiedDate);
+    const { modifiedSinceRes, unmodifiedSinceRes } =
+        checkDateModifiedHeaders(headers, lastModified);
     // If-Unmodified-Since condition evaluates to false and If-Match
     // is not present, then return the error. Otherwise, If-Unmodified-Since is
     // silent when If-Match match, and when If-Match does not match, it's the
@@ -120,5 +136,6 @@ module.exports = {
     _checkEtagNoneMatch,
     _checkModifiedSince,
     _checkUnmodifiedSince,
+    checkDateModifiedHeaders,
     validateConditionalHeaders,
 };

lib/s3routes/routes.js

@@ -10,6 +10,7 @@ const routeOPTIONS = require('./routes/routeOPTIONS');
 const routesUtils = require('./routesUtils');
 const routeWebsite = require('./routes/routeWebsite');
 
+const { objectKeyByteLimit } = require('../constants');
 const requestUtils = require('../../lib/policyEvaluator/requestUtils');
 
 const routeMap = {
@@ -42,7 +43,7 @@ function checkBucketAndKey(bucketName, objectKey, method, reqQuery,
         log.debug('empty bucket name', { method: 'routes' });
         return (method !== 'OPTIONS') ?
             errors.MethodNotAllowed : errors.AccessForbidden
-            .customizeDescription('CORSResponse: Bucket not found');
+                .customizeDescription('CORSResponse: Bucket not found');
     }
     if (bucketName !== undefined && routesUtils.isValidBucketName(bucketName,
         blacklistedPrefixes.bucket) === false) {
@@ -57,8 +58,14 @@ function checkBucketAndKey(bucketName, objectKey, method, reqQuery,
             blacklistedPrefixes.object);
         if (!result.isValid) {
             log.debug('invalid object key', { objectKey });
-            return errors.InvalidArgument.customizeDescription('Object key ' +
-            `must not start with "${result.invalidPrefix}".`);
+            if (result.invalidPrefix) {
+                return errors.InvalidArgument.customizeDescription('Invalid ' +
+                    'prefix - object key cannot start with ' +
+                    `"${result.invalidPrefix}".`);
+            }
+            return errors.KeyTooLong.customizeDescription('Object key is too ' +
+                'long. Maximum number of bytes allowed in keys is ' +
+                `${objectKeyByteLimit}.`);
         }
     }
     if ((reqQuery.partNumber || reqQuery.uploadId)
@@ -85,7 +92,7 @@ function checkTypes(req, res, params, logger, s3config) {
         'bad routes param: internalHandlers must be an object');
     if (params.statsClient) {
         assert.strictEqual(typeof params.statsClient, 'object',
-        'bad routes param: statsClient must be an object');
+            'bad routes param: statsClient must be an object');
     }
     assert(Array.isArray(params.allEndpoints),
         'bad routes param: allEndpoints must be an array');
@@ -93,13 +100,13 @@ function checkTypes(req, res, params, logger, s3config) {
         'bad routes param: allEndpoints must have at least one endpoint');
     params.allEndpoints.forEach(endpoint => {
         assert.strictEqual(typeof endpoint, 'string',
-        'bad routes param: each item in allEndpoints must be a string');
+            'bad routes param: each item in allEndpoints must be a string');
     });
     assert(Array.isArray(params.websiteEndpoints),
         'bad routes param: allEndpoints must be an array');
     params.websiteEndpoints.forEach(endpoint => {
         assert.strictEqual(typeof endpoint, 'string',
-        'bad routes param: each item in websiteEndpoints must be a string');
+            'bad routes param: each item in websiteEndpoints must be a string');
     });
     assert.strictEqual(typeof params.blacklistedPrefixes, 'object',
         'bad routes param: blacklistedPrefixes must be an object');
@@ -107,16 +114,16 @@ function checkTypes(req, res, params, logger, s3config) {
         'bad routes param: blacklistedPrefixes.bucket must be an array');
     params.blacklistedPrefixes.bucket.forEach(pre => {
         assert.strictEqual(typeof pre, 'string',
-        'bad routes param: each blacklisted bucket prefix must be a string');
+            'bad routes param: each blacklisted bucket prefix must be a string');
     });
     assert(Array.isArray(params.blacklistedPrefixes.object),
         'bad routes param: blacklistedPrefixes.object must be an array');
     params.blacklistedPrefixes.object.forEach(pre => {
         assert.strictEqual(typeof pre, 'string',
-        'bad routes param: each blacklisted object prefix must be a string');
+            'bad routes param: each blacklisted object prefix must be a string');
     });
-    assert.strictEqual(typeof params.dataRetrievalFn, 'function',
-        'bad routes param: dataRetrievalFn must be a defined function');
+    assert.strictEqual(typeof params.dataRetrievalParams, 'object',
+        'bad routes param: dataRetrievalParams must be a defined object');
     if (s3config) {
         assert.strictEqual(typeof s3config, 'object', 'bad routes param: s3config must be an object');
     }
@@ -138,7 +145,8 @@ function checkTypes(req, res, params, logger, s3config) {
  * @param {string[]} params.blacklistedPrefixes.object - object prefixes
  * @param {object} params.unsupportedQueries - object containing true/false
  *  values for whether queries are supported
- * @param {function} params.dataRetrievalFn - function to retrieve data
+ * @param {function} params.dataRetrievalParams - params to create instance of
+ * data retrieval function
  * @param {RequestLogger} logger - werelogs logger instance
  * @param {String} [s3config] - s3 configuration
  * @returns {undefined}
@@ -153,7 +161,7 @@ function routes(req, res, params, logger, s3config) {
         allEndpoints,
         websiteEndpoints,
         blacklistedPrefixes,
-        dataRetrievalFn,
+        dataRetrievalParams,
     } = params;
 
     const clientInfo = {
@@ -171,10 +179,11 @@ function routes(req, res, params, logger, s3config) {
         reqUids = undefined;
     }
     const log = (reqUids !== undefined ?
-                 logger.newRequestLoggerFromSerializedUids(reqUids) :
-                 logger.newRequestLogger());
+        logger.newRequestLoggerFromSerializedUids(reqUids) :
+        logger.newRequestLogger());
 
-    if (!req.url.startsWith('/_/healthcheck')) {
+    if (!req.url.startsWith('/_/healthcheck') &&
+        !req.url.startsWith('/_/report')) {
         log.info('received request', clientInfo);
     }
 
@@ -207,7 +216,7 @@ function routes(req, res, params, logger, s3config) {
         return routesUtils.responseXMLBody(
             errors.InvalidURI.customizeDescription('Could not parse the ' +
                 'specified URI. Check your restEndpoints configuration.'),
-                undefined, res, log);
+            undefined, res, log);
     }
 
     log.addDefaultFields({
@@ -229,16 +238,17 @@ function routes(req, res, params, logger, s3config) {
 
     if (bucketOrKeyError) {
         log.trace('error with bucket or key value',
-        { error: bucketOrKeyError });
+            { error: bucketOrKeyError });
         return routesUtils.responseXMLBody(bucketOrKeyError, null, res, log);
     }
 
     // bucket website request
     if (websiteEndpoints && websiteEndpoints.indexOf(req.parsedHost) > -1) {
-        return routeWebsite(req, res, api, log, statsClient, dataRetrievalFn);
+        return routeWebsite(req, res, api, log, statsClient,
+            dataRetrievalParams);
     }
 
-    return method(req, res, api, log, statsClient, dataRetrievalFn);
+    return method(req, res, api, log, statsClient, dataRetrievalParams);
 }
 
 module.exports = routes;

lib/s3routes/routes/routeDELETE.js

@@ -7,7 +7,7 @@ function routeDELETE(request, response, api, log, statsClient) {
     if (request.query.uploadId) {
         if (request.objectKey === undefined) {
             return routesUtils.responseNoBody(
-              errors.InvalidRequest.customizeDescription('A key must be ' +
+                errors.InvalidRequest.customizeDescription('A key must be ' +
               'specified'), null, response, 200, log);
         }
         api.callApiMethod('multipartDelete', request, response, log,
@@ -19,77 +19,77 @@ function routeDELETE(request, response, api, log, statsClient) {
     } else if (request.objectKey === undefined) {
         if (request.query.website !== undefined) {
             return api.callApiMethod('bucketDeleteWebsite', request,
-            response, log, (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 204, log);
-            });
+                response, log, (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 204, log);
+                });
         } else if (request.query.cors !== undefined) {
             return api.callApiMethod('bucketDeleteCors', request, response,
-            log, (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 204, log);
-            });
+                log, (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 204, log);
+                });
         } else if (request.query.replication !== undefined) {
             return api.callApiMethod('bucketDeleteReplication', request,
-            response, log, (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 204, log);
-            });
+                response, log, (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 204, log);
+                });
         } else if (request.query.lifecycle !== undefined) {
             return api.callApiMethod('bucketDeleteLifecycle', request,
-            response, log, (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 204, log);
-            });
+                response, log, (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 204, log);
+                });
         } else if (request.query.policy !== undefined) {
             return api.callApiMethod('bucketDeletePolicy', request,
-            response, log, (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 204, log);
-            });
+                response, log, (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 204, log);
+                });
         } else if (request.query.encryption !== undefined) {
             return api.callApiMethod('bucketDeleteEncryption', request,
-            response, log, (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 204, log);
-            });
+                response, log, (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 204, log);
+                });
         }
         api.callApiMethod('bucketDelete', request, response, log,
-        (err, corsHeaders) => {
-            routesUtils.statsReport500(err, statsClient);
-            return routesUtils.responseNoBody(err, corsHeaders, response,
-              204, log);
-        });
+            (err, corsHeaders) => {
+                routesUtils.statsReport500(err, statsClient);
+                return routesUtils.responseNoBody(err, corsHeaders, response,
+                    204, log);
+            });
     } else {
         if (request.query.tagging !== undefined) {
             return api.callApiMethod('objectDeleteTagging', request,
-            response, log, (err, resHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, resHeaders,
-                    response, 204, log);
-            });
+                response, log, (err, resHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, resHeaders,
+                        response, 204, log);
+                });
         }
         api.callApiMethod('objectDelete', request, response, log,
-          (err, corsHeaders) => {
-              /*
+            (err, corsHeaders) => {
+                /*
               * Since AWS expects a 204 regardless of the existence of
               the object, the errors NoSuchKey and NoSuchVersion should not
               * be sent back as a response.
               */
-              if (err && !err.NoSuchKey && !err.NoSuchVersion) {
-                  return routesUtils.responseNoBody(err, corsHeaders,
-                    response, null, log);
-              }
-              routesUtils.statsReport500(err, statsClient);
-              return routesUtils.responseNoBody(null, corsHeaders, response,
-                204, log);
-          });
+                if (err && !err.NoSuchKey && !err.NoSuchVersion) {
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, null, log);
+                }
+                routesUtils.statsReport500(err, statsClient);
+                return routesUtils.responseNoBody(null, corsHeaders, response,
+                    204, log);
+            });
     }
     return undefined;
 }

lib/s3routes/routes/routeGET.js

@@ -1,7 +1,8 @@
 const errors = require('../../errors');
 const routesUtils = require('../routesUtils');
 
-function routerGET(request, response, api, log, statsClient, dataRetrievalFn) {
+function routerGET(request, response, api, log, statsClient,
+    dataRetrievalParams) {
     log.debug('routing request', { method: 'routerGET' });
     if (request.bucketName === undefined && request.objectKey !== undefined) {
         routesUtils.responseXMLBody(errors.NoSuchBucket, null, response, log);
@@ -16,18 +17,18 @@ function routerGET(request, response, api, log, statsClient, dataRetrievalFn) {
         // GET bucket ACL
         if (request.query.acl !== undefined) {
             api.callApiMethod('bucketGetACL', request, response, log,
-            (err, xml, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseXMLBody(err, xml, response, log,
-                    corsHeaders);
-            });
+                (err, xml, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseXMLBody(err, xml, response, log,
+                        corsHeaders);
+                });
         } else if (request.query.replication !== undefined) {
             api.callApiMethod('bucketGetReplication', request, response, log,
-            (err, xml, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseXMLBody(err, xml, response, log,
-                    corsHeaders);
-            });
+                (err, xml, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseXMLBody(err, xml, response, log,
+                        corsHeaders);
+                });
         } else if (request.query.cors !== undefined) {
             api.callApiMethod('bucketGetCors', request, response, log,
                 (err, xml, corsHeaders) => {
@@ -69,7 +70,7 @@ function routerGET(request, response, api, log, statsClient, dataRetrievalFn) {
                 (err, xml, corsHeaders) => {
                     routesUtils.statsReport500(err, statsClient);
                     return routesUtils.responseXMLBody(err, xml, response, log,
-                      corsHeaders);
+                        corsHeaders);
                 });
         } else if (request.query.policy !== undefined) {
             api.callApiMethod('bucketGetPolicy', request, response, log,
@@ -94,11 +95,18 @@ function routerGET(request, response, api, log, statsClient, dataRetrievalFn) {
                 });
         } else if (request.query.encryption !== undefined) {
             api.callApiMethod('bucketGetEncryption', request, response, log,
-            (err, xml, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseXMLBody(err, xml, response,
-                    log, corsHeaders);
-            });
+                (err, xml, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseXMLBody(err, xml, response,
+                        log, corsHeaders);
+                });
+        } else if (request.query.search !== undefined) {
+            api.callApiMethod('metadataSearch', request, response, log,
+                (err, xml, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseXMLBody(err, xml, response,
+                        log, corsHeaders);
+                });
         } else {
             // GET bucket
             api.callApiMethod('bucketGet', request, response, log,
@@ -157,8 +165,8 @@ function routerGET(request, response, api, log, statsClient, dataRetrievalFn) {
                     log.end().addDefaultFields({ contentLength });
                     routesUtils.statsReport500(err, statsClient);
                     return routesUtils.responseStreamData(err, request.query,
-                        resMetaHeaders, dataGetInfo, dataRetrievalFn, response,
-                        range, log);
+                        resMetaHeaders, dataGetInfo, dataRetrievalParams,
+                        response, range, log);
                 });
         }
     }

lib/s3routes/routes/routeOPTIONS.js

@@ -21,11 +21,11 @@ function routeOPTIONS(request, response, api, log, statsClient) {
     }
 
     return api.callApiMethod('corsPreflight', request, response, log,
-    (err, resHeaders) => {
-        routesUtils.statsReport500(err, statsClient);
-        return routesUtils.responseNoBody(err, resHeaders, response, 200,
-            log);
-    });
+        (err, resHeaders) => {
+            routesUtils.statsReport500(err, statsClient);
+            return routesUtils.responseNoBody(err, resHeaders, response, 200,
+                log);
+        });
 }
 
 module.exports = routeOPTIONS;

lib/s3routes/routes/routePOST.js

@@ -27,28 +27,28 @@ function routePOST(request, response, api, log) {
     if (request.query.uploads !== undefined) {
         return api.callApiMethod('initiateMultipartUpload', request,
             response, log, (err, result, corsHeaders) =>
-            routesUtils.responseXMLBody(err, result, response, log,
-                corsHeaders));
+                routesUtils.responseXMLBody(err, result, response, log,
+                    corsHeaders));
     }
 
     // POST complete multipart upload
     if (request.query.uploadId !== undefined) {
         return api.callApiMethod('completeMultipartUpload', request,
             response, log, (err, result, resHeaders) =>
-            routesUtils.responseXMLBody(err, result, response, log,
-                resHeaders));
+                routesUtils.responseXMLBody(err, result, response, log,
+                    resHeaders));
     }
 
     // POST multiObjectDelete
     if (request.query.delete !== undefined) {
         return api.callApiMethod('multiObjectDelete', request, response,
             log, (err, xml, corsHeaders) =>
-            routesUtils.responseXMLBody(err, xml, response, log,
-                corsHeaders));
+                routesUtils.responseXMLBody(err, xml, response, log,
+                    corsHeaders));
     }
 
     return routesUtils.responseNoBody(errors.NotImplemented, null, response,
-                200, log);
+        200, log);
 }
 /* eslint-enable no-param-reassign */
 module.exports = routePOST;

lib/s3routes/routes/routePUT.js

@@ -14,16 +14,16 @@ function routePUT(request, response, api, log, statsClient) {
         || contentLength < 0)) || contentLength === '') {
             log.debug('invalid content-length header');
             return routesUtils.responseNoBody(
-              errors.BadRequest, null, response, null, log);
+                errors.BadRequest, null, response, null, log);
         }
         // PUT bucket ACL
         if (request.query.acl !== undefined) {
             api.callApiMethod('bucketPutACL', request, response, log,
-            (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 200, log);
-            });
+                (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 200, log);
+                });
         } else if (request.query.versioning !== undefined) {
             api.callApiMethod('bucketPutVersioning', request, response, log,
                 (err, corsHeaders) => {
@@ -82,11 +82,11 @@ function routePUT(request, response, api, log, statsClient) {
                 });
         } else if (request.query.encryption !== undefined) {
             api.callApiMethod('bucketPutEncryption', request, response, log,
-            (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders,
-                    response, 200, log);
-            });
+                (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, corsHeaders,
+                        response, 200, log);
+                });
         } else {
             // PUT bucket
             return api.callApiMethod('bucketPut', request, response, log,
@@ -110,7 +110,7 @@ function routePUT(request, response, api, log, statsClient) {
                 method: 'routePUT',
             });
             return routesUtils
-            .responseNoBody(errors.InvalidDigest, null, response, 200, log);
+                .responseNoBody(errors.InvalidDigest, null, response, 200, log);
         }
         if (request.headers['content-md5']) {
             request.contentMD5 = request.headers['content-md5'];
@@ -126,17 +126,17 @@ function routePUT(request, response, api, log, statsClient) {
                 });
                 return routesUtils
                     .responseNoBody(errors.InvalidDigest, null, response, 200,
-                                    log);
+                        log);
             }
         }
         if (request.query.partNumber) {
             if (request.headers['x-amz-copy-source']) {
                 api.callApiMethod('objectPutCopyPart', request, response, log,
-                (err, xml, additionalHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseXMLBody(err, xml, response, log,
+                    (err, xml, additionalHeaders) => {
+                        routesUtils.statsReport500(err, statsClient);
+                        return routesUtils.responseXMLBody(err, xml, response, log,
                             additionalHeaders);
-                });
+                    });
             } else {
                 api.callApiMethod('objectPutPart', request, response, log,
                     (err, calculatedHash, corsHeaders) => {
@@ -202,11 +202,11 @@ function routePUT(request, response, api, log, statsClient) {
                 contentLength: request.parsedContentLength,
             });
             api.callApiMethod('objectPut', request, response, log,
-            (err, resHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, resHeaders,
-                    response, 200, log);
-            });
+                (err, resHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseNoBody(err, resHeaders,
+                        response, 200, log);
+                });
         }
     }
     return undefined;

lib/s3routes/routes/routeWebsite.js

@@ -2,7 +2,7 @@ const errors = require('../../errors');
 const routesUtils = require('../routesUtils');
 
 function routerWebsite(request, response, api, log, statsClient,
-    dataRetrievalFn) {
+    dataRetrievalParams) {
     log.debug('routing request', { method: 'routerWebsite' });
     // website endpoint only supports GET and HEAD and must have a bucket
     // http://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteEndpoints.html
@@ -14,7 +14,7 @@ function routerWebsite(request, response, api, log, statsClient,
     if (request.method === 'GET') {
         return api.callApiMethod('websiteGet', request, response, log,
             (err, userErrorPageFailure, dataGetInfo, resMetaHeaders,
-            redirectInfo, key) => {
+                redirectInfo, key) => {
                 routesUtils.statsReport500(err, statsClient);
                 // request being redirected
                 if (redirectInfo) {
@@ -27,7 +27,7 @@ function routerWebsite(request, response, api, log, statsClient,
                 // user has their own error page
                 if (err && dataGetInfo) {
                     return routesUtils.streamUserErrorPage(err, dataGetInfo,
-                        dataRetrievalFn, response, resMetaHeaders, log);
+                        dataRetrievalParams, response, resMetaHeaders, log);
                 }
                 // send default error html response
                 if (err) {
@@ -37,27 +37,27 @@ function routerWebsite(request, response, api, log, statsClient,
                 }
                 // no error, stream data
                 return routesUtils.responseStreamData(null, request.query,
-                    resMetaHeaders, dataGetInfo, dataRetrievalFn, response,
+                    resMetaHeaders, dataGetInfo, dataRetrievalParams, response,
                     null, log);
             });
     }
     if (request.method === 'HEAD') {
         return api.callApiMethod('websiteHead', request, response, log,
-        (err, resMetaHeaders, redirectInfo, key) => {
-            routesUtils.statsReport500(err, statsClient);
-            if (redirectInfo) {
-                return routesUtils.redirectRequest(redirectInfo,
-                    key, request.connection.encrypted,
-                    response, request.headers.host, resMetaHeaders, log);
-            }
-            // could redirect on err so check for redirectInfo first
-            if (err) {
-                return routesUtils.errorHeaderResponse(err, response,
-                    resMetaHeaders, log);
-            }
-            return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
-                response, log);
-        });
+            (err, resMetaHeaders, redirectInfo, key) => {
+                routesUtils.statsReport500(err, statsClient);
+                if (redirectInfo) {
+                    return routesUtils.redirectRequest(redirectInfo,
+                        key, request.connection.encrypted,
+                        response, request.headers.host, resMetaHeaders, log);
+                }
+                // could redirect on err so check for redirectInfo first
+                if (err) {
+                    return routesUtils.errorHeaderResponse(err, response,
+                        resMetaHeaders, log);
+                }
+                return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
+                    response, log);
+            });
     }
     return undefined;
 }

lib/s3routes/routesUtils.js

@@ -4,6 +4,9 @@ const errors = require('../errors');
 const constants = require('../constants');
 const { eachSeries } = require('async');
 
+const DataWrapper = require('../storage/data/DataWrapper');
+const { objectKeyByteLimit } = require('../constants');
+
 const responseErr = new Error();
 responseErr.code = 'ResponseError';
 responseErr.message = 'response closed by client request before all data sent';
@@ -25,7 +28,7 @@ function setCommonResponseHeaders(headers, response, log) {
                 } catch (e) {
                     log.debug('header can not be added ' +
                       'to the response', { header: headers[key],
-                          error: e.stack, method: 'setCommonResponseHeaders' });
+                        error: e.stack, method: 'setCommonResponseHeaders' });
                 }
             }
         });
@@ -68,7 +71,7 @@ const XMLResponseBackend = {
      * @return {object} response - response object with additional headers
      */
     okResponse: function okXMLResponse(xml, response, log,
-                                       additionalHeaders) {
+        additionalHeaders) {
         const bytesSent = Buffer.byteLength(xml);
         log.trace('sending success xml response');
         log.addDefaultFields({
@@ -115,7 +118,7 @@ const XMLResponseBackend = {
             `<Message>${errCode.description}</Message>`,
             '<Resource></Resource>',
             `<RequestId>${log.getSerializedUids()}</RequestId>`,
-            '</Error>'
+            '</Error>',
         );
         const xmlStr = xml.join('');
         const bytesSent = Buffer.byteLength(xmlStr);
@@ -145,7 +148,7 @@ const JSONResponseBackend = {
      * @return {object} response - response object with additional headers
      */
     okResponse: function okJSONResponse(json, response, log,
-                                        additionalHeaders) {
+        additionalHeaders) {
         const bytesSent = Buffer.byteLength(json);
         log.trace('sending success json response');
         log.addDefaultFields({
@@ -163,7 +166,7 @@ const JSONResponseBackend = {
     },
 
     errorResponse: function errorJSONResponse(errCode, response, log,
-                                              corsHeaders) {
+        corsHeaders) {
         log.trace('sending error json response', { errCode });
         /*
          {
@@ -257,7 +260,7 @@ function okContentHeadersResponse(overrideParams, resHeaders,
     return response;
 }
 
-function retrieveDataAzure(locations, retrieveDataFn, response, logger) {
+function retrieveDataAzure(locations, retrieveDataParams, response, logger) {
     const errorHandlerFn = () => { response.connection.destroy(); };
     const current = locations.shift();
 
@@ -265,7 +268,18 @@ function retrieveDataAzure(locations, retrieveDataFn, response, logger) {
         logger.error('error piping data from source');
         errorHandlerFn(err);
     });
-    return retrieveDataFn(current, response, logger, err => {
+    const {
+        client,
+        implName,
+        config,
+        kms,
+        metadata,
+        locStorageCheckFn,
+        vault,
+    } = retrieveDataParams;
+    const data = new DataWrapper(
+        client, implName, config, kms, metadata, locStorageCheckFn, vault);
+    return data.get(current, response, logger, err => {
         if (err) {
             logger.error('failed to get object from source', {
                 error: err,
@@ -278,12 +292,12 @@ function retrieveDataAzure(locations, retrieveDataFn, response, logger) {
     });
 }
 
-function retrieveData(locations, retrieveDataFn, response, log) {
+function retrieveData(locations, retrieveDataParams, response, log) {
     if (locations.length === 0) {
         return response.end();
     }
     if (locations[0].azureStreamingOptions) {
-        return retrieveDataAzure(locations, retrieveDataFn, response, log);
+        return retrieveDataAzure(locations, retrieveDataParams, response, log);
     }
     // response is of type http.ServerResponse
     let responseDestroyed = false;
@@ -293,16 +307,33 @@ function retrieveData(locations, retrieveDataFn, response, log) {
         response.destroy();
         responseDestroyed = true;
     };
+
+    const _destroyReadable = readable => {
+        // s3-data sends Readable stream only which does not implement destroy
+        if (readable && readable.destroy) {
+            readable.destroy();
+        }
+    };
+
     // the S3-client might close the connection while we are processing it
     response.once('close', () => {
         responseDestroyed = true;
-        if (currentStream) {
-            currentStream.destroy();
-        }
+        _destroyReadable(currentStream);
     });
 
+    const {
+        client,
+        implName,
+        config,
+        kms,
+        metadata,
+        locStorageCheckFn,
+        vault,
+    } = retrieveDataParams;
+    const data = new DataWrapper(
+        client, implName, config, kms, metadata, locStorageCheckFn, vault);
     return eachSeries(locations,
-        (current, next) => retrieveDataFn(current, response, log,
+        (current, next) => data.get(current, response, log,
             (err, readable) => {
                 // NB: readable is of IncomingMessage type
                 if (err) {
@@ -319,7 +350,7 @@ function retrieveData(locations, retrieveDataFn, response, log) {
                 if (responseDestroyed || response.isclosed) {
                     log.debug(
                         'response destroyed before readable could stream');
-                    readable.destroy();
+                    _destroyReadable(readable);
                     return next(responseErr);
                 }
                 // readable stream successfully consumed
@@ -337,27 +368,27 @@ function retrieveData(locations, retrieveDataFn, response, log) {
                 currentStream = readable;
                 return readable.pipe(response, { end: false });
             }), err => {
-                currentStream = null;
-                if (err) {
-                    log.debug('abort response due to error', {
-                        error: err.code, errMsg: err.message });
-                }
-                // call end for all cases (error/success) per node.js docs
-                // recommendation
-                response.end();
+            currentStream = null;
+            if (err) {
+                log.debug('abort response due to error', {
+                    error: err.code, errMsg: err.message });
             }
+            // call end for all cases (error/success) per node.js docs
+            // recommendation
+            response.end();
+        },
     );
 }
 
 function _responseBody(responseBackend, errCode, payload, response, log,
-                       additionalHeaders) {
+    additionalHeaders) {
     if (errCode && !response.headersSent) {
         return responseBackend.errorResponse(errCode, response, log,
-                                             additionalHeaders);
+            additionalHeaders);
     }
     if (!response.headersSent) {
         return responseBackend.okResponse(payload, response, log,
-                                          additionalHeaders);
+            additionalHeaders);
     }
     return undefined;
 }
@@ -366,8 +397,8 @@ function _computeContentLengthFromLocation(dataLocations) {
     return dataLocations.reduce(
         (sum, location) => (sum !== undefined &&
                             (typeof location.size === 'number' || typeof location.size === 'string') ?
-                            sum + Number.parseInt(location.size, 10) :
-                            undefined), 0);
+            sum + Number.parseInt(location.size, 10) :
+            undefined), 0);
 }
 
 function _contentLengthMatchesLocations(contentLength, dataLocations) {
@@ -388,7 +419,7 @@ const routesUtils = {
      */
     responseXMLBody(errCode, xml, response, log, additionalHeaders) {
         return _responseBody(XMLResponseBackend, errCode, xml, response,
-                             log, additionalHeaders);
+            log, additionalHeaders);
     },
 
     /**
@@ -402,7 +433,7 @@ const routesUtils = {
      */
     responseJSONBody(errCode, json, response, log, additionalHeaders) {
         return _responseBody(JSONResponseBackend, errCode, json, response,
-                             log, additionalHeaders);
+            log, additionalHeaders);
     },
 
     /**
@@ -417,7 +448,7 @@ const routesUtils = {
     responseNoBody(errCode, resHeaders, response, httpCode = 200, log) {
         if (errCode && !response.headersSent) {
             return XMLResponseBackend.errorResponse(errCode, response, log,
-                                                    resHeaders);
+                resHeaders);
         }
         if (!response.headersSent) {
             return okHeaderResponse(resHeaders, response, httpCode, log);
@@ -435,10 +466,10 @@ const routesUtils = {
      * @return {object} - router's response object
      */
     responseContentHeaders(errCode, overrideParams, resHeaders, response,
-                           log) {
+        log) {
         if (errCode && !response.headersSent) {
             return XMLResponseBackend.errorResponse(errCode, response, log,
-                                                    resHeaders);
+                resHeaders);
         }
         if (!response.headersSent) {
             // Undefined added as an argument since need to send range to
@@ -461,7 +492,8 @@ const routesUtils = {
      * @param {array | null} dataLocations --
      *   - array of locations to get streams from sproxyd
      *   - null if no data for object and only metadata
-     * @param {function} retrieveDataFn - function to handle streaming data
+     * @param {object} retrieveDataParams - params to create instance of data
+     * retrieval function
      * @param {http.ServerResponse} response - response sent to the client
      * @param {array | undefined} range - range in format of [start, end]
      * if range header contained in request or undefined if not
@@ -469,10 +501,10 @@ const routesUtils = {
      * @return {undefined}
      */
     responseStreamData(errCode, overrideParams, resHeaders, dataLocations,
-        retrieveDataFn, response, range, log) {
+        retrieveDataParams, response, range, log) {
         if (errCode && !response.headersSent) {
             return XMLResponseBackend.errorResponse(errCode, response, log,
-                                                    resHeaders);
+                resHeaders);
         }
         if (dataLocations !== null && !response.headersSent) {
             // sanity check of content length against individual data
@@ -480,13 +512,13 @@ const routesUtils = {
             const contentLength = resHeaders && resHeaders['Content-Length'];
             if (contentLength !== undefined &&
                 !_contentLengthMatchesLocations(contentLength,
-                                                dataLocations)) {
+                    dataLocations)) {
                 log.error('logic error: total length of fetched data ' +
                           'locations does not match returned content-length',
-                          { contentLength, dataLocations });
+                { contentLength, dataLocations });
                 return XMLResponseBackend.errorResponse(errors.InternalError,
-                                                        response, log,
-                                                        resHeaders);
+                    response, log,
+                    resHeaders);
             }
         }
         if (!response.headersSent) {
@@ -505,20 +537,21 @@ const routesUtils = {
                 httpCode: response.statusCode,
             });
         });
-        return retrieveData(dataLocations, retrieveDataFn, response, log);
+        return retrieveData(dataLocations, retrieveDataParams, response, log);
     },
 
     /**
      * @param {object} err -- arsenal error object
      * @param {array} dataLocations --
      *   - array of locations to get streams from backend
-     * @param {function} retrieveDataFn - function to handle streaming data
+     * @param {object} retrieveDataParams - params to create instance of
+     * data retrieval function
      * @param {http.ServerResponse} response - response sent to the client
      * @param {object} corsHeaders - CORS-related response headers
      * @param {object} log - Werelogs logger
      * @return {undefined}
      */
-    streamUserErrorPage(err, dataLocations, retrieveDataFn, response,
+    streamUserErrorPage(err, dataLocations, retrieveDataParams, response,
         corsHeaders, log) {
         setCommonResponseHeaders(corsHeaders, response, log);
         response.writeHead(err.code, { 'Content-type': 'text/html' });
@@ -527,7 +560,7 @@ const routesUtils = {
                 httpCode: response.statusCode,
             });
         });
-        return retrieveData(dataLocations, retrieveDataFn, response, log);
+        return retrieveData(dataLocations, retrieveDataParams, response, log);
     },
 
     /**
@@ -558,7 +591,7 @@ const routesUtils = {
             `<h1>${err.code} ${response.statusMessage}</h1>`,
             '<ul>',
             `<li>Code: ${err.message}</li>`,
-            `<li>Message: ${err.description}</li>`
+            `<li>Message: ${err.description}</li>`,
         );
 
         if (!userErrorPageFailure && bucketName) {
@@ -568,7 +601,7 @@ const routesUtils = {
             `<li>RequestId: ${log.getSerializedUids()}</li>`,
             // AWS response contains HostId here.
             // TODO: consider adding
-            '</ul>'
+            '</ul>',
         );
         if (userErrorPageFailure) {
             html.push(
@@ -578,13 +611,13 @@ const routesUtils = {
                 '<ul>',
                 `<li>Code: ${err.message}</li>`,
                 `<li>Message: ${err.description}</li>`,
-                '</ul>'
+                '</ul>',
             );
         }
         html.push(
             '<hr/>',
             '</body>',
-            '</html>'
+            '</html>',
         );
 
         return response.end(html.join(''), 'utf8', () => {
@@ -806,7 +839,7 @@ const routesUtils = {
                     // most specific potential hostname
                     bucketName =
                         potentialBucketName.length < bucketName.length ?
-                        potentialBucketName : bucketName;
+                            potentialBucketName : bucketName;
                 }
             }
         }
@@ -814,7 +847,7 @@ const routesUtils = {
             return bucketName;
         }
         throw new Error(
-            `bad request: hostname ${host} is not in valid endpoints`
+            `bad request: hostname ${host} is not in valid endpoints`,
         );
     },
 
@@ -881,6 +914,9 @@ const routesUtils = {
         if (invalidPrefix) {
             return { isValid: false, invalidPrefix };
         }
+        if (Buffer.byteLength(objectKey, 'utf8') > objectKeyByteLimit) {
+            return { isValid: false };
+        }
         return { isValid: true };
     },
 

lib/storage/metadata/mongoclient/ListRecordStream.js

@@ -79,9 +79,9 @@ class ListRecordStream extends stream.Readable {
                     'did not encounter the last saved offset in oplog, ' +
                         'resuming processing right after the latest record ' +
                         'to date; some entries may have been skipped', {
-                            lastSavedID: this._lastSavedID,
-                            latestRecordID: this._latestOplogID,
-                        });
+                        lastSavedID: this._lastSavedID,
+                        latestRecordID: this._latestOplogID,
+                    });
                 this._unpublishedListing = true;
             }
             ++this._skipCount;

lib/storage/metadata/mongoclient/MongoClientInterface.js

@@ -552,11 +552,11 @@ class MongoClientInterface {
                     $exists: false,
                 },
             },
-                {
-                    'value.versionId': {
-                        $gt: objVal.versionId,
-                    },
+            {
+                'value.versionId': {
+                    $gt: objVal.versionId,
                 },
+            },
             ],
         };
         // values to update master

lib/storage/metadata/proxy/README.md

@@ -29,5 +29,4 @@ server.start(() => {
     logger.info('Metadata Proxy Server successfully started. ' +
                 `Using the ${metadataWrapper.implName} backend`);
 });
-
 ```

lib/testing/matrix.js

@@ -60,8 +60,8 @@ class TestMatrix {
         this.elementsToSpecialize = elementsToSpecialize;
         this.callback = callback;
         this.description = typeof description === 'undefined'
-        ? ''
-        : description;
+            ? ''
+            : description;
         return this;
     }
 
@@ -158,15 +158,15 @@ class TestMatrix {
                 const callFunction = (matrixFather, matrixChild, callback,
                     description) => {
                     const result = Object.keys(matrixChild.params)
-                    .every(currentKey =>
-                           Object.prototype.toString.call(
-                               matrixChild.params[currentKey]
+                        .every(currentKey =>
+                            Object.prototype.toString.call(
+                                matrixChild.params[currentKey],
                             ).indexOf('Array') === -1);
 
                     if (result === true) {
                         describe(matrixChild.serialize(), () => {
                             it(description,
-                               done => callback(matrixChild, done));
+                                done => callback(matrixChild, done));
                         });
                     } else {
                         describe(matrixChild.serialize(), () => {

lib/versioning/VersionID.js

@@ -247,7 +247,7 @@ function decode(str) {
 }
 
 module.exports = { generateVersionId, getInfVid,
-                   hexEncode, hexDecode,
-                   base62Encode, base62Decode,
-                   encode, decode,
-                   ENC_TYPE_HEX, ENC_TYPE_BASE62 };
+    hexEncode, hexDecode,
+    base62Encode, base62Decode,
+    encode, decode,
+    ENC_TYPE_HEX, ENC_TYPE_BASE62 };

lib/versioning/VersioningRequestProcessor.js

@@ -84,7 +84,7 @@ class VersioningRequestProcessor {
                 return callback(null, data);
             }
             logger.debug('master version is a PHD, getting the latest version',
-                         { db, key });
+                { db, key });
             // otherwise, need to search for the latest version
             return this.getByListing(request, logger, callback);
         });
@@ -187,7 +187,7 @@ class VersioningRequestProcessor {
                     return entry.callback(err, value);
                 }
                 return this.wgm.get(entry.request, entry.logger,
-                                    entry.callback);
+                    entry.callback);
             });
             delete this.queue[cacheKey];
         }
@@ -267,19 +267,19 @@ class VersioningRequestProcessor {
                 return callback(err);
             }
             return this.writeCache.batch({ db, array, options },
-                    logger, err => callback(err, `{"versionId":"${vid}"}`));
+                logger, err => callback(err, `{"versionId":"${vid}"}`));
         };
 
         if (versionId) {
             return this.processVersionSpecificPut(request, logger,
-                                                  versioningCb);
+                versioningCb);
         }
         if (versioning) {
             return this.processNewVersionPut(request, logger, versioningCb);
         }
         // no versioning or versioning configuration off
         return this.writeCache.batch({ db, array: [{ key, value }] },
-                                     logger, callback);
+            logger, callback);
     }
 
     /**
@@ -353,7 +353,7 @@ class VersioningRequestProcessor {
         if (!(options && options.versionId)) {
             return this.writeCache.batch({ db,
                 array: [{ key, type: 'del' }] },
-                                         logger, callback);
+            logger, callback);
         }
         // version specific DELETE
         return this.processVersionSpecificDelete(request, logger,
@@ -399,7 +399,7 @@ class VersioningRequestProcessor {
                 const cacheKey = formatCacheKey(db, key);
                 clearTimeout(this.repairing[cacheKey]);
                 this.repairing[cacheKey] = setTimeout(() =>
-                        this.getByListing(request, logger, () => {}), 15000);
+                    this.getByListing(request, logger, () => {}), 15000);
             }
             return callback(null, ops, versionId);
         });

package.json

@@ -3,14 +3,14 @@
   "engines": {
     "node": ">=16"
   },
-  "version": "7.10.15",
+  "version": "8.1.39",
   "description": "Common utilities for the S3 project components",
   "main": "build/index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/scality/Arsenal.git"
   },
-  "author": "Giorgio Regni",
+  "author": "Scality Inc.",
   "license": "Apache-2.0",
   "bugs": {
     "url": "https://github.com/scality/Arsenal/issues"
@@ -21,17 +21,18 @@
     "JSONStream": "^1.0.0",
     "agentkeepalive": "^4.1.3",
     "ajv": "6.12.2",
-    "async": "~2.1.5",
+    "async": "~2.6.1",
     "aws-sdk": "2.80.0",
     "azure-storage": "2.10.3",
     "backo": "^1.1.0",
     "base-x": "3.0.8",
     "base62": "2.0.1",
     "bson": "4.0.0",
-    "debug": "~2.6.9",
+    "debug": "~4.1.0",
     "diskusage": "^1.1.1",
     "fcntl": "github:scality/node-fcntl#0.2.0",
     "hdclient": "scality/hdclient#1.1.0",
+    "https-proxy-agent": "^2.2.0",
     "ioredis": "^4.28.5",
     "ipaddr.js": "1.9.1",
     "level": "~5.0.1",
@@ -39,11 +40,11 @@
     "mongodb": "^3.0.1",
     "node-forge": "^0.7.1",
     "prom-client": "10.2.3",
-    "simple-glob": "^0.2",
-    "socket.io": "~2.3.0",
-    "socket.io-client": "~2.3.0",
+    "simple-glob": "^0.2.0",
+    "socket.io": "2.4.1",
+    "socket.io-client": "2.4.0",
     "sproxydclient": "github:scality/sproxydclient#8.0.3",
-    "utf8": "2.1.2",
+    "utf8": "3.0.0",
     "uuid": "^3.0.1",
     "werelogs": "scality/werelogs#8.1.0",
     "xml2js": "~0.4.23"
@@ -57,13 +58,13 @@
     "@sinonjs/fake-timers": "^6.0.1",
     "@types/jest": "^27.4.1",
     "@types/node": "^17.0.21",
-    "eslint": "2.13.1",
+    "eslint": "^8.9.0",
     "eslint-config-airbnb": "6.2.0",
-    "eslint-config-scality": "scality/Guidelines#7.10.2",
+    "eslint-config-scality": "scality/Guidelines#ec33dfb",
     "eslint-plugin-react": "^4.3.0",
     "jest": "^27.5.1",
-    "mocha": "8.0.1",
     "mongodb-memory-server": "^6.0.2",
+    "nyc": "^15.1.0",
     "sinon": "^9.0.2",
     "temp": "0.9.1",
     "ts-jest": "^27.1.3",
@@ -77,11 +78,15 @@
     "test": "jest tests/unit",
     "build": "tsc",
     "prepare": "yarn build || true",
-    "ft_test": "jest tests/functional --testTimeout=120000 --forceExit"
+    "ft_test": "jest tests/functional --testTimeout=120000 --forceExit",
+    "coverage": "nyc --clean jest tests --coverage --testTimeout=120000 --forceExit"
   },
-  "private": true,
   "jest": {
+    "private": true,
     "maxWorkers": 1,
+    "coverageReporters": [
+      "json"
+    ],
     "collectCoverageFrom": [
       "lib/**/*.{js,ts}",
       "index.js"
@@ -94,5 +99,12 @@
         }
       }
     }
+  },
+  "nyc": {
+    "tempDirectory": "coverage",
+    "reporter": [
+      "lcov",
+      "text"
+    ]
   }
 }

tests/functional/kmip/highlevel.spec.js

@@ -43,28 +43,28 @@ describe('KMIP High Level Driver', () => {
             it('should work with' +
                ` x-name attribute: ${!!bucketNameAttributeName},` +
                ` compound creation: ${compoundCreateActivate}`,
-               done => {
-                   const kmipClient = new KMIPClient(options, TTLVCodec,
-                                                     LoopbackServerTransport);
-                   const plaintext = Buffer.from(crypto.randomBytes(32));
-                   async.waterfall([
-                       next => kmipClient.createBucketKey('plop', logger, next),
-                       (id, next) =>
-                           kmipClient.cipherDataKey(1, id, plaintext,
-                                                    logger, (err, ciphered) => {
-                                                        next(err, id, ciphered);
-                                                    }),
-                       (id, ciphered, next) =>
-                           kmipClient.decipherDataKey(
-                               1, id, ciphered, logger, (err, deciphered) => {
-                                   assert(plaintext
-                                          .compare(deciphered) === 0);
-                                   next(err, id);
-                               }),
-                       (id, next) =>
-                           kmipClient.destroyBucketKey(id, logger, next),
-                   ], done);
-               });
+            done => {
+                const kmipClient = new KMIPClient(options, TTLVCodec,
+                    LoopbackServerTransport);
+                const plaintext = Buffer.from(crypto.randomBytes(32));
+                async.waterfall([
+                    next => kmipClient.createBucketKey('plop', logger, next),
+                    (id, next) =>
+                        kmipClient.cipherDataKey(1, id, plaintext,
+                            logger, (err, ciphered) => {
+                                next(err, id, ciphered);
+                            }),
+                    (id, ciphered, next) =>
+                        kmipClient.decipherDataKey(
+                            1, id, ciphered, logger, (err, deciphered) => {
+                                assert(plaintext
+                                    .compare(deciphered) === 0);
+                                next(err, id);
+                            }),
+                    (id, next) =>
+                        kmipClient.destroyBucketKey(id, logger, next),
+                ], done);
+            });
         });
     });
     it('should succeed healthcheck with working KMIP client and server', done => {
@@ -84,7 +84,7 @@ describe('KMIP High Level Driver', () => {
             },
         };
         const kmipClient = new KMIPClient(options, TTLVCodec,
-                                          LoopbackServerTransport);
+            LoopbackServerTransport);
         kmipClient.healthcheck(logger, err => {
             assert.ifError(err);
             done();

tests/functional/kmip/lowlevel.spec.js

@@ -36,17 +36,17 @@ describe('KMIP Low Level Driver', () => {
             const kmip = new KMIP(TTLVCodec, MirrorTransport, options);
             const requestPayload = fixture.payload(kmip);
             kmip.request(logger, fixture.operation,
-                         requestPayload, (err, response) => {
-                             if (err) {
-                                 return done(err);
-                             }
-                             const responsePayload = response.lookup(
-                                 'Response Message/Batch Item/Response Payload'
-                             )[0];
-                             assert.deepStrictEqual(responsePayload,
-                                                    requestPayload);
-                             return done();
-                         });
+                requestPayload, (err, response) => {
+                    if (err) {
+                        return done(err);
+                    }
+                    const responsePayload = response.lookup(
+                        'Response Message/Batch Item/Response Payload',
+                    )[0];
+                    assert.deepStrictEqual(responsePayload,
+                        requestPayload);
+                    return done();
+                });
         });
     });
 });

tests/functional/kmip/transport.spec.js

@@ -14,68 +14,68 @@ describe('KMIP Transport Template Class', () => {
         requestNumbers.forEach(iterations => {
             it(`should survive ${iterations} iterations` +
                ` with ${pipelineDepth}way pipeline`,
-               done => {
-                   const transport = new TransportTemplate(
-                       new EchoChannel,
-                       {
-                           pipelineDepth,
-                           tls: {
-                               port: 5696,
-                           },
-                       });
-                   const request = Buffer.alloc(10).fill(6);
-                   async.times(iterations, (n, next) => {
-                       transport.send(logger, request,
-                                      (err, conversation, response) => {
-                                          if (err) {
-                                              return next(err);
-                                          }
-                                          if (request.compare(response) !== 0) {
-                                              return next(Error('arg'));
-                                          }
-                                          return next();
-                                      });
-                   }, err => {
-                       transport.end();
-                       done(err);
-                   });
-               });
+            done => {
+                const transport = new TransportTemplate(
+                    new EchoChannel,
+                    {
+                        pipelineDepth,
+                        tls: {
+                            port: 5696,
+                        },
+                    });
+                const request = Buffer.alloc(10).fill(6);
+                async.times(iterations, (n, next) => {
+                    transport.send(logger, request,
+                        (err, conversation, response) => {
+                            if (err) {
+                                return next(err);
+                            }
+                            if (request.compare(response) !== 0) {
+                                return next(Error('arg'));
+                            }
+                            return next();
+                        });
+                }, err => {
+                    transport.end();
+                    done(err);
+                });
+            });
 
             [true, false].forEach(doEmit => {
                 it('should report errors to outstanding requests.' +
                    ` w:${pipelineDepth}, i:${iterations}, e:${doEmit}`,
-                   done => {
-                       const echoChannel = new EchoChannel;
-                       echoChannel.clog();
-                       const transport = new TransportTemplate(
-                           echoChannel,
-                           {
-                               pipelineDepth,
-                               tls: {
-                                   port: 5696,
-                               },
-                           });
-                       const request = Buffer.alloc(10).fill(6);
-                       /* Using a for loop here instead of anything
+                done => {
+                    const echoChannel = new EchoChannel;
+                    echoChannel.clog();
+                    const transport = new TransportTemplate(
+                        echoChannel,
+                        {
+                            pipelineDepth,
+                            tls: {
+                                port: 5696,
+                            },
+                        });
+                    const request = Buffer.alloc(10).fill(6);
+                    /* Using a for loop here instead of anything
                         * asynchronous, the callbacks get stuck in
                         * the conversation queue and are unwind with
                         * an error. It is the purpose of this test */
-                       for (let i = 0; i < iterations; ++i) {
-                           transport.send(
-                               logger, request,
-                               (err, conversation, response) => {
-                                   assert(err);
-                                   assert(!response);
-                               });
-                       }
-                       if (doEmit) {
-                           echoChannel.emit('error', new Error('awesome'));
-                       } else {
-                           transport.abortPipeline(echoChannel);
-                       }
-                       transport.end();
-                       done();
-                   });
+                    for (let i = 0; i < iterations; ++i) {
+                        transport.send(
+                            logger, request,
+                            (err, conversation, response) => {
+                                assert(err);
+                                assert(!response);
+                            });
+                    }
+                    if (doEmit) {
+                        echoChannel.emit('error', new Error('awesome'));
+                    } else {
+                        transport.abortPipeline(echoChannel);
+                    }
+                    transport.end();
+                    done();
+                });
             });
         });
     });

tests/functional/metadata/mongodb/delObject.spec.js

@@ -0,0 +1,463 @@
+const async = require('async');
+const assert = require('assert');
+const sinon = require('sinon');
+const werelogs = require('werelogs');
+const { MongoMemoryReplSet } = require('mongodb-memory-server');
+const { errors, versioning } = require('../../../../index');
+const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug');
+const BucketInfo = require('../../../../lib/models/BucketInfo');
+const MetadataWrapper =
+    require('../../../../lib/storage/metadata/MetadataWrapper');
+const genVID = require('../../../../lib/versioning/VersionID').generateVersionId;
+const { BucketVersioningKeyFormat } = versioning.VersioningConstants;
+
+const IMPL_NAME = 'mongodb';
+const DB_NAME = 'metadata';
+const BUCKET_NAME = 'test-bucket';
+const replicationGroupId = 'RG001';
+
+const mongoserver = new MongoMemoryReplSet({
+    debug: false,
+    instanceOpts: [
+        { port: 27018 },
+    ],
+    replSet: {
+        name: 'rs0',
+        count: 1,
+        DB_NAME,
+        storageEngine: 'ephemeralForTest',
+    },
+});
+
+let uidCounter = 0;
+function generateVersionId() {
+    return genVID(`${process.pid}.${uidCounter++}`,
+        replicationGroupId);
+}
+
+const variations = [
+    { it: '(v0)', vFormat: BucketVersioningKeyFormat.v0 },
+    { it: '(v1)', vFormat: BucketVersioningKeyFormat.v1 },
+];
+describe('MongoClientInterface::metadata.deleteObjectMD', () => {
+    let metadata;
+    let collection;
+
+    function getObjectCount(cb) {
+        collection.countDocuments((err, count) => {
+            if (err) {
+                cb(err);
+            }
+            cb(null, count);
+        });
+    }
+
+    function getObject(key, cb) {
+        collection.findOne({
+            _id: key,
+        }, {}, (err, doc) => {
+            if (err) {
+                return cb(err);
+            }
+            if (!doc) {
+                return cb(errors.NoSuchKey);
+            }
+            return cb(null, doc.value);
+        });
+    }
+
+    beforeAll(done => {
+        mongoserver.waitUntilRunning().then(() => {
+            const opts = {
+                mongodb: {
+                    replicaSetHosts: 'localhost:27018',
+                    writeConcern: 'majority',
+                    replicaSet: 'rs0',
+                    readPreference: 'primary',
+                    database: DB_NAME,
+                },
+            };
+            metadata = new MetadataWrapper(IMPL_NAME, opts, null, logger);
+            metadata.setup(done);
+        });
+    });
+
+    afterAll(done => {
+        async.series([
+            next => metadata.close(next),
+            next => mongoserver.stop()
+                .then(() => next())
+                .catch(next),
+        ], done);
+    });
+
+    variations.forEach(variation => {
+        const itOnlyInV1 = variation.vFormat === 'v1' ? it : it.skip;
+        describe(`vFormat : ${variation.vFormat}`, () => {
+            beforeEach(done => {
+                const bucketMD = BucketInfo.fromObj({
+                    _name: BUCKET_NAME,
+                    _owner: 'testowner',
+                    _ownerDisplayName: 'testdisplayname',
+                    _creationDate: new Date().toJSON(),
+                    _acl: {
+                        Canned: 'private',
+                        FULL_CONTROL: [],
+                        WRITE: [],
+                        WRITE_ACP: [],
+                        READ: [],
+                        READ_ACP: [],
+                    },
+                    _mdBucketModelVersion: 10,
+                    _transient: false,
+                    _deleted: false,
+                    _serverSideEncryption: null,
+                    _versioningConfiguration: null,
+                    _locationConstraint: 'us-east-1',
+                    _readLocationConstraint: null,
+                    _cors: null,
+                    _replicationConfiguration: null,
+                    _lifecycleConfiguration: null,
+                    _uid: '',
+                    _isNFS: null,
+                    ingestion: null,
+                });
+                async.series([
+                    next => {
+                        metadata.client.defaultBucketKeyFormat = variation.vFormat;
+                        return next();
+                    },
+                    next => metadata.createBucket(BUCKET_NAME, bucketMD, logger, err => {
+                        if (err) {
+                            return next(err);
+                        }
+                        collection = metadata.client.getCollection(BUCKET_NAME);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            afterEach(done => {
+                metadata.deleteBucket(BUCKET_NAME, logger, done);
+            });
+
+            it(`Should delete non versioned object ${variation.vFormat}`, done => {
+                const params = {
+                    objName: 'non-deleted-object',
+                    objVal: {
+                        key: 'non-deleted-object',
+                        versionId: 'null',
+                    },
+                };
+                const versionParams = {
+                    versioning: false,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                return async.series([
+                    next => {
+                        // we put the master version of object
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, next);
+                    },
+                    next => {
+                        // we put the master version of a second object
+                        params.objName = 'object-to-deleted';
+                        params.objVal.key = 'object-to-deleted';
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, next);
+                    },
+                    next => {
+                        // We delete the first object
+                        metadata.deleteObjectMD(BUCKET_NAME, params.objName, null, logger, next);
+                    },
+                    next => {
+                        // Object must be removed
+                        metadata.getObjectMD(BUCKET_NAME, params.objName, null, logger, err => {
+                            assert.deepStrictEqual(err, errors.NoSuchKey);
+                            return next();
+                        });
+                    },
+                    next => {
+                        // only 1 object remaining in db
+                        getObjectCount((err, count) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(count, 1);
+                            return next();
+                        });
+                    },
+                ], done);
+            });
+
+            it(`Should not throw error when object non existent ${variation.vFormat}`, done => {
+                const objName = 'non-existent-object';
+                metadata.deleteObjectMD(BUCKET_NAME, objName, null, logger, err => {
+                    assert.deepStrictEqual(err, null);
+                    return done();
+                });
+            });
+
+            it(`Should not throw error when bucket non existent ${variation.vFormat}`, done => {
+                const objName = 'non-existent-object';
+                metadata.deleteObjectMD(BUCKET_NAME, objName, null, logger, err => {
+                    assert.deepStrictEqual(err, null);
+                    return done();
+                });
+            });
+
+            it(`Master should not be updated when non lastest version is deleted ${variation.vFormat}`, done => {
+                let versionId1 = null;
+                const params = {
+                    objName: 'test-object',
+                    objVal: {
+                        key: 'test-object',
+                        versionId: 'null',
+                    },
+                    vFormat: 'v0',
+                };
+                const versionParams = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                return async.series([
+                    next => {
+                        // we start by creating a new version and master
+                        versionId1 = generateVersionId(this.replicationGroupId);
+                        params.versionId = versionId1;
+                        params.objVal.versionId = versionId1;
+                        versionParams.versionId = versionId1;
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, next);
+                    },
+                    next => {
+                        // we create a second version of the same object (master is updated)
+                        params.objVal.versionId = 'version2';
+                        versionParams.versionId = null;
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, next);
+                    },
+                    next => {
+                        // we delete the first version
+                        metadata.deleteObjectMD(BUCKET_NAME, params.objName, { versionId: versionId1 },
+                            logger, next);
+                    },
+                    next => {
+                        // the first version should no longer be available
+                        metadata.getObjectMD(BUCKET_NAME, params.objName, { versionId: versionId1 }, logger, err => {
+                            assert.deepStrictEqual(err, errors.NoSuchKey);
+                            return next();
+                        });
+                    },
+                    next => {
+                        // master must be containing second version metadata
+                        metadata.getObjectMD(BUCKET_NAME, params.objName, null, logger, (err, data) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.notStrictEqual(data.versionId, versionId1);
+                            return next();
+                        });
+                    },
+                    next => {
+                        // master and one version remaining in db
+                        getObjectCount((err, count) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(count, 2);
+                            return next();
+                        });
+                    },
+                ], done);
+            });
+
+            it(`Master should be updated when last version is deleted ${variation.vFormat}`, done => {
+                let versionId1;
+                let versionId2;
+
+                const params = {
+                    objName: 'test-object',
+                    objVal: {
+                        key: 'test-object',
+                        versionId: 'null',
+                        isLast: false,
+                    },
+                };
+                const versionParams = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                return async.series([
+                    next => {
+                        // we start by creating a new version and master
+                        versionId1 = generateVersionId(this.replicationGroupId);
+                        params.versionId = versionId1;
+                        params.objVal.versionId = versionId1;
+                        versionParams.versionId = versionId1;
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, next);
+                    },
+                    next => {
+                        // we create a second version of the same object (master is updated)
+                        // params.objVal.versionId = 'version2';
+                        // versionParams.versionId = null;
+                        versionId2 = generateVersionId(this.replicationGroupId);
+                        params.versionId = versionId2;
+                        params.objVal.versionId = versionId2;
+                        versionParams.versionId = versionId2;
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, next);
+                    },
+                    next => {
+                        // deleting latest version
+                        metadata.deleteObjectMD(BUCKET_NAME, params.objName, { versionId: versionId2 },
+                            logger, next);
+                    },
+                    next => {
+                        // latest version must be removed
+                        metadata.getObjectMD(BUCKET_NAME, params.objName, { versionId: versionId2 }, logger, err => {
+                            assert.deepStrictEqual(err, errors.NoSuchKey);
+                            return next();
+                        });
+                    },
+                    next => {
+                        // master must be updated to contain first version data
+                        metadata.getObjectMD(BUCKET_NAME, params.objName, null, logger, (err, data) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(data.versionId, versionId1);
+                            return next();
+                        });
+                    },
+                    next => {
+                        // one master and version in the db
+                        getObjectCount((err, count) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(count, 2);
+                            return next();
+                        });
+                    },
+                ], done);
+            });
+
+            it(`Should fail when version id non existent ${variation.vFormat}`, done => {
+                const versionId = generateVersionId(this.replicationGroupId);
+                const objName = 'test-object';
+                metadata.deleteObjectMD(BUCKET_NAME, objName, { versionId }, logger, err => {
+                    assert.deepStrictEqual(err, errors.NoSuchKey);
+                    return done();
+                });
+            });
+
+            itOnlyInV1(`Should create master when delete marker removed ${variation.vFormat}`, done => {
+                const objVal = {
+                    key: 'test-object',
+                    isDeleteMarker: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                let firstVersionVersionId;
+                let deleteMarkerVersionId;
+                async.series([
+                    // We first create a new version and master
+                    next => metadata.putObjectMD(BUCKET_NAME, 'test-object', objVal, params, logger, (err, res) => {
+                        if (err) {
+                            return next(err);
+                        }
+                        firstVersionVersionId = JSON.parse(res).versionId;
+                        return next();
+                    }),
+                    // putting a delete marker as last version
+                    next => {
+                        objVal.isDeleteMarker = true;
+                        params.versionId = null;
+                        return metadata.putObjectMD(BUCKET_NAME, 'test-object', objVal, params, logger, (err, res) => {
+                            if (err) {
+                                return next(err);
+                            }
+                            deleteMarkerVersionId = JSON.parse(res).versionId;
+                            return next();
+                        });
+                    },
+                    next => {
+                        // using fake clock to override the setTimeout used by the repair
+                        const clock = sinon.useFakeTimers();
+                        return metadata.deleteObjectMD(BUCKET_NAME, 'test-object', { versionId: deleteMarkerVersionId },
+                            logger, () => {
+                                // running the repair callback
+                                clock.runAll();
+                                clock.restore();
+                                return next();
+                            });
+                    },
+                    // waiting for the repair callback to finish
+                    next => setTimeout(next, 100),
+                    // master should be created
+                    next => {
+                        getObject('\x7fMtest-object', (err, object) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(object.key, 'test-object');
+                            assert.strictEqual(object.versionId, firstVersionVersionId);
+                            assert.strictEqual(object.isDeleteMarker, false);
+                            return next();
+                        });
+                    },
+                ], done);
+            });
+
+            itOnlyInV1(`Should delete master when delete marker becomes last version ${variation.vFormat}`, done => {
+                const objVal = {
+                    key: 'test-object',
+                    isDeleteMarker: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                let versionId;
+                async.series([
+                    // We first create a new version and master
+                    next => metadata.putObjectMD(BUCKET_NAME, 'test-object', objVal, params, logger, next),
+                    // putting a delete marker as last version
+                    next => {
+                        objVal.isDeleteMarker = true;
+                        params.versionId = null;
+                        return metadata.putObjectMD(BUCKET_NAME, 'test-object', objVal, params, logger, next);
+                    },
+                    // putting new version on top of delete marker
+                    next => {
+                        objVal.isDeleteMarker = false;
+                        return metadata.putObjectMD(BUCKET_NAME, 'test-object', objVal, params, logger, (err, res) => {
+                            if (err) {
+                                return next(err);
+                            }
+                            versionId = JSON.parse(res).versionId;
+                            return next();
+                        });
+                    },
+                    next => {
+                        // using fake clock to override the setTimeout used by the repair
+                        const clock = sinon.useFakeTimers();
+                        return metadata.deleteObjectMD(BUCKET_NAME, 'test-object', { versionId },
+                            logger, () => {
+                                // running the repair callback
+                                clock.runAll();
+                                clock.restore();
+                                return next();
+                            });
+                    },
+                    // waiting for the repair callback to finish
+                    next => setTimeout(next, 100),
+                    // master must be deleted
+                    next => {
+                        getObject('\x7fMtest-object', err => {
+                            assert.deepStrictEqual(err, errors.NoSuchKey);
+                            return next();
+                        });
+                    },
+                ], done);
+            });
+        });
+    });
+});

tests/functional/metadata/mongodb/getObject.spec.js

@@ -0,0 +1,283 @@
+const async = require('async');
+const assert = require('assert');
+const werelogs = require('werelogs');
+const { MongoMemoryReplSet } = require('mongodb-memory-server');
+const { errors, versioning } = require('../../../../index');
+const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug');
+const BucketInfo = require('../../../../lib/models/BucketInfo');
+const MetadataWrapper =
+    require('../../../../lib/storage/metadata/MetadataWrapper');
+const genVID = versioning.VersionID.generateVersionId;
+const { BucketVersioningKeyFormat } = versioning.VersioningConstants;
+const { formatMasterKey } = require('../../../../lib/storage/metadata/mongoclient/utils');
+
+const IMPL_NAME = 'mongodb';
+const DB_NAME = 'metadata';
+const BUCKET_NAME = 'test-bucket';
+const replicationGroupId = 'RG001';
+
+const mongoserver = new MongoMemoryReplSet({
+    debug: false,
+    instanceOpts: [
+        { port: 27019 },
+    ],
+    replSet: {
+        name: 'rs0',
+        count: 1,
+        DB_NAME,
+        storageEngine: 'ephemeralForTest',
+    },
+});
+
+let uidCounter = 0;
+function generateVersionId() {
+    return genVID(`${process.pid}.${uidCounter++}`,
+        replicationGroupId);
+}
+
+const variations = [
+    { it: '(v0)', vFormat: BucketVersioningKeyFormat.v0 },
+    { it: '(v1)', vFormat: BucketVersioningKeyFormat.v1 },
+];
+
+describe('MongoClientInterface::metadata.getObjectMD', () => {
+    let metadata;
+    let collection;
+    let versionId1;
+    let versionId2;
+
+    let params = {
+        objName: 'pfx1-test-object',
+        objVal: {
+            key: 'pfx1-test-object',
+            versionId: 'null',
+        },
+    };
+
+    function updateMasterObject(objName, versionId, objVal, vFormat, cb) {
+        const mKey = formatMasterKey(objName, vFormat);
+        collection.updateOne(
+            {
+                _id: mKey,
+                $or: [{
+                    'value.versionId': {
+                        $exists: false,
+                    },
+                },
+                {
+                    'value.versionId': {
+                        $gt: versionId,
+                    },
+                },
+                ],
+            },
+            {
+                $set: { _id: mKey, value: objVal },
+            },
+            { upsert: true },
+            err => {
+                if (err) {
+                    return cb(err);
+                }
+                return cb(null);
+            });
+    }
+
+    beforeAll(done => {
+        mongoserver.waitUntilRunning().then(() => {
+            const opts = {
+                mongodb: {
+                    replicaSetHosts: 'localhost:27019',
+                    writeConcern: 'majority',
+                    replicaSet: 'rs0',
+                    readPreference: 'primary',
+                    database: DB_NAME,
+                },
+            };
+            metadata = new MetadataWrapper(IMPL_NAME, opts, null, logger);
+            metadata.setup(done);
+        });
+    });
+
+    afterAll(done => {
+        async.series([
+            next => metadata.close(next),
+            next => mongoserver.stop()
+                .then(() => next())
+                .catch(next),
+        ], done);
+    });
+
+    variations.forEach(variation => {
+        const itOnlyInV1 = variation.vFormat === 'v1' ? it : it.skip;
+        describe(`vFormat : ${variation.vFormat}`, () => {
+            beforeEach(done => {
+                const bucketMD = BucketInfo.fromObj({
+                    _name: BUCKET_NAME,
+                    _owner: 'testowner',
+                    _ownerDisplayName: 'testdisplayname',
+                    _creationDate: new Date().toJSON(),
+                    _acl: {
+                        Canned: 'private',
+                        FULL_CONTROL: [],
+                        WRITE: [],
+                        WRITE_ACP: [],
+                        READ: [],
+                        READ_ACP: [],
+                    },
+                    _mdBucketModelVersion: 10,
+                    _transient: false,
+                    _deleted: false,
+                    _serverSideEncryption: null,
+                    _versioningConfiguration: null,
+                    _locationConstraint: 'us-east-1',
+                    _readLocationConstraint: null,
+                    _cors: null,
+                    _replicationConfiguration: null,
+                    _lifecycleConfiguration: null,
+                    _uid: '',
+                    _isNFS: null,
+                    ingestion: null,
+                });
+                const versionParams = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                async.series([
+                    next => {
+                        metadata.client.defaultBucketKeyFormat = variation.vFormat;
+                        return next();
+                    },
+                    next => metadata.createBucket(BUCKET_NAME, bucketMD, logger, err => {
+                        if (err) {
+                            return next(err);
+                        }
+                        collection = metadata.client.getCollection(BUCKET_NAME);
+                        return next();
+                    }),
+                    next => {
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, (err, res) => {
+                                if (err) {
+                                    return next(err);
+                                }
+                                versionId1 = JSON.parse(res).versionId;
+                                return next(null);
+                            });
+                    },
+                    next => {
+                        metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal,
+                            versionParams, logger, (err, res) => {
+                                if (err) {
+                                    return next(err);
+                                }
+                                versionId2 = JSON.parse(res).versionId;
+                                return next(null);
+                            });
+                    },
+                ], done);
+            });
+
+            afterEach(done => {
+                // reset params
+                params = {
+                    objName: 'pfx1-test-object',
+                    objVal: {
+                        key: 'pfx1-test-object',
+                        versionId: 'null',
+                    },
+                };
+                metadata.deleteBucket(BUCKET_NAME, logger, done);
+            });
+
+            it(`Should return latest version of object ${variation.it}`, done =>
+                metadata.getObjectMD(BUCKET_NAME, params.objName, null, logger, (err, object) => {
+                    assert.deepStrictEqual(err, null);
+                    assert.strictEqual(object.key, params.objName);
+                    assert.strictEqual(object.versionId, versionId2);
+                    return done();
+                }));
+
+            it(`Should return the specified version of object ${variation.it}`, done =>
+                metadata.getObjectMD(BUCKET_NAME, params.objName, { versionId: versionId1 }, logger, (err, object) => {
+                    assert.deepStrictEqual(err, null);
+                    assert.strictEqual(object.key, params.objName);
+                    assert.strictEqual(object.versionId, versionId1);
+                    return done();
+                }));
+
+            it(`Should throw error when version non existent ${variation.it}`, done => {
+                const versionId = '1234567890';
+                return metadata.getObjectMD(BUCKET_NAME, params.objName, { versionId }, logger, (err, object) => {
+                    assert.deepStrictEqual(object, undefined);
+                    assert.deepStrictEqual(err, errors.NoSuchKey);
+                    return done();
+                });
+            });
+
+            it(`Should throw error when object non existent ${variation.it}`, done => {
+                const objName = 'non-existent-object';
+                return metadata.getObjectMD(BUCKET_NAME, objName, null, logger, err => {
+                    assert.deepStrictEqual(err, errors.NoSuchKey);
+                    return done();
+                });
+            });
+
+            it(`Should throw error when object non existent ${variation.it}`, done => {
+                const bucketName = 'non-existent-bucket';
+                return metadata.getObjectMD(bucketName, params.objName, null, logger, (err, object) => {
+                    assert.deepStrictEqual(object, undefined);
+                    assert.deepStrictEqual(err, errors.NoSuchKey);
+                    return done();
+                });
+            });
+
+            it(`Should return latest version when master is PHD ${variation.it}`, done => {
+                async.series([
+                    next => {
+                        const objectName = variation.vFormat === 'v0' ? 'pfx1-test-object' : '\x7fMpfx1-test-object';
+                        // adding isPHD flag to master
+                        const phdVersionId = generateVersionId();
+                        params.objVal.versionId = phdVersionId;
+                        params.objVal.isPHD = true;
+                        updateMasterObject(objectName, phdVersionId, params.objVal,
+                            variation.vFormat, next);
+                    },
+                    // Should return latest object version
+                    next => metadata.getObjectMD(BUCKET_NAME, params.objName, null, logger, (err, object) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(object.key, params.objName);
+                        assert.strictEqual(object.versionId, versionId2);
+                        delete params.isPHD;
+                        return next();
+                    }),
+                ], done);
+            });
+
+            itOnlyInV1(`Should return last version when master deleted ${variation.vFormat}`, done => {
+                const versioningParams = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                async.series([
+                    // putting a delete marker as last version
+                    next => {
+                        params.versionId = null;
+                        params.objVal.isDeleteMarker = true;
+                        return metadata.putObjectMD(BUCKET_NAME, params.objName, params.objVal, versioningParams,
+                            logger, next);
+                    },
+                    next => metadata.getObjectMD(BUCKET_NAME, params.objName, null, logger, (err, object) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(object.key, params.objName);
+                        assert.strictEqual(object.isDeleteMarker, true);
+                        params.objVal.isDeleteMarker = null;
+                        return next();
+                    }),
+                ], done);
+            });
+        });
+    });
+});

tests/functional/metadata/mongodb/listObject.spec.js

@@ -0,0 +1,412 @@
+const async = require('async');
+const assert = require('assert');
+const werelogs = require('werelogs');
+const { MongoMemoryReplSet } = require('mongodb-memory-server');
+const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug');
+const BucketInfo = require('../../../../lib/models/BucketInfo');
+const MetadataWrapper =
+require('../../../../lib/storage/metadata/MetadataWrapper');
+const { versioning } = require('../../../../index');
+const { BucketVersioningKeyFormat } = versioning.VersioningConstants;
+
+const IMPL_NAME = 'mongodb';
+const DB_NAME = 'metadata';
+const BUCKET_NAME = 'test-bucket';
+
+const mongoserver = new MongoMemoryReplSet({
+    debug: false,
+    instanceOpts: [
+        { port: 27020 },
+    ],
+    replSet: {
+        name: 'rs0',
+        count: 1,
+        DB_NAME,
+        storageEngine: 'ephemeralForTest',
+    },
+});
+
+const variations = [
+    { it: '(v0)', vFormat: BucketVersioningKeyFormat.v0 },
+    { it: '(v1)', vFormat: BucketVersioningKeyFormat.v1 },
+];
+
+describe('MongoClientInterface::metadata.listObject', () => {
+    let metadata;
+
+    function putBulkObjectVersions(bucketName, objName, objVal, params, versionNb, cb) {
+        let count = 0;
+        async.whilst(
+            () => count < versionNb,
+            cbIterator => {
+                count++;
+                // eslint-disable-next-line
+                return metadata.putObjectMD(bucketName, objName, objVal, params,
+                    logger, cbIterator);
+            },
+            err => {
+                if (err) {
+                    return cb(err);
+                }
+                return cb(null);
+            },
+        );
+    }
+
+    beforeAll(done => {
+        mongoserver.waitUntilRunning().then(() => {
+            const opts = {
+                mongodb: {
+                    replicaSetHosts: 'localhost:27020',
+                    writeConcern: 'majority',
+                    replicaSet: 'rs0',
+                    readPreference: 'primary',
+                    database: DB_NAME,
+                },
+            };
+            metadata = new MetadataWrapper(IMPL_NAME, opts, null, logger);
+            metadata.setup(done);
+        });
+    });
+
+    afterAll(done => {
+        async.series([
+            next => metadata.close(next),
+            next => mongoserver.stop()
+                .then(() => next())
+                .catch(next),
+        ], done);
+    });
+
+    variations.forEach(variation => {
+        beforeEach(done => {
+            const bucketMD = BucketInfo.fromObj({
+                _name: BUCKET_NAME,
+                _owner: 'testowner',
+                _ownerDisplayName: 'testdisplayname',
+                _creationDate: new Date().toJSON(),
+                _acl: {
+                    Canned: 'private',
+                    FULL_CONTROL: [],
+                    WRITE: [],
+                    WRITE_ACP: [],
+                    READ: [],
+                    READ_ACP: [],
+                },
+                _mdBucketModelVersion: 10,
+                _transient: false,
+                _deleted: false,
+                _serverSideEncryption: null,
+                _versioningConfiguration: null,
+                _locationConstraint: 'us-east-1',
+                _readLocationConstraint: null,
+                _cors: null,
+                _replicationConfiguration: null,
+                _lifecycleConfiguration: null,
+                _uid: '',
+                _isNFS: null,
+                ingestion: null,
+            });
+            const versionParams = {
+                versioning: true,
+                versionId: null,
+                repairMaster: null,
+            };
+            async.series([
+                next => {
+                    metadata.client.defaultBucketKeyFormat = variation.vFormat;
+                    return next();
+                },
+                next => metadata.createBucket(BUCKET_NAME, bucketMD, logger, err => {
+                    if (err) {
+                        return next(err);
+                    }
+                    return next();
+                }),
+                next => {
+                    const params = {
+                        objName: 'pfx1-test-object',
+                        objVal: {
+                            key: 'pfx1-test-object',
+                            versionId: 'null',
+                        },
+                        nbVersions: 5,
+                    };
+                    putBulkObjectVersions(BUCKET_NAME, params.objName, params.objVal, versionParams,
+                        params.nbVersions, next);
+                },
+                next => {
+                    const params = {
+                        objName: 'pfx2-test-object',
+                        objVal: {
+                            key: 'pfx2-test-object',
+                            versionId: 'null',
+                        },
+                        nbVersions: 5,
+                    };
+                    putBulkObjectVersions(BUCKET_NAME, params.objName, params.objVal, versionParams,
+                        params.nbVersions, next);
+                },
+                next => {
+                    const params = {
+                        objName: 'pfx3-test-object',
+                        objVal: {
+                            key: 'pfx3-test-object',
+                            versionId: 'null',
+                        },
+                        nbVersions: 5,
+                    };
+                    putBulkObjectVersions(BUCKET_NAME, params.objName, params.objVal, versionParams,
+                        params.nbVersions, next);
+                },
+            ], done);
+        });
+
+        afterEach(done => {
+            metadata.deleteBucket(BUCKET_NAME, logger, done);
+        });
+
+        it(`Should list master versions of objects ${variation.it}`, done => {
+            const params = {
+                listingType: 'DelimiterMaster',
+                maxKeys: 100,
+            };
+            return metadata.listObject(BUCKET_NAME, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert.strictEqual(data.Contents.length, 3);
+                assert.strictEqual(data.Contents[0].key, 'pfx1-test-object');
+                assert.strictEqual(data.Contents[1].key, 'pfx2-test-object');
+                assert.strictEqual(data.Contents[2].key, 'pfx3-test-object');
+                return done();
+            });
+        });
+
+        it(`Should truncate list of master versions of objects ${variation.it}`, done => {
+            const params = {
+                listingType: 'DelimiterMaster',
+                maxKeys: 2,
+            };
+            return metadata.listObject(BUCKET_NAME, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert.strictEqual(data.Contents.length, 2);
+                assert.strictEqual(data.Contents[0].key, 'pfx1-test-object');
+                assert.strictEqual(data.Contents[1].key, 'pfx2-test-object');
+                return done();
+            });
+        });
+
+        it(`Should list master versions of objects that start with prefix ${variation.it}`, done => {
+            const bucketName = BUCKET_NAME;
+            const params = {
+                listingType: 'DelimiterMaster',
+                maxKeys: 100,
+                prefix: 'pfx2',
+            };
+            return metadata.listObject(bucketName, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert.strictEqual(data.Contents.length, 1);
+                assert.strictEqual(data.Contents[0].key, 'pfx2-test-object');
+                return done();
+            });
+        });
+
+        it(`Should return empty results when bucket non existent (master) ${variation.it}`, done => {
+            const bucketName = 'non-existent-bucket';
+            const params = {
+                listingType: 'DelimiterMaster',
+                maxKeys: 100,
+            };
+            return metadata.listObject(bucketName, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert(data);
+                assert.strictEqual(data.Contents.length, 0);
+                return done();
+            });
+        });
+
+        it(`Should list all versions of objects ${variation.it}`, done => {
+            const bucketName = BUCKET_NAME;
+            const params = {
+                listingType: 'DelimiterVersions',
+                maxKeys: 1000,
+            };
+            const versionsPerKey = {};
+            return metadata.listObject(bucketName, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert.strictEqual(data.Versions.length, 15);
+                data.Versions.forEach(version => {
+                    versionsPerKey[version.key] = (versionsPerKey[version.key] || 0) + 1;
+                });
+                assert.strictEqual(versionsPerKey['pfx1-test-object'], 5);
+                assert.strictEqual(versionsPerKey['pfx2-test-object'], 5);
+                assert.strictEqual(versionsPerKey['pfx3-test-object'], 5);
+                return done();
+            });
+        });
+
+        it(`Should truncate list of master versions of objects ${variation.it}`, done => {
+            const params = {
+                listingType: 'DelimiterVersions',
+                maxKeys: 5,
+            };
+            const versionsPerKey = {};
+            return metadata.listObject(BUCKET_NAME, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert.strictEqual(data.Versions.length, 5);
+                data.Versions.forEach(version => {
+                    versionsPerKey[version.key] = (versionsPerKey[version.key] || 0) + 1;
+                });
+                assert.strictEqual(versionsPerKey['pfx1-test-object'], 5);
+                return done();
+            });
+        });
+
+        it(`Should list versions of objects that start with prefix ${variation.it}`, done => {
+            const params = {
+                listingType: 'DelimiterVersions',
+                maxKeys: 100,
+                prefix: 'pfx2',
+            };
+            const versionsPerKey = {};
+            return metadata.listObject(BUCKET_NAME, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert.strictEqual(data.Versions.length, 5);
+                data.Versions.forEach(version => {
+                    versionsPerKey[version.key] = (versionsPerKey[version.key] || 0) + 1;
+                });
+                assert.strictEqual(versionsPerKey['pfx2-test-object'], 5);
+                return done();
+            });
+        });
+
+        it(`Should return empty results when bucket not existing (version) ${variation.it}`, done => {
+            const bucketName = 'non-existent-bucket';
+            const params = {
+                listingType: 'DelimiterVersions',
+                maxKeys: 100,
+            };
+            return metadata.listObject(bucketName, params, logger, (err, data) => {
+                assert.deepStrictEqual(err, null);
+                assert(data);
+                assert.strictEqual(data.Versions.length, 0);
+                return done();
+            });
+        });
+
+        it(`should check entire list with pagination (version) ${variation.it}`, done => {
+            const versionsPerKey = {};
+            const bucketName = BUCKET_NAME;
+            const get = (maxKeys, keyMarker, versionIdMarker, cb) => metadata.listObject(bucketName, {
+                listingType: 'DelimiterVersions',
+                maxKeys,
+                keyMarker,
+                versionIdMarker,
+            }, logger, (err, res) => {
+                if (err) {
+                    return cb(err);
+                }
+                res.Versions.forEach(version => {
+                    versionsPerKey[version.key] = (versionsPerKey[version.key] || 0) + 1;
+                });
+                if (res.IsTruncated) {
+                    return get(maxKeys, res.NextKeyMarker, res.NextVersionIdMarker, cb);
+                }
+                return cb(null);
+            });
+            return get(3, null, null, err => {
+                assert.deepStrictEqual(err, null);
+                assert.strictEqual(Object.keys(versionsPerKey).length, 3);
+                assert.strictEqual(versionsPerKey['pfx1-test-object'], 5);
+                assert.strictEqual(versionsPerKey['pfx2-test-object'], 5);
+                assert.strictEqual(versionsPerKey['pfx3-test-object'], 5);
+                done();
+            });
+        });
+
+        it(`should not list phd master key when listing masters ${variation.it}`, done => {
+            const objVal = {
+                key: 'pfx1-test-object',
+                versionId: 'null',
+            };
+            const versionParams = {
+                versioning: true,
+            };
+            const params = {
+                listingType: 'DelimiterMaster',
+                prefix: 'pfx1',
+            };
+            let versionId;
+            let lastVersionId;
+            async.series([
+                next => metadata.putObjectMD(BUCKET_NAME, 'pfx1-test-object', objVal, versionParams,
+                    logger, (err, res) => {
+                        if (err) {
+                            return next(err);
+                        }
+                        versionId = JSON.parse(res).versionId;
+                        return next(null);
+                    }),
+                next => metadata.putObjectMD(BUCKET_NAME, 'pfx1-test-object', objVal, versionParams,
+                    logger, (err, res) => {
+                        if (err) {
+                            return next(err);
+                        }
+                        lastVersionId = JSON.parse(res).versionId;
+                        return next(null);
+                    }),
+                // when deleting the last version of an object a PHD master is created
+                // and kept for 15s before it's repaired
+                next => metadata.deleteObjectMD(BUCKET_NAME, 'pfx1-test-object', { versionId: lastVersionId },
+                    logger, next),
+                next => metadata.listObject(BUCKET_NAME, params, logger, (err, data) => {
+                    assert.ifError(err);
+                    assert.strictEqual(data.Contents[0].value.VersionId, versionId);
+                    return next();
+                }),
+            ], done);
+        });
+
+        it(`should not list phd master key when listing versions ${variation.it}`, done => {
+            const objVal = {
+                key: 'pfx1-test-object',
+                versionId: 'null',
+            };
+            const versionParams = {
+                versioning: true,
+            };
+            const params = {
+                listingType: 'DelimiterVersions',
+                prefix: 'pfx1',
+            };
+            let lastVersionId;
+            let versionIds;
+            async.series([
+                next => metadata.listObject(BUCKET_NAME, params, logger, (err, data) => {
+                    assert.ifError(err);
+                    assert.strictEqual(data.Versions.length, 5);
+                    versionIds = data.Versions.map(version => version.VersionId);
+                    return next();
+                }),
+                next => metadata.putObjectMD(BUCKET_NAME, 'pfx1-test-object', objVal, versionParams,
+                    logger, (err, res) => {
+                        if (err) {
+                            return next(err);
+                        }
+                        lastVersionId = JSON.parse(res).versionId;
+                        return next(null);
+                    }),
+                // when deleting the last version of an object a PHD master is created
+                // and kept for 15s before it's repaired
+                next => metadata.deleteObjectMD(BUCKET_NAME, 'pfx1-test-object', { versionId: lastVersionId },
+                    logger, next),
+                next => metadata.listObject(BUCKET_NAME, params, logger, (err, data) => {
+                    assert.ifError(err);
+                    const newVersionIds = data.Versions.map(version => version.VersionId);
+                    assert.strictEqual(data.Versions.length, 5);
+                    assert(versionIds.every(version => newVersionIds.includes(version)));
+                    return next();
+                }),
+            ], done);
+        });
+    });
+});

tests/functional/metadata/mongodb/putObject.spec.js

@@ -0,0 +1,429 @@
+const async = require('async');
+const assert = require('assert');
+const werelogs = require('werelogs');
+const { MongoMemoryReplSet } = require('mongodb-memory-server');
+const { errors, versioning } = require('../../../../index');
+const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug');
+const BucketInfo = require('../../../../lib/models/BucketInfo');
+const MetadataWrapper =
+    require('../../../../lib/storage/metadata/MetadataWrapper');
+const { BucketVersioningKeyFormat } = versioning.VersioningConstants;
+
+const IMPL_NAME = 'mongodb';
+const DB_NAME = 'metadata';
+const BUCKET_NAME = 'test-bucket';
+const OBJECT_NAME = 'test-object';
+const VERSION_ID = '98451712418844999999RG001  22019.0';
+
+const mongoserver = new MongoMemoryReplSet({
+    debug: false,
+    instanceOpts: [
+        { port: 27021 },
+    ],
+    replSet: {
+        name: 'rs0',
+        count: 1,
+        DB_NAME,
+        storageEngine: 'ephemeralForTest',
+    },
+});
+
+const variations = [
+    { it: '(v0)', vFormat: BucketVersioningKeyFormat.v0 },
+    { it: '(v1)', vFormat: BucketVersioningKeyFormat.v1 },
+];
+
+describe('MongoClientInterface:metadata.putObjectMD', () => {
+    let metadata;
+    let collection;
+
+    function getObject(key, cb) {
+        collection.findOne({
+            _id: key,
+        }, {}, (err, doc) => {
+            if (err) {
+                return cb(err);
+            }
+            if (!doc) {
+                return cb(errors.NoSuchKey);
+            }
+            return cb(null, doc.value);
+        });
+    }
+
+    function getObjectCount(cb) {
+        collection.countDocuments((err, count) => {
+            if (err) {
+                cb(err);
+            }
+            cb(null, count);
+        });
+    }
+
+    beforeAll(done => {
+        mongoserver.waitUntilRunning().then(() => {
+            const opts = {
+                mongodb: {
+                    replicaSetHosts: 'localhost:27021',
+                    writeConcern: 'majority',
+                    replicaSet: 'rs0',
+                    readPreference: 'primary',
+                    database: DB_NAME,
+                },
+            };
+            metadata = new MetadataWrapper(IMPL_NAME, opts, null, logger);
+            metadata.setup(done);
+        });
+    });
+
+    afterAll(done => {
+        async.series([
+            next => metadata.close(next),
+            next => mongoserver.stop()
+                .then(() => next())
+                .catch(next),
+        ], done);
+    });
+
+    variations.forEach(variation => {
+        const itOnlyInV1 = variation.vFormat === 'v1' ? it : it.skip;
+        describe(`vFormat : ${variation.vFormat}`, () => {
+            beforeEach(done => {
+                const bucketMD = BucketInfo.fromObj({
+                    _name: BUCKET_NAME,
+                    _owner: 'testowner',
+                    _ownerDisplayName: 'testdisplayname',
+                    _creationDate: new Date().toJSON(),
+                    _acl: {
+                        Canned: 'private',
+                        FULL_CONTROL: [],
+                        WRITE: [],
+                        WRITE_ACP: [],
+                        READ: [],
+                        READ_ACP: [],
+                    },
+                    _mdBucketModelVersion: 10,
+                    _transient: false,
+                    _deleted: false,
+                    _serverSideEncryption: null,
+                    _versioningConfiguration: null,
+                    _locationConstraint: 'us-east-1',
+                    _readLocationConstraint: null,
+                    _cors: null,
+                    _replicationConfiguration: null,
+                    _lifecycleConfiguration: null,
+                    _uid: '',
+                    _isNFS: null,
+                    ingestion: null,
+                });
+                async.series([
+                    next => {
+                        metadata.client.defaultBucketKeyFormat = variation.vFormat;
+                        return next();
+                    },
+                    next => metadata.createBucket(BUCKET_NAME, bucketMD, logger, err => {
+                        if (err) {
+                            return next(err);
+                        }
+                        collection = metadata.client.getCollection(BUCKET_NAME);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            afterEach(done => {
+                metadata.deleteBucket(BUCKET_NAME, logger, done);
+            });
+
+            it(`Should put a new non versionned object ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: 'null',
+                    updated: false,
+                };
+                const params = {
+                    versioning: null,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                async.series([
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next),
+                    next => {
+                        const key = variation.vFormat === 'v0' ? 'test-object' : '\x7fMtest-object';
+                        getObject(key, (err, object) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(object.key, OBJECT_NAME);
+                            return next();
+                        });
+                    },
+                    // When versionning not active only one document is created (master)
+                    next => getObjectCount((err, count) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(count, 1);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            it(`Should update the metadata ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: 'null',
+                    updated: false,
+                };
+                const params = {
+                    versioning: null,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                async.series([
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next),
+                    next => {
+                        objVal.updated = true;
+                        metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next);
+                    },
+                    // object metadata must be updated
+                    next => {
+                        const key = variation.vFormat === 'v0' ? 'test-object' : '\x7fMtest-object';
+                        getObject(key, (err, object) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(object.key, OBJECT_NAME);
+                            assert.strictEqual(object.updated, true);
+                            return next();
+                        });
+                    },
+                    // Only a master version should be created
+                    next => getObjectCount((err, count) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(count, 1);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            it(`Should put versionned object with the specific versionId ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: VERSION_ID,
+                    updated: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: VERSION_ID,
+                    repairMaster: null,
+                };
+                async.series([
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next),
+                    // checking if metadata corresponds to what was given to the function
+                    next => {
+                        const key = variation.vFormat === 'v0' ? 'test-object' : '\x7fMtest-object';
+                        getObject(key, (err, object) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(object.key, OBJECT_NAME);
+                            assert.strictEqual(object.versionId, VERSION_ID);
+                            return next();
+                        });
+                    },
+                    // We'll have one master and one version
+                    next => getObjectCount((err, count) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(count, 2);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            it(`Should put new version and update master ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: VERSION_ID,
+                    updated: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                let versionId = null;
+
+                async.series([
+                    // We first create a master and a version
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, (err, data) => {
+                        assert.deepStrictEqual(err, null);
+                        versionId = JSON.parse(data).versionId;
+                        return next();
+                    }),
+                    // We put another version of the object
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next),
+                    // Master must be updated
+                    next => {
+                        const key = variation.vFormat === 'v0' ? 'test-object' : '\x7fMtest-object';
+                        getObject(key, (err, object) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(object.key, OBJECT_NAME);
+                            assert.notStrictEqual(object.versionId, versionId);
+                            return next();
+                        });
+                    },
+                    // we'll have two versions and one master
+                    next => getObjectCount((err, count) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(count, 3);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            it(`Should update master when versionning is disabled ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: VERSION_ID,
+                    updated: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: null,
+                    repairMaster: null,
+                };
+                let versionId = null;
+                async.series([
+                    // We first create a new version and master
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, (err, data) => {
+                        assert.deepStrictEqual(err, null);
+                        versionId = JSON.parse(data).versionId;
+                        return next();
+                    }),
+                    next => {
+                        // Disabling versionning and putting new version
+                        params.versioning = false;
+                        params.versionId = '';
+                        return metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next);
+                    },
+                    // Master must be updated
+                    next => {
+                        const key = variation.vFormat === 'v0' ? 'test-object' : '\x7fMtest-object';
+                        getObject(key, (err, object) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(object.key, OBJECT_NAME);
+                            assert.notStrictEqual(object.versionId, versionId);
+                            return next();
+                        });
+                    },
+                    // The second put shouldn't create a new version
+                    next => getObjectCount((err, count) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(count, 2);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            it(`Should update latest version and repair master ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: VERSION_ID,
+                    updated: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: VERSION_ID,
+                    repairMaster: null,
+                };
+                async.series([
+                    // We first create a new version and master
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next),
+                    next => {
+                        // Updating the version and repairing master
+                        params.repairMaster = true;
+                        objVal.updated = true;
+                        return metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next);
+                    },
+                    // Master must be updated
+                    next => {
+                        const key = variation.vFormat === 'v0' ? 'test-object' : '\x7fMtest-object';
+                        getObject(key, (err, object) => {
+                            assert.deepStrictEqual(err, null);
+                            assert.strictEqual(object.key, OBJECT_NAME);
+                            assert.strictEqual(object.versionId, VERSION_ID);
+                            assert.strictEqual(object.updated, true);
+                            return next();
+                        });
+                    },
+                    // The second put shouldn't create a new version
+                    next => getObjectCount((err, count) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(count, 2);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            itOnlyInV1(`Should delete master when last version is delete marker ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: VERSION_ID,
+                    updated: false,
+                    isDeleteMarker: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: VERSION_ID,
+                    repairMaster: null,
+                };
+                async.series([
+                    // We first create a new version and master
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next),
+                    // putting a delete marker as last version
+                    next => {
+                        objVal.isDeleteMarker = true;
+                        params.versionId = null;
+                        return metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next);
+                    },
+                    // master must be deleted
+                    next => getObject('\x7fMtest-object', err => {
+                        assert.deepStrictEqual(err, errors.NoSuchKey);
+                        return next();
+                    }),
+                ], done);
+            });
+
+            itOnlyInV1(`Should create master when new version is put on top of delete marker ${variation.it}`, done => {
+                const objVal = {
+                    key: OBJECT_NAME,
+                    versionId: VERSION_ID,
+                    updated: false,
+                    isDeleteMarker: false,
+                };
+                const params = {
+                    versioning: true,
+                    versionId: VERSION_ID,
+                    repairMaster: null,
+                };
+                async.series([
+                    // We first create a new version and master
+                    next => metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next),
+                    // putting a delete marker as last version
+                    next => {
+                        objVal.isDeleteMarker = true;
+                        params.versionId = null;
+                        return metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next);
+                    },
+                    // We put a new version on top of delete marker
+                    next => {
+                        objVal.isDeleteMarker = false;
+                        objVal.updated = true;
+                        return metadata.putObjectMD(BUCKET_NAME, OBJECT_NAME, objVal, params, logger, next);
+                    },
+                    // master must be created
+                    next => getObject('\x7fMtest-object', (err, object) => {
+                        assert.deepStrictEqual(err, null);
+                        assert.strictEqual(object.key, OBJECT_NAME);
+                        assert.strictEqual(object.updated, true);
+                        assert.strictEqual(object.isDeleteMarker, false);
+                        return next();
+                    }),
+                ], done);
+            });
+        });
+    });
+});

tests/functional/metadata/mongodb/withCond.spec.js

@@ -0,0 +1,330 @@
+const async = require('async');
+const assert = require('assert');
+const werelogs = require('werelogs');
+const { MongoMemoryReplSet } = require('mongodb-memory-server');
+const { errors, versioning } = require('../../../../index');
+const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug');
+const BucketInfo = require('../../../../lib/models/BucketInfo');
+const MetadataWrapper =
+    require('../../../../lib/storage/metadata/MetadataWrapper');
+const { BucketVersioningKeyFormat } = versioning.VersioningConstants;
+
+const IMP_NAME = 'mongodb';
+const DB_NAME = 'metadata';
+const BUCKET_NAME = 'testbucket';
+
+const mongoserver = new MongoMemoryReplSet({
+    debug: false,
+    instanceOpts: [
+        { port: 27022 },
+    ],
+    replSet: {
+        name: 'rs0',
+        count: 1,
+        DB_NAME,
+        storageEngine: 'ephemeralForTest',
+    },
+});
+
+describe('MongoClientInterface:withCond', () => {
+    let metadata;
+
+    const variations = [
+        { it: '(v0)', vFormat: BucketVersioningKeyFormat.v0 },
+        { it: '(v1)', vFormat: BucketVersioningKeyFormat.v1 },
+    ];
+
+    beforeAll(done => {
+        mongoserver.waitUntilRunning().then(() => {
+            const opts = {
+                mongodb: {
+                    replicaSetHosts: 'localhost:27022',
+                    writeConcern: 'majority',
+                    replicaSet: 'rs0',
+                    readPreference: 'primary',
+                    database: DB_NAME,
+                },
+            };
+            metadata = new MetadataWrapper(IMP_NAME, opts, null, logger);
+            metadata.setup(done);
+        });
+    });
+
+    afterAll(done => {
+        async.series([
+            next => metadata.close(next),
+            next => mongoserver.stop()
+                .then(() => next())
+                .catch(next),
+        ], done);
+    });
+
+    variations.forEach(variation => {
+        describe('::putObjectWithCond', () => {
+            beforeEach(done => {
+                const bucketMD = BucketInfo.fromObj({
+                    _name: BUCKET_NAME,
+                    _owner: 'testowner',
+                    _ownerDisplayName: 'testdisplayname',
+                    _creationDate: new Date().toJSON(),
+                    _acl: {
+                        Canned: 'private',
+                        FULL_CONTROL: [],
+                        WRITE: [],
+                        WRITE_ACP: [],
+                        READ: [],
+                        READ_ACP: [],
+                    },
+                    _mdBucketModelVersion: 10,
+                    _transient: false,
+                    _deleted: false,
+                    _serverSideEncryption: null,
+                    _versioningConfiguration: null,
+                    _locationConstraint: 'us-east-1',
+                    _readLocationConstraint: null,
+                    _cors: null,
+                    _replicationConfiguration: null,
+                    _lifecycleConfiguration: null,
+                    _uid: '',
+                    _isNFS: null,
+                    ingestion: null,
+                });
+                async.series([
+                    next => {
+                        metadata.client.defaultBucketKeyFormat = variation.vFormat;
+                        return next();
+                    },
+                    next => metadata.createBucket(BUCKET_NAME, bucketMD, logger, next),
+                ], done);
+            });
+
+            afterEach(done => {
+                metadata.deleteBucket(BUCKET_NAME, logger, done);
+            });
+
+            const tests = [
+                [
+                    `should upsert object if an existing object does not exist ${variation.it}`,
+                    {
+                        initVal: null,
+                        upsertVal: { value: { number: 42, string: 'forty-two' } },
+                        conditions: { value: { number: 24 } },
+                        expectedVal: { value: { number: 42, string: 'forty-two' } },
+                        error: null,
+                    },
+                ],
+                [
+                    `should not update an existing object if the conditions fails ${variation.it}`,
+                    {
+                        initVal: { value: { number: 0, string: 'zero' } },
+                        upsertVal: { value: { number: 42, string: 'forty-two' } },
+                        conditions: { value: { number: 24 } },
+                        expectedVal: { value: { number: 0, string: 'zero' } },
+                        error: errors.InternalError,
+                    },
+                ],
+                [
+                    `should not update an existing object if the conditions fails ${variation.it}`,
+                    {
+                        initVal: { value: { number: 0, string: 'zero' } },
+                        upsertVal: { value: { number: 42, string: 'forty-two' } },
+                        conditions: { value: { string: { $eq: 'twenty-four' } } },
+                        expectedVal: { value: { number: 0, string: 'zero' } },
+                        error: errors.InternalError,
+                    },
+                ],
+                [
+                    `should not update an existing object if the conditions fails ${variation.it}`,
+                    {
+                        initVal: { value: { number: 0, string: 'zero' } },
+                        upsertVal: { value: { number: 42, string: 'forty-two' } },
+                        conditions: {
+                            value: {
+                                string: { $eq: 'twenty-four' },
+                                number: { $eq: 0 },
+                            },
+                        },
+                        expectedVal: { value: { number: 0, string: 'zero' } },
+                        error: errors.InternalError,
+                    },
+                ],
+                [
+                    `should update an existing object if the conditions passes ${variation.it}`,
+                    {
+                        initVal: { value: { number: 24, string: 'twenty-four' } },
+                        upsertVal: { value: { number: 42, string: 'forty-two' } },
+                        conditions: { value: { number: 24 } },
+                        expectedVal: { value: { number: 42, string: 'forty-two' } },
+                        error: null,
+                    },
+                ],
+                [
+                    `should update an existing object if the conditions passes ${variation.it}`,
+                    {
+                        initVal: { value: { number: 24, string: 'twenty-four' } },
+                        upsertVal: { value: { number: 42, string: 'forty-two' } },
+                        conditions: { value: { string: { $eq: 'twenty-four' } } },
+                        expectedVal: { value: { number: 42, string: 'forty-two' } },
+                        error: null,
+                    },
+                ],
+                [
+                    `should update an existing object if the conditions passes ${variation.it}`,
+                    {
+                        initVal: { value: { number: 24, string: 'twenty-four' } },
+                        upsertVal: { value: { number: 42, string: 'forty-two' } },
+                        conditions: {
+                            value: {
+                                string: { $eq: 'twenty-four' },
+                                number: { $eq: 24 },
+                            },
+                        },
+                        expectedVal: { value: { number: 42, string: 'forty-two' } },
+                        error: null,
+                    },
+                ],
+            ];
+            tests.forEach(([msg, testCase]) => it(msg, done => {
+                const objectKey = 'testkey';
+                const {
+                    initVal, upsertVal, conditions, expectedVal, error,
+                } = testCase;
+                const params = { conditions };
+                async.series([
+                    next => {
+                        if (!initVal) {
+                            return next();
+                        }
+                        return metadata.putObjectMD(BUCKET_NAME, objectKey, initVal,
+                            {}, logger, next);
+                    },
+                    next => metadata.putObjectWithCond(BUCKET_NAME, objectKey,
+                        upsertVal, params, logger, err => {
+                            if (error) {
+                                assert.deepStrictEqual(err, error);
+                                return next();
+                            }
+                            assert(!err);
+                            return next();
+                        }),
+                    next => metadata.getObjectMD(BUCKET_NAME, objectKey, {}, logger,
+                        (err, res) => {
+                            assert(!err);
+                            assert.deepStrictEqual(res, expectedVal);
+                            next();
+                        }),
+                ], done);
+            }));
+        });
+
+        describe('::deleteObjectWithCond', () => {
+            const tests = [
+                [
+                    `should return no such key if the object does not exist ${variation.it}`,
+                    {
+                        initVal: null,
+                        conditions: { value: { number: 24 } },
+                        expectedVal: null,
+                        error: errors.NoSuchKey,
+                    },
+                ],
+                [
+                    `should return no such key if the conditions fails ${variation.it}`,
+                    {
+                        initVal: { value: { number: 0, string: 'zero' } },
+                        conditions: { value: { number: { $eq: 24 } } },
+                        expectedVal: { value: { number: 0, string: 'zero' } },
+                        error: errors.NoSuchKey,
+                    },
+                ],
+                [
+                    `should return no such key if the conditions fails ${variation.it}`,
+                    {
+                        initVal: { value: { number: 0, string: 'zero' } },
+                        conditions: { value: { string: 'twenty-four' } },
+                        expectedVal: { value: { number: 0, string: 'zero' } },
+                        error: errors.NoSuchKey,
+                    },
+                ],
+                [
+                    `should return no such key if the conditions fails ${variation.it}`,
+                    {
+                        initVal: { value: { number: 0, string: 'zero' } },
+                        conditions: {
+                            value: {
+                                string: 'twenty-four',
+                                number: { $eq: 0 },
+                            },
+                        },
+                        expectedVal: { value: { number: 0, string: 'zero' } },
+                        error: errors.NoSuchKey,
+                    },
+                ],
+                [
+                    `should successfully delete matched object ${variation.it}`,
+                    {
+                        initVal: { value: { number: 24, string: 'twenty-four' } },
+                        conditions: { value: { number: 24 } },
+                        expectedVal: null,
+                        error: null,
+                    },
+                ],
+                [
+                    `should successfully delete matched object ${variation.it}`,
+                    {
+                        initVal: { value: { number: 24, string: 'twenty-four' } },
+                        conditions: { value: { string: { $eq: 'twenty-four' } } },
+                        expectedVal: null,
+                        error: null,
+                    },
+                ],
+                [
+                    `should successfully delete matched object ${variation.it}`,
+                    {
+                        initVal: { value: { number: 24, string: 'twenty-four' } },
+                        conditions: {
+                            value: {
+                                string: { $eq: 'twenty-four' },
+                                number: { $eq: 24 },
+                            },
+                        },
+                        expectedVal: null,
+                        error: null,
+                    },
+                ],
+            ];
+            tests.forEach(([msg, testCase]) => it(msg, done => {
+                const objectKey = 'testkey';
+                const { initVal, conditions, expectedVal, error } = testCase;
+                const params = { conditions };
+                async.series([
+                    next => {
+                        if (!initVal) {
+                            return next();
+                        }
+                        return metadata.putObjectMD(BUCKET_NAME, objectKey, initVal,
+                            {}, logger, next);
+                    },
+                    next => metadata.deleteObjectWithCond(BUCKET_NAME, objectKey,
+                        params, logger, err => {
+                            if (error) {
+                                assert.deepStrictEqual(err, error);
+                                return next();
+                            }
+                            assert(!err);
+                            return next();
+                        }),
+                    next => metadata.getObjectMD(BUCKET_NAME, objectKey, {}, logger,
+                        (err, res) => {
+                            if (expectedVal) {
+                                assert.deepStrictEqual(res, expectedVal);
+                            } else {
+                                assert.deepStrictEqual(err, errors.NoSuchKey);
+                            }
+                            return next();
+                        }),
+                ], done);
+            }));
+        });
+    });
+});

tests/functional/metadataProxy/routesToMem.spec.js

@@ -0,0 +1,319 @@
+'use strict'; // eslint-disable-line strict
+
+const werelogs = require('werelogs');
+const assert = require('assert');
+const async = require('async');
+
+const logger = new werelogs.Logger('MetadataProxyServer', 'debug', 'debug');
+const MetadataWrapper =
+      require('../../../lib/storage/metadata/MetadataWrapper');
+const BucketRoutes =
+      require('../../../lib/storage/metadata/proxy/BucketdRoutes');
+const metadataWrapper = new MetadataWrapper('mem', {}, null, logger);
+const { RequestDispatcher } = require('../../utils/mdProxyUtils');
+
+const routes = new BucketRoutes(metadataWrapper, logger);
+const dispatcher = new RequestDispatcher(routes);
+
+const Bucket = 'test';
+const bucketInfo = {
+    acl: {
+        Canned: 'private',
+        FULL_CONTROL: [],
+        WRITE: [],
+        WRITE_ACP: [],
+        READ: [],
+        READ_ACP: [],
+    },
+    name: Bucket,
+    owner: '9d8fe19a78974c56dceb2ea4a8f01ed0f5fecb9d29f80e9e3b84104e4a3ea520',
+    ownerDisplayName: 'anonymousCoward',
+    creationDate: '2018-06-04T17:45:42.592Z',
+    mdBucketModelVersion: 8,
+    transient: false,
+    deleted: false,
+    serverSideEncryption: null,
+    versioningConfiguration: null,
+    locationConstraint: 'us-east-1',
+    readLocationConstraint: 'us-east-1',
+    cors: null,
+    replicationConfiguration: null,
+    lifecycleConfiguration: null,
+    uid: 'fea97818-6a9a-11e8-9777-e311618cc5d4',
+    isNFS: null,
+};
+
+const objects = [
+    'aaa',
+    'bbb/xaa',
+    'bbb/xbb',
+    'bbb/xcc',
+    'ccc',
+    'ddd',
+];
+
+function _getExpectedListing(prefix, objects) {
+    const filtered = objects.map(key => {
+        const deprefixed = key.slice(prefix.length);
+        return deprefixed.replace(/[/].*/, '/');
+    });
+    const keySet = {};
+    return filtered.filter(key => {
+        if (keySet[key]) {
+            return false;
+        }
+        if (key === '') {
+            return false;
+        }
+        keySet[key] = true;
+        return true;
+    });
+}
+
+function _listingURL(prefix, marker) {
+    const reSlash = /[/]/g;
+    const escapedPrefix = prefix.replace(reSlash, '%2F');
+    const escapedMarker = marker.replace(reSlash, '%2F');
+    return `/default/bucket/${Bucket}?delimiter=%2F&prefix=` +
+        `${escapedPrefix}&maxKeys=1&marker=${escapedMarker}`;
+}
+
+function _listObjects(prefix, objects, cb) {
+    const keys = _getExpectedListing(prefix, objects);
+    const markers = keys.slice(0);
+    markers.unshift(undefined);
+    const lastKey = keys[keys.length - 1];
+    const listing = keys.map((key, index) => ({
+        key,
+        marker: markers[index],
+        NextMarker: markers[index + 1],
+        IsTruncated: key !== lastKey,
+        isPrefix: key.endsWith('/'),
+    }));
+    async.mapLimit(listing, 5, (obj, next) => {
+        const currentMarker = obj.marker === undefined ? '' : obj.marker;
+        dispatcher.get(_listingURL(prefix, prefix + currentMarker),
+            (err, response, body) => {
+                if (err) {
+                    return next(err);
+                }
+                if (obj.isPrefix) {
+                    assert.strictEqual(body.Contents.length, 0);
+                    assert.strictEqual(body.CommonPrefixes.length,
+                        1);
+                    assert.strictEqual(body.CommonPrefixes[0],
+                        prefix + obj.key);
+                } else {
+                    assert.strictEqual(body.Contents.length, 1);
+                    assert.strictEqual(body.CommonPrefixes.length,
+                        0);
+                    assert.strictEqual(body.Contents[0].key,
+                        prefix + obj.key);
+                }
+                assert.strictEqual(body.IsTruncated,
+                    obj.IsTruncated);
+                if (body.IsTruncated) {
+                    assert.strictEqual(body.NextMarker,
+                        prefix + obj.NextMarker);
+                }
+                return next();
+            });
+    }, err => cb(err));
+}
+
+function _createObjects(objects, cb) {
+    async.mapLimit(objects, 5, (key, next) => {
+        dispatcher.post(`/default/bucket/${Bucket}/${key}`,
+            { key }, next);
+    }, err => {
+        cb(err);
+    });
+}
+
+function _readObjects(objects, cb) {
+    async.mapLimit(objects, 5, (key, next) => {
+        dispatcher.get(`/default/bucket/${Bucket}/${key}`,
+            (err, response, body) => {
+                assert.deepStrictEqual(body.key, key);
+                next(err);
+            });
+    }, err => {
+        cb(err);
+    });
+}
+
+function _deleteObjects(objects, cb) {
+    async.mapLimit(objects, 5, (key, next) => {
+        dispatcher.delete(`/default/bucket/${Bucket}/${key}`,
+            err => next(err));
+    }, err => {
+        cb(err);
+    });
+}
+
+describe('Basic Metadata Proxy Server test',
+    () => {
+        jest.setTimeout(10000);
+        it('Shoud get the metadataInformation', done => {
+            dispatcher.get('/default/metadataInformation',
+                (err, response, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepStrictEqual(
+                        body, { metadataVersion: 2 });
+                    return done();
+                });
+        });
+    });
+
+describe('Basic Metadata Proxy Server CRUD test', () => {
+    jest.setTimeout(10000);
+
+    beforeEach(done => {
+        dispatcher.post(`/default/bucket/${Bucket}`, bucketInfo,
+            done);
+    });
+
+    afterEach(done => {
+        dispatcher.delete(`/default/bucket/${Bucket}`, done);
+    });
+
+    it('Should get the bucket attributes', done => {
+        dispatcher.get(`/default/attributes/${Bucket}`,
+            (err, response, body) => {
+                if (err) {
+                    return done(err);
+                }
+                assert.deepStrictEqual(body.name,
+                    bucketInfo.name);
+                return done();
+            });
+    });
+
+    it('Should crud an object', done => {
+        async.waterfall([
+            next => dispatcher.post(`/default/bucket/${Bucket}/test1`,
+                { foo: 'gabu' }, err => next(err)),
+            next => dispatcher.get(`/default/bucket/${Bucket}/test1`,
+                (err, response, body) => {
+                    if (!err) {
+                        assert.deepStrictEqual(body.foo,
+                            'gabu');
+                        next(err);
+                    }
+                }),
+            next => dispatcher.post(`/default/bucket/${Bucket}/test1`,
+                { foo: 'zome' }, err => next(err)),
+            next => dispatcher.get(`/default/bucket/${Bucket}/test1`,
+                (err, response, body) => {
+                    if (!err) {
+                        assert.deepStrictEqual(body.foo,
+                            'zome');
+                        next(err);
+                    }
+                }),
+            next => dispatcher.delete(`/default/bucket/${Bucket}/test1`,
+                err => next(err)),
+        ], err => done(err));
+    });
+
+    it('Should list objects', done => {
+        async.waterfall([
+            next => _createObjects(objects, next),
+            next => _readObjects(objects, next),
+            next => _listObjects('', objects, next),
+            next => _listObjects('bbb/', objects, next),
+            next => _deleteObjects(objects, next),
+        ], err => {
+            done(err);
+        });
+    });
+
+    it('Should update bucket properties', done => {
+        dispatcher.get(
+            `/default/attributes/${Bucket}`, (err, response, body) => {
+                assert.strictEqual(err, null);
+                const bucketInfo = body;
+                const newOwnerDisplayName = 'divertedfrom';
+                bucketInfo.ownerDisplayName = newOwnerDisplayName;
+                dispatcher.post(
+                    `/default/attributes/${Bucket}`, bucketInfo, err => {
+                        assert.strictEqual(err, null);
+                        dispatcher.get(
+                            `/default/attributes/${Bucket}`,
+                            (err, response, body) => {
+                                assert.strictEqual(err, null);
+                                const newBucketInfo = body;
+                                assert.strictEqual(
+                                    newBucketInfo.ownerDisplayName,
+                                    newOwnerDisplayName);
+                                done(null);
+                            });
+                    });
+            });
+    });
+
+    it('Should fail to list a non existing bucket', done => {
+        dispatcher.get('/default/bucket/nonexisting',
+            (err, response) => {
+                assert.strictEqual(
+                    response.responseHead.statusCode,
+                    404);
+                done(err);
+            });
+    });
+
+    it('Should fail to get attributes from a non existing bucket', done => {
+        dispatcher.get('/default/attributes/nonexisting',
+            (err, response) => {
+                assert.strictEqual(
+                    response.responseHead.statusCode,
+                    404);
+                done(err);
+            });
+    });
+
+    it('should succeed a health check', done => {
+        dispatcher.get('/_/healthcheck', (err, response, body) => {
+            if (err) {
+                return done(err);
+            }
+            const expectedResponse = {
+                memorybucket: {
+                    code: 200,
+                    message: 'OK',
+                },
+            };
+            assert.strictEqual(response.responseHead.statusCode, 200);
+            assert.deepStrictEqual(body, expectedResponse);
+            return done(err);
+        });
+    });
+
+    it('should work with parallel route', done => {
+        const objectName = 'theObj';
+        async.waterfall([
+            next => _createObjects([objectName], next),
+            next => {
+                dispatcher.get(
+                    `/default/parallel/${Bucket}/${objectName}`,
+                    (err, response, body) => {
+                        if (err) {
+                            return next(err);
+                        }
+                        assert.strictEqual(response.responseHead.statusCode,
+                            200);
+                        const bucketMD = JSON.parse(body.bucket);
+                        const objectMD = JSON.parse(body.obj);
+                        const expectedObjectMD = { key: objectName };
+                        assert.deepStrictEqual(bucketMD.name,
+                            bucketInfo.name);
+                        assert.deepStrictEqual(objectMD, expectedObjectMD);
+                        return next(err);
+                    });
+            },
+            next => _deleteObjects([objectName], next),
+        ], done);
+    });
+});

tests/functional/metrics/StatsClient.spec.js

@@ -31,28 +31,28 @@ describe('StatsClient class', () => {
     afterAll(() => redisClient.disconnect());
 
     it('should correctly record a new request by default one increment',
-    done => {
-        async.series([
-            next => {
-                statsClient.reportNewRequest(id, (err, res) => {
-                    assert.ifError(err);
+        done => {
+            async.series([
+                next => {
+                    statsClient.reportNewRequest(id, (err, res) => {
+                        assert.ifError(err);
 
-                    const expected = [[null, 1], [null, 1]];
-                    assert.deepEqual(res, expected);
-                    next();
-                });
-            },
-            next => {
-                statsClient.reportNewRequest(id, (err, res) => {
-                    assert.ifError(err);
+                        const expected = [[null, 1], [null, 1]];
+                        assert.deepEqual(res, expected);
+                        next();
+                    });
+                },
+                next => {
+                    statsClient.reportNewRequest(id, (err, res) => {
+                        assert.ifError(err);
 
-                    const expected = [[null, 2], [null, 1]];
-                    assert.deepEqual(res, expected);
-                    next();
-                });
-            },
-        ], done);
-    });
+                        const expected = [[null, 2], [null, 1]];
+                        assert.deepEqual(res, expected);
+                        next();
+                    });
+                },
+            ], done);
+        });
 
     it('should record new requests by defined amount increments', done => {
         function noop() {}

tests/functional/metrics/StatsModel.spec.js

@@ -0,0 +1,318 @@
+'use strict'; // eslint-disable-line strict
+
+const assert = require('assert');
+const async = require('async');
+
+const RedisClient = require('../../../lib/metrics/RedisClient');
+const StatsModel = require('../../../lib/metrics/StatsModel');
+
+// setup redis client
+const config = {
+    host: '127.0.0.1',
+    port: 6379,
+    enableOfflineQueue: true,
+};
+const fakeLogger = {
+    trace: () => {},
+    error: () => {},
+};
+const redisClient = new RedisClient(config, fakeLogger);
+
+// setup stats model
+const STATS_INTERVAL = 300; // 5 minutes
+const STATS_EXPIRY = 86400; // 24 hours
+const statsModel = new StatsModel(redisClient, STATS_INTERVAL, STATS_EXPIRY);
+
+function setExpectedStats(expected) {
+    return expected.concat(
+        Array((STATS_EXPIRY / STATS_INTERVAL) - expected.length).fill(0));
+}
+
+// Since many methods were overwritten, these tests should validate the changes
+// made to the original methods
+describe('StatsModel class', () => {
+    const id = 'arsenal-test';
+    const id2 = 'test-2';
+    const id3 = 'test-3';
+
+    afterEach(() => redisClient.clear(() => {}));
+
+    it('should convert a 2d array columns into rows and vice versa using _zip',
+        () => {
+            const arrays = [
+                [1, 2, 3],
+                [4, 5, 6],
+                [7, 8, 9],
+            ];
+
+            const res = statsModel._zip(arrays);
+            const expected = [
+                [1, 4, 7],
+                [2, 5, 8],
+                [3, 6, 9],
+            ];
+
+            assert.deepStrictEqual(res, expected);
+        });
+
+    it('_zip should return an empty array if given an invalid array', () => {
+        const arrays = [];
+
+        const res = statsModel._zip(arrays);
+
+        assert.deepStrictEqual(res, []);
+    });
+
+    it('_getCount should return a an array of all valid integer values',
+        () => {
+            const res = statsModel._getCount([
+                [null, '1'],
+                [null, '2'],
+                [null, null],
+            ]);
+            assert.deepStrictEqual(res, setExpectedStats([1, 2, 0]));
+        });
+
+    it('should correctly record a new request by default one increment',
+        done => {
+            async.series([
+                next => {
+                    statsModel.reportNewRequest(id, (err, res) => {
+                        assert.ifError(err);
+
+                        const expected = [[null, 1], [null, 1]];
+                        assert.deepStrictEqual(res, expected);
+                        next();
+                    });
+                },
+                next => {
+                    statsModel.reportNewRequest(id, (err, res) => {
+                        assert.ifError(err);
+
+                        const expected = [[null, 2], [null, 1]];
+                        assert.deepStrictEqual(res, expected);
+                        next();
+                    });
+                },
+            ], done);
+        });
+
+    it('should record new requests by defined amount increments', done => {
+        function noop() {}
+
+        async.series([
+            next => {
+                statsModel.reportNewRequest(id, 9);
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepStrictEqual(res.requests, setExpectedStats([9]));
+                    next();
+                });
+            },
+            next => {
+                statsModel.reportNewRequest(id);
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepStrictEqual(res.requests,
+                        setExpectedStats([10]));
+                    next();
+                });
+            },
+            next => {
+                statsModel.reportNewRequest(id, noop);
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepStrictEqual(res.requests,
+                        setExpectedStats([11]));
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should correctly record a 500 on the server', done => {
+        statsModel.report500(id, (err, res) => {
+            assert.ifError(err);
+
+            const expected = [[null, 1], [null, 1]];
+            assert.deepStrictEqual(res, expected);
+            done();
+        });
+    });
+
+    it('should respond back with total requests as an array', done => {
+        async.series([
+            next => {
+                statsModel.reportNewRequest(id, err => {
+                    assert.ifError(err);
+                    next();
+                });
+            },
+            next => {
+                statsModel.report500(id, err => {
+                    assert.ifError(err);
+                    next();
+                });
+            },
+            next => {
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    const expected = {
+                        'requests': setExpectedStats([1]),
+                        '500s': setExpectedStats([1]),
+                        'sampleDuration': STATS_EXPIRY,
+                    };
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should not crash on empty results', done => {
+        async.series([
+            next => {
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+                    const expected = {
+                        'requests': setExpectedStats([]),
+                        '500s': setExpectedStats([]),
+                        'sampleDuration': STATS_EXPIRY,
+                    };
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+            next => {
+                statsModel.getAllStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+                    const expected = {
+                        'requests': setExpectedStats([]),
+                        '500s': setExpectedStats([]),
+                        'sampleDuration': STATS_EXPIRY,
+                    };
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should return a zero-filled array if no ids are passed to getAllStats',
+        done => {
+            statsModel.getAllStats(fakeLogger, [], (err, res) => {
+                assert.ifError(err);
+
+                assert.deepStrictEqual(res.requests, setExpectedStats([]));
+                assert.deepStrictEqual(res['500s'], setExpectedStats([]));
+                done();
+            });
+        });
+
+    it('should get accurately reported data for given id from getAllStats',
+        done => {
+            statsModel.reportNewRequest(id, 9);
+            statsModel.reportNewRequest(id2, 2);
+            statsModel.reportNewRequest(id3, 3);
+            statsModel.report500(id);
+
+            async.series([
+                next => {
+                    statsModel.getAllStats(fakeLogger, [id], (err, res) => {
+                        assert.ifError(err);
+
+                        assert.equal(res.requests[0], 9);
+                        assert.equal(res['500s'][0], 1);
+                        next();
+                    });
+                },
+                next => {
+                    statsModel.getAllStats(fakeLogger, [id, id2, id3],
+                        (err, res) => {
+                            assert.ifError(err);
+
+                            assert.equal(res.requests[0], 14);
+                            assert.deepStrictEqual(res.requests,
+                                setExpectedStats([14]));
+                            next();
+                        });
+                },
+            ], done);
+        });
+
+    it('should normalize to the nearest hour using normalizeTimestampByHour',
+        () => {
+            const date = new Date('2018-09-13T23:30:59.195Z');
+            const newDate = new Date(statsModel.normalizeTimestampByHour(date));
+
+            assert.strictEqual(date.getHours(), newDate.getHours());
+            assert.strictEqual(newDate.getMinutes(), 0);
+            assert.strictEqual(newDate.getSeconds(), 0);
+            assert.strictEqual(newDate.getMilliseconds(), 0);
+        });
+
+    it('should get previous hour using _getDatePreviousHour', () => {
+        const date = new Date('2018-09-13T23:30:59.195Z');
+        const newDate = statsModel._getDatePreviousHour(new Date(date));
+
+        const millisecondsInOneHour = 3600000;
+        assert.strictEqual(date - newDate, millisecondsInOneHour);
+    });
+
+    it('should get an array of hourly timestamps using getSortedSetHours',
+        () => {
+            const epoch = 1536882476501;
+            const millisecondsInOneHour = 3600000;
+
+            const expected = [];
+            let dateInMilliseconds = statsModel.normalizeTimestampByHour(
+                new Date(epoch));
+
+            for (let i = 0; i < 24; i++) {
+                expected.push(dateInMilliseconds);
+                dateInMilliseconds -= millisecondsInOneHour;
+            }
+            const res = statsModel.getSortedSetHours(epoch);
+
+            assert.deepStrictEqual(res, expected);
+        });
+
+    it('should apply TTL on a new sorted set using addToSortedSet', done => {
+        const key = 'a-test-key';
+        const score = 100;
+        const value = 'a-value';
+
+        const now = Date.now();
+        const nearestHour = statsModel.normalizeTimestampByHour(new Date(now));
+
+        statsModel.addToSortedSet(key, score, value, (err, res) => {
+            assert.ifError(err);
+            // check both a "zadd" and "expire" occurred
+            assert.equal(res, 1);
+            redisClient.ttl(key, (err, res) => {
+                assert.ifError(err);
+                // assert this new set has a ttl applied
+                assert(res > 0);
+
+                const adjustmentSecs = now - nearestHour;
+                const msInADay = 24 * 60 * 60 * 1000;
+                const msInAnHour = 60 * 60 * 1000;
+                const upperLimitSecs =
+                    Math.ceil((msInADay - adjustmentSecs) / 1000);
+                const lowerLimitSecs =
+                    Math.floor((msInADay - adjustmentSecs - msInAnHour) / 1000);
+
+                // assert new ttl is between 23 and 24 hours adjusted by time
+                // elapsed since normalized hourly time
+                assert(res >= lowerLimitSecs);
+                assert(res <= upperLimitSecs);
+
+                done();
+            });
+        });
+    });
+});

tests/unit/auth/chainBackend.spec.js

@@ -0,0 +1,326 @@
+const assert = require('assert');
+
+const ChainBackend = require('../../../lib/auth/auth').backends.chainBackend;
+const BaseBackend = require('../../../lib/auth/auth').backends.baseBackend;
+const errors = require('../../../lib/errors');
+
+
+const testError = new Error('backend error');
+
+const backendWithAllMethods = {
+    verifySignatureV2: () => {},
+    verifySignatureV4: () => {},
+    getCanonicalIds: () => {},
+    getEmailAddresses: () => {},
+    checkPolicies: () => {},
+    healthcheck: () => {},
+};
+
+function getBackendWithMissingMethod(methodName) {
+    const backend = Object.assign({}, backendWithAllMethods);
+    delete backend[methodName];
+    return backend;
+}
+
+class TestBackend extends BaseBackend {
+    constructor(service, error, result) {
+        super(service);
+        this._error = error;
+        this._result = result;
+    }
+
+    verifySignatureV2(stringToSign, signatureFromRequest, accessKey, options, callback) {
+        return callback(this._error, this._result);
+    }
+
+    verifySignatureV4(stringToSign, signatureFromRequest, accessKey, region, scopeDate, options, callback) {
+        return callback(this._error, this._result);
+    }
+
+    getCanonicalIds(emailAddresses, options, callback) {
+        return callback(this._error, this._result);
+    }
+
+    getEmailAddresses(canonicalIDs, options, callback) {
+        return callback(this._error, this._result);
+    }
+
+    checkPolicies(requestContextParams, userArn, options, callback) {
+        return callback(this._error, this._result);
+    }
+
+    healthcheck(reqUid, callback) {
+        return callback(this._error, this._result);
+    }
+}
+
+describe('Auth Backend: Chain Backend', () => {
+    [
+        ['should throw an error if client list is not an array', null],
+        ['should throw an error if client list empty', []],
+        ['should throw an error if a client is missing the verifySignatureV2 method', [
+            new TestBackend(),
+            getBackendWithMissingMethod('verifySignatureV2'),
+        ]],
+        ['should throw an error if a client is missing the verifySignatureV4 auth method', [
+            new TestBackend(),
+            getBackendWithMissingMethod('verifySignatureV4'),
+        ]],
+        ['should throw an error if a client is missing the getCanonicalId method', [
+            new TestBackend(),
+            getBackendWithMissingMethod('getCanonicalIds'),
+        ]],
+        ['should throw an error if a client is missing the getEmailAddresses method', [
+            new TestBackend(),
+            getBackendWithMissingMethod('getEmailAddresses'),
+        ]],
+        ['should throw an error if a client is missing the checkPolicies method', [
+            new TestBackend(),
+            getBackendWithMissingMethod('checkPolicies'),
+        ]],
+        ['should throw an error if a client is missing the healthcheck method', [
+            new TestBackend(),
+            getBackendWithMissingMethod('healthcheck'),
+        ]],
+    ].forEach(([msg, input]) => it(msg, () => {
+        assert.throws(() => {
+            new ChainBackend('chain', input); // eslint-disable-line no-new
+        });
+    }));
+
+    [
+        // function name, function args
+        ['verifySignatureV2', [null, null, null, null]],
+        ['verifySignatureV4', [null, null, null, null, null, null]],
+    ].forEach(([fn, fnArgs]) =>
+        describe(`::${fn}`, () => {
+            it('should return an error if none of the clients returns a result', done => {
+                const backend = new ChainBackend('chain', [
+                    new TestBackend('test1', testError, null),
+                    new TestBackend('test2', testError, null),
+                    new TestBackend('test3', testError, null),
+                ]);
+
+                backend[fn](...fnArgs, err => {
+                    assert.deepStrictEqual(err, testError);
+                    done();
+                });
+            });
+
+            [
+                [
+                    'should return result of the first successful client (multiple successful client)',
+                    'expectedResult',
+                    // backend constructor args
+                    [
+                        ['test1', null, 'expectedResult'],
+                        ['test2', null, 'test2'],
+                        ['test3', testError, null],
+                    ],
+                ],
+                [
+                    'should return result of successful client',
+                    'expectedResult',
+                    // backend constructor args
+                    [
+                        ['test1', testError, null],
+                        ['test2', null, 'expectedResult'],
+                        ['test3', testError, null],
+                    ],
+                ],
+                [
+                    'should return result of successful client',
+                    'expectedResult',
+                    // backend constructor args
+                    [
+                        ['test1', testError, null],
+                        ['test1', testError, null],
+                        ['test3', null, 'expectedResult'],
+                    ],
+                ],
+            ].forEach(([msg, expected, backendArgs]) => {
+                it(msg, done => {
+                    const backend = new ChainBackend('chain',
+                        backendArgs.map((args) => new TestBackend(...args)));
+                    backend[fn](...fnArgs, (err, res) => {
+                        assert.ifError(err);
+                        assert.strictEqual(res, expected);
+                        done();
+                    });
+                });
+            });
+        }));
+
+    [
+        // function name, function args
+        ['getCanonicalIds', [null, null]],
+        ['getEmailAddresses', [null, null]],
+    ].forEach(([fn, fnArgs]) =>
+        describe(`::${fn}`, () => {
+            it('should return an error if any of the clients fails', done => {
+                const backend = new ChainBackend('chain', [
+                    new TestBackend('test1', null, { message: { body: { test1: 'aaa' } } }),
+                    new TestBackend('test2', testError, null),
+                    new TestBackend('test3', null, { message: { body: { test2: 'bbb' } } }),
+                ]);
+
+                backend[fn](...fnArgs, err => {
+                    assert.deepStrictEqual(err, testError);
+                    done();
+                });
+            });
+
+            it('should merge results from clients into a single response object', done => {
+                const backend = new ChainBackend('chain', [
+                    new TestBackend('test1', null, { message: { body: { test1: 'aaa' } } }),
+                    new TestBackend('test2', null, { message: { body: { test2: 'bbb' } } }),
+                ]);
+
+                backend[fn](...fnArgs, (err, res) => {
+                    assert.ifError(err);
+                    assert.deepStrictEqual(res, {
+                        message: { body: {
+                            test1: 'aaa',
+                            test2: 'bbb',
+                        } },
+                    });
+                    done();
+                });
+            });
+        }));
+
+    describe('::checkPolicies', () => {
+        it('should return an error if any of the clients fails', done => {
+            const backend = new ChainBackend('chain', [
+                new TestBackend('test1', null, {
+                    message: { body: [{ isAllowed: false, arn: 'arn:aws:s3:::policybucket/obj1' }] },
+                }),
+                new TestBackend('test2', testError, null),
+                new TestBackend('test3', null, {
+                    message: { body: [{ isAllowed: true, arn: 'arn:aws:s3:::policybucket/obj1' }] },
+                }),
+            ]);
+
+            backend.checkPolicies(null, null, null, err => {
+                assert.deepStrictEqual(err, testError);
+                done();
+            });
+        });
+
+        it('should merge results from clients into a single response object', done => {
+            const backend = new ChainBackend('chain', [
+                new TestBackend('test1', null, {
+                    message: { body: [{ isAllowed: false, arn: 'arn:aws:s3:::policybucket/obj1' }] },
+                }),
+                new TestBackend('test2', null, {
+                    message: { body: [{ isAllowed: true, arn: 'arn:aws:s3:::policybucket/obj2' }] },
+                }),
+                new TestBackend('test3', null, {
+                    message: { body: [{ isAllowed: true, arn: 'arn:aws:s3:::policybucket/obj1' }] },
+                }),
+            ]);
+
+            backend.checkPolicies(null, null, null, (err, res) => {
+                assert.ifError(err);
+                assert.deepStrictEqual(res, {
+                    message: { body: [
+                        { isAllowed: true, arn: 'arn:aws:s3:::policybucket/obj1' },
+                        { isAllowed: true, arn: 'arn:aws:s3:::policybucket/obj2' },
+                    ] },
+                });
+                done();
+            });
+        });
+    });
+
+
+    describe('::_mergeObject', () => {
+        it('should correctly merge reponses', () => {
+            const objectResps = [
+                { message: { body: {
+                    id1: 'email1@test.com',
+                    wrongformatcanid: 'WrongFormat',
+                    id4: 'email4@test.com',
+                } } },
+                { message: { body: {
+                    id2: 'NotFound',
+                    id3: 'email3@test.com',
+                    id4: 'email5@test.com',
+                } } },
+            ];
+            assert.deepStrictEqual(
+                ChainBackend._mergeObjects(objectResps),
+                {
+                    id1: 'email1@test.com',
+                    wrongformatcanid: 'WrongFormat',
+                    id2: 'NotFound',
+                    id3: 'email3@test.com',
+                    // id4 should be overwritten
+                    id4: 'email5@test.com',
+                },
+            );
+        });
+    });
+
+    describe('::_mergePolicies', () => {
+        it('should correctly merge policies', () => {
+            const policyResps = [
+                { message: { body: [
+                    { isAllowed: false, arn: 'arn:aws:s3:::policybucket/true1' },
+                    { isAllowed: true, arn: 'arn:aws:s3:::policybucket/true2' },
+                    { isAllowed: false, arn: 'arn:aws:s3:::policybucket/false1' },
+                ] } },
+                { message: { body: [
+                    { isAllowed: true, arn: 'arn:aws:s3:::policybucket/true1' },
+                    { isAllowed: false, arn: 'arn:aws:s3:::policybucket/true2' },
+                    { isAllowed: false, arn: 'arn:aws:s3:::policybucket/false2' },
+                ] } },
+            ];
+            assert.deepStrictEqual(
+                ChainBackend._mergePolicies(policyResps),
+                [
+                    { isAllowed: true, arn: 'arn:aws:s3:::policybucket/true1' },
+                    { isAllowed: true, arn: 'arn:aws:s3:::policybucket/true2' },
+                    { isAllowed: false, arn: 'arn:aws:s3:::policybucket/false1' },
+                    { isAllowed: false, arn: 'arn:aws:s3:::policybucket/false2' },
+                ],
+            );
+        });
+    });
+
+    describe('::checkhealth', () => {
+        it('should return error if a single client is unhealthy', done => {
+            const backend = new ChainBackend('chain', [
+                new TestBackend('test1', null, { code: 200 }),
+                new TestBackend('test2', testError, { code: 503 }),
+                new TestBackend('test3', null, { code: 200 }),
+            ]);
+            backend.healthcheck(null, (err, res) => {
+                assert.deepStrictEqual(err, errors.InternalError);
+                assert.deepStrictEqual(res, [
+                    { error: null, status: { code: 200 } },
+                    { error: testError, status: { code: 503 } },
+                    { error: null, status: { code: 200 } },
+                ]);
+                done();
+            });
+        });
+
+        it('should return result if all clients are healthy', done => {
+            const backend = new ChainBackend('chain', [
+                new TestBackend('test1', null, { msg: 'test1', code: 200 }),
+                new TestBackend('test2', null, { msg: 'test2', code: 200 }),
+                new TestBackend('test3', null, { msg: 'test3', code: 200 }),
+            ]);
+            backend.healthcheck(null, (err, res) => {
+                assert.ifError(err);
+                assert.deepStrictEqual(res, [
+                    { error: null, status: { msg: 'test1', code: 200 } },
+                    { error: null, status: { msg: 'test2', code: 200 } },
+                    { error: null, status: { msg: 'test3', code: 200 } },
+                ]);
+                done();
+            });
+        });
+    });
+});

tests/unit/auth/in_memory/AuthLoader.spec.js

@@ -89,10 +89,10 @@ describe('AuthLoader class', () => {
             // Check a failure when the type of field is different than
             // expected
             it(`should fail when modified field ${test[0]} ${test[1]}`,
-            done => {
-                should._exec = shouldFail;
-                should.modifiedField(obj, test[0], test[1], done);
-            });
+                done => {
+                    should._exec = shouldFail;
+                    should.modifiedField(obj, test[0], test[1], done);
+                });
         }
     });