Merge branch 'w/7.5/improvement/S3C-2352-install-yarn' into tmp/octopus/w/8.0/improvement/S3C-2352-install-yarn

Fix mongoclient.ListRecordStream to properly pass on bucket names with
periods (like foo.bar), instead of truncating the name after the first
period.

Down the line, that fixes replication for objects contained in such
buckets.

errors/arsenalErrors.json

@@ -387,6 +387,10 @@
     "code": 409,
     "description": "The request was rejected because it attempted to create a resource that already exists."
   },
+  "KeyAlreadyExists": {
+    "code": 409,
+    "description": "The request was rejected because it attempted to create a resource that already exists."
+  },
   "ServiceFailure": {
     "code": 500,
     "description": "Server error: the request processing has failed because of an unknown error, exception or failure."

index.js

@@ -3,6 +3,7 @@ module.exports = {
     constants: require('./lib/constants'),
     db: require('./lib/db'),
     errors: require('./lib/errors.js'),
+    errorUtils: require('./lib/errorUtils'),
     shuffle: require('./lib/shuffle'),
     stringHash: require('./lib/stringHash'),
     ipCheck: require('./lib/ipCheck'),
@@ -12,15 +13,7 @@ module.exports = {
         dhparam: require('./lib/https/dh2048.js'),
     },
     algorithms: {
-        list: {
-            Basic: require('./lib/algos/list/basic').List,
-            Delimiter: require('./lib/algos/list/delimiter').Delimiter,
-            DelimiterVersions: require('./lib/algos/list/delimiterVersions')
-                .DelimiterVersions,
-            DelimiterMaster: require('./lib/algos/list/delimiterMaster')
-                .DelimiterMaster,
-            MPU: require('./lib/algos/list/MPU').MultipartUploads,
-        },
+        list: require('./lib/algos/list/exportAlgos'),
         listTools: {
             DelimiterTools: require('./lib/algos/list/tools'),
         },
@@ -53,6 +46,12 @@ module.exports = {
             RESTClient: require('./lib/network/rest/RESTClient'),
         },
         RoundRobin: require('./lib/network/RoundRobin'),
+        probe: {
+            HealthProbeServer:
+                require('./lib/network/probe/HealthProbeServer.js'),
+        },
+        kmip: require('./lib/network/kmip'),
+        kmipClient: require('./lib/network/kmip/Client'),
     },
     s3routes: {
         routes: require('./lib/s3routes/routes'),
@@ -63,6 +62,9 @@ module.exports = {
         convertToXml: require('./lib/s3middleware/convertToXml'),
         escapeForXml: require('./lib/s3middleware/escapeForXml'),
         tagging: require('./lib/s3middleware/tagging'),
+        checkDateModifiedHeaders:
+            require('./lib/s3middleware/validateConditionalHeaders')
+            .checkDateModifiedHeaders,
         validateConditionalHeaders:
             require('./lib/s3middleware/validateConditionalHeaders')
             .validateConditionalHeaders,
@@ -80,12 +82,39 @@ module.exports = {
     },
     storage: {
         metadata: {
+            MetadataWrapper: require('./lib/storage/metadata/MetadataWrapper'),
+            bucketclient: {
+                BucketClientInterface:
+                require('./lib/storage/metadata/bucketclient/' +
+                    'BucketClientInterface'),
+                LogConsumer:
+                require('./lib/storage/metadata/bucketclient/LogConsumer'),
+            },
+            file: {
+                BucketFileInterface:
+                require('./lib/storage/metadata/file/BucketFileInterface'),
                 MetadataFileServer:
                 require('./lib/storage/metadata/file/MetadataFileServer'),
                 MetadataFileClient:
                 require('./lib/storage/metadata/file/MetadataFileClient'),
+            },
+            inMemory: {
+                metastore:
+                require('./lib/storage/metadata/in_memory/metastore'),
+                metadata: require('./lib/storage/metadata/in_memory/metadata'),
+                bucketUtilities:
+                require('./lib/storage/metadata/in_memory/bucket_utilities'),
+            },
+            mongoclient: {
+                MongoClientInterface:
+                require('./lib/storage/metadata/mongoclient/' +
+                    'MongoClientInterface'),
                 LogConsumer:
-            require('./lib/storage/metadata/bucketclient/LogConsumer'),
+                require('./lib/storage/metadata/mongoclient/LogConsumer'),
+            },
+            proxy: {
+                Server: require('./lib/storage/metadata/proxy/Server'),
+            },
         },
         data: {
             file: {
@@ -114,4 +143,8 @@ module.exports = {
     pensieve: {
         credentialUtils: require('./lib/executables/pensieveCreds/utils'),
     },
+    backbeat: {
+        Metrics: require('./lib/backbeat/Metrics'),
+        routes: require('./lib/backbeat/routes'),
+    },
 };

lib/algos/list/exportAlgos.js

@@ -0,0 +1,9 @@
+module.exports = {
+    Basic: require('./basic').List,
+    Delimiter: require('./delimiter').Delimiter,
+    DelimiterVersions: require('./delimiterVersions')
+        .DelimiterVersions,
+    DelimiterMaster: require('./delimiterMaster')
+        .DelimiterMaster,
+    MPU: require('./MPU').MultipartUploads,
+};

lib/algos/list/skip.js

@@ -0,0 +1,88 @@
+const assert = require('assert');
+
+const { FILTER_END, FILTER_SKIP, SKIP_NONE } = require('./tools');
+
+
+const MAX_STREAK_LENGTH = 100;
+
+/**
+ * Handle the filtering and the skip mechanism of a listing result.
+ */
+class Skip {
+    /**
+     * @param {Object} params           - skip parameters
+     * @param {Object} params.extension - delimiter extension used (required)
+     * @param {String} params.gte       - current range gte (greater than or
+     *                                    equal) used by the client code
+     */
+    constructor(params) {
+        assert(params.extension);
+
+        this.extension = params.extension;
+        this.gteParams = params.gte;
+
+        this.listingEndCb = null;
+        this.skipRangeCb = null;
+
+        /* Used to count consecutive FILTER_SKIP returned by the extension
+         * filter method. Once this counter reaches MAX_STREAK_LENGTH, the
+         * filter function tries to skip unwanted values by defining a new
+         * range. */
+        this.streakLength = 0;
+    }
+
+    setListingEndCb(cb) {
+        this.listingEndCb = cb;
+    }
+
+    setSkipRangeCb(cb) {
+        this.skipRangeCb = cb;
+    }
+
+    /**
+     * Filter an entry.
+     * @param {Object} entry - entry to filter.
+     * @return {undefined}
+     *
+     * This function calls the listing end or the skip range callbacks if
+     * needed.
+     */
+    filter(entry) {
+        assert(this.listingEndCb);
+        assert(this.skipRangeCb);
+
+        const filteringResult = this.extension.filter(entry);
+        const skippingRange = this.extension.skipping();
+
+        if (filteringResult === FILTER_END) {
+            this.listingEndCb();
+        } else if (filteringResult === FILTER_SKIP
+                   && skippingRange !== SKIP_NONE) {
+            if (++this.streakLength >= MAX_STREAK_LENGTH) {
+                const newRange = this._inc(skippingRange);
+
+                /* Avoid to loop on the same range again and again. */
+                if (newRange === this.gteParams) {
+                    this.streakLength = 1;
+                } else {
+                    this.skipRangeCb(newRange);
+                }
+            }
+        } else {
+            this.streakLength = 0;
+        }
+    }
+
+    _inc(str) {
+        if (!str) {
+            return str;
+        }
+        const lastCharValue = str.charCodeAt(str.length - 1);
+        const lastCharNewValue = String.fromCharCode(lastCharValue + 1);
+
+        return `${str.slice(0, str.length - 1)}${lastCharNewValue}`;
+    }
+}
+
+
+module.exports = Skip;

lib/auth/v4/headerAuthCheck.js

@@ -127,6 +127,17 @@ function check(request, log, data, awsService) {
         return { err: errors.RequestTimeTooSkewed };
     }
 
+    let proxyPath = null;
+    if (request.headers.proxy_path) {
+        try {
+            proxyPath = decodeURIComponent(request.headers.proxy_path);
+        } catch (err) {
+            log.debug('invalid proxy_path header', { proxyPath, err });
+            return { err: errors.InvalidArgument.customizeDescription(
+              'invalid proxy_path header') };
+        }
+    }
+
     const stringToSign = constructStringToSign({
         log,
         request,
@@ -136,6 +147,7 @@ function check(request, log, data, awsService) {
         timestamp,
         payloadChecksum,
         awsService: service,
+        proxyPath,
     });
     log.trace('constructed stringToSign', { stringToSign });
     if (stringToSign instanceof Error) {

lib/auth/v4/queryAuthCheck.js

@@ -62,6 +62,17 @@ function check(request, log, data) {
         return { err: errors.RequestTimeTooSkewed };
     }
 
+    let proxyPath = null;
+    if (request.headers.proxy_path) {
+        try {
+            proxyPath = decodeURIComponent(request.headers.proxy_path);
+        } catch (err) {
+            log.debug('invalid proxy_path header', { proxyPath });
+            return { err: errors.InvalidArgument.customizeDescription(
+              'invalid proxy_path header') };
+        }
+    }
+
     // In query v4 auth, the canonical request needs
     // to include the query params OTHER THAN
     // the signature so create a
@@ -87,6 +98,7 @@ function check(request, log, data) {
         credentialScope:
             `${scopeDate}/${region}/${service}/${requestType}`,
         awsService: service,
+        proxyPath,
     });
     if (stringToSign instanceof Error) {
         return { err: stringToSign };

lib/backbeat/Metrics.js

@@ -0,0 +1,539 @@
+const async = require('async');
+
+const errors = require('../../lib/errors');
+const RedisClient = require('../../lib/metrics/RedisClient');
+const StatsModel = require('../../lib/metrics/StatsModel');
+const INTERVAL = 300; // 5 minutes
+const EXPIRY = 86400; // 24 hours
+const THROUGHPUT_EXPIRY = 900; // 15 minutes
+const isTest = process.env.CI === 'true';
+
+class Metrics {
+    constructor(config, logger) {
+        const { redisConfig, validSites, internalStart } = config;
+        this._logger = logger;
+        this._redisClient = new RedisClient(redisConfig, this._logger);
+        // Redis expiry increased by an additional interval so we can reference
+        // the immediate older data for average throughput calculation
+        this._statsClient = new StatsModel(this._redisClient, INTERVAL, EXPIRY);
+        this._validSites = validSites;
+        this._internalStart = internalStart;
+    }
+
+    /**
+     * Query StatsClient for all ops given
+     * @param {array} ops - array of redis key names to query
+     * @param {string} site - site name or '*' wildcard
+     * @param {string} bucketName - the name of the bucket
+     * @param {string} objectKey - the object key name
+     * @param {string} versionId - the object version ID
+     * @param {function} cb - callback(err, res)
+     * @return {undefined}
+     */
+    _queryStats(ops, site, bucketName, objectKey, versionId, cb) {
+        return async.map(ops, (op, done) => {
+            const hasGlobalKey = this._hasGlobalKey(op);
+            if (site === 'all') {
+                const queryStrings = this._validSites.map(s => {
+                    if (bucketName && objectKey && versionId) {
+                        return `${s}:${bucketName}:${objectKey}:` +
+                               `${versionId}:${op}`;
+                    }
+                    return `${s}:${op}`;
+                });
+                if (hasGlobalKey) {
+                    return this._statsClient.getAllGlobalStats(queryStrings,
+                        this._logger, done);
+                }
+                return this._statsClient.getAllStats(this._logger, queryStrings,
+                    done);
+            }
+            // Query only a single given site or storage class
+            // First, validate the site or storage class
+            if (!this._validSites.includes(site)) {
+                // escalate error to log later
+                return done({
+                    message: 'invalid site name provided',
+                    type: errors.RouteNotFound,
+                    method: 'Metrics._queryStats',
+                });
+            }
+            let queryString;
+            if (bucketName && objectKey && versionId) {
+                queryString =
+                    `${site}:${bucketName}:${objectKey}:${versionId}:${op}`;
+            } else {
+                queryString = `${site}:${op}`;
+            }
+            if (hasGlobalKey) {
+                return this._redisClient.get(queryString, (err, res) => {
+                    if (err) {
+                        return done({
+                            message: `Redis error: ${err.message}`,
+                            type: errors.InternalError,
+                            method: 'Metrics._queryStats',
+                        });
+                    }
+                    return done(null, { requests: [res || 0] });
+                });
+            }
+            return this._statsClient.getStats(this._logger, queryString, done);
+        }, cb);
+    }
+
+    /**
+     * Determines whether the Redis op uses a global counter or interval key.
+     * @param {String} op - The Redis operation
+     * @return {Boolean} true if a global counter, false otherwise
+     */
+    _hasGlobalKey(op) {
+        if (isTest) {
+            return op.includes('test:bb:bytespending') ||
+                op.includes('test:bb:opspending');
+        }
+        return op.includes('bb:crr:bytespending') ||
+            op.includes('bb:crr:opspending');
+    }
+
+    /**
+     * Get data points which are the keys used to query Redis
+     * @param {object} details - route details from lib/backbeat/routes.js
+     * @param {array} data - provides already fetched data in order of
+     *   dataPoints mentioned for each route in lib/backbeat/routes.js. This can
+     *   be undefined.
+     * @param {function} cb - callback(error, data), where data returns
+     *   data stored in Redis.
+     * @return {array} dataPoints array defined in lib/backbeat/routes.js
+     */
+    _getData(details, data, cb) {
+        if (!data) {
+            const { dataPoints, site, bucketName, objectKey,
+                versionId } = details;
+            return this._queryStats(dataPoints, site, bucketName, objectKey,
+                versionId, cb);
+        }
+        return cb(null, data);
+    }
+
+    /**
+     * Uptime of server based on this._internalStart up to max of expiry
+     * @param {number} expiry - max expiry
+     * @return {number} uptime of server up to expiry time
+     */
+    _getMaxUptime(expiry) {
+        let secondsSinceStart = (Date.now() - this._internalStart) / 1000;
+        // allow only a minimum value of 1 for uptime
+        if (secondsSinceStart < 1) {
+            secondsSinceStart = 1;
+        }
+        return secondsSinceStart < expiry ? secondsSinceStart : expiry;
+    }
+
+    /**
+     * Get replication backlog in ops count and size in bytes
+     * @param {object} details - route details from lib/backbeat/routes.js
+     * @param {function} cb - callback(error, data)
+     * @param {array} data - optional field providing already fetched data in
+     *   order of dataPoints mentioned for each route in lib/backbeat/routes.js
+     * @return {undefined}
+     */
+    getBacklog(details, cb, data) {
+        this._getData(details, data, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: backlog', {
+                    origin: err.method,
+                    method: 'Metrics.getBacklog',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            if (err || res.length !== details.dataPoints.length) {
+                this._logger.error('error getting metrics: backlog', {
+                    method: 'Metrics.getBacklog',
+                });
+                return cb(errors.InternalError);
+            }
+            const count = Number.parseInt(res[0].requests, 10);
+            const size = Number.parseInt(res[1].requests, 10);
+            const response = {
+                backlog: {
+                    description: 'Number of incomplete replication ' +
+                        'operations (count) and number of incomplete bytes ' +
+                        'transferred (size)',
+                    results: {
+                        count: count < 0 ? 0 : count,
+                        size: size < 0 ? 0 : size,
+                    },
+                },
+            };
+            return cb(null, response);
+        });
+    }
+
+    /**
+     * Get completed replicated stats by ops count and size in bytes
+     * @param {object} details - route details from lib/backbeat/routes.js
+     * @param {function} cb - callback(error, data)
+     * @param {array} data - optional field providing already fetched data in
+     *   order of dataPoints mentioned for each route in lib/backbeat/routes.js
+     * @return {undefined}
+     */
+    getCompletions(details, cb, data) {
+        this._getData(details, data, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: completions', {
+                    origin: err.method,
+                    method: 'Metrics.getCompletions',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            if (err || res.length !== details.dataPoints.length) {
+                this._logger.error('error getting metrics: completions', {
+                    method: 'Metrics.getCompletions',
+                });
+                return cb(errors.InternalError);
+            }
+            const uptime = this._getMaxUptime(EXPIRY);
+            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const [opsDone, bytesDone] = res.map(r => (
+                r.requests.slice(0, numOfIntervals).reduce((acc, i) =>
+                    acc + i, 0)
+            ));
+
+            const response = {
+                completions: {
+                    description: 'Number of completed replication operations ' +
+                        '(count) and number of bytes transferred (size) in ' +
+                        `the last ${Math.floor(uptime)} seconds`,
+                    results: {
+                        count: opsDone,
+                        size: bytesDone,
+                    },
+                },
+            };
+            return cb(null, response);
+        });
+    }
+
+    /**
+     * Get failed replication stats by ops count and size in bytes
+     * @param {object} details - route details from lib/backbeat/routes.js
+     * @param {function} cb - callback(error, data)
+     * @param {array} data - optional field providing already fetched data in
+     *   order of dataPoints mentioned for each route in lib/backbeat/routes.js
+     * @return {undefined}
+     */
+    getFailedMetrics(details, cb, data) {
+        this._getData(details, data, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: failures', {
+                    origin: err.emthod,
+                    method: 'Metrics.getFailedMetrics',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            if (err || res.length !== details.dataPoints.length) {
+                this._logger.error('error getting metrics: failures', {
+                    method: 'Metrics.getFailedMetrics',
+                });
+                return cb(errors.InternalError);
+            }
+            const uptime = this._getMaxUptime(EXPIRY);
+            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const [opsFail, bytesFail] = res.map(r => (
+                r.requests.slice(0, numOfIntervals).reduce((acc, i) =>
+                    acc + i, 0)
+            ));
+
+            const response = {
+                failures: {
+                    description: 'Number of failed replication operations ' +
+                        '(count) and bytes (size) in the last ' +
+                        `${Math.floor(uptime)} seconds`,
+                    results: {
+                        count: opsFail,
+                        size: bytesFail,
+                    },
+                },
+            };
+            return cb(null, response);
+        });
+    }
+
+    /**
+     * Get current throughput in ops/sec and bytes/sec up to max of 15 minutes
+     * Throughput is the number of units processed in a given time
+     * @param {object} details - route details from lib/backbeat/routes.js
+     * @param {function} cb - callback(error, data)
+     * @param {array} data - optional field providing already fetched data in
+     *   order of dataPoints mentioned for each route in lib/backbeat/routes.js
+     * @return {undefined}
+     */
+    getThroughput(details, cb, data) {
+        this._getData(details, data, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: throughput', {
+                    origin: err.method,
+                    method: 'Metrics.getThroughput',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            if (err) {
+                this._logger.error('error getting metrics: throughput', {
+                    method: 'Metrics.getThroughput',
+                });
+                return cb(errors.InternalError);
+            }
+            const now = new Date();
+            const uptime = this._getMaxUptime(THROUGHPUT_EXPIRY);
+            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const [opsThroughput, bytesThroughput] = res.map(r => {
+                let total = r.requests.slice(0, numOfIntervals).reduce(
+                    (acc, i) => acc + i, 0);
+
+                // if uptime !== THROUGHPUT_EXPIRY, use internal timer and
+                // do not include the extra 4th interval
+                if (uptime === THROUGHPUT_EXPIRY) {
+                    // all intervals apply, including 4th interval
+                    const lastInterval =
+                        this._statsClient._normalizeTimestamp(now);
+                    // in seconds
+                    const diff = (now - lastInterval) / 1000;
+                    // Get average for last interval depending on time
+                    // surpassed so far for newest interval
+                    total += ((INTERVAL - diff) / INTERVAL) *
+                        r.requests[numOfIntervals];
+                }
+
+                // Divide total by uptime to determine data per second
+                return (total / uptime);
+            });
+
+            const response = {
+                throughput: {
+                    description: 'Current throughput for replication ' +
+                        'operations in ops/sec (count) and bytes/sec (size) ' +
+                        `in the last ${Math.floor(uptime)} seconds`,
+                    results: {
+                        count: opsThroughput.toFixed(2),
+                        size: bytesThroughput.toFixed(2),
+                    },
+                },
+            };
+            return cb(null, response);
+        });
+    }
+
+    /**
+     * Get current throughput for an object in bytes/sec. Throughput is the
+     * number of bytes transferred in a given time.
+     * @param {object} details - route details from lib/api/routes.js
+     * @param {function} cb - callback(error, data)
+     * @return {undefined}
+     */
+    getObjectThroughput(details, cb) {
+        this._getData(details, undefined, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: object throughput', {
+                    origin: err.method,
+                    method: 'Metrics.getObjectThroughput',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            if (err) {
+                this._logger.error('error getting metrics: object throughput', {
+                    method: 'Metrics.getObjectThroughput',
+                    error: err.message,
+                });
+                return cb(errors.InternalError);
+            }
+            const now = new Date();
+            const uptime = this._getMaxUptime(THROUGHPUT_EXPIRY);
+            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const { requests } = res[0]; // Bytes done
+            let total = requests.slice(0, numOfIntervals)
+                .reduce((acc, i) => acc + i, 0);
+            // if uptime !== THROUGHPUT_EXPIRY, use internal timer
+            // and do not include the extra 4th interval
+            if (uptime === THROUGHPUT_EXPIRY) {
+                // all intervals apply, including 4th interval
+                const lastInterval =
+                    this._statsClient._normalizeTimestamp(now);
+                // in seconds
+                const diff = (now - lastInterval) / 1000;
+                // Get average for last interval depending on time passed so
+                // far for newest interval
+                total += ((INTERVAL - diff) / INTERVAL) *
+                    requests[numOfIntervals];
+            }
+            // Divide total by timeDisplay to determine data per second
+            const response = {
+                description: 'Current throughput for object replication in ' +
+                    'bytes/sec (throughput)',
+                throughput: (total / uptime).toFixed(2),
+            };
+            return cb(null, response);
+        });
+    }
+
+    /**
+     * Get CRR progress for an object in bytes. Progress is the percentage of
+     * the object that has completed replication.
+     * @param {object} details - route details from lib/api/routes.js
+     * @param {function} cb - callback(error, data)
+     * @return {undefined}
+     */
+    getObjectProgress(details, cb) {
+        this._getData(details, undefined, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: object progress', {
+                    origin: err.method,
+                    method: 'Metrics.getObjectProgress',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            if (err || res.length !== details.dataPoints.length) {
+                this._logger.error('error getting metrics: object progress', {
+                    method: 'Metrics.getObjectProgress',
+                    error: err.message,
+                });
+                return cb(errors.InternalError);
+            }
+            // Find if time since start is less than EXPIRY time
+            const uptime = this._getMaxUptime(EXPIRY);
+            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const [totalBytesToComplete, bytesComplete] = res.map(r => (
+                r.requests.slice(0, numOfIntervals).reduce((acc, i) =>
+                    acc + i, 0)
+            ));
+            const ratio = totalBytesToComplete === 0 ? 0 :
+                bytesComplete / totalBytesToComplete;
+            const percentage = (ratio * 100).toFixed();
+            const response = {
+                description: 'Number of bytes to be replicated ' +
+                    '(pending), number of bytes transferred to the ' +
+                    'destination (completed), and percentage of the ' +
+                    'object that has completed replication (progress)',
+                pending: totalBytesToComplete - bytesComplete,
+                completed: bytesComplete,
+                progress: `${percentage}%`,
+            };
+            return cb(null, response);
+        });
+    }
+
+    /**
+     * Get pending replication stats by ops count and size in bytes
+     * @param {object} details - route details from lib/backbeat/routes.js
+     * @param {function} cb - callback(error, data)
+     * @param {array} data - optional field providing already fetched data in
+     *   order of dataPoints mentioned for each route in lib/backbeat/routes.js
+     * @return {undefined}
+     */
+    getPending(details, cb, data) {
+        this._getData(details, data, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: pending', {
+                    origin: err.method,
+                    method: 'Metrics.getPending',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            const { dataPoints } = details;
+            if (err || res.length !== dataPoints.length) {
+                this._logger.error('error getting metrics: pending', {
+                    method: 'Metrics.getPending',
+                    error: err,
+                    dataPoints,
+                    res,
+                });
+                return cb(errors.InternalError
+                    .customizeDescription('error getting metrics: pending'));
+            }
+            const count = Number.parseInt(res[0].requests, 10);
+            const size = Number.parseInt(res[1].requests, 10);
+            const response = {
+                pending: {
+                    description: 'Number of pending replication ' +
+                        'operations (count) and bytes (size)',
+                    results: {
+                        count: count < 0 ? 0 : count,
+                        size: size < 0 ? 0 : size,
+                    },
+                },
+            };
+            return cb(null, response);
+        });
+    }
+
+    /**
+     * Get all metrics
+     * @param {object} details - route details from lib/backbeat/routes.js
+     * @param {function} cb = callback(error, data)
+     * @param {array} data - optional field providing already fetched data in
+     *   order of dataPoints mentioned for each route in lib/backbeat/routes.js
+     * @return {undefined}
+     */
+    getAllMetrics(details, cb, data) {
+        this._getData(details, data, (err, res) => {
+            if (err && err.type) {
+                this._logger.error('error getting metric: all', {
+                    origin: err.method,
+                    method: 'Metrics.getAllMetrics',
+                });
+                return cb(err.type.customizeDescription(err.message));
+            }
+            if (err || res.length !== details.dataPoints.length) {
+                this._logger.error('error getting metrics: all', {
+                    method: 'Metrics.getAllMetrics',
+                });
+                return cb(errors.InternalError);
+            }
+            // NOTE: Edited to fit failed metrics
+            const failMetricsDetails = Object.assign({}, details,
+                { dataPoints: new Array(2) });
+            // res = [ ops, ops_done, ops_fail, bytes, bytes_done, bytes_fail,
+            // opsPending, bytesPending ]
+            return async.parallel([
+                done => this.getBacklog({ dataPoints: new Array(2) }, done,
+                    [res[6], res[7]]),
+                done => this.getCompletions({ dataPoints: new Array(2) }, done,
+                    [res[1], res[4]]),
+                done => this.getFailedMetrics(failMetricsDetails, done,
+                    [res[2], res[5]]),
+                done => this.getThroughput({ dataPoints: new Array(2) }, done,
+                    [res[1], res[4]]),
+                done => this.getPending({ dataPoints: new Array(2) }, done,
+                    [res[6], res[7]]),
+            ], (err, results) => {
+                if (err) {
+                    this._logger.error('error getting metrics: all', {
+                        method: 'Metrics.getAllMetrics',
+                    });
+                    return cb(errors.InternalError);
+                }
+                const store = Object.assign({}, ...results);
+                return cb(null, store);
+            });
+        });
+    }
+
+    /**
+     * Close redis client
+     * @param {function} cb - callback(error, data)
+     * @return {undefined}
+     */
+    disconnect(cb) {
+        return this._redisClient.disconnect(cb);
+    }
+
+    /**
+     * Retrieve the list of redis client connectiosn
+     * @param {function} cb - callback(error, data)
+     * @return {undefined}
+     */
+    listClients(cb) {
+        return this._redisClient.listClients(cb);
+    }
+}
+
+module.exports = Metrics;

lib/backbeat/routes.js

@@ -0,0 +1,167 @@
+/*
+    This file contains Backbeat API routes and route details
+*/
+
+/**
+ * The metrics route model.
+ * @param {Object} redisKeys - The Redis keys used for Backbeat metrics
+ * @param {Array} allLocations - The list of replication location names
+ * @return {Array} The array of route objects
+ */
+function routes(redisKeys, allLocations) {
+    return [
+        // Route: /_/healthcheck
+        {
+            httpMethod: 'GET',
+            category: 'healthcheck',
+            type: 'basic',
+            method: 'getHealthcheck',
+            extensions: {},
+        },
+        // Route: /_/metrics/crr/<location>/pending
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'pending',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getPending',
+            dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
+        },
+        // Route: /_/metrics/crr/<location>/backlog
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'backlog',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getBacklog',
+            dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
+        },
+        // Route: /_/metrics/crr/<location>/completions
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'completions',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getCompletions',
+            dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
+        },
+        // Route: /_/metrics/crr/<location>/failures
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'failures',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getFailedMetrics',
+            dataPoints: [redisKeys.opsFail, redisKeys.bytesFail],
+        },
+        // Route: /_/metrics/crr/<location>/throughput
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'throughput',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getThroughput',
+            dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
+        },
+        // Route: /_/metrics/crr/<location>/all
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'all',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getAllMetrics',
+            dataPoints: [redisKeys.ops, redisKeys.opsDone, redisKeys.opsFail,
+                redisKeys.bytes, redisKeys.bytesDone, redisKeys.bytesFail,
+                redisKeys.opsPending, redisKeys.bytesPending],
+        },
+        // Route: /_/metrics/crr/<site>/progress/<bucket>/<key>
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'progress',
+            level: 'object',
+            extensions: { crr: [...allLocations] },
+            method: 'getObjectProgress',
+            dataPoints: [redisKeys.objectBytes, redisKeys.objectBytesDone],
+        },
+        // Route: /_/metrics/crr/<site>/throughput/<bucket>/<key>
+        {
+            httpMethod: 'GET',
+            category: 'metrics',
+            type: 'throughput',
+            level: 'object',
+            extensions: { crr: [...allLocations] },
+            method: 'getObjectThroughput',
+            dataPoints: [redisKeys.objectBytesDone],
+        },
+        // Route: /_/crr/failed?site=<site>&marker=<marker>
+        {
+            httpMethod: 'GET',
+            type: 'all',
+            extensions: { crr: ['failed'] },
+            method: 'getSiteFailedCRR',
+        },
+        // Route: /_/crr/failed/<bucket>/<key>/<versionId>
+        {
+            httpMethod: 'GET',
+            type: 'specific',
+            extensions: { crr: ['failed'] },
+            method: 'getFailedCRR',
+        },
+        // Route: /_/crr/failed
+        {
+            httpMethod: 'POST',
+            type: 'all',
+            extensions: { crr: ['failed'] },
+            method: 'retryFailedCRR',
+        },
+        // Route: /_/monitoring/metrics
+        {
+            httpMethod: 'GET',
+            category: 'monitoring',
+            type: 'metrics',
+            extensions: {},
+            method: 'monitoringHandler',
+        },
+        // Route: /_/crr/pause/<location>
+        // Where <location> is an optional field
+        {
+            httpMethod: 'POST',
+            type: 'pause',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'pauseCRRService',
+        },
+        // Route: /_/crr/resume/<location>
+        // Route: /_/crr/resume/<location>/schedule
+        // Where <location> is an optional field unless "schedule" route
+        {
+            httpMethod: 'POST',
+            type: 'resume',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'resumeCRRService',
+        },
+        {
+            httpMethod: 'DELETE',
+            type: 'resume',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'deleteScheduledResumeService',
+        },
+        // Route: /_/crr/resume/<location>
+        {
+            httpMethod: 'GET',
+            type: 'resume',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getResumeCRRSchedule',
+        },
+        // Route: /_/crr/status/<location>
+        // Where <location> is an optional field
+        {
+            httpMethod: 'GET',
+            type: 'status',
+            extensions: { crr: [...allLocations, 'all'] },
+            method: 'getCRRServiceStatus',
+        },
+    ];
+}
+
+module.exports = routes;

lib/constants.js

@@ -72,4 +72,10 @@ module.exports = {
     permittedCapitalizedBuckets: {
         METADATA: true,
     },
+    // Setting a lower object key limit to account for:
+    // - Mongo key limit of 1012 bytes
+    // - Version ID in Mongo Key if versioned of 33
+    // - Max bucket name length if bucket match false of 63
+    // - Extra prefix slash for bucket prefix if bucket match of 1
+    objectKeyByteLimit: 915,
 };

lib/errorUtils.js

@@ -0,0 +1,13 @@
+function reshapeExceptionError(error) {
+    const { message, code, stack, name } = error;
+    return {
+        message,
+        code,
+        stack,
+        name,
+    };
+}
+
+module.exports = {
+    reshapeExceptionError,
+};

lib/metrics/RedisClient.js

@@ -22,6 +22,28 @@ class RedisClient {
         return this;
     }
 
+    /**
+    * scan a pattern and return matching keys
+    * @param {string} pattern - string pattern to match with all existing keys
+    * @param {number} [count=10] - scan count
+    * @param {callback} cb - callback (error, result)
+    * @return {undefined}
+    */
+    scan(pattern, count = 10, cb) {
+        const params = { match: pattern, count };
+        const keys = [];
+
+        const stream = this._client.scanStream(params);
+        stream.on('data', resultKeys => {
+            for (let i = 0; i < resultKeys.length; i++) {
+                keys.push(resultKeys[i]);
+            }
+        });
+        stream.on('end', () => {
+            cb(null, keys);
+        });
+    }
+
     /**
     * increment value of a key by 1 and set a ttl
     * @param {string} key - key holding the value
@@ -35,6 +57,17 @@ class RedisClient {
             .exec(cb);
     }
 
+    /**
+     * increment value of a key by a given amount
+     * @param {string} key - key holding the value
+     * @param {number} amount - amount to increase by
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    incrby(key, amount, cb) {
+        return this._client.incrby(key, amount, cb);
+    }
+
     /**
      * increment value of a key by a given amount and set a ttl
      * @param {string} key - key holding the value
@@ -50,13 +83,24 @@ class RedisClient {
     }
 
     /**
-    * execute a batch of commands
-    * @param {string[]} cmds - list of commands
+     * decrement value of a key by a given amount
+     * @param {string} key - key holding the value
+     * @param {number} amount - amount to increase by
      * @param {callback} cb - callback
      * @return {undefined}
      */
-    batch(cmds, cb) {
-        return this._client.pipeline(cmds).exec(cb);
+    decrby(key, amount, cb) {
+        return this._client.decrby(key, amount, cb);
+    }
+
+    /**
+     * get value stored at key
+     * @param {string} key - key holding the value
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    get(key, cb) {
+        return this._client.get(key, cb);
     }
 
     /**
@@ -71,6 +115,16 @@ class RedisClient {
         return this._client.exists(key, cb);
     }
 
+    /**
+    * execute a batch of commands
+    * @param {string[]} cmds - list of commands
+    * @param {callback} cb - callback
+    * @return {undefined}
+    */
+    batch(cmds, cb) {
+        return this._client.pipeline(cmds).exec(cb);
+    }
+
     /**
      * Add a value and its score to a sorted set. If no sorted set exists, this
      * will create a new one for the given key.
@@ -150,9 +204,27 @@ class RedisClient {
         return this._client.zrangebyscore(key, min, max, cb);
     }
 
+    /**
+     * get TTL or expiration in seconds
+     * @param {string} key - name of key
+     * @param {function} cb - callback
+     * @return {undefined}
+     */
+    ttl(key, cb) {
+        return this._client.ttl(key, cb);
+    }
+
     clear(cb) {
         return this._client.flushdb(cb);
     }
+
+    disconnect(cb) {
+        return this._client.quit(cb);
+    }
+
+    listClients(cb) {
+        return this._client.client('list', cb);
+    }
 }
 
 module.exports = RedisClient;

lib/metrics/StatsClient.js

@@ -41,11 +41,11 @@ class StatsClient {
     /**
     * build redis key to get total number of occurrences on the server
     * @param {string} name - key name identifier
-    * @param {object} d - Date instance
+    * @param {Date} date - Date instance
     * @return {string} key - key for redis
     */
-    _buildKey(name, d) {
-        return `${name}:${this._normalizeTimestamp(d)}`;
+    buildKey(name, date) {
+        return `${name}:${this._normalizeTimestamp(date)}`;
     }
 
     /**
@@ -85,11 +85,35 @@ class StatsClient {
             amount = (typeof incr === 'number') ? incr : 1;
         }
 
-        const key = this._buildKey(`${id}:requests`, new Date());
+        const key = this.buildKey(`${id}:requests`, new Date());
 
         return this._redis.incrbyEx(key, amount, this._expiry, callback);
     }
 
+    /**
+     * Increment the given key by the given value.
+     * @param {String} key - The Redis key to increment
+     * @param {Number} incr - The value to increment by
+     * @param {function} [cb] - callback
+     * @return {undefined}
+     */
+    incrementKey(key, incr, cb) {
+        const callback = cb || this._noop;
+        return this._redis.incrby(key, incr, callback);
+    }
+
+    /**
+     * Decrement the given key by the given value.
+     * @param {String} key - The Redis key to decrement
+     * @param {Number} decr - The value to decrement by
+     * @param {function} [cb] - callback
+     * @return {undefined}
+     */
+    decrementKey(key, decr, cb) {
+        const callback = cb || this._noop;
+        return this._redis.decrby(key, decr, callback);
+    }
+
     /**
     * report/record a request that ended up being a 500 on the server
     * @param {string} id - service identifier
@@ -101,10 +125,54 @@ class StatsClient {
             return undefined;
         }
         const callback = cb || this._noop;
-        const key = this._buildKey(`${id}:500s`, new Date());
+        const key = this.buildKey(`${id}:500s`, new Date());
         return this._redis.incrEx(key, this._expiry, callback);
     }
 
+    /**
+    * wrapper on `getStats` that handles a list of keys
+    * @param {object} log - Werelogs request logger
+    * @param {array} ids - service identifiers
+    * @param {callback} cb - callback to call with the err/result
+    * @return {undefined}
+    */
+    getAllStats(log, ids, cb) {
+        if (!this._redis) {
+            return cb(null, {});
+        }
+
+        const statsRes = {
+            'requests': 0,
+            '500s': 0,
+            'sampleDuration': this._expiry,
+        };
+        let requests = 0;
+        let errors = 0;
+
+        // for now set concurrency to default of 10
+        return async.eachLimit(ids, 10, (id, done) => {
+            this.getStats(log, id, (err, res) => {
+                if (err) {
+                    return done(err);
+                }
+                requests += res.requests;
+                errors += res['500s'];
+                return done();
+            });
+        }, error => {
+            if (error) {
+                log.error('error getting stats', {
+                    error,
+                    method: 'StatsClient.getAllStats',
+                });
+                return cb(null, statsRes);
+            }
+            statsRes.requests = requests;
+            statsRes['500s'] = errors;
+            return cb(null, statsRes);
+        });
+    }
+
     /**
     * get stats for the last x seconds, x being the sampling duration
     * @param {object} log - Werelogs request logger
@@ -121,8 +189,8 @@ class StatsClient {
         const reqsKeys = [];
         const req500sKeys = [];
         for (let i = 0; i < totalKeys; i++) {
-            reqsKeys.push(['get', this._buildKey(`${id}:requests`, d)]);
-            req500sKeys.push(['get', this._buildKey(`${id}:500s`, d)]);
+            reqsKeys.push(['get', this.buildKey(`${id}:requests`, d)]);
+            req500sKeys.push(['get', this.buildKey(`${id}:500s`, d)]);
             this._setPrevInterval(d);
         }
         return async.parallel([

lib/metrics/StatsModel.js

@@ -1,4 +1,7 @@
+const async = require('async');
+
 const StatsClient = require('./StatsClient');
+
 /**
  * @class StatsModel
  *
@@ -6,6 +9,141 @@ const StatsClient = require('./StatsClient');
  * rather than by seconds
  */
 class StatsModel extends StatsClient {
+
+    /**
+    * Utility method to convert 2d array rows to columns, and vice versa
+    * See also: https://docs.ruby-lang.org/en/2.0.0/Array.html#method-i-zip
+    * @param {array} arrays - 2d array of integers
+    * @return {array} converted array
+    */
+    _zip(arrays) {
+        if (arrays.length > 0 && arrays.every(a => Array.isArray(a))) {
+            return arrays[0].map((_, i) => arrays.map(a => a[i]));
+        }
+        return [];
+    }
+
+    /**
+    * normalize to the nearest interval
+    * @param {object} d - Date instance
+    * @return {number} timestamp - normalized to the nearest interval
+    */
+    _normalizeTimestamp(d) {
+        const m = d.getMinutes();
+        return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
+    }
+
+    /**
+    * override the method to get the count as an array of integers separated
+    * by each interval
+    * typical input looks like [[null, '1'], [null, '2'], [null, null]...]
+    * @param {array} arr - each index contains the result of each batch command
+    *   where index 0 signifies the error and index 1 contains the result
+    * @return {array} array of integers, ordered from most recent interval to
+    *   oldest interval with length of (expiry / interval)
+    */
+    _getCount(arr) {
+        const size = Math.floor(this._expiry / this._interval);
+        const array = arr.reduce((store, i) => {
+            let num = parseInt(i[1], 10);
+            num = Number.isNaN(num) ? 0 : num;
+            store.push(num);
+            return store;
+        }, []);
+
+        if (array.length < size) {
+            array.push(...Array(size - array.length).fill(0));
+        }
+        return array;
+    }
+
+    /**
+    * wrapper on `getStats` that handles a list of keys
+    * override the method to reduce the returned 2d array from `_getCount`
+    * @param {object} log - Werelogs request logger
+    * @param {array} ids - service identifiers
+    * @param {callback} cb - callback to call with the err/result
+    * @return {undefined}
+    */
+    getAllStats(log, ids, cb) {
+        if (!this._redis) {
+            return cb(null, {});
+        }
+
+        const size = Math.floor(this._expiry / this._interval);
+        const statsRes = {
+            'requests': Array(size).fill(0),
+            '500s': Array(size).fill(0),
+            'sampleDuration': this._expiry,
+        };
+        const requests = [];
+        const errors = [];
+
+        if (ids.length === 0) {
+            return cb(null, statsRes);
+        }
+
+        // for now set concurrency to default of 10
+        return async.eachLimit(ids, 10, (id, done) => {
+            this.getStats(log, id, (err, res) => {
+                if (err) {
+                    return done(err);
+                }
+                requests.push(res.requests);
+                errors.push(res['500s']);
+                return done();
+            });
+        }, error => {
+            if (error) {
+                log.error('error getting stats', {
+                    error,
+                    method: 'StatsModel.getAllStats',
+                });
+                return cb(null, statsRes);
+            }
+
+            statsRes.requests = this._zip(requests).map(arr =>
+                arr.reduce((acc, i) => acc + i), 0);
+            statsRes['500s'] = this._zip(errors).map(arr =>
+                arr.reduce((acc, i) => acc + i), 0);
+
+            return cb(null, statsRes);
+        });
+    }
+
+    /**
+     * Handles getting a list of global keys.
+     * @param {array} ids - Service identifiers
+     * @param {object} log - Werelogs request logger
+     * @param {function} cb - Callback
+     * @return {undefined}
+     */
+    getAllGlobalStats(ids, log, cb) {
+        const reqsKeys = ids.map(key => (['get', key]));
+        return this._redis.batch(reqsKeys, (err, res) => {
+            const statsRes = { requests: 0 };
+            if (err) {
+                log.error('error getting metrics', {
+                    error: err,
+                    method: 'StatsClient.getAllGlobalStats',
+                });
+                return cb(null, statsRes);
+            }
+            statsRes.requests = res.reduce((sum, curr) => {
+                const [cmdErr, val] = curr;
+                if (cmdErr) {
+                    // Log any individual request errors from the batch request.
+                    log.error('error getting metrics', {
+                        error: cmdErr,
+                        method: 'StatsClient.getAllGlobalStats',
+                    });
+                }
+                return sum + (Number.parseInt(val, 10) || 0);
+            }, 0);
+            return cb(null, statsRes);
+        });
+    }
+
     /**
      * normalize date timestamp to the nearest hour
      * @param {Date} d - Date instance
@@ -24,34 +162,6 @@ class StatsModel extends StatsClient {
         return d.setHours(d.getHours() - 1);
     }
 
-    /**
-     * normalize to the nearest interval
-     * @param {object} d - Date instance
-     * @return {number} timestamp - normalized to the nearest interval
-     */
-    _normalizeTimestamp(d) {
-        const m = d.getMinutes();
-        return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
-    }
-
-    /**
-     * override the method to get the result as an array of integers separated
-     * by each interval
-     * typical input looks like [[null, '1'], [null, '2'], [null, null]...]
-     * @param {array} arr - each index contains the result of each batch command
-     *   where index 0 signifies the error and index 1 contains the result
-     * @return {array} array of integers, ordered from most recent interval to
-     *   oldest interval
-     */
-    _getCount(arr) {
-        return arr.reduce((store, i) => {
-            let num = parseInt(i[1], 10);
-            num = Number.isNaN(num) ? 0 : num;
-            store.push(num);
-            return store;
-        }, []);
-    }
-
     /**
      * get list of sorted set key timestamps
      * @param {number} epoch - epoch time

lib/models/BucketInfo.js

@@ -1,10 +1,13 @@
 const assert = require('assert');
+const uuid = require('uuid/v4');
+
 const { WebsiteConfiguration } = require('./WebsiteConfiguration');
 const ReplicationConfiguration = require('./ReplicationConfiguration');
 const LifecycleConfiguration = require('./LifecycleConfiguration');
 
 // WHEN UPDATING THIS NUMBER, UPDATE MODELVERSION.MD CHANGELOG
-const modelVersion = 6;
+// MODELVERSION.MD can be found in S3 repo: lib/metadata/ModelVersion.md
+const modelVersion = 9;
 
 class BucketInfo {
     /**
@@ -47,12 +50,17 @@ class BucketInfo {
     * @param {string[]} [cors[].exposeHeaders] - headers expose to applications
     * @param {object} [replicationConfiguration] - replication configuration
     * @param {object} [lifecycleConfiguration] - lifecycle configuration
+    * @param {string} [uid] - unique identifier for the bucket, necessary
+    * @param {string} readLocationConstraint - readLocationConstraint for bucket
+    * addition for use with lifecycle operations
+    * @param {boolean} [isNFS] - whether the bucket is on NFS
     */
     constructor(name, owner, ownerDisplayName, creationDate,
                 mdBucketModelVersion, acl, transient, deleted,
                 serverSideEncryption, versioningConfiguration,
                 locationConstraint, websiteConfiguration, cors,
-                replicationConfiguration, lifecycleConfiguration) {
+                replicationConfiguration, lifecycleConfiguration, uid,
+                readLocationConstraint, isNFS) {
         assert.strictEqual(typeof name, 'string');
         assert.strictEqual(typeof owner, 'string');
         assert.strictEqual(typeof ownerDisplayName, 'string');
@@ -90,6 +98,9 @@ class BucketInfo {
         if (locationConstraint) {
             assert.strictEqual(typeof locationConstraint, 'string');
         }
+        if (readLocationConstraint) {
+            assert.strictEqual(typeof readLocationConstraint, 'string');
+        }
         if (websiteConfiguration) {
             assert(websiteConfiguration instanceof WebsiteConfiguration);
             const { indexDocument, errorDocument, redirectAllRequestsTo,
@@ -112,6 +123,10 @@ class BucketInfo {
         if (lifecycleConfiguration) {
             LifecycleConfiguration.validateConfig(lifecycleConfiguration);
         }
+        if (uid) {
+            assert.strictEqual(typeof uid, 'string');
+            assert.strictEqual(uid.length, 36);
+        }
         const aclInstance = acl || {
             Canned: 'private',
             FULL_CONTROL: [],
@@ -133,10 +148,13 @@ class BucketInfo {
         this._serverSideEncryption = serverSideEncryption || null;
         this._versioningConfiguration = versioningConfiguration || null;
         this._locationConstraint = locationConstraint || null;
+        this._readLocationConstraint = readLocationConstraint || null;
         this._websiteConfiguration = websiteConfiguration || null;
         this._replicationConfiguration = replicationConfiguration || null;
         this._cors = cors || null;
         this._lifecycleConfiguration = lifecycleConfiguration || null;
+        this._uid = uid || uuid();
+        this._isNFS = isNFS || null;
         return this;
     }
     /**
@@ -156,10 +174,13 @@ class BucketInfo {
             serverSideEncryption: this._serverSideEncryption,
             versioningConfiguration: this._versioningConfiguration,
             locationConstraint: this._locationConstraint,
+            readLocationConstraint: this._readLocationConstraint,
             websiteConfiguration: undefined,
             cors: this._cors,
             replicationConfiguration: this._replicationConfiguration,
             lifecycleConfiguration: this._lifecycleConfiguration,
+            uid: this._uid,
+            isNFS: this._isNFS,
         };
         if (this._websiteConfiguration) {
             bucketInfos.websiteConfiguration =
@@ -180,7 +201,8 @@ class BucketInfo {
             obj.creationDate, obj.mdBucketModelVersion, obj.acl,
             obj.transient, obj.deleted, obj.serverSideEncryption,
             obj.versioningConfiguration, obj.locationConstraint, websiteConfig,
-            obj.cors, obj.replicationConfiguration, obj.lifecycleConfiguration);
+            obj.cors, obj.replicationConfiguration, obj.lifecycleConfiguration,
+            obj.uid, obj.readLocationConstraint, obj.isNFS);
     }
 
     /**
@@ -203,7 +225,8 @@ class BucketInfo {
             data._transient, data._deleted, data._serverSideEncryption,
             data._versioningConfiguration, data._locationConstraint,
             data._websiteConfiguration, data._cors,
-            data._replicationConfiguration, data._lifecycleConfiguration);
+            data._replicationConfiguration, data._lifecycleConfiguration,
+            data._uid, data._readLocationConstraint, data._isNFS);
     }
 
     /**
@@ -449,6 +472,17 @@ class BucketInfo {
         return this._locationConstraint;
     }
 
+    /**
+    * Get read location constraint.
+    * @return {string} - bucket read location constraint
+    */
+    getReadLocationConstraint() {
+        if (this._readLocationConstraint) {
+            return this._readLocationConstraint;
+        }
+        return this._locationConstraint;
+    }
+
     /**
      * Set Bucket model version
      *
@@ -521,6 +555,29 @@ class BucketInfo {
         return this._versioningConfiguration &&
             this._versioningConfiguration.Status === 'Enabled';
     }
+    /**
+     * Get unique id of bucket.
+     * @return {string} - unique id
+     */
+    getUid() {
+        return this._uid;
+    }
+    /**
+     * Check if the bucket is an NFS bucket.
+     * @return {boolean} - Wether the bucket is NFS or not
+     */
+    isNFS() {
+        return this._isNFS;
+    }
+    /**
+     * Set whether the bucket is an NFS bucket.
+     * @param {boolean} isNFS - Wether the bucket is NFS or not
+     * @return {BucketInfo} - bucket info instance
+     */
+    setIsNFS(isNFS) {
+        this._isNFS = isNFS;
+        return this;
+    }
 }
 
 module.exports = BucketInfo;

lib/models/LifecycleConfiguration.js

@@ -269,7 +269,7 @@ class LifecycleConfiguration {
             return filterObj;
         }
         if (filter.Tag) {
-            const tagObj = this._parseTags(filter.Tag[0]);
+            const tagObj = this._parseTags(filter.Tag);
             if (tagObj.error) {
                 filterObj.error = tagObj.error;
                 return filterObj;
@@ -287,7 +287,7 @@ class LifecycleConfiguration {
             if (andF.Prefix && andF.Prefix.length >= 1) {
                 filterObj.rulePrefix = andF.Prefix.pop();
             }
-            const tagObj = this._parseTags(andF.Tag[0]);
+            const tagObj = this._parseTags(andF.Tag);
             if (tagObj.error) {
                 filterObj.error = tagObj.error;
                 return filterObj;
@@ -320,31 +320,28 @@ class LifecycleConfiguration {
         // reset _tagKeys to empty because keys cannot overlap within a rule,
         // but different rules can have the same tag keys
         this._tagKeys = [];
-        if (!tags.Key || !tags.Value) {
-            tagObj.error = errors.MissingRequiredParameter.customizeDescription(
+        for (let i = 0; i < tags.length; i++) {
+            if (!tags[i].Key || !tags[i].Value) {
+                tagObj.error =
+                    errors.MissingRequiredParameter.customizeDescription(
                     'Tag XML does not contain both Key and Value');
-            return tagObj;
+                break;
             }
-        if (tags.Key.length !== tags.Value.length) {
-            tagObj.error = errors.MalformedXML.customizeDescription(
-                'Tag XML should contain same number of Keys and Values');
-            return tagObj;
-        }
-        for (let i = 0; i < tags.Key.length; i++) {
-            if (tags.Key[i].length < 1 || tags.Key[i].length > 128) {
+
+            if (tags[i].Key[0].length < 1 || tags[i].Key[0].length > 128) {
                 tagObj.error = errors.InvalidRequest.customizeDescription(
                     'Tag Key must be a length between 1 and 128 char');
                 break;
             }
-            if (this._tagKeys.includes(tags.Key[i])) {
+            if (this._tagKeys.includes(tags[i].Key[0])) {
                 tagObj.error = errors.InvalidRequest.customizeDescription(
                     'Tag Keys must be unique');
                 break;
             }
-            this._tagKeys.push(tags.Key[i]);
+            this._tagKeys.push(tags[i].Key[0]);
             const tag = {
-                key: tags.Key[i],
-                val: tags.Value[i],
+                key: tags[i].Key[0],
+                val: tags[i].Value[0],
             };
             tagObj.tags.push(tag);
         }
@@ -677,13 +674,12 @@ class LifecycleConfiguration {
             const Prefix = rulePrefix ? `<Prefix>${rulePrefix}</Prefix>` : '';
             let tagXML = '';
             if (tags) {
-                const keysVals = tags.map(t => {
+                tagXML = tags.map(t => {
                     const { key, val } = t;
-                    const Tag = `<Key>${key}</Key>` +
-                        `<Value>${val}</Value>`;
+                    const Tag = `<Tag><Key>${key}</Key>` +
+                        `<Value>${val}</Value></Tag>`;
                     return Tag;
                 }).join('');
-                tagXML = `<Tag>${keysVals}</Tag>`;
             }
             let Filter;
             if (rulePrefix && !tags) {

lib/models/ObjectMD.js

@@ -120,6 +120,7 @@ class ObjectMD {
                 role: '',
                 storageType: '',
                 dataStoreVersionId: '',
+                isNFS: null,
             },
             'dataStoreName': '',
         };
@@ -648,6 +649,19 @@ class ObjectMD {
         return this._data.isDeleteMarker;
     }
 
+    /**
+     * Get if the object is a multipart upload (MPU)
+     *
+     * The function checks the "content-md5" field: if it contains a
+     * dash ('-') it is a MPU, as the content-md5 string ends with
+     * "-[nbparts]" for MPUs.
+     *
+     * @return {boolean} Whether object is a multipart upload
+     */
+    isMultipartUpload() {
+        return this.getContentMd5().includes('-');
+    }
+
     /**
      * Set metadata versionId value
      *
@@ -675,8 +689,11 @@ class ObjectMD {
      * @return {string} The encoded object versionId
      */
     getEncodedVersionId() {
+        if (this.getVersionId()) {
             return VersionIDUtils.encode(this.getVersionId());
         }
+        return undefined;
+    }
 
     /**
      * Set tags
@@ -706,7 +723,7 @@ class ObjectMD {
      */
     setReplicationInfo(replicationInfo) {
         const { status, backends, content, destination, storageClass, role,
-            storageType, dataStoreVersionId } = replicationInfo;
+            storageType, dataStoreVersionId, isNFS } = replicationInfo;
         this._data.replicationInfo = {
             status,
             backends,
@@ -716,6 +733,7 @@ class ObjectMD {
             role,
             storageType: storageType || '',
             dataStoreVersionId: dataStoreVersionId || '',
+            isNFS: isNFS || null,
         };
         return this;
     }
@@ -734,6 +752,24 @@ class ObjectMD {
         return this;
     }
 
+    /**
+     * Set whether the replication is occurring from an NFS bucket.
+     * @param {Boolean} isNFS - Whether replication from an NFS bucket
+     * @return {ObjectMD} itself
+     */
+    setReplicationIsNFS(isNFS) {
+        this._data.replicationInfo.isNFS = isNFS;
+        return this;
+    }
+
+    /**
+     * Get whether the replication is occurring from an NFS bucket.
+     * @return {Boolean} Whether replication from an NFS bucket
+     */
+    getReplicationIsNFS() {
+        return this._data.replicationInfo.isNFS;
+    }
+
     setReplicationSiteStatus(site, status) {
         const backend = this._data.replicationInfo.backends
             .find(o => o.site === site);

lib/models/ReplicationConfiguration.js

@@ -59,6 +59,7 @@ class ReplicationConfiguration {
         this._rules = null;
         this._prevStorageClass = null;
         this._hasScalityDestination = null;
+        this._preferredReadLocation = null;
     }
 
     /**
@@ -85,6 +86,18 @@ class ReplicationConfiguration {
         return this._rules;
     }
 
+    /**
+     * The preferred read location
+     * @return {string|null} - The preferred read location if defined,
+     * otherwise null
+     *
+     * FIXME ideally we should be able to specify one preferred read
+     * location for each rule
+     */
+    getPreferredReadLocation() {
+        return this._preferredReadLocation;
+    }
+
     /**
      * Get the replication configuration
      * @return {object} - The replication configuration
@@ -94,6 +107,7 @@ class ReplicationConfiguration {
             role: this.getRole(),
             destination: this.getDestination(),
             rules: this.getRules(),
+            preferredReadLocation: this.getPreferredReadLocation(),
         };
     }
 
@@ -292,6 +306,14 @@ class ReplicationConfiguration {
             return undefined;
         }
         const storageClasses = destination.StorageClass[0].split(',');
+        const prefReadIndex = storageClasses.findIndex(storageClass =>
+            storageClass.endsWith(':preferred_read'));
+        if (prefReadIndex !== -1) {
+            const prefRead = storageClasses[prefReadIndex].split(':')[0];
+            // remove :preferred_read tag from storage class name
+            storageClasses[prefReadIndex] = prefRead;
+            this._preferredReadLocation = prefRead;
+        }
         const isValidStorageClass = storageClasses.every(storageClass => {
             if (validStorageClasses.includes(storageClass)) {
                 this._hasScalityDestination =

lib/network/http/server.js

@@ -352,6 +352,8 @@ class Server {
             error: err.stack || err,
             address: sock.address(),
         });
+        // socket is not systematically destroyed
+        sock.destroy();
     }
 
     /**

lib/network/kmip/Client.js

@@ -0,0 +1,579 @@
+'use strict'; // eslint-disable-line
+/* eslint new-cap: "off" */
+
+const async = require('async');
+
+const errors = require('../../errors');
+const TTLVCodec = require('./codec/ttlv.js');
+const TlsTransport = require('./transport/tls.js');
+const KMIP = require('.');
+
+const CRYPTOGRAPHIC_OBJECT_TYPE = 'Symmetric Key';
+const CRYPTOGRAPHIC_ALGORITHM = 'AES';
+const CRYPTOGRAPHIC_CIPHER_MODE = 'CBC';
+const CRYPTOGRAPHIC_PADDING_METHOD = 'PKCS5';
+const CRYPTOGRAPHIC_LENGTH = 256;
+const CRYPTOGRAPHIC_USAGE_MASK = ['Encrypt', 'Decrypt'];
+const CRYPTOGRAPHIC_DEFAULT_IV = Buffer.alloc(16).fill(0);
+
+const searchFilter = {
+    protocolVersionMajor:
+    'Response Message/Batch Item/' +
+        'Response Payload/Protocol Version/' +
+        'Protocol Version Major',
+    protocolVersionMinor:
+    'Response Message/Batch Item/' +
+        'Response Payload/Protocol Version/' +
+        'Protocol Version Minor',
+    extensionName:
+    'Response Message/Batch Item/Response Payload' +
+        '/Extension Information/Extension Name',
+    extensionTag:
+    'Response Message/Batch Item/Response Payload' +
+        '/Extension Information/Extension Tag',
+    vendorIdentification:
+    'Response Message/Batch Item/Response Payload/Vendor Identification',
+    serverInformation:
+    'Response Message/Batch Item/Response Payload/Server Information',
+    operation:
+    'Response Message/Batch Item/Response Payload/Operation',
+    objectType:
+    'Response Message/Batch Item/Response Payload/Object Type',
+    uniqueIdentifier:
+    'Response Message/Batch Item/Response Payload/Unique Identifier',
+    data:
+    'Response Message/Batch Item/Response Payload/Data',
+};
+
+/**
+ * Normalize errors according to arsenal definitions
+ * @param {string | Error} err - an Error instance or a message string
+ * @returns {arsenal.errors} - arsenal error
+ */
+function _arsenalError(err) {
+    const messagePrefix = 'KMIP:';
+    if (typeof err === 'string') {
+        return errors.InternalError
+            .customizeDescription(`${messagePrefix} ${err}`);
+    } else if (err instanceof Error) {
+        return errors.InternalError
+            .customizeDescription(`${messagePrefix} ${err.message}`);
+    }
+    return errors.InternalError
+        .customizeDescription(`${messagePrefix} Unspecified error`);
+}
+
+
+/**
+ * Negotiate with the server the use of a recent version of the protocol and
+ * update the low level driver with this new knowledge.
+ * @param {Object} client - The Client instance
+ * @param {Object} logger - Werelog logger object
+ * @param {Function} cb - The callback triggered after the negotiation.
+ * @returns {undefined}
+ */
+function _negotiateProtocolVersion(client, logger, cb) {
+    return client.kmip.request(logger, 'Discover Versions', [
+        KMIP.Structure('Protocol Version', [
+            KMIP.Integer('Protocol Version Major', 1),
+            KMIP.Integer('Protocol Version Minor', 4),
+        ]),
+        KMIP.Structure('Protocol Version', [
+            KMIP.Integer('Protocol Version Major', 1),
+            KMIP.Integer('Protocol Version Minor', 3),
+        ]),
+        KMIP.Structure('Protocol Version', [
+            KMIP.Integer('Protocol Version Major', 1),
+            KMIP.Integer('Protocol Version Minor', 2),
+        ]),
+    ], (err, response) => {
+        if (err) {
+            const error = _arsenalError(err);
+            logger.error('KMIP::negotiateProtocolVersion',
+                         { error,
+                           vendorIdentification: client.vendorIdentification });
+            return cb(error);
+        }
+        const majorVersions =
+              response.lookup(searchFilter.protocolVersionMajor);
+        const minorVersions =
+              response.lookup(searchFilter.protocolVersionMinor);
+        if (majorVersions.length === 0 ||
+            majorVersions.length !== minorVersions.length) {
+            const error = _arsenalError('No suitable protocol version');
+            logger.error('KMIP::negotiateProtocolVersion',
+                         { error,
+                           vendorIdentification: client.vendorIdentification });
+            return cb(error);
+        }
+        client.kmip.changeProtocolVersion(majorVersions[0], minorVersions[0]);
+        return cb();
+    });
+}
+
+/**
+ * Obtain from the server the various extensions defined by the vendor
+ * and update the low level driver with this new knowledge.
+ * @param {Object} client - The Client instance
+ * @param {Object} logger - Werelog logger object
+ * @param {Function} cb - The callback triggered after the extension mapping
+ * @returns {undefined}
+ */
+function _mapExtensions(client, logger, cb) {
+    return client.kmip.request(logger, 'Query', [
+        KMIP.Enumeration('Query Function', 'Query Extension Map'),
+    ], (err, response) => {
+        if (err) {
+            const error = _arsenalError(err);
+            logger.error('KMIP::mapExtensions',
+                         { error,
+                           vendorIdentification: client.vendorIdentification });
+            return cb(error);
+        }
+        const extensionNames = response.lookup(searchFilter.extensionName);
+        const extensionTags = response.lookup(searchFilter.extensionTag);
+        if (extensionNames.length !== extensionTags.length) {
+            const error = _arsenalError('Inconsistent extension list');
+            logger.error('KMIP::mapExtensions',
+                         { error,
+                           vendorIdentification: client.vendorIdentification });
+            return cb(error);
+        }
+        extensionNames.forEach((extensionName, idx) => {
+            client.kmip.mapExtension(extensionName, extensionTags[idx]);
+        });
+        return cb();
+    });
+}
+
+/**
+ * Query the Server information and identify its vendor
+ * @param {Object} client - The Client instance
+ * @param {Object} logger - Werelog logger object
+ * @param {Function} cb - The callback triggered after the information discovery
+ * @returns {undefined}
+ */
+function _queryServerInformation(client, logger, cb) {
+    client.kmip.request(logger, 'Query', [
+        KMIP.Enumeration('Query Function', 'Query Server Information'),
+    ], (err, response) => {
+        if (err) {
+            const error = _arsenalError(err);
+            logger.warn('KMIP::queryServerInformation',
+                        { error });
+            /* no error returned, caller can keep going */
+            return cb();
+        }
+        client._setVendorIdentification(
+            response.lookup(searchFilter.vendorIdentification)[0]);
+        client._setServerInformation(
+            JSON.stringify(response.lookup(searchFilter.serverInformation)[0]));
+
+        logger.info('KMIP Server identified',
+                    { vendorIdentification: client.vendorIdentification,
+                      serverInformation: client.serverInformation,
+                      negotiatedProtocolVersion: client.kmip.protocolVersion });
+        return cb();
+    });
+}
+
+/**
+ * Query the Server for the supported operations and managed object types.
+ * The fact that a server doesn't announce the support for a required feature
+ * is not a show stopper because some vendor support more or less what they
+ * announce. If a subsequent request fails, this information can be used to
+ * figure out the reason for the failure.
+ * @param {Object} client - The Client instance
+ * @param {Object} logger - Werelog logger object
+ * @param {Function} cb - The callback triggered after the information discovery
+ * @returns {undefined}
+ */
+function _queryOperationsAndObjects(client, logger, cb) {
+    return client.kmip.request(logger, 'Query', [
+        KMIP.Enumeration('Query Function', 'Query Operations'),
+        KMIP.Enumeration('Query Function', 'Query Objects'),
+    ], (err, response) => {
+        if (err) {
+            const error = _arsenalError(err);
+            logger.error('KMIP::queryOperationsAndObjects',
+                         { error,
+                           vendorIdentification: client.vendorIdentification });
+            return cb(error);
+        }
+        const supportedOperations = response.lookup(searchFilter.operation);
+        const supportedObjectTypes = response.lookup(searchFilter.objectType);
+        const supportsEncrypt = supportedOperations.includes('Encrypt');
+        const supportsDecrypt = supportedOperations.includes('Decrypt');
+        const supportsActivate = supportedOperations.includes('Activate');
+        const supportsRevoke = supportedOperations.includes('Revoke');
+        const supportsCreate = supportedOperations.includes('Create');
+        const supportsDestroy = supportedOperations.includes('Destroy');
+        const supportsQuery = supportedOperations.includes('Query');
+        const supportsSymmetricKeys =
+              supportedObjectTypes.includes('Symmetric Key');
+        if (!supportsEncrypt || !supportsDecrypt ||
+            !supportsActivate || !supportsRevoke ||
+            !supportsCreate || !supportsDestroy ||
+            !supportsQuery || !supportsSymmetricKeys) {
+            /* This should not be considered as an error since some vendors
+             * are not consistent between what they really support and what
+             * they announce to support.
+             */
+            logger.warn('KMIP::queryOperationsAndObjects: ' +
+                        'The KMIP Server announces that it ' +
+                        'does not support all of the required features',
+                        { vendorIdentification: client.vendorIdentification,
+                          serverInformation: client.serverInformation,
+                          supportsEncrypt, supportsDecrypt,
+                          supportsActivate, supportsRevoke,
+                          supportsCreate, supportsDestroy,
+                          supportsQuery, supportsSymmetricKeys });
+        } else {
+            logger.info('KMIP Server provides the necessary feature set',
+                        { vendorIdentification: client.vendorIdentification });
+        }
+        return cb();
+    });
+}
+
+
+class Client {
+    /**
+     * Construct a high level KMIP driver suitable for cloudserver
+     * @param {Object} options - Instance options
+     * @param {Object} options.kmip - Low level driver options
+     * @param {Object} options.kmip.client - This high level driver options
+     * @param {Object} options.kmip.client.compoundCreateActivate -
+     *                 Depends on the server's ability. False offers the best
+     *                 compatibility. True does not offer a significant
+     *                 performance gain, but can be useful in case of unreliable
+     *                 time synchronization between the client and the server.
+     * @param {Object} options.kmip.client.bucketNameAttributeName -
+     *                 Depends on the server's ability. Not specifying this
+     *                 offers the best compatibility and disable the attachement
+     *                 of the bucket name as a key attribute.
+     * @param {Object} options.kmip.codec - KMIP Codec options
+     * @param {Object} options.kmip.transport - KMIP Transport options
+     * @param {Class} CodecClass - diversion for the Codec class,
+     *                             defaults to TTLVCodec
+     * @param {Class} TransportClass - diversion for the Transport class,
+     *                                 defaults to TlsTransport
+     */
+    constructor(options, CodecClass, TransportClass) {
+        this.options = options.kmip.client || {};
+        this.vendorIdentification = '';
+        this.serverInformation = [];
+        this.kmip = new KMIP(CodecClass || TTLVCodec,
+                             TransportClass || TlsTransport,
+                             options);
+        this.kmip.registerHandshakeFunction((logger, cb) => {
+            this._kmipHandshake(logger, cb);
+        });
+    }
+
+    /**
+     * Update this client with the vendor identification of the server
+     * @param {String} vendorIdentification - Vendor identification string
+     * @returns {undefined}
+     */
+    _setVendorIdentification(vendorIdentification) {
+        this.vendorIdentification = vendorIdentification;
+    }
+
+    /**
+     * Update this client with the information about the server
+     * @param {Object} serverInformation - Server information object
+     * @returns {undefined}
+     */
+    _setServerInformation(serverInformation) {
+        this.serverInformation = serverInformation;
+    }
+
+    /**
+     * Perform the KMIP level handshake with the server
+     * @param {Object} logger - Werelog logger object
+     * @param {Function} cb - Callback to be triggered at the end of the
+     *                        handshake. cb(err: Error)
+     * @returns {undefined}
+     */
+    _kmipHandshake(logger, cb) {
+        return async.waterfall([
+            next => _negotiateProtocolVersion(this, logger, next),
+            next => _mapExtensions(this, logger, next),
+            next => _queryServerInformation(this, logger, next),
+            next => _queryOperationsAndObjects(this, logger, next),
+        ], cb);
+    }
+
+
+    /**
+     * Activate a cryptographic key managed by the server,
+     * for a specific bucket. This is a required action to perform after
+     * the key creation.
+     * @param {string} keyIdentifier - The bucket key Id
+     * @param {object} logger - Werelog logger object
+     * @param {function} cb - The callback(err: Error)
+     * @returns {undefined}
+     */
+    _activateBucketKey(keyIdentifier, logger, cb) {
+        return this.kmip.request(logger, 'Activate', [
+            KMIP.TextString('Unique Identifier', keyIdentifier),
+        ], (err, response) => {
+            if (err) {
+                const error = _arsenalError(err);
+                logger.error('KMIP::_activateBucketKey',
+                             { error,
+                               serverInformation: this.serverInformation });
+                return cb(error);
+            }
+            const uniqueIdentifier =
+                  response.lookup(searchFilter.uniqueIdentifier)[0];
+            if (uniqueIdentifier !== keyIdentifier) {
+                const error = _arsenalError(
+                    'Server did not return the expected identifier');
+                logger.error('KMIP::cipherDataKey',
+                             { error, uniqueIdentifier });
+                return cb(error);
+            }
+            return cb(null, keyIdentifier);
+        });
+    }
+
+    /**
+     * Create a new cryptographic key managed by the server,
+     * for a specific bucket
+     * @param {string} bucketName - The bucket name
+     * @param {object} logger - Werelog logger object
+     * @param {function} cb - The callback(err: Error, bucketKeyId: String)
+     * @returns {undefined}
+     */
+    createBucketKey(bucketName, logger, cb) {
+        const attributes = [];
+        if (!!this.options.bucketNameAttributeName) {
+            attributes.push(KMIP.Attribute('TextString',
+                                           this.options.bucketNameAttributeName,
+                                           bucketName));
+        }
+        attributes.push(...[
+            KMIP.Attribute('Enumeration', 'Cryptographic Algorithm',
+                           CRYPTOGRAPHIC_ALGORITHM),
+            KMIP.Attribute('Integer', 'Cryptographic Length',
+                           CRYPTOGRAPHIC_LENGTH),
+            KMIP.Attribute('Integer', 'Cryptographic Usage Mask',
+                           this.kmip.encodeMask('Cryptographic Usage Mask',
+                                                CRYPTOGRAPHIC_USAGE_MASK))]);
+        if (this.options.compoundCreateActivate) {
+            attributes.push(KMIP.Attribute('Date-Time', 'Activation Date',
+                                           new Date(Date.UTC())));
+        }
+
+        return this.kmip.request(logger, 'Create', [
+            KMIP.Enumeration('Object Type', CRYPTOGRAPHIC_OBJECT_TYPE),
+            KMIP.Structure('Template-Attribute', attributes),
+        ], (err, response) => {
+            if (err) {
+                const error = _arsenalError(err);
+                logger.error('KMIP::createBucketKey',
+                             { error,
+                               serverInformation: this.serverInformation });
+                return cb(error);
+            }
+            const createdObjectType =
+                  response.lookup(searchFilter.objectType)[0];
+            const uniqueIdentifier =
+                  response.lookup(searchFilter.uniqueIdentifier)[0];
+            if (createdObjectType !== CRYPTOGRAPHIC_OBJECT_TYPE) {
+                const error = _arsenalError(
+                    'Server created an object of wrong type');
+                logger.error('KMIP::createBucketKey',
+                             { error, createdObjectType });
+                return cb(error);
+            }
+            if (!this.options.compoundCreateActivate) {
+                return this._activateBucketKey(uniqueIdentifier, logger, cb);
+            }
+            return cb(null, uniqueIdentifier);
+        });
+    }
+
+    /**
+     * Revoke a cryptographic key managed by the server, for a specific bucket.
+     * This is a required action to perform before being able to destroy the
+     * managed key.
+     * @param {string} bucketKeyId - The bucket key Id
+     * @param {object} logger - Werelog logger object
+     * @param {function} cb - The callback(err: Error)
+     * @returns {undefined}
+     */
+    _revokeBucketKey(bucketKeyId, logger, cb) {
+        // maybe revoke first
+        return this.kmip.request(logger, 'Revoke', [
+            KMIP.TextString('Unique Identifier', bucketKeyId),
+            KMIP.Structure('Revocation Reason', [
+                KMIP.Enumeration('Revocation Reason Code',
+                                 'Cessation of Operation'),
+                KMIP.TextString('Revocation Message',
+                                'About to be deleted'),
+            ]),
+        ], (err, response) => {
+            if (err) {
+                const error = _arsenalError(err);
+                logger.error('KMIP::_revokeBucketKey',
+                             { error,
+                               serverInformation: this.serverInformation });
+                return cb(error);
+            }
+            const uniqueIdentifier =
+                  response.lookup(searchFilter.uniqueIdentifier)[0];
+            if (uniqueIdentifier !== bucketKeyId) {
+                const error = _arsenalError(
+                    'Server did not return the expected identifier');
+                logger.error('KMIP::_revokeBucketKey',
+                             { error, uniqueIdentifier });
+                return cb(error);
+            }
+            return cb();
+        });
+    }
+
+    /**
+     * Destroy a cryptographic key managed by the server, for a specific bucket.
+     * @param {string} bucketKeyId - The bucket key Id
+     * @param {object} logger - Werelog logger object
+     * @param {function} cb - The callback(err: Error)
+     * @returns {undefined}
+     */
+    destroyBucketKey(bucketKeyId, logger, cb) {
+        return this._revokeBucketKey(bucketKeyId, logger, err => {
+            if (err) {
+                const error = _arsenalError(err);
+                logger.error('KMIP::destroyBucketKey: revocation failed',
+                             { error,
+                               serverInformation: this.serverInformation });
+                return cb(error);
+            }
+            return this.kmip.request(logger, 'Destroy', [
+                KMIP.TextString('Unique Identifier', bucketKeyId),
+            ], (err, response) => {
+                if (err) {
+                    const error = _arsenalError(err);
+                    logger.error('KMIP::destroyBucketKey',
+                                 { error,
+                                   serverInformation: this.serverInformation });
+                    return cb(error);
+                }
+                const uniqueIdentifier =
+                      response.lookup(searchFilter.uniqueIdentifier)[0];
+                if (uniqueIdentifier !== bucketKeyId) {
+                    const error = _arsenalError(
+                        'Server did not return the expected identifier');
+                    logger.error('KMIP::destroyBucketKey',
+                                 { error, uniqueIdentifier });
+                    return cb(error);
+                }
+                return cb();
+            });
+        });
+    }
+
+    /**
+     *
+     * @param {number} cryptoScheme - crypto scheme version number
+     * @param {string} masterKeyId - key to retrieve master key
+     * @param {buffer} plainTextDataKey - data key
+     * @param {object} logger - werelog logger object
+     * @param {function} cb - callback
+     * @returns {undefined}
+     * @callback called with (err, cipheredDataKey: Buffer)
+     */
+    cipherDataKey(cryptoScheme,
+                  masterKeyId,
+                  plainTextDataKey,
+                  logger,
+                  cb) {
+        return this.kmip.request(logger, 'Encrypt', [
+            KMIP.TextString('Unique Identifier', masterKeyId),
+            KMIP.Structure('Cryptographic Parameters', [
+                KMIP.Enumeration('Block Cipher Mode',
+                                CRYPTOGRAPHIC_CIPHER_MODE),
+                KMIP.Enumeration('Padding Method',
+                                CRYPTOGRAPHIC_PADDING_METHOD),
+                KMIP.Enumeration('Cryptographic Algorithm',
+                               CRYPTOGRAPHIC_ALGORITHM),
+            ]),
+            KMIP.ByteString('Data', plainTextDataKey),
+            KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
+        ], (err, response) => {
+            if (err) {
+                const error = _arsenalError(err);
+                logger.error('KMIP::cipherDataKey',
+                             { error,
+                               serverInformation: this.serverInformation });
+                return cb(error);
+            }
+            const uniqueIdentifier =
+                  response.lookup(searchFilter.uniqueIdentifier)[0];
+            const data = response.lookup(searchFilter.data)[0];
+            if (uniqueIdentifier !== masterKeyId) {
+                const error = _arsenalError(
+                    'Server did not return the expected identifier');
+                logger.error('KMIP::cipherDataKey',
+                             { error, uniqueIdentifier });
+                return cb(error);
+            }
+            return cb(null, data);
+        });
+    }
+
+    /**
+     *
+     * @param {number} cryptoScheme - crypto scheme version number
+     * @param {string} masterKeyId - key to retrieve master key
+     * @param {buffer} cipheredDataKey - data key
+     * @param {object} logger - werelog logger object
+     * @param {function} cb - callback
+     * @returns {undefined}
+     * @callback called with (err, plainTextDataKey: Buffer)
+     */
+    decipherDataKey(cryptoScheme,
+                    masterKeyId,
+                    cipheredDataKey,
+                    logger,
+                    cb) {
+        return this.kmip.request(logger, 'Decrypt', [
+            KMIP.TextString('Unique Identifier', masterKeyId),
+            KMIP.Structure('Cryptographic Parameters', [
+                KMIP.Enumeration('Block Cipher Mode',
+                                CRYPTOGRAPHIC_CIPHER_MODE),
+                KMIP.Enumeration('Padding Method',
+                                 CRYPTOGRAPHIC_PADDING_METHOD),
+                KMIP.Enumeration('Cryptographic Algorithm',
+                                 CRYPTOGRAPHIC_ALGORITHM),
+            ]),
+            KMIP.ByteString('Data', cipheredDataKey),
+            KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
+        ], (err, response) => {
+            if (err) {
+                const error = _arsenalError(err);
+                logger.error('KMIP::decipherDataKey',
+                             { error,
+                               serverInformation: this.serverInformation });
+                return cb(error);
+            }
+            const uniqueIdentifier =
+                  response.lookup(searchFilter.uniqueIdentifier)[0];
+            const data = response.lookup(searchFilter.data)[0];
+            if (uniqueIdentifier !== masterKeyId) {
+                const error = _arsenalError(
+                    'Server did not return the right identifier');
+                logger.error('KMIP::decipherDataKey',
+                             { error, uniqueIdentifier });
+                return cb(error);
+            }
+            return cb(null, data);
+        });
+    }
+
+}
+
+module.exports = Client;

lib/network/kmip/Message.js

@@ -0,0 +1,54 @@
+'use strict'; // eslint-disable-line
+
+const assert = require('assert');
+
+
+function _lookup(decodedTTLV, path) {
+    const xpath = path.split('/').filter(word => word.length > 0);
+    const canonicalPath = xpath.join('/');
+    const obj = decodedTTLV;
+    let res = [];
+    assert(Array.isArray(obj));
+    for (let current = xpath.shift(); current; current = xpath.shift()) {
+        for (let i = 0; i < obj.length; ++i) {
+            const cell = obj[i];
+            if (cell[current]) {
+                if (xpath.length === 0) {
+                    /* Skip if the search path has not been
+                     * completely consumed yet */
+                    res.push(cell[current].value);
+                } else {
+                    const subPath = xpath.join('/');
+                    assert(current.length + 1 + subPath.length ===
+                           canonicalPath.length);
+                    const intermediate =
+                          _lookup(cell[current].value, subPath);
+                    res = res.concat(intermediate);
+                }
+            }
+        }
+    }
+    return res;
+}
+
+class Message {
+    /**
+     * Construct a new abstract Message
+     * @param {Object} content - the content  of the message
+     */
+    constructor(content) {
+        this.content = content;
+    }
+
+    /**
+     * Lookup the values corresponding to the provided path
+     * @param {String} path - the path in the hierarchy of the values
+     *                        of interest
+     * @return {Object} - an array of the values matching the provided path
+     */
+    lookup(path) {
+        return _lookup(this.content, path);
+    }
+}
+
+module.exports = Message;

lib/network/kmip/README.md

@@ -0,0 +1,105 @@
+# KMIP
+
+Key Management Interoperability Protocol
+
+## Preliminary usage example
+
+```javascript
+const {
+    kmipServerHostName,
+    clientKey,
+    clientCert,
+    serverCert,
+    rootCa
+} = require('./myconfiguration.js');
+
+const assert = require('assert');
+const fs = require('fs');
+const tls = require('tls');
+const werelogs = require('werelogs');
+const KMIP = require('arsenal').network.kmip;
+
+const logger = new werelogs.Logger('kmiptest');
+const kmip = new KMIP;
+const options = {
+    host: kmipServerHostName,
+    key: fs.readFileSync(clientKey),
+    cert: fs.readFileSync(clientCert),
+    ca: [ fs.readFileSync(serverCert),
+          fs.readFileSync(rootCa), ],
+    checkServerIdentity: console.log,
+};
+
+const message = KMIP.Message([
+    KMIP.Structure('Request Message', [
+        KMIP.Structure('Request Header', [
+            KMIP.Structure('Protocol Version', [
+                KMIP.Integer('Protocol Version Major', 1),
+                KMIP.Integer('Protocol Version Minor', 3),
+            ]),
+            KMIP.Integer('Maximum Response Size', 3456),
+            KMIP.Integer('Batch Count', 1),
+        ]),
+        KMIP.Structure('Batch Item', [
+            KMIP.Enumeration('Operation', 'Query'),
+            KMIP.Structure('Request Payload', [
+                KMIP.Enumeration('Query Function', 'Query Operations'),
+                KMIP.Enumeration('Query Function', 'Query Objects'),
+                KMIP.Enumeration('Query Function', 'Query Server Information'),
+                KMIP.Enumeration('Query Function', 'Query Extension Map'),
+            ]),
+        ]),
+    ])
+]);
+
+const encodedMessage = kmip.encodeMessage(logger, message);
+
+const socket = tls.connect(5696, options, () => {
+    socket.write(encodedMessage);
+});
+
+socket.on('data', (data) => {
+    const decodedMessage = kmip.decodeMessage(logger, data);
+    const summary = {
+        major: decodedMessage.lookup(
+            'Response Message/Response Header/' +
+                'Protocol Version/Protocol Version Major')[0],
+        minor: decodedMessage.lookup(
+            'Response Message/Response Header/' +
+                'Protocol Version/Protocol Version Minor')[0],
+        supportedOperations: decodedMessage.lookup(
+            'Response Message/Batch Item/Response Payload/Operation'),
+        supportedObjectTypes: decodedMessage.lookup(
+            'Response Message/Batch Item/Response Payload/Object Type'),
+        serverInformation: decodedMessage.lookup(
+            'Response Message/Batch Item/Response Payload/Server Information'),
+    };
+
+    console.log(JSON.stringify(summary));
+    //console.log(JSON.stringify(decodedMessage.content));
+    //console.log(data.toString('hex'));
+
+    const protocolVersionMajor =
+        decodedMessage.lookup('Response Message/Response Header/' +
+                              'Protocol Version/Protocol Version Major');
+    const protocolVersionMinor =
+        decodedMessage.lookup('Response Message/Response Header/' +
+                              'Protocol Version/Protocol Version Minor');
+
+    assert(summary.supportedOperations.includes('Encrypt'));
+    assert(summary.supportedOperations.includes('Decrypt'));
+    assert(summary.supportedOperations.includes('Create'));
+    assert(summary.supportedOperations.includes('Destroy'));
+    assert(summary.supportedOperations.includes('Query'));
+    assert(summary.supportedObjectTypes.includes('Symmetric Key'));
+    assert(protocolVersionMajor[0] >= 2 ||
+           (protocolVersionMajor[0] === 1 &&
+            protocolVersionMinor[0] >= 2));
+
+    socket.end();
+});
+
+socket.on('end', () => {
+    console.log('server ends connection');
+});
+```

lib/network/kmip/codec/README.md

@@ -0,0 +1,367 @@
+# KMIP codecs
+
+The KMIP protocol is based on the exchange of structured messages between a
+client and a server.
+
+About the structure of the messages:
+
+* It is composed of fields and nested fields
+* It varies depending on the context of the request and who emits the message.
+* It follows the same encoding rules for both the client and the server
+* The set of primitive types is the cornerstone of the protocol, the structure
+  description is contained within the messages along with the actual payload.
+* The set of defined tags is the keystone of the protocol. It permits to
+  attribute a meaning to the fields of a structured message.
+
+The role of the codec is twofold.
+
+* To decode a message from a particular encoding, to an abstract
+  representation of the KMIP structured messages.
+* To encode a message from its abstract representation to the particular
+  encoding.
+
+The codecs are not responsible for sending the messages on the wire.
+This task is devoted to the transport layer.
+
+## Abstract representation
+
+The primitive data types defined by the protocol are represented internally
+as data structures following the form
+
+```javascript
+    const abstractKmipField = {
+        [tagName]: {
+            type,
+            value
+        }
+    };
+```
+
+The tag name `tagName` is a string. It is decoded from the tag value
+using the KMIP nomenclature and identify the meaning of the field in the message.
+
+The type name `type` is a string and is one of the primitive types
+defined by the KMIP protocol. This element of a field also implicitly carries
+the information of length for fixed size data types.
+
+The value `value` is decoded from the payload of the KMIP field. This
+element carries the length information for varying data types.
+
+## Constructing an abstract Message
+
+```javascript
+    const msg = KMIP.Message(content);
+```
+
+The static method `KMIP.Message` instantiates an object of the class
+`Message`. Message objects wrap the content of the message without
+alteration and offer a `lookup` method to search the message for
+named fields.
+
+### Structure
+
+```javascript
+    const field =
+        KMIP.Structure('Request Header', [
+            field_1,
+            ...,
+            field_n,
+        ]);
+    console.log(field);
+    {
+        'Request Header': {
+            type: 'Structure',
+            value: [
+                field_1,
+                ...,
+                field_n
+            ]
+        }
+    }
+```
+
+Fields in the array parameter must be provided in the order defined by the
+specification for the considered structure name.
+
+### Integer
+
+```javascript
+    const field = KMIP.Integer('Protocol Version Minor', 3);
+    console.log(field);
+    {
+        'Protocol Version Minor': {
+            type: "Integer",
+            value: 3
+        }
+    }
+```
+
+Integers are encoded as four-byte long (32 bit) binary signed numbers in 2's
+complement notation, transmitted big-endian.
+
+### LongInteger
+
+```javascript
+    const field = KMIP.LongInteger('Usage Limits Total', 10 ** 42);
+    console.log(field);
+    {
+        'Usage Limits Total': {
+            type: 'LongInteger',
+            value: 1e+42
+        }
+    }
+```
+
+Long Integers are encoded as eight-byte long (64 bit) binary signed numbers in
+2's complement notation, transmitted big-endian.
+
+Due to an accuracy limitation of number representation, `LongInteger` values
+cannot exceed 2^53. It's expected from the codec to throw an error when
+attempting to transcode a LongInteger greater than this value.
+
+### BigInteger
+
+```javascript
+    const field = KMIP.BigInteger('Usage Limits Total', value);
+    console.log(field);
+    {
+        'Usage Limits Total': {
+            type: 'LongInteger',
+            value: <Buffer ab cd ef ...>
+        }
+    }
+```
+
+Big Integers are encoded as a sequence of eight-bit bytes, in two's complement
+notation, transmitted big-endian. If the length of the sequence is not a
+multiple of eight bytes, then Big Integers SHALL be padded with the minimal
+number of leading sign-extended bytes to make the length a multiple of eight
+bytes. These padding bytes are part of the Item Value and SHALL be counted in
+the Item Length.
+
+### Enumeration
+
+```javascript
+    const field = KMIP.Enumeration('Operation', 'Discover Versions');
+    console.log(field);
+    {
+        'Operation': {
+            type: 'Enumeration',
+            value: 'Discover Versions'
+        }
+    }
+```
+
+### Boolean
+
+```javascript
+    const field = KMIP.Boolean('Asynchronous Indicator', false);
+    console.log(field);
+    {
+        'Asynchronous Indicator': {
+            type: 'Boolean',
+            value: false
+        }
+    }
+```
+
+### TextString
+
+```javascript
+    const field = KMIP.TextString('Username', 'alice');
+    console.log(field);
+    {
+        'Username': {
+            type: 'TextString',
+            value: 'alice'
+        }
+    }
+```
+
+Text Strings are sequences of bytes that encode character values according to
+the UTF-8 encoding standard. There SHALL NOT be null-termination at the end of
+such strings.
+
+### ByteString
+
+```javascript
+    const field = KMIP.ByteString('Asynchronous Correlation Value', buffer);
+    console.log(field);
+    {
+        'Username': {
+            type: 'ByteString',
+            value: <Buffer ab cd ef ...>
+        }
+    }
+```
+
+Byte Strings are sequences of bytes containing individual unspecified eight-bit
+binary values, and are interpreted in the same sequence order.
+
+### DateTime
+
+```javascript
+    const field = KMIP.DateTime('Activation Date', new Date);
+    console.log(field);
+    {
+        'Username': {
+            type: 'ByteString',
+            value: <Date 2019-01-10T20:41:36.914Z>
+        }
+    }
+```
+
+DateTime takes a Date object as its second parameter. The millisecond part of
+the date is silently discarded and not sent through the Network.
+
+For this particular example, the 'Activation Date' tag is used for illustration
+purpose. This is not the appropriate way to instanciate this attribute value and
+the special function `KMIP.Attribute` must be used instead of `KMIP.DateTime`.
+
+### Interval
+
+```javascript
+    const field = KMIP.Interval('Lease Time', 42);
+    console.log(field);
+    {
+        'Lease Time': {
+            type: "Interval",
+            value: 42
+        }
+    }
+```
+
+Intervals are encoded as four-byte long (32 bit) binary unsigned numbers,
+transmitted big-endian. They have a resolution of one second.
+
+### Special types
+
+#### Bit Mask
+
+Bit masks are encoded using the `Integer` primitive type relative to an instance
+of the KMIP class (e.g. `encodeMask` and `decodemask` are not static class
+function but regular methods).
+
+```javascript
+    const kmip = new KMIP;
+    const mask = ['Encrypt', 'Decrypt'];
+    const bitMask = kmip.encodeMask('Cryptographic Usage Mask', mask);
+    const decodedMask = kmip.decodeMask('Cryptographic Usage Mask', bitMask);
+    assert.deepStrictEqual(decodedMask, mask);
+    assert(bitMask === 12);
+```
+
+#### Attribute
+
+Attribute names and values are managed in a way that deviates from the general
+rule. Particularly when it comes associate the value of an enumeration to its
+tag. In the nominal form, the value of an enumeration in a field is retrieved
+from the tag of this field. For the case of an Attribute, the tag of the
+enumeration is referenced in the `Attribute Name` as a `TextString` and the
+encoded enumeration value is stored in the `Attribute value`, hence
+disconnecting the value from its tag.
+
+```javascript
+    const cryptographicAlgorithm =
+        KMIP.Attribute('Enumeration', 'Cryptographic Algorithm', 'AES'),
+    const requestPayload =
+        KMIP.Structure('Request Payload', [
+            KMIP.Enumeration('Object Type', 'Symmetric Key'),
+            KMIP.Structure('Template-Attribute', [
+                KMIP.Attribute('TextString', 'x-Name', 's3-thekey'),
+                cryptographicAlgorithm,
+                KMIP.Attribute('Integer', 'Cryptographic Length', 256),
+                KMIP.Attribute('Integer', 'Cryptographic Usage Mask',
+                               kmip.encodeMask('Cryptographic Usage Mask',
+                                               ['Encrypt', 'Decrypt'])),
+                KMIP.Attribute('Date-Time', 'Activation Date', new Date),
+            ]),
+        ]);
+    console.log(cryptographicAlgorithm);
+    {
+        'Attribute': {
+            type: 'Structure',
+            value: [
+                {
+                    'Attribute Name': {
+                        type: 'TextString',
+                        value: 'Cryptographic Algorithm'
+                    }
+                },
+                {
+                    'Attribute Value': {
+                        type: 'Enumeration'
+                        value: 'AES',
+                        diversion: 'Cryptographic Algorithm'
+                    }
+                }
+            ]
+        }
+    }
+```
+
+The `diversion` attribute in the `Attribute Value` structure is used by the
+codec to identify the `Enumeration` the value relates to.
+
+## Codec Interface
+
+```javascript
+class MyCodec {
+    /**
+     * Construct a new instance of the codec
+     */
+    constructor() {}
+
+    /**
+     * Encode a bitmask
+     * @param {String} tagName - name of the bit mask defining tag
+     * @param {Array of Strings} value - array of named bits to set in the mask
+     * @return {Integer} Integer encoded bitmask
+     */
+    encodeMask(tagName, value) {}
+
+    /**
+     * Decode a bitmask
+     * @param {string} tagName - name of the bit mask defining tag
+     * @param {Integer} givenMask - bit mask to decode
+     * @return {Array of Strings} array of named bits set in the given bit mask
+     */
+    decodeMask(tagName, givenMask) {}
+
+    /**
+     * Encode an abstract message
+     * @param {Object} message - Instance of a KMIP.Message
+     * @return {Buffer} the encoded message suitable for the transport layer
+     */
+    encode(message) {}
+
+    /**
+     * Decode a raw message, usually received from the transport layer
+     * @param {Object} logger - a Logger instance
+     * @param {Buffer} rawMessage - the message to decode
+     * @return {Object} the decoded message as an instance of KMIP.Message
+     */
+    decode(logger, rawMessage) {}
+
+    /**
+     * Amend the tag nomenclature with a vendor specific extension
+     * @param {String} tagName - Name of the tag to record
+     * @param {Integer} tagValue - Tag value represented as an integer
+     */
+    mapExtension(tagName, tagValue) {}
+}
+```
+
+## Encoding specification links
+
+### TTLV Encoding Baseline Profile
+
+[TTLV Encoding Specification](http://docs.oasis-open.org/kmip/spec/v1.4/os/kmip-spec-v1.4-os.html#_Toc490660911)
+
+### XML Encoding Profile
+
+[XML Encoding Profile Specification](http://docs.oasis-open.org/kmip/profiles/v1.4/csprd01/kmip-profiles-v1.4-csprd01.html#_Toc479342078)
+
+### JSON Encoding Profile
+
+[JSON Encoding Profile Specification](http://docs.oasis-open.org/kmip/profiles/v1.4/csprd01/kmip-profiles-v1.4-csprd01.html#_Toc479342090)

lib/network/kmip/codec/ttlv.js

@@ -0,0 +1,434 @@
+'use strict'; // eslint-disable-line
+/* eslint dot-notation: "off" */
+
+const KMIPTags = require('../tags.json');
+const KMIPMessage = require('../Message.js');
+
+const UINT32_MAX = Math.pow(2, 32);
+
+function _ttlvPadVector(vec) {
+    let length = 0;
+    vec.forEach(buf => {
+        if (!(buf instanceof Buffer)) {
+            throw Error('Not a Buffer');
+        }
+        length += buf.length;
+    });
+    const paddingLength = (Math.ceil(length / 8) * 8) - length;
+    if (paddingLength > 0) {
+        vec.push(Buffer.alloc(paddingLength).fill(0));
+    }
+    return vec;
+}
+
+function _throwError(logger, msg, data) {
+    logger.error(msg, data);
+    throw Error(msg);
+}
+
+function TTLVCodec() {
+    if (!new.target) {
+        return new TTLVCodec();
+    }
+
+    const TagDecoder = JSON.parse(JSON.stringify(KMIPTags));
+    const TagEncoder = {};
+
+    const TypeDecoder = {};
+    const TypeEncoder = {};
+
+    const PrimitiveTypes = {
+        '01': {
+            name: 'Structure',
+            decode: (logger, unusedTag, value) => {
+                const funcName = 'Structure::decode';
+                const length = value.length;
+                let i = 0;
+                const result = [];
+                let diversion = null;
+                while (i < length) {
+                    const element = {};
+                    const elementTag = value.slice(i, i + 3).toString('hex');
+                    const elementType =
+                          value.slice(i + 3, i + 4).toString('hex');
+                    const elementLength = value.readUInt32BE(i + 4);
+                    const property = {};
+                    if (!TypeDecoder[elementType]) {
+                        _throwError(logger,
+                                    'Unknown element type',
+                                    { funcName, elementTag, elementType });
+                    }
+                    const elementValue = value.slice(i + 8,
+                                                   i + 8 + elementLength);
+                    if (elementValue.length !== elementLength) {
+                        _throwError(logger, 'BUG: Wrong buffer size',
+                                    { funcName, elementLength,
+                                      bufferLength: elementValue.length });
+                    }
+                    property.type = TypeDecoder[elementType].name;
+                    property.value = TypeDecoder[elementType]
+                        .decode(logger, elementTag, elementValue, diversion);
+                    if (diversion) {
+                        property.diversion = diversion;
+                        diversion = null;
+                    }
+                    const tagInfo = TagDecoder[elementTag];
+                    if (!tagInfo) {
+                        logger.debug('Unknown element tag',
+                                     { funcName, elementTag });
+                        property.tag = elementTag;
+                        element['Unknown Tag'] = property;
+                    } else {
+                        element[tagInfo.name] = property;
+                        if (tagInfo.name === 'Attribute Name') {
+                            if (property.type !== 'TextString') {
+                                _throwError(logger,
+                                            'Invalide type',
+                                            { funcName, type: property.type });
+                            }
+                            diversion = property.value;
+                        }
+                    }
+                    i += Math.ceil((8 + elementLength) / 8.0) * 8;
+                    result.push(element);
+                }
+                return result;
+            },
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type = Buffer.from(TypeEncoder['Structure'].value, 'hex');
+                const length = Buffer.alloc(4);
+                let vectorLength = 0;
+                let encodedValue = [];
+                value.forEach(item => {
+                    Object.keys(item).forEach(key => {
+                        const itemTagName = key;
+                        const itemType = item[key].type;
+                        const itemValue = item[key].value;
+                        const itemDiversion = item[key].diversion;
+                        if (!TagEncoder[itemTagName]) {
+                            throw Error(`Unknown Tag '${itemTagName}'`);
+                        }
+                        if (!TypeEncoder[itemType]) {
+                            throw Error(`Unknown Type '${itemType}'`);
+                        }
+                        const itemResult =
+                              TypeEncoder[itemType].encode(itemTagName,
+                                                           itemValue,
+                                                           itemDiversion);
+                        encodedValue = encodedValue
+                            .concat(_ttlvPadVector(itemResult));
+                    });
+                });
+                encodedValue = _ttlvPadVector(encodedValue);
+                encodedValue.forEach(buf => { vectorLength += buf.length; });
+                length.writeUInt32BE(vectorLength);
+                return _ttlvPadVector([tag, type, length, ...encodedValue]);
+            },
+        },
+        '02': {
+            name: 'Integer',
+            decode: (logger, tag, value) => {
+                const funcName = 'Integer::decode';
+                const fixedLength = 4;
+                if (fixedLength !== value.length) {
+                    _throwError(logger,
+                                'Length mismatch',
+                                { funcName, fixedLength,
+                                  bufferLength: value.length });
+                }
+                return value.readUInt32BE(0);
+            },
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type = Buffer.from(TypeEncoder['Integer'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(4);
+                const encodedValue = Buffer.alloc(4);
+                encodedValue.writeUInt32BE(value);
+                return _ttlvPadVector([tag, type, length, encodedValue]);
+            },
+        },
+        '03': {
+            name: 'LongInteger',
+            decode: (logger, tag, value) => {
+                const funcName = 'LongInteger::decode';
+                const fixedLength = 8;
+                if (fixedLength !== value.length) {
+                    _throwError(logger,
+                                'Length mismatch',
+                                { funcName, fixedLength,
+                                  bufferLength: value.length });
+                }
+                const longUInt = UINT32_MAX * value.readUInt32BE(0) +
+                      value.readUInt32BE(4);
+                if (longUInt > Number.MAX_SAFE_INTEGER) {
+                    _throwError(logger,
+                                '53-bit overflow',
+                                { funcName, longUInt });
+                }
+                return longUInt;
+            },
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type =
+                      Buffer.from(TypeEncoder['LongInteger'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(8);
+                const encodedValue = Buffer.alloc(8);
+                encodedValue.writeUInt32BE(Math.floor(value / UINT32_MAX), 0);
+                encodedValue.writeUInt32BE(value % UINT32_MAX, 4);
+                return _ttlvPadVector([tag, type, length, encodedValue]);
+            },
+        },
+        '04': {
+            name: 'BigInteger',
+            decode: (logger, tag, value) => value,
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type =
+                      Buffer.from(TypeEncoder['BigInteger'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(value.length);
+                return _ttlvPadVector([tag, type, length, value]);
+            },
+        },
+        '05': {
+            name: 'Enumeration',
+            decode: (logger, tag, value, diversion) => {
+                const funcName = 'Enumeration::decode';
+                const fixedLength = 4;
+                if (fixedLength !== value.length) {
+                    _throwError(logger,
+                                'Length mismatch',
+                                { funcName, fixedLength,
+                                  bufferLength: value.length });
+                }
+                const enumValue = value.toString('hex');
+                const actualTag = diversion ? TagEncoder[diversion].value : tag;
+                const enumInfo = TagDecoder[actualTag];
+                if (!enumInfo ||
+                    !enumInfo.enumeration ||
+                    !enumInfo.enumeration[enumValue]) {
+                    return { tag,
+                             value: enumValue,
+                             message: 'Unknown enumeration value',
+                             diversion,
+                           };
+                }
+                return enumInfo.enumeration[enumValue];
+            },
+            encode: (tagName, value, diversion) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type =
+                      Buffer.from(TypeEncoder['Enumeration'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(4);
+                const actualTag = diversion || tagName;
+                const encodedValue =
+                      Buffer.from(TagEncoder[actualTag].enumeration[value],
+                                  'hex');
+                return _ttlvPadVector([tag, type, length, encodedValue]);
+            },
+        },
+        '06': {
+            name: 'Boolean',
+            decode: (logger, tag, value) => {
+                const funcName = 'Boolean::decode';
+                const fixedLength = 8;
+                if (fixedLength !== value.length) {
+                    _throwError(logger,
+                                'Length mismatch',
+                                { funcName, fixedLength,
+                                  bufferLength: value.length });
+                }
+                const msUInt = value.readUInt32BE(0);
+                const lsUInt = value.readUInt32BE(4);
+                return !!(msUInt | lsUInt);
+            },
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type = Buffer.from(TypeEncoder['Boolean'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(8);
+                const encodedValue = Buffer.alloc(8);
+                encodedValue.writeUInt32BE(0, 0);
+                encodedValue.writeUInt32BE(value ? 1 : 0, 4);
+                return _ttlvPadVector([tag, type, length, encodedValue]);
+            },
+        },
+        '07': {
+            name: 'TextString',
+            decode: (logger, tag, value) => value.toString('utf8'),
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type =
+                      Buffer.from(TypeEncoder['TextString'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(value.length);
+                return _ttlvPadVector([tag, type, length,
+                               Buffer.from(value, 'utf8')]);
+            },
+        },
+        '08': {
+            name: 'ByteString',
+            decode: (logger, tag, value) => value,
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type =
+                      Buffer.from(TypeEncoder['ByteString'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(value.length);
+                return _ttlvPadVector([tag, type, length, value]);
+            },
+        },
+        '09': {
+            name: 'Date-Time',
+            decode: (logger, tag, value) => {
+                const funcName = 'Date-Time::decode';
+                const fixedLength = 8;
+                if (fixedLength !== value.length) {
+                    _throwError(logger,
+                                'Length mismatch',
+                                { funcName, fixedLength,
+                                  bufferLength: value.length });
+                }
+                const d = new Date(0);
+                const utcSeconds = UINT32_MAX * value.readUInt32BE(0) +
+                      value.readUInt32BE(4);
+                if (utcSeconds > Number.MAX_SAFE_INTEGER) {
+                    _throwError(logger,
+                                '53-bit overflow',
+                                { funcName, utcSeconds });
+                }
+                d.setUTCSeconds(utcSeconds);
+                return d;
+            },
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type = Buffer.from(TypeEncoder['Date-Time'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(8);
+                const encodedValue = Buffer.alloc(8);
+                const ts = value.getTime() / 1000;
+                encodedValue.writeUInt32BE(Math.floor(ts / UINT32_MAX), 0);
+                encodedValue.writeUInt32BE(ts % UINT32_MAX, 4);
+                return _ttlvPadVector([tag, type, length, encodedValue]);
+            },
+        },
+        '0a': {
+            name: 'Interval',
+            decode: (logger, tag, value) => {
+                const funcName = 'Interval::decode';
+                const fixedLength = 4;
+                if (fixedLength !== value.length) {
+                    _throwError(logger,
+                                'Length mismatch',
+                                { funcName, fixedLength,
+                                  bufferLength: value.length });
+                }
+                return value.readInt32BE(0);
+            },
+            encode: (tagName, value) => {
+                const tag = Buffer.from(TagEncoder[tagName].value, 'hex');
+                const type = Buffer.from(TypeEncoder['Interval'].value, 'hex');
+                const length = Buffer.alloc(4);
+                length.writeUInt32BE(4);
+                const encodedValue = Buffer.alloc(4);
+                encodedValue.writeUInt32BE(value);
+                return _ttlvPadVector([tag, type, length, encodedValue]);
+            },
+        },
+    };
+
+
+    /* Construct TagDecoder */
+    Object.keys(TagDecoder).forEach(key => {
+        const element = {};
+        element.value = key;
+        if (TagDecoder[key]['enumeration']) {
+            const enumeration = {};
+            Object.keys(TagDecoder[key]['enumeration']).forEach(enumValue => {
+                const enumKey = TagDecoder[key]['enumeration'][enumValue];
+                enumeration[enumKey] = enumValue;
+            });
+            element.enumeration = enumeration;
+        }
+        TagEncoder[TagDecoder[key].name] = element;
+    });
+
+
+    /* Construct TypeDecoder and TypeEncoder */
+    Object.keys(PrimitiveTypes).forEach(value => {
+        const name = PrimitiveTypes[value].name;
+        const encode = PrimitiveTypes[value].encode;
+        const decode = PrimitiveTypes[value].decode;
+        TypeDecoder[value] = { name, decode };
+        TypeEncoder[name] = { value, encode };
+    });
+
+
+    /* Public Methods Definition */
+
+    this.encodeMask = (tagName, value) => {
+        let mask = 0;
+        value.forEach(item => {
+            const enumValue = TagEncoder[tagName].enumeration[item];
+            if (!enumValue) {
+                throw Error('Invalid bit name');
+            }
+            mask |= parseInt(enumValue, 16);
+        });
+        return mask;
+    };
+
+    this.decodeMask = (tagName, givenMask) => {
+        let mask = givenMask;
+        const value = [];
+        const tag = TagEncoder[tagName].value;
+        Object.keys(TagDecoder[tag].enumeration).forEach(key => {
+            const bit = Buffer.from(key, 'hex').readUInt32BE(0);
+            if (bit & mask) {
+                mask &= ~bit;
+                value.push(TagDecoder[tag].enumeration[key]);
+            }
+        });
+        return value;
+    };
+
+    this.decode = (logger, rawMessage) => {
+        const messageContent =
+              TypeDecoder['01'].decode(logger, null, rawMessage);
+        return new KMIPMessage(messageContent);
+    };
+
+    this.encode = message => {
+        const value = message.content;
+        let result = [];
+        value.forEach(item => {
+            Object.keys(item).forEach(key => {
+                if (!TagEncoder[key]) {
+                    throw Error(`Unknown Tag '${key}'`);
+                }
+                const type = item[key].type;
+                if (!TypeEncoder[type]) {
+                    throw Error(`Unknown Type '${type}'`);
+                }
+                const itemValue = TypeEncoder[type].encode(key,
+                                                        item[key].value,
+                                                        item[key].diversion);
+                result = result.concat(_ttlvPadVector(itemValue));
+            });
+        });
+        return Buffer.concat(_ttlvPadVector(result));
+    };
+
+    this.mapExtension = (tagName, tagValue) => {
+        const tagValueStr = tagValue.toString(16);
+        TagDecoder[tagValueStr] = { name: tagName };
+        TagEncoder[tagName] = { value: tagValueStr };
+    };
+    return this;
+}
+
+module.exports = TTLVCodec;

lib/network/kmip/index.js

@@ -0,0 +1,350 @@
+'use strict'; // eslint-disable-line
+/* eslint new-cap: "off" */
+
+const uuidv4 = require('uuid/v4');
+
+const Message = require('./Message.js');
+
+/* This client requires at least a KMIP 1.2 compatible server */
+const DEFAULT_PROTOCOL_VERSION_MAJOR = 1;
+const DEFAULT_PROTOCOL_VERSION_MINOR = 2;
+/* Response is for one operation, consider raising this value if
+ * compounding ops */
+const DEFAULT_MAXIMUM_RESPONSE_SIZE = 8000;
+
+function _uniqueBatchItemID() {
+    const theUUID = Buffer.alloc(16);
+    return uuidv4(null, theUUID);
+}
+
+
+function _PrimitiveType(tagName, type, value) {
+    return { [tagName]: { type, value } };
+}
+
+class KMIP {
+    /**
+     * Construct a new KMIP Object
+     * @param {Class} Codec -
+     * @param {Class} Transport -
+     * @param {Object} options -
+     * @param {Function} cb -
+     */
+    constructor(Codec, Transport, options) {
+        this.protocolVersion = {
+            major: DEFAULT_PROTOCOL_VERSION_MAJOR,
+            minor: DEFAULT_PROTOCOL_VERSION_MINOR,
+        };
+        this.maximumResponseSize = DEFAULT_MAXIMUM_RESPONSE_SIZE;
+        this.options = options.kmip;
+        this.codec = new Codec(options.kmip.codec);
+        this.transport = new Transport(options.kmip.transport);
+    }
+
+    /* Static class methods */
+
+    /**
+     * create a new abstract message instance
+     * @param {Object} content - Most likely a call to KMIP.Structure
+     *                           with 'Request Message' as tagName
+     * @returns {Object} an instance of Message
+     */
+    static Message(content) {
+        return new Message(content);
+    }
+
+    /**
+     * Create a KMIP Structure field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Array} value - array of KMIP fields
+     * @returns {Object} an abstract KMIP field
+     */
+    static Structure(tagName, value) {
+        return _PrimitiveType(tagName, 'Structure', value);
+    }
+
+    /**
+     * Create a KMIP Integer field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Number} value - a number
+     * @returns {Object} an abstract KMIP field
+     */
+    static Integer(tagName, value) {
+        return _PrimitiveType(tagName, 'Integer', value);
+    }
+
+    /**
+     * Create a KMIP Long Integer field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Number} value - a number (beware of the 53-bit limitation)
+     * @returns {Object} an abstract KMIP field
+     */
+    static LongInteger(tagName, value) {
+        return _PrimitiveType(tagName, 'LongInteger', value);
+    }
+
+    /**
+     * Create a KMIP Big Integer field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Buffer} value - buffer containing the big integer
+     * @returns {Object} an abstract KMIP field
+     */
+    static BigInteger(tagName, value) {
+        if (value.length % 8 !== 0) {
+            throw Error('Big Integer value length must be a multiple of 8');
+        }
+        return _PrimitiveType(tagName, 'BigInteger', value);
+    }
+
+    /**
+     * Create a KMIP Enumeration field instance
+     * @param {String} tagName - Name of the KMIP Enumeration
+     * @param {String} value - Name of the KMIP Enumeration value
+     * @returns {Object} an abstract KMIP field
+     */
+    static Enumeration(tagName, value) {
+        return _PrimitiveType(tagName, 'Enumeration', value);
+    }
+
+    /**
+     * Create a KMIP Boolean field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Boolean} value - anything falsey or not (converted to a Boolean)
+     * @returns {Object} an abstract KMIP field
+     */
+    static Boolean(tagName, value) {
+        return _PrimitiveType(tagName, 'Boolean', !!value);
+    }
+
+    /**
+     * Create a KMIP Text String field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {String} value - the text string
+     * @returns {Object} an abstract KMIP field
+     */
+    static TextString(tagName, value) {
+        return _PrimitiveType(tagName, 'TextString', value);
+    }
+
+    /**
+     * Create a KMIP Byte String field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Buffer} value - buffer containing the byte string
+     * @returns {Object} an abstract KMIP field
+     */
+    static ByteString(tagName, value) {
+        return _PrimitiveType(tagName, 'ByteString', value);
+    }
+
+    /**
+     * Create a KMIP Date-Time field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Date} value - instance of a Date (ms are discarded)
+     * @returns {Object} an abstract KMIP field
+     */
+    static DateTime(tagName, value) {
+        value.setMilliseconds(0);
+        return _PrimitiveType(tagName, 'Date-Time', value);
+    }
+
+    /**
+     * Create a KMIP Interval field instance
+     * @param {String} tagName - Name of the KMIP field
+     * @param {Integer} value - number of seconds of the interval
+     * @returns {Object} an abstract KMIP field
+     */
+    static Interval(tagName, value) {
+        return _PrimitiveType(tagName, 'Interval', value);
+    }
+
+    /**
+     * Create a KMIP Attribute field instance
+     * @param {String} type - type of the attribute value
+     * @param {String} name - Name of the attribute or KMIP field
+     * @param {Object} value - value of the field suitable for the
+     *                         specified type
+     * @returns {Object} an abstract KMIP field
+     */
+    static Attribute(type, name, value) {
+        if (type === 'Date-Time') {
+            value.setMilliseconds(0);
+        }
+        return {
+            Attribute: {
+                type: 'Structure',
+                value: [
+                    {
+                        'Attribute Name': {
+                            type: 'TextString',
+                            value: name,
+                        },
+                    },
+                    {
+                        'Attribute Value': {
+                            type,
+                            value,
+                            diversion: name,
+                        },
+                    },
+                ],
+            },
+        };
+    }
+
+    /* Object methods */
+
+    /**
+     * Register a higher level handshake function to be called
+     * after the connection is initialized and before the first
+     * message is sent.
+     * @param {Function} handshakeFunction - (logger: Object, cb: Function(err))
+     * @returns {undefined}
+     */
+    registerHandshakeFunction(handshakeFunction) {
+        this.transport.registerHandshakeFunction(handshakeFunction);
+    }
+
+    /**
+     * Decode a raw message, usually received from the transport layer
+     * @param {Object} logger - a Logger instance
+     * @param {Buffer} rawMessage - the message to decode
+     * @returns {Object} the decoded message as an instance of KMIP.Message
+     */
+    _decodeMessage(logger, rawMessage) {
+        return this.codec.decode(logger, rawMessage);
+    }
+
+    /**
+     * Encode an message
+     * @param {Object} message - Instance of a KMIP.Message
+     * @returns {Buffer} the encoded message suitable for the transport layer
+     */
+    _encodeMessage(message) {
+        return this.codec.encode(message);
+    }
+
+    /**
+     * Decode a bitmask
+     * @param {string} tagName - name of the bit mask defining tag
+     * @param {Integer} givenMask - bit mask to decode
+     * @returns {Array} array of named bits set in the given bit mask
+     */
+    decodeMask(tagName, givenMask) {
+        return this.codec.decodeMask(tagName, givenMask);
+    }
+
+    /**
+     * Encode a bitmask
+     * @param {String} tagName - name of the bit mask defining tag
+     * @param {Array} value - array of named bits to set in the mask
+     * @returns {Integer} Integer encoded bitmask
+     */
+    encodeMask(tagName, value) {
+        return this.codec.encodeMask(tagName, value);
+    }
+
+    /**
+     * Amend the tag nomenclature with a vendor specific extension
+     * @param {String} extensionName - Name of the tag to record
+     * @param {Integer} extensionTag - Tag value represented as an integer
+     * @returns {undefined}
+     */
+    mapExtension(extensionName, extensionTag) {
+        return this.codec.mapExtension(extensionName, extensionTag);
+    }
+
+    changeProtocolVersion(major, minor) {
+        this.protocolVersion = { major, minor };
+    }
+
+    /**
+     * Send an operation request message to the KMIP Server
+     * @param {Object} logger - Werelog logger object
+     * @param {String} operation - The name of the operation as defined in
+     *                             the KMIP protocol specification.
+     * @param {Object} payload - payload of the operation request. Specifically
+     *                           the content of the Request Payload as defined
+     *                           by the KMIP protocol specification.
+     * @param {Function} cb - The callback(error: Object, response: Object)
+     * @returns {undefined}
+     */
+    request(logger, operation, payload, cb) {
+        const uuid = _uniqueBatchItemID();
+        const message = KMIP.Message([
+            KMIP.Structure('Request Message', [
+                KMIP.Structure('Request Header', [
+                    KMIP.Structure('Protocol Version', [
+                        KMIP.Integer('Protocol Version Major',
+                                     this.protocolVersion.major),
+                        KMIP.Integer('Protocol Version Minor',
+                                     this.protocolVersion.minor)]),
+                    KMIP.Integer('Maximum Response Size',
+                                 this.maximumResponseSize),
+                    KMIP.Integer('Batch Count', 1)]),
+                KMIP.Structure('Batch Item', [
+                    KMIP.Enumeration('Operation', operation),
+                    KMIP.ByteString('Unique Batch Item ID', uuid),
+                    KMIP.Structure('Request Payload', payload),
+                ])])]);
+        const encodedMessage = this._encodeMessage(message);
+        this.transport.send(
+            logger, encodedMessage,
+            (err, conversation, rawResponse) => {
+                if (err) {
+                    logger.error('KMIP::request: Failed to encode message',
+                                 { error: err });
+                    return cb(err);
+                }
+                const response = this._decodeMessage(logger, rawResponse);
+                const performedOperation =
+                      response.lookup('Response Message/' +
+                                      'Batch Item/Operation')[0];
+                const resultStatus =
+                      response.lookup('Response Message/' +
+                                      'Batch Item/Result Status')[0];
+                const resultUniqueBatchItemID =
+                      response.lookup('Response Message/' +
+                                      'Batch Item/Unique Batch Item ID')[0];
+
+                if (!resultUniqueBatchItemID ||
+                    resultUniqueBatchItemID.compare(uuid) !== 0) {
+                    this.transport.abortPipeline(conversation);
+                    const error = Error('Invalid batch item ID returned');
+                    logger.error('KMIP::request: failed',
+                                 { resultUniqueBatchItemID, uuid, error });
+                    return cb(error);
+                }
+                if (performedOperation !== operation) {
+                    this.transport.abortPipeline(conversation);
+                    const error = Error('Operation mismatch',
+                                        { got: performedOperation,
+                                         expected: operation });
+                    logger.error('KMIP::request: Operation mismatch',
+                                 { error });
+                    return cb(error);
+                }
+                if (resultStatus !== 'Success') {
+                    const resultReason =
+                          response.lookup(
+                              'Response Message/Batch Item/Result Reason')[0];
+                    const resultMessage =
+                          response.lookup(
+                              'Response Message/Batch Item/Result Message')[0];
+                    const error = Error('KMIP request failure',
+                                        { resultStatus,
+                                          resultReason,
+                                          resultMessage });
+                    logger.error('KMIP::request: request failed',
+                                 { error, resultStatus,
+                                   resultReason, resultMessage });
+                    return cb(error);
+                }
+                return cb(null, response);
+            });
+    }
+
+
+}
+
+
+module.exports = KMIP;

lib/network/kmip/tags.json

@@ -0,0 +1,579 @@
+{
+    "420006": {
+        "name": "Asynchronous Correlation Value"
+    },
+    "420007": {
+        "name": "Asynchronous Indicator"
+    },
+    "420008": {
+        "name": "Attribute"
+    },
+    "420009": {
+        "name": "Attribute Index"
+    },
+    "42000a": {
+        "name": "Attribute Name"
+    },
+    "42000b": {
+        "name": "Attribute Value"
+    },
+    "42000c": {
+        "name": "Authentication"
+    },
+    "42000d": {
+        "name": "Batch Count"
+    },
+    "42000f": {
+        "name": "Batch Item"
+    },
+    "420011": {
+        "name": "Block Cipher Mode",
+        "enumeration": {
+            "00000001": "CBC",
+            "00000002": "ECB",
+            "00000003": "PCBC",
+            "00000004": "CFB",
+            "00000005": "OFB",
+            "00000006": "CTR",
+            "00000007": "CMAC",
+            "00000008": "CCM",
+            "00000009": "GCM",
+            "0000000a": "CBC-MAC",
+            "0000000b": "XTS",
+            "0000000c": "AESKeyWrapPadding",
+            "0000000d": "NISTKeyWrap",
+            "0000000e": "X9.102 AESKW",
+            "0000000f": "X9.102 TDKW",
+            "00000010": "X9.102 AKW1",
+            "00000011": "X9.102 AKW2",
+            "00000012": "AEAD"
+        }
+    },
+    "420028": {
+        "name": "Cryptographic Algorithm",
+        "enumeration": {
+            "00000001": "DES",
+            "00000002": "3DES",
+            "00000003": "AES",
+            "00000004": "RSA",
+            "00000005": "DSA",
+            "00000006": "ECDSA",
+            "00000007": "HMAC-SHA1",
+            "00000008": "HMAC-SHA224",
+            "00000009": "HMAC-SHA256",
+            "0000000a": "HMAC-SHA384",
+            "0000000b": "HMAC-SHA512",
+            "0000000c": "HMAC-MD5",
+            "0000000d": "DH",
+            "0000000e": "ECDH",
+            "0000000f": "ECMQV",
+            "00000010": "Blowfish",
+            "00000011": "Camellia",
+            "00000012": "CAST5",
+            "00000013": "IDEA",
+            "00000014": "MARS",
+            "00000015": "RC2",
+            "00000016": "RC4",
+            "00000017": "RC5",
+            "00000018": "SKIPJACK",
+            "00000019": "Twofish",
+            "0000001a": "EC",
+            "0000001b": "One Time Pad",
+            "0000001c": "ChaCha20",
+            "0000001d": "Poly1305",
+            "0000001e": "ChaCha20Poly1305",
+            "0000001f": "SHA3-224",
+            "00000020": "SHA3-256",
+            "00000021": "SHA3-384",
+            "00000022": "SHA3-512",
+            "00000023": "HMAC-SHA3-224",
+            "00000024": "HMAC-SHA3-256",
+            "00000025": "HMAC-SHA3-384",
+            "00000026": "HMAC-SHA3-512",
+            "00000027": "SHAKE-128",
+            "00000028": "SHAKE-256"
+        }
+    },
+    "42002b": {
+        "name": "Cryptographic Parameters"
+    },
+    "42002c": {
+        "name": "Cryptographic Usage Mask",
+        "enumeration": {
+            "00000001": "Sign",
+            "00000002": "Verify",
+            "00000004": "Encrypt",
+            "00000008": "Decrypt",
+            "00000010": "Wrap Key",
+            "00000020": "Unwrap Key",
+            "00000040": "Export",
+            "00000080": "MAC Generate",
+            "00000100": "MAC Verify",
+            "00000200": "Derive Key",
+            "00000400": "Content Commitment",
+            "00000800": "Key Agreement",
+            "00001000": "Certificate Sign",
+            "00002000": "CRL Sign",
+            "00004000": "Generate Cryptogram",
+            "00008000": "Validate Cryptogram",
+            "00010000": "Translate Encrypt",
+            "00020000": "Translate Decrypt",
+            "00040000": "Translate Wrap",
+            "00080000": "Translate Unwrap"
+        }
+    },
+    "42003d": {
+        "name": "IV/Counter/Nonce"
+    },
+    "420050": {
+        "name": "Maximum Response Size"
+    },
+    "420054": {
+        "name": "Name Type",
+        "enumeration": {
+            "00000001": "Uninterpreted Text String",
+            "00000002": "URI"
+        }
+    },
+    "420057": {
+        "name": "Object Type",
+        "enumeration": {
+            "00000001": "Certificate",
+            "00000002": "Symmetric Key",
+            "00000003": "Public Key",
+            "00000004": "Private Key",
+            "00000005": "Split Key",
+            "00000006": "Template",
+            "00000007": "Secret Data",
+            "00000008": "Opaque Object",
+            "00000009": "PGP Key"
+        }
+    },
+    "42005c": {
+        "name": "Operation",
+        "enumeration": {
+            "00000001": "Create",
+            "00000002": "Create Key Pair",
+            "00000003": "Register",
+            "00000004": "Re-key",
+            "00000005": "Derive Key",
+            "00000006": "Certify",
+            "00000007": "Re-certify",
+            "00000008": "Locate",
+            "00000009": "Check",
+            "0000000a": "Get",
+            "0000000b": "Get Attributes",
+            "0000000c": "Get Attribute List",
+            "0000000d": "Add Attribute",
+            "0000000e": "Modify Attribute",
+            "0000000f": "Delete Attribute",
+            "00000010": "Obtain Lease",
+            "00000011": "Get Usage Allocation",
+            "00000012": "Activate",
+            "00000013": "Revoke",
+            "00000014": "Destroy",
+            "00000015": "Archive",
+            "00000016": "Recover",
+            "00000017": "Validate",
+            "00000018": "Query",
+            "00000019": "Cancel",
+            "0000001a": "Poll",
+            "0000001b": "Notify",
+            "0000001c": "Put",
+            "0000001d": "Re-key Key Pair",
+            "0000001e": "Discover Versions",
+            "0000001f": "Encrypt",
+            "00000020": "Decrypt",
+            "00000021": "Sign",
+            "00000022": "Signature Verify",
+            "00000023": "MAC",
+            "00000024": "MAC Verify",
+            "00000025": "RNG Retrieve",
+            "00000026": "RNG Seed",
+            "00000027": "Hash",
+            "00000028": "Create Split Key",
+            "00000029": "Join Split Key",
+            "0000002a": "Import",
+            "0000002b": "Export"
+        }
+    },
+    "42005f": {
+        "name": "Padding Method",
+        "enumeration": {
+            "00000001": "None",
+            "00000002": "OAEP",
+            "00000003": "PKCS5",
+            "00000004": "SSL3",
+            "00000005": "Zeros",
+            "00000006": "ANSI X9.23",
+            "00000007": "ISO 10126",
+            "00000008": "PKCS1 v1.5",
+            "00000009": "X9.31",
+            "0000000a": "PSS"
+        }
+    },
+    "420069": {
+        "name": "Protocol Version"
+    },
+    "42006a": {
+        "name": "Protocol Version Major"
+    },
+    "42006b": {
+        "name": "Protocol Version Minor"
+    },
+    "420074": {
+        "name": "Query Function",
+        "enumeration": {
+            "00000001": "Query Operations",
+            "00000002": "Query Objects",
+            "00000003": "Query Server Information",
+            "00000004": "Query Application Namespaces",
+            "00000005": "Query Extension List",
+            "00000006": "Query Extension Map",
+            "00000007": "Query Attestation Types",
+            "00000008": "Query RNGs",
+            "00000009": "Query Validations",
+            "0000000a": "Query Profiles",
+            "0000000b": "Query Capabilities",
+            "0000000c": "Query Client Registration Methods"
+        }
+    },
+    "420077": {
+        "name": "Request Header"
+    },
+    "420078": {
+        "name": "Request Message"
+    },
+    "420079": {
+        "name": "Request Payload"
+    },
+    "42007a": {
+        "name": "Response Header"
+    },
+    "42007b": {
+        "name": "Response Message"
+    },
+    "42007c": {
+        "name": "Response Payload"
+    },
+    "42007d": {
+        "name": "Result Message"
+    },
+    "42007e": {
+        "name": "Result Reason",
+        "enumeration": {
+            "00000001": "Item Not Found",
+            "00000002": "Response Too Large",
+            "00000003": "Authentication Not Successful",
+            "00000004": "Invalid Message",
+            "00000005": "Operation Not Supported",
+            "00000006": "Missing Data",
+            "00000007": "Invalid Field",
+            "00000008": "Feature Not Supported",
+            "00000009": "Operation Canceled By Requester",
+            "0000000a": "Cryptographic Failure",
+            "0000000b": "Illegal Operation",
+            "0000000c": "Permission Denied",
+            "0000000d": "Object archived",
+            "0000000e": "Index Out of Bounds",
+            "0000000f": "Application Namespace Not Supported",
+            "00000010": "Key Format Type Not Supported",
+            "00000011": "Key Compression Type Not Supported",
+            "00000012": "Encoding Option Error",
+            "00000013": "Key Value Not Present",
+            "00000014": "Attestation Required",
+            "00000015": "Attestation Failed",
+            "00000016": "Sensitive",
+            "00000017": "Not Extractable",
+            "00000018": "Object Already Exists",
+            "00000100": "General Failure"
+        }
+    },
+    "42007f": {
+        "name": "Result Status",
+        "enumeration": {
+            "00000000": "Success",
+            "00000001": "Operation Failed",
+            "00000002": "Operation Pending",
+            "00000003": "Operation Undone"
+        }
+    },
+    "420080": {
+        "name": "Revocation Message"
+    },
+    "420081": {
+        "name": "Revocation Reason"
+    },
+    "420082": {
+        "name": "Revocation Reason Code",
+        "enumeration": {
+            "00000001": "Unspecified",
+            "00000002": "Key Compromise",
+            "00000003": "CA Compromise",
+            "00000004": "Affiliation Changed",
+            "00000005": "Superseded",
+            "00000006": "Cessation of Operation",
+            "00000007": "Privilege Withdrawn"
+        }
+    },
+    "420088": {
+        "name": "Server Information"
+    },
+    "420091": {
+        "name": "Template-Attribute"
+    },
+    "420092": {
+        "name": "Time Stamp"
+    },
+    "420093": {
+        "name": "Unique Batch Item ID"
+    },
+    "420094": {
+        "name": "Unique Identifier"
+    },
+    "42009d": {
+        "name": "Vendor Identification"
+    },
+    "4200a4": {
+        "name": "Extension Information"
+    },
+    "4200a5": {
+        "name": "Extension Name"
+    },
+    "4200a6": {
+        "name": "Extension Tag"
+    },
+    "4200a7": {
+        "name": "Extension Type"
+    },
+    "4200c2": {
+        "name": "Data"
+    },
+    "4200eb": {
+        "name": "Profile Information"
+    },
+    "4200ec": {
+        "name": "Profile Name",
+        "enumeration": {
+            "00000001": "Baseline Server Basic KMIP v1.2",
+            "00000002": "Baseline Server TLS v1.2 KMIP v1.2",
+            "00000003": "Baseline Client Basic KMIP v1.2",
+            "00000004": "Baseline Client TLS v1.2 KMIP v1.2",
+            "00000005": "Complete Server Basic KMIP v1.2",
+            "00000006": "Complete Server TLS v1.2 KMIP v1.2",
+            "00000007": "Tape Library Client KMIP v1.0",
+            "00000008": "Tape Library Client KMIP v1.1",
+            "00000009": "Tape Library Client KMIP v1.2",
+            "0000000a": "Tape Library Server KMIP v1.0",
+            "0000000b": "Tape Library Server KMIP v1.1",
+            "0000000c": "Tape Library Server KMIP v1.2",
+            "0000000d": "Symmetric Key Lifecycle Client KMIP v1.0",
+            "0000000e": "Symmetric Key Lifecycle Client KMIP v1.1",
+            "0000000f": "Symmetric Key Lifecycle Client KMIP v1.2",
+            "00000010": "Symmetric Key Lifecycle Server KMIP v1.0",
+            "00000011": "Symmetric Key Lifecycle Server KMIP v1.1",
+            "00000012": "Symmetric Key Lifecycle Server KMIP v1.2",
+            "00000013": "Asymmetric Key Lifecycle Client KMIP v1.0",
+            "00000014": "Asymmetric Key Lifecycle Client KMIP v1.1",
+            "00000015": "Asymmetric Key Lifecycle Client KMIP v1.2",
+            "00000016": "Asymmetric Key Lifecycle Server KMIP v1.0",
+            "00000017": "Asymmetric Key Lifecycle Server KMIP v1.1",
+            "00000018": "Asymmetric Key Lifecycle Server KMIP v1.2",
+            "00000019": "Basic Cryptographic Client KMIP v1.2",
+            "0000001a": "Basic Cryptographic Server KMIP v1.2",
+            "0000001b": "Advanced Cryptographic Client KMIP v1.2",
+            "0000001c": "Advanced Cryptographic Server KMIP v1.2",
+            "0000001d": "RNG Cryptographic Client KMIP v1.2",
+            "0000001e": "RNG Cryptographic Server KMIP v1.2",
+            "0000001f": "Basic Symmetric Key Foundry Client KMIP v1.0",
+            "00000020": "Intermediate Symmetric Key Foundry Client KMIP v1.0",
+            "00000021": "Advanced Symmetric Key Foundry Client KMIP v1.0",
+            "00000022": "Basic Symmetric Key Foundry Client KMIP v1.1",
+            "00000023": "Intermediate Symmetric Key Foundry Client KMIP v1.1",
+            "00000024": "Advanced Symmetric Key Foundry Client KMIP v1.1",
+            "00000025": "Basic Symmetric Key Foundry Client KMIP v1.2",
+            "00000026": "Intermediate Symmetric Key Foundry Client KMIP v1.2",
+            "00000027": "Advanced Symmetric Key Foundry Client KMIP v1.2",
+            "00000028": "Symmetric Key Foundry Server KMIP v1.0",
+            "00000029": "Symmetric Key Foundry Server KMIP v1.1",
+            "0000002a": "Symmetric Key Foundry Server KMIP v1.2",
+            "0000002b": "Opaque Managed Object Store Client KMIP v1.0",
+            "0000002c": "Opaque Managed Object Store Client KMIP v1.1",
+            "0000002d": "Opaque Managed Object Store Client KMIP v1.2",
+            "0000002e": "Opaque Managed Object Store Server KMIP v1.0",
+            "0000002f": "Opaque Managed Object Store Server KMIP v1.1",
+            "00000030": "Opaque Managed Object Store Server KMIP v1.2",
+            "00000031": "Suite B minLOS_128 Client KMIP v1.0",
+            "00000032": "Suite B minLOS_128 Client KMIP v1.1",
+            "00000033": "Suite B minLOS_128 Client KMIP v1.2",
+            "00000034": "Suite B minLOS_128 Server KMIP v1.0",
+            "00000035": "Suite B minLOS_128 Server KMIP v1.1",
+            "00000036": "Suite B minLOS_128 Server KMIP v1.2",
+            "00000037": "Suite B minLOS_192 Client KMIP v1.0",
+            "00000038": "Suite B minLOS_192 Client KMIP v1.1",
+            "00000039": "Suite B minLOS_192 Client KMIP v1.2",
+            "0000003a": "Suite B minLOS_192 Server KMIP v1.0",
+            "0000003b": "Suite B minLOS_192 Server KMIP v1.1",
+            "0000003c": "Suite B minLOS_192 Server KMIP v1.2",
+            "0000003d": "Storage Array with Self Encrypting Drive Client KMIP v1.0",
+            "0000003e": "Storage Array with Self Encrypting Drive Client KMIP v1.1",
+            "0000003f": "Storage Array with Self Encrypting Drive Client KMIP v1.2",
+            "00000040": "Storage Array with Self Encrypting Drive Server KMIP v1.0",
+            "00000041": "Storage Array with Self Encrypting Drive Server KMIP v1.1",
+            "00000042": "Storage Array with Self Encrypting Drive Server KMIP v1.2",
+            "00000043": "HTTPS Client KMIP v1.0",
+            "00000044": "HTTPS Client KMIP v1.1",
+            "00000045": "HTTPS Client KMIP v1.2",
+            "00000046": "HTTPS Server KMIP v1.0",
+            "00000047": "HTTPS Server KMIP v1.1",
+            "00000048": "HTTPS Server KMIP v1.2",
+            "00000049": "JSON Client KMIP v1.0",
+            "0000004a": "JSON Client KMIP v1.1",
+            "0000004b": "JSON Client KMIP v1.2",
+            "0000004c": "JSON Server KMIP v1.0",
+            "0000004d": "JSON Server KMIP v1.1",
+            "0000004e": "JSON Server KMIP v1.2",
+            "0000004f": "XML Client KMIP v1.0",
+            "00000050": "XML Client KMIP v1.1",
+            "00000051": "XML Client KMIP v1.2",
+            "00000052": "XML Server KMIP v1.0",
+            "00000053": "XML Server KMIP v1.1",
+            "00000054": "XML Server KMIP v1.2",
+            "00000055": "Baseline Server Basic KMIP v1.3",
+            "00000056": "Baseline Server TLS v1.2 KMIP v1.3",
+            "00000057": "Baseline Client Basic KMIP v1.3",
+            "00000058": "Baseline Client TLS v1.2 KMIP v1.3",
+            "00000059": "Complete Server Basic KMIP v1.3",
+            "0000005a": "Complete Server TLS v1.2 KMIP v1.3",
+            "0000005b": "Tape Library Client KMIP v1.3",
+            "0000005c": "Tape Library Server KMIP v1.3",
+            "0000005d": "Symmetric Key Lifecycle Client KMIP v1.3",
+            "0000005e": "Symmetric Key Lifecycle Server KMIP v1.3",
+            "0000005f": "Asymmetric Key Lifecycle Client KMIP v1.3",
+            "00000060": "Asymmetric Key Lifecycle Server KMIP v1.3",
+            "00000061": "Basic Cryptographic Client KMIP v1.3",
+            "00000062": "Basic Cryptographic Server KMIP v1.3",
+            "00000063": "Advanced Cryptographic Client KMIP v1.3",
+            "00000064": "Advanced Cryptographic Server KMIP v1.3",
+            "00000065": "RNG Cryptographic Client KMIP v1.3",
+            "00000066": "RNG Cryptographic Server KMIP v1.3",
+            "00000067": "Basic Symmetric Key Foundry Client KMIP v1.3",
+            "00000068": "Intermediate Symmetric Key Foundry Client KMIP v1.3",
+            "00000069": "Advanced Symmetric Key Foundry Client KMIP v1.3",
+            "0000006a": "Symmetric Key Foundry Server KMIP v1.3",
+            "0000006b": "Opaque Managed Object Store Client KMIP v1.3",
+            "0000006c": "Opaque Managed Object Store Server KMIP v1.3",
+            "0000006d": "Suite B minLOS_128 Client KMIP v1.3",
+            "0000006e": "Suite B minLOS_128 Server KMIP v1.3",
+            "0000006f": "Suite B minLOS_192 Client KMIP v1.3",
+            "00000070": "Suite B minLOS_192 Server KMIP v1.3",
+            "00000071": "Storage Array with Self Encrypting Drive Client KMIP v1.3",
+            "00000072": "Storage Array with Self Encrypting Drive Server KMIP v1.3",
+            "00000073": "HTTPS Client KMIP v1.3",
+            "00000074": "HTTPS Server KMIP v1.3",
+            "00000075": "JSON Client KMIP v1.3",
+            "00000076": "JSON Server KMIP v1.3",
+            "00000077": "XML Client KMIP v1.3",
+            "00000078": "XML Server KMIP v1.3",
+            "00000079": "Baseline Server Basic KMIP v1.4",
+            "0000007a": "Baseline Server TLS v1.2 KMIP v1.4",
+            "0000007b": "Baseline Client Basic KMIP v1.4",
+            "0000007c": "Baseline Client TLS v1.2 KMIP v1.4",
+            "0000007d": "Complete Server Basic KMIP v1.4",
+            "0000007e": "Complete Server TLS v1.2 KMIP v1.4",
+            "0000007f": "Tape Library Client KMIP v1.4",
+            "00000080": "Tape Library Server KMIP v1.4",
+            "00000081": "Symmetric Key Lifecycle Client KMIP v1.4",
+            "00000082": "Symmetric Key Lifecycle Server KMIP v1.4",
+            "00000083": "Asymmetric Key Lifecycle Client KMIP v1.4",
+            "00000084": "Asymmetric Key Lifecycle Server KMIP v1.4",
+            "00000085": "Basic Cryptographic Client KMIP v1.4",
+            "00000086": "Basic Cryptographic Server KMIP v1.4",
+            "00000087": "Advanced Cryptographic Client KMIP v1.4",
+            "00000088": "Advanced Cryptographic Server KMIP v1.4",
+            "00000089": "RNG Cryptographic Client KMIP v1.4",
+            "0000008a": "RNG Cryptographic Server KMIP v1.4",
+            "0000008b": "Basic Symmetric Key Foundry Client KMIP v1.4",
+            "0000008c": "Intermediate Symmetric Key Foundry Client KMIP v1.4",
+            "0000008d": "Advanced Symmetric Key Foundry Client KMIP v1.4",
+            "0000008e": "Symmetric Key Foundry Server KMIP v1.4",
+            "0000008f": "Opaque Managed Object Store Client KMIP v1.4",
+            "00000090": "Opaque Managed Object Store Server KMIP v1.4",
+            "00000091": "Suite B minLOS_128 Client KMIP v1.4",
+            "00000092": "Suite B minLOS_128 Server KMIP v1.4",
+            "00000093": "Suite B minLOS_192 Client KMIP v1.4",
+            "00000094": "Suite B minLOS_192 Server KMIP v1.4",
+            "00000095": "Storage Array with Self Encrypting Drive Client KMIP v1.4",
+            "00000096": "Storage Array with Self Encrypting Drive Server KMIP v1.4",
+            "00000097": "HTTPS Client KMIP v1.4",
+            "00000098": "HTTPS Server KMIP v1.4",
+            "00000099": "JSON Client KMIP v1.4",
+            "0000009a": "JSON Server KMIP v1.4",
+            "0000009b": "XML Client KMIP v1.4",
+            "0000009c": "XML Server KMIP v1.4"
+        }
+    },
+    "4200ed": {
+        "name": "Server URI"
+    },
+    "4200ee": {
+        "name": "Server Port"
+    },
+    "4200ef": {
+        "name": "Streaming Capability"
+    },
+    "4200f0": {
+        "name": "Asynchronous Capability"
+    },
+    "4200f1": {
+        "name": "Attestation Capability"
+    },
+    "4200f2": {
+        "name": "Unwrap Mode",
+        "enumeration": {
+            "00000001": "Unspecified",
+            "00000002": "Processed",
+            "00000003": "Not Processed"
+        }
+    },
+    "4200f3": {
+        "name": "Destroy Action",
+        "enumeration": {
+            "00000001": "Unspecified",
+            "00000002": "Key Material Deleted",
+            "00000003": "Key Material Shredded",
+            "00000004": "Meta Data Deleted",
+            "00000005": "Meta Data Shredded",
+            "00000006": "Deleted",
+            "00000007": "Shredded"
+        }
+    },
+    "4200f4": {
+        "name": "Shredding Algorithm",
+        "enumeration": {
+            "00000001": "Unspecified",
+            "00000002": "Cryptographic",
+            "00000003": "Unsupported"
+        }
+    },
+    "4200f5": {
+        "name": "RNG Mode",
+        "enumeration": {
+            "00000001": "Unspecified",
+            "00000002": "Shared Instantiation",
+            "00000003": "Non-Shared Instantiation"
+        }
+    },
+    "4200f6": {
+        "name": "Client Registration Method"
+    },
+    "4200f7": {
+        "name": "Capability Information"
+    },
+    "420105": {
+        "name": "Client Correlation Value"
+    },
+    "420106": {
+        "name": "Server Correlation Value"
+    }
+}

lib/network/kmip/transport/TransportTemplate.js

@@ -0,0 +1,181 @@
+'use strict'; // eslint-disable-line
+
+const assert = require('assert');
+
+const DEFAULT_PIPELINE_DEPTH = 8;
+const DEFAULT_KMIP_PORT = 5696;
+
+class TransportTemplate {
+    /**
+     * Construct a new object of the TransportTemplate class
+     * @param {Object} channel - Typically the tls object
+     * @param {Object} options - Instance options
+     * @param {Number} options.pipelineDepth - depth of the pipeline
+     * @param {Object} options.tls - Standard TLS socket initialization
+     *                               parameters
+     * @param {Number} options.tls.port - TLS server port to connect to
+     */
+    constructor(channel, options) {
+        this.channel = channel;
+        this.options = options;
+        this.pipelineDepth = Math.max(1, options.pipelineDepth ||
+                                      DEFAULT_PIPELINE_DEPTH);
+        this.callbackPipeline = [];
+        this.deferedRequests = [];
+        this.pipelineDrainedCallback = null;
+        this.handshakeFunction = null;
+        this.socket = null;
+    }
+
+    /**
+     * Drain the outstanding and defered request queues by
+     * calling the associated callback with an error
+     * @param {Error} error - the error to call the callback function with.
+     * @returns {undefined}
+     */
+    _drainQueuesWithError(error) {
+        this.callbackPipeline.forEach(queuedCallback => {
+            queuedCallback(error);
+        });
+        this.deferedRequests.forEach(deferedRequest => {
+            deferedRequest.cb(error);
+        });
+        this.callbackPipeline = [];
+        this.deferedRequests = [];
+    }
+
+    /**
+     * Register a higher level handshake function to be called
+     * after the connection is initialized and before the first
+     * message is sent.
+     * @param {Function} handshakeFunction - (logger: Object, cb: Function(err))
+     * @returns {undefined}
+     */
+    registerHandshakeFunction(handshakeFunction) {
+        this.handshakeFunction = handshakeFunction;
+    }
+
+    /**
+     * Create a new conversation (e.g. a socket) between the client
+     * and the server.
+     * @param {Object} logger - Werelogs logger object
+     * @param {Function} readyCallback - callback function to call when the
+     *                                   conversation is ready to be initiated
+     *                                   func(err: Error)
+     * @returns {undefined}
+     */
+    _createConversation(logger, readyCallback) {
+        try {
+            const socket = this.channel.connect(
+                this.options.tls.port || DEFAULT_KMIP_PORT,
+                this.options.tls,
+                () => {
+                    socket.on('data', data => {
+                        const queuedCallback = this.callbackPipeline.shift();
+                        queuedCallback(null, socket, data);
+
+                        if (this.callbackPipeline.length <
+                            this.pipelineDepth &&
+                            this.deferedRequests.length > 0) {
+                            const deferedRequest = this.deferedRequests.shift();
+                            process.nextTick(() => {
+                                this.send(logger,
+                                          deferedRequest.encodedMessage,
+                                          deferedRequest.cb);
+                            });
+                        } else if (this.callbackPipeline.length === 0 &&
+                                   this.deferedRequests.length === 0 &&
+                                   this.pipelineDrainedCallback) {
+                            this.pipelineDrainedCallback();
+                            this.pipelineDrainedCallback = null;
+                        }
+                    });
+                    socket.on('end', () => {
+                        const error = Error('Conversation interrupted');
+                        this._drainQueuesWithError(error);
+                        this.socket = null;
+                    });
+                    socket.on('error', err => {
+                        this._drainQueuesWithError(err);
+                    });
+                    if (this.handshakeFunction) {
+                        this.handshakeFunction(logger, readyCallback);
+                    } else {
+                        readyCallback(null);
+                    }
+                });
+            this.socket = socket;
+        } catch (err) {
+            logger.error();
+            readyCallback(err);
+        }
+    }
+
+    _doSend(logger, encodedMessage, cb) {
+        const socket = this.socket;
+        if (!socket) {
+            const error = new Error('Socket to server not available');
+            logger.error('TransportTemplate::_doSend', { error });
+            return cb(error);
+        }
+        this.callbackPipeline.push(cb);
+        socket.cork();
+        socket.write(encodedMessage);
+        socket.uncork();
+        return undefined;
+    }
+
+    /**
+     * Send an encoded message to the server
+     * @param {Object} logger - Werelogs logger object
+     * @param {Buffer} encodedMessage - the encoded message to send to the
+     *                                  server
+     * @param {Function} cb - (err, conversation, rawResponse)
+     * @returns {undefined}
+     */
+    send(logger, encodedMessage, cb) {
+        if (this.callbackPipeline.length >= this.pipelineDepth) {
+            return this.deferedRequests.push({ encodedMessage, cb });
+        }
+        assert(encodedMessage.length !== 0);
+        if (this.socket === null) {
+            return this._createConversation(logger, err => {
+                if (err) {
+                    return cb(err);
+                }
+                return this._doSend(logger, encodedMessage, cb);
+            });
+        }
+        return this._doSend(logger, encodedMessage, cb);
+    }
+
+    /**
+     * Gracefuly interrupt the conversation. If the caller keeps sending
+     * message after calling this function, the conversation won't
+     * converge to its end.
+     * @returns {undefined}
+     */
+    end() {
+        if (!this.socket) {
+            return;
+        }
+        if (this.callbackPipeline.length !== 0 ||
+            this.deferedRequests.length !== 0) {
+            this.pipelineDrainedCallback = this.socket.end.bind(this.socket);
+        } else {
+            this.socket.end();
+        }
+    }
+
+    /**
+     * Abruptly interrupt the conversation and cancel the outstanding and
+     * defered requests
+     * @param {Object} conversation - the conversation to abort
+     * @returns {undefined}
+     */
+    abortPipeline(conversation) {
+        conversation.end();
+    }
+}
+
+module.exports = TransportTemplate;

lib/network/kmip/transport/tls.js

@@ -0,0 +1,12 @@
+'use strict'; // eslint-disable-line
+
+const tls = require('tls');
+const TransportTemplate = require('./TransportTemplate.js');
+
+class TlsTransport extends TransportTemplate {
+    constructor(options) {
+        super(tls, options);
+    }
+}
+
+module.exports = TlsTransport;

lib/network/probe/HealthProbeServer.js

@@ -0,0 +1,97 @@
+const httpServer = require('../http/server');
+const werelogs = require('werelogs');
+const errors = require('../../errors');
+
+function sendError(res, log, error, optMessage) {
+    res.writeHead(error.code);
+    let message;
+    if (optMessage) {
+        message = optMessage;
+    } else {
+        message = error.description || '';
+    }
+    log.debug('sending back error response', { httpCode: error.code,
+        errorType: error.message,
+        error: message });
+    res.end(`${JSON.stringify({ errorType: error.message,
+        errorMessage: message })}\n`);
+}
+
+function sendSuccess(res, log, msg) {
+    res.writeHead(200);
+    log.debug('replying with success');
+    const message = msg || 'OK';
+    res.end(message);
+}
+
+function constructEndpoints(ns, path) {
+    return `/${ns}/${path}`;
+}
+
+function checkStub(log) { // eslint-disable-line
+    return true;
+}
+
+class HealthProbeServer extends httpServer {
+    constructor(params) {
+        const logging = new werelogs.Logger('HealthProbeServer');
+        super(params.port, logging);
+        this.logging = logging;
+        this.setBindAddress(params.bindAddress || 'localhost');
+        this._namespace = params.namespace || '_/health';
+        const livenessURI = constructEndpoints(this._namespace,
+            params.livenessURI || 'liveness');
+        const readinessURI = constructEndpoints(this._namespace,
+            params.readinessURI || 'readiness');
+        // hooking our request processing function by calling the
+        // parent's method for that
+        this.onRequest(this._onRequest);
+        this._reqHandlers = {};
+        this._reqHandlers[livenessURI] = this._onLiveness.bind(this);
+        this._reqHandlers[readinessURI] = this._onReadiness.bind(this);
+        this._livenessCheck = params.livenessCheck || checkStub;
+        this._readinessCheck = params.readinessCheck || checkStub;
+    }
+
+    onLiveCheck(f) {
+        this._livenessCheck = f;
+    }
+
+    onReadyCheck(f) {
+        this._readinessCheck = f;
+    }
+
+    _onRequest(req, res) {
+        const log = this.logging.newRequestLogger();
+        log.debug('request received', { method: req.method,
+            url: req.url });
+        if (req.method !== 'GET') {
+            sendError(res, log, errors.MethodNotAllowed);
+        }
+        if (req.url.startsWith(`/${this._namespace}`) &&
+            req.url in this._reqHandlers) {
+            this._reqHandlers[req.url](req, res, log);
+        } else {
+            sendError(res, log, errors.InvalidURI);
+        }
+    }
+
+    _onLiveness(req, res, log) {
+        if (this._livenessCheck(log)) {
+            sendSuccess(res, log);
+        } else {
+            sendError(res, log, errors.ServiceUnavailable);
+        }
+    }
+
+    _onReadiness(req, res, log) {
+        if (this._readinessCheck(log)) {
+            sendSuccess(res, log);
+        } else {
+            sendError(res, log, errors.ServiceUnavailable);
+        }
+    }
+
+}
+
+module.exports = HealthProbeServer;

lib/network/rest/RESTServer.js

@@ -214,7 +214,7 @@ class RESTServer extends httpServer {
         if (req.url.startsWith(`${constants.dataFileURL}?`)) {
             const queryParam = url.parse(req.url).query;
             if (queryParam === 'diskUsage') {
-                this.dataStore.getDiskUsage((err, result) => {
+                return this.dataStore.getDiskUsage((err, result) => {
                     if (err) {
                         return sendError(res, log, err);
                     }

lib/s3middleware/validateConditionalHeaders.js

@@ -68,6 +68,31 @@ function _checkUnmodifiedSince(ifUnmodifiedSinceTime, lastModified) {
     return res;
 }
 
+/**
+ * checks 'if-modified-since' and 'if-unmodified-since' headers if included in
+ * request against last-modified date of object
+ * @param {object} headers - headers from request object
+ * @param {string} lastModified - last modified date of object
+ * @return {object} contains modifiedSince and unmodifiedSince res objects
+ */
+function checkDateModifiedHeaders(headers, lastModified) {
+    let lastModifiedDate = new Date(lastModified);
+    lastModifiedDate.setMilliseconds(0);
+    lastModifiedDate = lastModifiedDate.getTime();
+
+    const ifModifiedSinceHeader = headers['if-modified-since'] ||
+        headers['x-amz-copy-source-if-modified-since'];
+    const ifUnmodifiedSinceHeader = headers['if-unmodified-since'] ||
+        headers['x-amz-copy-source-if-unmodified-since'];
+
+    const modifiedSinceRes = _checkModifiedSince(ifModifiedSinceHeader,
+        lastModifiedDate);
+    const unmodifiedSinceRes = _checkUnmodifiedSince(ifUnmodifiedSinceHeader,
+        lastModifiedDate);
+
+    return { modifiedSinceRes, unmodifiedSinceRes };
+}
+
 /**
  * validateConditionalHeaders - validates 'if-modified-since',
  * 'if-unmodified-since', 'if-match' or 'if-none-match' headers if included in
@@ -79,23 +104,14 @@ function _checkUnmodifiedSince(ifUnmodifiedSinceTime, lastModified) {
  * empty object if no error
  */
 function validateConditionalHeaders(headers, lastModified, contentMD5) {
-    let lastModifiedDate = new Date(lastModified);
-    lastModifiedDate.setMilliseconds(0);
-    lastModifiedDate = lastModifiedDate.getTime();
     const ifMatchHeader = headers['if-match'] ||
         headers['x-amz-copy-source-if-match'];
     const ifNoneMatchHeader = headers['if-none-match'] ||
         headers['x-amz-copy-source-if-none-match'];
-    const ifModifiedSinceHeader = headers['if-modified-since'] ||
-        headers['x-amz-copy-source-if-modified-since'];
-    const ifUnmodifiedSinceHeader = headers['if-unmodified-since'] ||
-        headers['x-amz-copy-source-if-unmodified-since'];
     const etagMatchRes = _checkEtagMatch(ifMatchHeader, contentMD5);
     const etagNoneMatchRes = _checkEtagNoneMatch(ifNoneMatchHeader, contentMD5);
-    const modifiedSinceRes = _checkModifiedSince(ifModifiedSinceHeader,
-        lastModifiedDate);
-    const unmodifiedSinceRes = _checkUnmodifiedSince(ifUnmodifiedSinceHeader,
-        lastModifiedDate);
+    const { modifiedSinceRes, unmodifiedSinceRes } =
+        checkDateModifiedHeaders(headers, lastModified);
     // If-Unmodified-Since condition evaluates to false and If-Match
     // is not present, then return the error. Otherwise, If-Unmodified-Since is
     // silent when If-Match match, and when If-Match does not match, it's the
@@ -120,5 +136,6 @@ module.exports = {
     _checkEtagNoneMatch,
     _checkModifiedSince,
     _checkUnmodifiedSince,
+    checkDateModifiedHeaders,
     validateConditionalHeaders,
 };

lib/s3routes/routes.js

@@ -10,6 +10,8 @@ const routeOPTIONS = require('./routes/routeOPTIONS');
 const routesUtils = require('./routesUtils');
 const routeWebsite = require('./routes/routeWebsite');
 
+const { objectKeyByteLimit } = require('../constants');
+
 const routeMap = {
     GET: routeGET,
     PUT: routePUT,
@@ -55,8 +57,14 @@ function checkBucketAndKey(bucketName, objectKey, method, reqQuery,
             blacklistedPrefixes.object);
         if (!result.isValid) {
             log.debug('invalid object key', { objectKey });
-            return errors.InvalidArgument.customizeDescription('Object key ' +
-            `must not start with "${result.invalidPrefix}".`);
+            if (result.invalidPrefix) {
+                return errors.InvalidArgument.customizeDescription('Invalid ' +
+                    'prefix - object key cannot start with ' +
+                    `"${result.invalidPrefix}".`);
+            }
+            return errors.KeyTooLong.customizeDescription('Object key is too ' +
+                'long. Maximum number of bytes allowed in keys is ' +
+                `${objectKeyByteLimit}.`);
         }
     }
     if ((reqQuery.partNumber || reqQuery.uploadId)
@@ -167,7 +175,8 @@ function routes(req, res, params, logger) {
                  logger.newRequestLoggerFromSerializedUids(reqUids) :
                  logger.newRequestLogger());
 
-    if (!req.url.startsWith('/_/healthcheck')) {
+    if (!req.url.startsWith('/_/healthcheck') &&
+        !req.url.startsWith('/_/report')) {
         log.info('received request', clientInfo);
     }
 

lib/s3routes/routes/routeDELETE.js

@@ -45,6 +45,13 @@ function routeDELETE(request, response, api, log, statsClient) {
                 return routesUtils.responseNoBody(err, corsHeaders,
                     response, 204, log);
             });
+        } else if (request.query.policy !== undefined) {
+            return api.callApiMethod('bucketDeletePolicy', request,
+            response, log, (err, corsHeaders) => {
+                routesUtils.statsReport500(err, statsClient);
+                return routesUtils.responseNoBody(err, corsHeaders,
+                    response, 204, log);
+            });
         }
         api.callApiMethod('bucketDelete', request, response, log,
         (err, corsHeaders) => {

lib/s3routes/routes/routeGET.js

@@ -71,6 +71,13 @@ function routerGET(request, response, api, log, statsClient, dataRetrievalFn) {
                     return routesUtils.responseXMLBody(err, xml, response, log,
                       corsHeaders);
                 });
+        } else if (request.query.policy !== undefined) {
+            api.callApiMethod('bucketGetPolicy', request, response, log,
+                (err, json, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    return routesUtils.responseJSONBody(err, json, response,
+                        log, corsHeaders);
+                });
         } else {
             // GET bucket
             api.callApiMethod('bucketGet', request, response, log,

lib/s3routes/routes/routePUT.js

@@ -60,6 +60,13 @@ function routePUT(request, response, api, log, statsClient) {
                     routesUtils.responseNoBody(err, corsHeaders, response, 200,
                         log);
                 });
+        } else if (request.query.policy !== undefined) {
+            api.callApiMethod('bucketPutPolicy', request, response, log,
+                (err, corsHeaders) => {
+                    routesUtils.statsReport500(err, statsClient);
+                    routesUtils.responseNoBody(err, corsHeaders, response, 200,
+                        log);
+                });
         } else {
             // PUT bucket
             return api.callApiMethod('bucketPut', request, response, log,

lib/s3routes/routesUtils.js

@@ -4,6 +4,8 @@ const errors = require('../errors');
 const constants = require('../constants');
 const { eachSeries } = require('async');
 
+const { objectKeyByteLimit } = require('../constants');
+
 const responseErr = new Error();
 responseErr.code = 'ResponseError';
 responseErr.message = 'response closed by client request before all data sent';
@@ -284,13 +286,19 @@ function retrieveData(locations, retrieveDataFn, response, log) {
         response.destroy();
         responseDestroyed = true;
     };
+
+    const _destroyReadable = readable => {
+        // s3-data sends Readable stream only which does not implement destroy
+        if (readable && readable.destroy) {
+            readable.destroy();
+        }
+    };
+
     // the S3-client might close the connection while we are processing it
     response.once('close', () => {
         log.debug('received close event before response end');
         responseDestroyed = true;
-        if (currentStream) {
-            currentStream.destroy();
-        }
+        _destroyReadable(currentStream);
     });
 
     return eachSeries(locations,
@@ -311,7 +319,7 @@ function retrieveData(locations, retrieveDataFn, response, log) {
                 if (responseDestroyed || response.isclosed) {
                     log.debug(
                         'response destroyed before readable could stream');
-                    readable.destroy();
+                    _destroyReadable(readable);
                     return next(responseErr);
                 }
                 // readable stream successfully consumed
@@ -868,6 +876,9 @@ const routesUtils = {
         if (invalidPrefix) {
             return { isValid: false, invalidPrefix };
         }
+        if (Buffer.byteLength(objectKey, 'utf8') > objectKeyByteLimit) {
+            return { isValid: false };
+        }
         return { isValid: true };
     },
 

lib/storage/data/file/DataFileStore.js

@@ -10,6 +10,7 @@ const errors = require('../../../errors');
 const stringHash = require('../../../stringHash');
 const jsutil = require('../../../jsutil');
 const storageUtils = require('../../utils');
+const releasePageCacheSync = require('./utils');
 
 // The FOLDER_HASH constant refers to the number of base directories
 // used for directory hashing of stored objects.
@@ -43,6 +44,8 @@ class DataFileStore {
      *   sync calls that ensure files and directories are fully
      *   written on the physical drive before returning an
      *   answer. Used to speed up unit tests, may have other uses.
+     * @param {Boolean} [dataConfig.noCache=false] - If true, attempt
+     *   to free page caches associated with the managed files
      * @param {werelogs.API} [logApi] - object providing a constructor function
      *                                for the Logger object
      */
@@ -50,6 +53,7 @@ class DataFileStore {
         this.logger = new (logApi || werelogs).Logger('DataFileStore');
         this.dataPath = dataConfig.dataPath;
         this.noSync = dataConfig.noSync || false;
+        this.noCache = dataConfig.noCache || false;
     }
 
     /**
@@ -149,10 +153,29 @@ class DataFileStore {
                     return cbOnce(null, key);
                 }
                 if (this.noSync) {
+                    /*
+                     * It's not guaranteed that the Kernel will release page
+                     * caches when this.noSync is true. If you want to ensure
+                     * this behavior, set this.noSync to false.
+                     */
+                    if (this.noCache) {
+                        releasePageCacheSync(filePath, fd, log);
+                    }
                     fs.closeSync(fd);
                     return ok();
                 }
                 fs.fsync(fd, err => {
+                    /*
+                     * Disabling the caching of stored files is
+                     * temporary fix for
+                     * https://github.com/kubernetes/kubernetes/issues/43916
+                     * that causes cache memory to be accounted as RSS memory
+                     * for the pod and can potentially cause the pod
+                     * to be killed under memory pressure:
+                     */
+                    if (this.noCache) {
+                        releasePageCacheSync(filePath, fd, log);
+                    }
                     fs.close(fd, err => {
                         if (err) {
                             log.error('error closing fd after write',
@@ -193,6 +216,15 @@ class DataFileStore {
                 return cbOnce(errors.InternalError.customizeDescription(
                     `read stream error: ${err.code}`));
             });
+            dataStream.on('close', () => {
+                // this means the underlying socket has been closed
+                log.debug('Client closed socket while streaming',
+                          { method: 'put', key, filePath });
+                // destroying the write stream forces a close(fd)
+                fileStream.destroy();
+                // we need to unlink the file ourselves
+                fs.unlinkSync(filePath);
+            });
             return undefined;
         });
     }
@@ -241,7 +273,7 @@ class DataFileStore {
             flags: 'r',
             encoding: null,
             fd: null,
-            autoClose: true,
+            autoClose: false,
         };
         if (byteRange) {
             readStreamOptions.start = byteRange[0];
@@ -256,13 +288,26 @@ class DataFileStore {
                         return cbOnce(errors.ObjNotFound);
                     }
                     log.error('error retrieving file',
-                          { method: 'get', key, filePath,
+                        { method: 'DataFileStore.get', key, filePath,
                             error: err });
                     return cbOnce(
                         errors.InternalError.customizeDescription(
                             `filesystem read error: ${err.code}`));
                 })
-                  .on('open', () => { cbOnce(null, rs); });
+                .on('open', () => { cbOnce(null, rs); })
+                .on('end', () => {
+                    if (this.noCache) {
+                        releasePageCacheSync(filePath, rs.fd, log);
+                    }
+                    fs.close(rs.fd, err => {
+                        if (err) {
+                            log.error('unable to close file descriptor', {
+                                method: 'DataFileStore.get', key, filePath,
+                                error: err,
+                            });
+                        }
+                    });
+                });
     }
 
     /**

lib/storage/data/file/utils.js

@@ -0,0 +1,19 @@
+const posixFadvise = require('fcntl');
+
+/**
+ * Release free cached pages associated with a file
+ *
+ * @param {String} filePath - absolute path of the associated file
+ * @param {Int} fd - file descriptor of the associated file
+ * @param {werelogs.RequestLogger} log - logging object
+ * @return {undefined}
+ */
+function releasePageCacheSync(filePath, fd, log) {
+    const ret = posixFadvise(fd, 0, 0, 4);
+    if (ret !== 0) {
+        log.warning(
+        `error fadv_dontneed ${filePath} returned ${ret}`);
+    }
+}
+
+module.exports = releasePageCacheSync;

lib/storage/metadata/MetadataWrapper.js

@@ -87,6 +87,7 @@ class MetadataWrapper {
                 database: params.mongodb.database,
                 replicationGroupId: params.replicationGroupId,
                 path: params.mongodb.path,
+                config: params.config,
                 logger,
             });
             this.implName = 'mongoclient';
@@ -109,7 +110,7 @@ class MetadataWrapper {
         if (this.client.setup) {
             return this.client.setup(done);
         }
-        return process.nextTick(() => done);
+        return process.nextTick(done);
     }
 
     createBucket(bucketName, bucketMD, log, cb) {

lib/storage/metadata/file/BucketFileInterface.js

@@ -0,0 +1,421 @@
+const cluster = require('cluster');
+const async = require('async');
+
+const errors = require('../../../errors');
+
+const BucketInfo = require('../../../models/BucketInfo');
+
+const list = require('../../../algos/list/exportAlgos');
+
+const MetadataFileClient = require('./MetadataFileClient');
+const versionSep =
+    require('../../../versioning/constants')
+    .VersioningConstants.VersionId.Separator;
+
+const METASTORE = '__metastore';
+
+const itemScanRefreshDelay = 1000 * 30 * 60; // 30 minutes
+
+class BucketFileInterface {
+
+    /**
+     * @constructor
+     * @param {object} [params] - constructor params
+     * @param {boolean} [params.noDbOpen=false] - true to skip DB open
+     * @param {object} logger - logger
+     *   (for unit tests only)
+     */
+    constructor(params, logger) {
+        this.logger = logger;
+        const { host, port } = params.metadataClient;
+        this.constants = params.constants;
+        this.mdClient = new MetadataFileClient({ host, port });
+        if (params && params.noDbOpen) {
+            return;
+        }
+        this.lastItemScanTime = null;
+        this.lastItemScanResult = null;
+    }
+
+    setup(done) {
+        return this.mdClient.openDB((err, value) => {
+            if (err) {
+                throw err;
+            }
+            // the metastore sublevel is used to store bucket attributes
+            this.mdDB = value;
+            this.metastore = this.mdDB.openSub(METASTORE);
+            if (cluster.isMaster) {
+                this.setupMetadataServer(done);
+            }
+        });
+    }
+
+    setupMetadataServer(done) {
+        /* Since the bucket creation API is expecting the
+           usersBucket to have attributes, we pre-create the
+           usersBucket attributes here */
+        this.mdClient.logger.debug('setting up metadata server');
+        const usersBucketAttr = new BucketInfo(this.constants.usersBucket,
+            'admin', 'admin', new Date().toJSON(),
+            BucketInfo.currentModelVersion());
+        return this.metastore.put(
+            this.constants.usersBucket,
+            usersBucketAttr.serialize(), {}, err => {
+                if (err) {
+                    this.logger.fatal('error writing usersBucket ' +
+                                      'attributes to metadata',
+                                      { error: err });
+                    throw (errors.InternalError);
+                }
+                return done();
+            });
+    }
+
+    /**
+     * Load DB if exists
+     * @param {String} bucketName - name of bucket
+     * @param {Object} log - logger
+     * @param {function} cb - callback(err, db, attr)
+     * @return {undefined}
+     */
+    loadDBIfExists(bucketName, log, cb) {
+        this.getBucketAttributes(bucketName, log, (err, attr) => {
+            if (err) {
+                return cb(err);
+            }
+            try {
+                const db = this.mdDB.openSub(bucketName);
+                return cb(null, db, attr);
+            } catch (err) {
+                return cb(errors.InternalError);
+            }
+        });
+        return undefined;
+    }
+
+    createBucket(bucketName, bucketMD, log, cb) {
+        this.getBucketAttributes(bucketName, log, err => {
+            if (err && err !== errors.NoSuchBucket) {
+                return cb(err);
+            }
+            if (err === undefined) {
+                return cb(errors.BucketAlreadyExists);
+            }
+            this.lastItemScanTime = null;
+            this.putBucketAttributes(bucketName,
+                                     bucketMD,
+                                     log, cb);
+            return undefined;
+        });
+    }
+
+    getBucketAttributes(bucketName, log, cb) {
+        this.metastore
+            .withRequestLogger(log)
+            .get(bucketName, {}, (err, data) => {
+                if (err) {
+                    if (err.ObjNotFound) {
+                        return cb(errors.NoSuchBucket);
+                    }
+                    const logObj = {
+                        rawError: err,
+                        error: err.message,
+                        errorStack: err.stack,
+                    };
+                    log.error('error getting db attributes', logObj);
+                    return cb(errors.InternalError);
+                }
+                return cb(null, BucketInfo.deSerialize(data));
+            });
+        return undefined;
+    }
+
+    getBucketAndObject(bucketName, objName, params, log, cb) {
+        this.loadDBIfExists(bucketName, log, (err, db, bucketAttr) => {
+            if (err) {
+                return cb(err);
+            }
+            db.withRequestLogger(log)
+                .get(objName, params, (err, objAttr) => {
+                    if (err) {
+                        if (err.ObjNotFound) {
+                            return cb(null, {
+                                bucket: bucketAttr.serialize(),
+                            });
+                        }
+                        const logObj = {
+                            rawError: err,
+                            error: err.message,
+                            errorStack: err.stack,
+                        };
+                        log.error('error getting object', logObj);
+                        return cb(errors.InternalError);
+                    }
+                    return cb(null, {
+                        bucket: bucketAttr.serialize(),
+                        obj: objAttr,
+                    });
+                });
+            return undefined;
+        });
+        return undefined;
+    }
+
+    putBucketAttributes(bucketName, bucketMD, log, cb) {
+        this.metastore
+            .withRequestLogger(log)
+            .put(bucketName, bucketMD.serialize(), {}, err => {
+                if (err) {
+                    const logObj = {
+                        rawError: err,
+                        error: err.message,
+                        errorStack: err.stack,
+                    };
+                    log.error('error putting db attributes', logObj);
+                    return cb(errors.InternalError);
+                }
+                return cb();
+            });
+        return undefined;
+    }
+
+    deleteBucket(bucketName, log, cb) {
+        this.metastore
+            .withRequestLogger(log)
+            .del(bucketName, {}, err => {
+                if (err) {
+                    const logObj = {
+                        rawError: err,
+                        error: err.message,
+                        errorStack: err.stack,
+                    };
+                    log.error('error deleting bucket',
+                              logObj);
+                    return cb(errors.InternalError);
+                }
+                this.lastItemScanTime = null;
+                return cb();
+            });
+        return undefined;
+    }
+
+    putObject(bucketName, objName, objVal, params, log, cb) {
+        this.loadDBIfExists(bucketName, log, (err, db) => {
+            if (err) {
+                return cb(err);
+            }
+            db.withRequestLogger(log)
+                .put(objName, JSON.stringify(objVal), params, (err, data) => {
+                    if (err) {
+                        const logObj = {
+                            rawError: err,
+                            error: err.message,
+                            errorStack: err.stack,
+                        };
+                        log.error('error putting object', logObj);
+                        return cb(errors.InternalError);
+                    }
+                    return cb(err, data);
+                });
+            return undefined;
+        });
+    }
+
+    getObject(bucketName, objName, params, log, cb) {
+        this.loadDBIfExists(bucketName, log, (err, db) => {
+            if (err) {
+                return cb(err);
+            }
+            db.withRequestLogger(log).get(objName, params, (err, data) => {
+                if (err) {
+                    if (err.ObjNotFound) {
+                        return cb(errors.NoSuchKey);
+                    }
+                    const logObj = {
+                        rawError: err,
+                        error: err.message,
+                        errorStack: err.stack,
+                    };
+                    log.error('error getting object', logObj);
+                    return cb(errors.InternalError);
+                }
+                return cb(null, JSON.parse(data));
+            });
+            return undefined;
+        });
+    }
+
+    deleteObject(bucketName, objName, params, log, cb) {
+        this.loadDBIfExists(bucketName, log, (err, db) => {
+            if (err) {
+                return cb(err);
+            }
+            db.withRequestLogger(log).del(objName, params, err => {
+                if (err) {
+                    const logObj = {
+                        rawError: err,
+                        error: err.message,
+                        errorStack: err.stack,
+                    };
+                    log.error('error deleting object', logObj);
+                    return cb(errors.InternalError);
+                }
+                return cb();
+            });
+            return undefined;
+        });
+    }
+
+    /**
+     *  This complex function deals with different extensions of bucket listing:
+     *  Delimiter based search or MPU based search.
+     *  @param {String} bucketName - The name of the bucket to list
+     *  @param {Object} params - The params to search
+     *  @param {Object} log - The logger object
+     *  @param {function} cb - Callback when done
+     *  @return {undefined}
+     */
+    internalListObject(bucketName, params, log, cb) {
+        const extName = params.listingType;
+        const extension = new list[extName](params, log);
+        const requestParams = extension.genMDParams();
+        this.loadDBIfExists(bucketName, log, (err, db) => {
+            if (err) {
+                return cb(err);
+            }
+            let cbDone = false;
+            db.withRequestLogger(log)
+                .createReadStream(requestParams, (err, stream) => {
+                    if (err) {
+                        return cb(err);
+                    }
+                    stream
+                        .on('data', e => {
+                            if (extension.filter(e) < 0) {
+                                stream.emit('end');
+                                stream.destroy();
+                            }
+                        })
+                        .on('error', err => {
+                            if (!cbDone) {
+                                cbDone = true;
+                                const logObj = {
+                                    rawError: err,
+                                    error: err.message,
+                                    errorStack: err.stack,
+                                };
+                                log.error('error listing objects', logObj);
+                                cb(errors.InternalError);
+                            }
+                        })
+                        .on('end', () => {
+                            if (!cbDone) {
+                                cbDone = true;
+                                const data = extension.result();
+                                cb(null, data);
+                            }
+                        });
+                    return undefined;
+                });
+            return undefined;
+        });
+    }
+
+    listObject(bucketName, params, log, cb) {
+        return this.internalListObject(bucketName, params, log, cb);
+    }
+
+    listMultipartUploads(bucketName, params, log, cb) {
+        return this.internalListObject(bucketName, params, log, cb);
+    }
+
+    getUUID(log, cb) {
+        return this.mdDB.getUUID(cb);
+    }
+
+    getDiskUsage(cb) {
+        return this.mdDB.getDiskUsage(cb);
+    }
+
+    countItems(log, cb) {
+        if (this.lastItemScanTime !== null &&
+            (Date.now() - this.lastItemScanTime) <= itemScanRefreshDelay) {
+            return process.nextTick(cb, null, this.lastItemScanResult);
+        }
+
+        const params = {};
+        const extension = new list.Basic(params, log);
+        const requestParams = extension.genMDParams();
+
+        const res = {
+            objects: 0,
+            versions: 0,
+            buckets: 0,
+            bucketList: [],
+        };
+        let cbDone = false;
+
+        this.mdDB.rawListKeys(requestParams, (err, stream) => {
+            if (err) {
+                return cb(err);
+            }
+            stream
+                .on('data', e => {
+                    if (!e.includes(METASTORE)) {
+                        if (e.includes(this.constants.usersBucket)) {
+                            res.buckets++;
+                            res.bucketList.push({
+                                name: e.split(this.constants.splitter)[1],
+                            });
+                        } else if (e.includes(versionSep)) {
+                            res.versions++;
+                        } else if (!e.includes('..recordLogs#s3-recordlog')) {
+                            res.objects++;
+                        }
+                    }
+                })
+                .on('error', err => {
+                    if (!cbDone) {
+                        cbDone = true;
+                        const logObj = {
+                            error: err,
+                            errorMessage: err.message,
+                            errorStack: err.stack,
+                        };
+                        log.error('error listing objects', logObj);
+                        cb(errors.InternalError);
+                    }
+                })
+                .on('end', () => {
+                    if (!cbDone) {
+                        cbDone = true;
+                        async.eachSeries(res.bucketList, (bucket, cb) => {
+                            this.getBucketAttributes(bucket.name, log,
+                            (err, bucketInfo) => {
+                                if (err) {
+                                    return cb(err);
+                                }
+                                /* eslint-disable no-param-reassign */
+                                bucket.location =
+                                    bucketInfo.getLocationConstraint();
+                                /* eslint-enable no-param-reassign */
+                                return cb();
+                            });
+                        }, err => {
+                            if (!err) {
+                                this.lastItemScanTime = Date.now();
+                                this.lastItemScanResult = res;
+                            }
+                            return cb(err, res);
+                        });
+                    }
+                    return undefined;
+                });
+            return undefined;
+        });
+        return undefined;
+    }
+}
+
+module.exports = BucketFileInterface;

lib/storage/metadata/file/MetadataFileClient.js

@@ -58,8 +58,7 @@ class MetadataFileClient {
             logger: this.logger,
             callTimeoutMs: this.callTimeoutMs,
         });
-        dbClient.connect(done);
-        return dbClient;
+        return dbClient.connect(() => done(null, dbClient));
     }
 
     /**

lib/storage/metadata/in_memory/ListMultipartUploadsResult.js

@@ -0,0 +1,32 @@
+const ListResult = require('./ListResult');
+
+class ListMultipartUploadsResult extends ListResult {
+    constructor() {
+        super();
+        this.Uploads = [];
+        this.NextKeyMarker = undefined;
+        this.NextUploadIdMarker = undefined;
+    }
+
+    addUpload(uploadInfo) {
+        this.Uploads.push({
+            key: decodeURIComponent(uploadInfo.key),
+            value: {
+                UploadId: uploadInfo.uploadId,
+                Initiator: {
+                    ID: uploadInfo.initiatorID,
+                    DisplayName: uploadInfo.initiatorDisplayName,
+                },
+                Owner: {
+                    ID: uploadInfo.ownerID,
+                    DisplayName: uploadInfo.ownerDisplayName,
+                },
+                StorageClass: uploadInfo.storageClass,
+                Initiated: uploadInfo.initiated,
+            },
+        });
+        this.MaxKeys += 1;
+    }
+}
+
+module.exports = ListMultipartUploadsResult;

lib/storage/metadata/in_memory/ListResult.js

@@ -0,0 +1,27 @@
+class ListResult {
+    constructor() {
+        this.IsTruncated = false;
+        this.NextMarker = undefined;
+        this.CommonPrefixes = [];
+        /*
+        Note:  this.MaxKeys will get incremented as
+        keys are added so that when response is returned,
+        this.MaxKeys will equal total keys in response
+        (with each CommonPrefix counting as 1 key)
+        */
+        this.MaxKeys = 0;
+    }
+
+    addCommonPrefix(prefix) {
+        if (!this.hasCommonPrefix(prefix)) {
+            this.CommonPrefixes.push(prefix);
+            this.MaxKeys += 1;
+        }
+    }
+
+    hasCommonPrefix(prefix) {
+        return (this.CommonPrefixes.indexOf(prefix) !== -1);
+    }
+}
+
+module.exports = ListResult;

lib/storage/metadata/in_memory/bucket_mem.js.Design.md

@@ -0,0 +1,62 @@
+# bucket_mem design
+
+## RATIONALE
+
+The bucket API will be used for managing buckets behind the S3 interface.
+
+We plan to have only 2 backends using this interface:
+
+* One production backend
+* One debug backend purely in memory
+
+One important remark here is that we don't want an abstraction but a
+duck-typing style interface (different classes MemoryBucket and Bucket having
+the same methods putObjectMD(), getObjectMD(), etc).
+
+Notes about the memory backend: The backend is currently a simple key/value
+store in memory. The functions actually use nextTick() to emulate the future
+asynchronous behavior of the production backend.
+
+## BUCKET API
+
+The bucket API is a very simple API with 5 functions:
+
+- putObjectMD(): put metadata for an object in the bucket
+- getObjectMD(): get metadata from the bucket
+- deleteObjectMD(): delete metadata for an object from the bucket
+- deleteBucketMD(): delete a bucket
+- getBucketListObjects(): perform the complex bucket listing AWS search
+  function with various flavors. This function returns a response in a
+  ListBucketResult object.
+
+getBucketListObjects(prefix, marker, delimiter, maxKeys, callback) behavior is
+the following:
+
+prefix (not required): Limits the response to keys that begin with the
+specified prefix. You can use prefixes to separate a bucket into different
+groupings of keys. (You can think of using prefix to make groups in the same
+way you'd use a folder in a file system.)
+
+marker (not required): Specifies the key to start with when listing objects in
+a bucket. Amazon S3 returns object keys in alphabetical order, starting with
+key after the marker in order.
+
+delimiter (not required): A delimiter is a character you use to group keys.
+All keys that contain the same string between the prefix, if specified, and the
+first occurrence of the delimiter after the prefix are grouped under a single
+result element, CommonPrefixes. If you don't specify the prefix parameter, then
+the substring starts at the beginning of the key. The keys that are grouped
+under CommonPrefixes are not returned elsewhere in the response.
+
+maxKeys: Sets the maximum number of keys returned in the response body. You can
+add this to your request if you want to retrieve fewer than the default 1000
+keys.  The response might contain fewer keys but will never contain more. If
+there are additional keys that satisfy the search criteria but were not
+returned because maxKeys was exceeded, the response contains an attribute of
+IsTruncated set to true and a NextMarker. To return the additional keys, call
+the function again using NextMarker as your marker argument in the function.
+
+Any key that does not contain the delimiter will be returned individually in
+Contents rather than in CommonPrefixes.
+
+If there is an error, the error subfield is returned in the response.

lib/storage/metadata/in_memory/bucket_utilities.js

@@ -0,0 +1,34 @@
+function markerFilterMPU(allMarkers, array) {
+    const { keyMarker, uploadIdMarker } = allMarkers;
+
+    // 1. if the item key matches the keyMarker and an uploadIdMarker exists,
+    // find the first uploadId in the array that is alphabetically after
+    // uploadIdMarker
+    // 2. if the item key does not match the keyMarker, find the first uploadId
+    // in the array that is alphabetically after keyMarker
+    const firstUnfilteredIndex = array.findIndex(
+        item => (uploadIdMarker && item.key === keyMarker ?
+             item.uploadId > uploadIdMarker :
+             item.key > keyMarker));
+    return firstUnfilteredIndex !== -1 ? array.slice(firstUnfilteredIndex) : [];
+}
+
+function prefixFilter(prefix, array) {
+    for (let i = 0; i < array.length; i++) {
+        if (array[i].indexOf(prefix) !== 0) {
+            array.splice(i, 1);
+            i--;
+        }
+    }
+    return array;
+}
+
+function isKeyInContents(responseObject, key) {
+    return responseObject.Contents.some(val => val.key === key);
+}
+
+module.exports = {
+    markerFilterMPU,
+    prefixFilter,
+    isKeyInContents,
+};

lib/storage/metadata/in_memory/getMultipartUploadListing.js

@@ -0,0 +1,148 @@
+const errors = require('../../../errors');
+
+const { markerFilterMPU, prefixFilter } = require('./bucket_utilities');
+const ListMultipartUploadsResult = require('./ListMultipartUploadsResult');
+const { metadata } = require('./metadata');
+
+const defaultMaxKeys = 1000;
+function getMultipartUploadListing(bucket, params, callback) {
+    const { delimiter, keyMarker,
+        uploadIdMarker, prefix, queryPrefixLength, splitter } = params;
+    const splitterLen = splitter.length;
+    const maxKeys = params.maxKeys !== undefined ?
+        Number.parseInt(params.maxKeys, 10) : defaultMaxKeys;
+    const response = new ListMultipartUploadsResult();
+    const keyMap = metadata.keyMaps.get(bucket.getName());
+    if (prefix) {
+        response.Prefix = prefix;
+        if (typeof prefix !== 'string') {
+            return callback(errors.InvalidArgument);
+        }
+    }
+
+    if (keyMarker) {
+        response.KeyMarker = keyMarker;
+        if (typeof keyMarker !== 'string') {
+            return callback(errors.InvalidArgument);
+        }
+    }
+
+    if (uploadIdMarker) {
+        response.UploadIdMarker = uploadIdMarker;
+        if (typeof uploadIdMarker !== 'string') {
+            return callback(errors.InvalidArgument);
+        }
+    }
+
+    if (delimiter) {
+        response.Delimiter = delimiter;
+        if (typeof delimiter !== 'string') {
+            return callback(errors.InvalidArgument);
+        }
+    }
+
+    if (maxKeys && typeof maxKeys !== 'number') {
+        return callback(errors.InvalidArgument);
+    }
+
+    // Sort uploads alphatebetically by objectKey and if same objectKey,
+    // then sort in ascending order by time initiated
+    let uploads = [];
+    keyMap.forEach((val, key) => {
+        uploads.push(key);
+    });
+    uploads.sort((a, b) => {
+        const aIndex = a.indexOf(splitter);
+        const bIndex = b.indexOf(splitter);
+        const aObjectKey = a.substring(aIndex + splitterLen);
+        const bObjectKey = b.substring(bIndex + splitterLen);
+        const aInitiated = keyMap.get(a).initiated;
+        const bInitiated = keyMap.get(b).initiated;
+        if (aObjectKey === bObjectKey) {
+            if (Date.parse(aInitiated) >= Date.parse(bInitiated)) {
+                return 1;
+            }
+            if (Date.parse(aInitiated) < Date.parse(bInitiated)) {
+                return -1;
+            }
+        }
+        return (aObjectKey < bObjectKey) ? -1 : 1;
+    });
+    // Edit the uploads array so it only
+    // contains keys that contain the prefix
+    uploads = prefixFilter(prefix, uploads);
+    uploads = uploads.map(stringKey => {
+        const index = stringKey.indexOf(splitter);
+        const index2 = stringKey.indexOf(splitter, index + splitterLen);
+        const storedMD = keyMap.get(stringKey);
+        return {
+            key: stringKey.substring(index + splitterLen, index2),
+            uploadId: stringKey.substring(index2 + splitterLen),
+            bucket: storedMD.eventualStorageBucket,
+            initiatorID: storedMD.initiator.ID,
+            initiatorDisplayName: storedMD.initiator.DisplayName,
+            ownerID: storedMD['owner-id'],
+            ownerDisplayName: storedMD['owner-display-name'],
+            storageClass: storedMD['x-amz-storage-class'],
+            initiated: storedMD.initiated,
+        };
+    });
+    // If keyMarker specified, edit the uploads array so it
+    // only contains keys that occur alphabetically after the marker.
+    // If there is also an uploadIdMarker specified, filter to eliminate
+    // any uploads that share the keyMarker and have an uploadId before
+    //  the uploadIdMarker.
+    if (keyMarker) {
+        const allMarkers = {
+            keyMarker,
+            uploadIdMarker,
+        };
+        uploads = markerFilterMPU(allMarkers, uploads);
+    }
+
+    // Iterate through uploads and filter uploads
+    // with keys containing delimiter
+    // into response.CommonPrefixes and filter remaining uploads
+    // into response.Uploads
+    for (let i = 0; i < uploads.length; i++) {
+        const currentUpload = uploads[i];
+        // If hit maxKeys, stop adding keys to response
+        if (response.MaxKeys >= maxKeys) {
+            response.IsTruncated = true;
+            break;
+        }
+        // If a delimiter is specified, find its
+        // index in the current key AFTER THE OCCURRENCE OF THE PREFIX
+        // THAT WAS SENT IN THE QUERY (not the prefix including the splitter
+        // and other elements)
+        let delimiterIndexAfterPrefix = -1;
+        const currentKeyWithoutPrefix =
+            currentUpload.key.slice(queryPrefixLength);
+        let sliceEnd;
+        if (delimiter) {
+            delimiterIndexAfterPrefix = currentKeyWithoutPrefix
+                .indexOf(delimiter);
+            sliceEnd = delimiterIndexAfterPrefix + queryPrefixLength;
+        }
+        // If delimiter occurs in current key, add key to
+        // response.CommonPrefixes.
+        // Otherwise add upload to response.Uploads
+        if (delimiterIndexAfterPrefix > -1) {
+            const keySubstring = currentUpload.key.slice(0, sliceEnd + 1);
+            response.addCommonPrefix(keySubstring);
+        } else {
+            response.NextKeyMarker = currentUpload.key;
+            response.NextUploadIdMarker = currentUpload.uploadId;
+            response.addUpload(currentUpload);
+        }
+    }
+    // `response.MaxKeys` should be the value from the original `MaxUploads`
+    // parameter specified by the user (or else the default 1000). Redefine it
+    // here, so it does not equal the value of `uploads.length`.
+    response.MaxKeys = maxKeys;
+    // If `response.MaxKeys` is 0, `response.IsTruncated` should be `false`.
+    response.IsTruncated = maxKeys === 0 ? false : response.IsTruncated;
+    return callback(null, response);
+}
+
+module.exports = getMultipartUploadListing;

lib/storage/metadata/in_memory/metadata.js

@@ -0,0 +1,8 @@
+const metadata = {
+    buckets: new Map,
+    keyMaps: new Map,
+};
+
+module.exports = {
+    metadata,
+};

lib/storage/metadata/in_memory/metastore.js

@@ -0,0 +1,333 @@
+const errors = require('../../../errors');
+const list = require('../../../algos/list/exportAlgos');
+const genVID =
+    require('../../../versioning/VersionID').generateVersionId;
+
+const getMultipartUploadListing = require('./getMultipartUploadListing');
+const { metadata } = require('./metadata');
+
+// const genVID = versioning.VersionID.generateVersionId;
+
+const defaultMaxKeys = 1000;
+let uidCounter = 0;
+
+function generateVersionId(replicationGroupId) {
+    return genVID(uidCounter++, replicationGroupId);
+}
+
+function formatVersionKey(key, versionId) {
+    return `${key}\0${versionId}`;
+}
+
+function inc(str) {
+    return str ? (str.slice(0, str.length - 1) +
+            String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str;
+}
+
+const metastore = {
+    createBucket: (bucketName, bucketMD, log, cb) => {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
+                // TODO Check whether user already owns the bucket,
+                // if so return "BucketAlreadyOwnedByYou"
+                // If not owned by user, return "BucketAlreadyExists"
+                if (bucket) {
+                    return cb(errors.BucketAlreadyExists);
+                }
+                metadata.buckets.set(bucketName, bucketMD);
+                metadata.keyMaps.set(bucketName, new Map);
+                return cb();
+            });
+        });
+    },
+
+    putBucketAttributes: (bucketName, bucketMD, log, cb) => {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, err => {
+                if (err) {
+                    return cb(err);
+                }
+                metadata.buckets.set(bucketName, bucketMD);
+                return cb();
+            });
+        });
+    },
+
+    getBucketAttributes: (bucketName, log, cb) => {
+        process.nextTick(() => {
+            if (!metadata.buckets.has(bucketName)) {
+                return cb(errors.NoSuchBucket);
+            }
+            return cb(null, metadata.buckets.get(bucketName));
+        });
+    },
+
+    deleteBucket: (bucketName, log, cb) => {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, err => {
+                if (err) {
+                    return cb(err);
+                }
+                if (metadata.keyMaps.has(bucketName)
+                    && metadata.keyMaps.get(bucketName).length > 0) {
+                    return cb(errors.BucketNotEmpty);
+                }
+                metadata.buckets.delete(bucketName);
+                metadata.keyMaps.delete(bucketName);
+                return cb(null);
+            });
+        });
+    },
+
+    putObject: (bucketName, objName, objVal, params, log, cb) => {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, err => {
+                if (err) {
+                    return cb(err);
+                }
+                /*
+                    valid combinations of versioning options:
+                    - !versioning && !versionId: normal non-versioning put
+                    - versioning && !versionId: create a new version
+                    - versionId: update (PUT/DELETE) an existing version,
+                        and also update master version in case the put
+                        version is newer or same version than master.
+                        if versionId === '' update master version
+                */
+
+                if (params && params.versionId) {
+                    objVal.versionId = params.versionId; // eslint-disable-line
+                    const mst = metadata.keyMaps.get(bucketName).get(objName);
+                    if (mst && mst.versionId === params.versionId || !mst) {
+                        metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    }
+                    // eslint-disable-next-line
+                    objName = formatVersionKey(objName, params.versionId);
+                    metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
+                }
+                if (params && params.versioning) {
+                    const versionId = generateVersionId();
+                    objVal.versionId = versionId; // eslint-disable-line
+                    metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    // eslint-disable-next-line
+                    objName = formatVersionKey(objName, versionId);
+                    metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    return cb(null, `{"versionId":"${versionId}"}`);
+                }
+                if (params && params.versionId === '') {
+                    const versionId = generateVersionId();
+                    objVal.versionId = versionId; // eslint-disable-line
+                    metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
+                }
+                metadata.keyMaps.get(bucketName).set(objName, objVal);
+                return cb(null);
+            });
+        });
+    },
+
+    getBucketAndObject: (bucketName, objName, params, log, cb) => {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
+                if (err) {
+                    return cb(err, { bucket });
+                }
+                if (params && params.versionId) {
+                    // eslint-disable-next-line
+                    objName = formatVersionKey(objName, params.versionId);
+                }
+                if (!metadata.keyMaps.has(bucketName)
+                    || !metadata.keyMaps.get(bucketName).has(objName)) {
+                    return cb(null, { bucket: bucket.serialize() });
+                }
+                return cb(null, {
+                    bucket: bucket.serialize(),
+                    obj: JSON.stringify(
+                        metadata.keyMaps.get(bucketName).get(objName)
+                    ),
+                });
+            });
+        });
+    },
+
+    getObject: (bucketName, objName, params, log, cb) => {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, err => {
+                if (err) {
+                    return cb(err);
+                }
+                if (params && params.versionId) {
+                    // eslint-disable-next-line
+                    objName = formatVersionKey(objName, params.versionId);
+                }
+                if (!metadata.keyMaps.has(bucketName)
+                    || !metadata.keyMaps.get(bucketName).has(objName)) {
+                    return cb(errors.NoSuchKey);
+                }
+                return cb(null, metadata.keyMaps.get(bucketName).get(objName));
+            });
+        });
+    },
+
+    deleteObject: (bucketName, objName, params, log, cb) => {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, err => {
+                if (err) {
+                    return cb(err);
+                }
+                if (!metadata.keyMaps.get(bucketName).has(objName)) {
+                    return cb(errors.NoSuchKey);
+                }
+                if (params && params.versionId) {
+                    const baseKey = inc(formatVersionKey(objName, ''));
+                    const vobjName = formatVersionKey(objName,
+                            params.versionId);
+                    metadata.keyMaps.get(bucketName).delete(vobjName);
+                    const mst = metadata.keyMaps.get(bucketName).get(objName);
+                    if (mst.versionId === params.versionId) {
+                        const keys = [];
+                        metadata.keyMaps.get(bucketName).forEach((val, key) => {
+                            if (key < baseKey && key > vobjName) {
+                                keys.push(key);
+                            }
+                        });
+                        if (keys.length === 0) {
+                            metadata.keyMaps.get(bucketName).delete(objName);
+                            return cb();
+                        }
+                        const key = keys.sort()[0];
+                        const value = metadata.keyMaps.get(bucketName).get(key);
+                        metadata.keyMaps.get(bucketName).set(objName, value);
+                    }
+                    return cb();
+                }
+                metadata.keyMaps.get(bucketName).delete(objName);
+                return cb();
+            });
+        });
+    },
+
+    _hasDeleteMarker(key, keyMap) {
+        const objectMD = keyMap.get(key);
+        if (objectMD['x-amz-delete-marker'] !== undefined) {
+            return (objectMD['x-amz-delete-marker'] === true);
+        }
+        return false;
+    },
+
+    listObject(bucketName, params, log, cb) {
+        process.nextTick(() => {
+            const {
+                prefix,
+                marker,
+                delimiter,
+                maxKeys,
+                continuationToken,
+                startAfter,
+            } = params;
+            if (prefix && typeof prefix !== 'string') {
+                return cb(errors.InvalidArgument);
+            }
+
+            if (marker && typeof marker !== 'string') {
+                return cb(errors.InvalidArgument);
+            }
+
+            if (delimiter && typeof delimiter !== 'string') {
+                return cb(errors.InvalidArgument);
+            }
+
+            if (maxKeys && typeof maxKeys !== 'number') {
+                return cb(errors.InvalidArgument);
+            }
+
+            if (continuationToken && typeof continuationToken !== 'string') {
+                return cb(errors.InvalidArgument);
+            }
+
+            if (startAfter && typeof startAfter !== 'string') {
+                return cb(errors.InvalidArgument);
+            }
+
+            // If paramMaxKeys is undefined, the default parameter will set it.
+            // However, if it is null, the default parameter will not set it.
+            let numKeys = maxKeys;
+            if (numKeys === null) {
+                numKeys = defaultMaxKeys;
+            }
+
+            if (!metadata.keyMaps.has(bucketName)) {
+                return cb(errors.NoSuchBucket);
+            }
+
+            // If marker specified, edit the keys array so it
+            // only contains keys that occur alphabetically after the marker
+            const listingType = params.listingType;
+            const extension = new list[listingType](params, log);
+            const listingParams = extension.genMDParams();
+
+            const keys = [];
+            metadata.keyMaps.get(bucketName).forEach((val, key) => {
+                if (listingParams.gt && listingParams.gt >= key) {
+                    return null;
+                }
+                if (listingParams.gte && listingParams.gte > key) {
+                    return null;
+                }
+                if (listingParams.lt && key >= listingParams.lt) {
+                    return null;
+                }
+                if (listingParams.lte && key > listingParams.lte) {
+                    return null;
+                }
+                return keys.push(key);
+            });
+            keys.sort();
+
+            // Iterate through keys array and filter keys containing
+            // delimiter into response.CommonPrefixes and filter remaining
+            // keys into response.Contents
+            for (let i = 0; i < keys.length; ++i) {
+                const currentKey = keys[i];
+                // Do not list object with delete markers
+                if (this._hasDeleteMarker(currentKey,
+                    metadata.keyMaps.get(bucketName))) {
+                    continue;
+                }
+                const objMD = metadata.keyMaps.get(bucketName).get(currentKey);
+                const value = JSON.stringify(objMD);
+                const obj = {
+                    key: currentKey,
+                    value,
+                };
+                // calling Ext.filter(obj) adds the obj to the Ext result if
+                // not filtered.
+                // Also, Ext.filter returns false when hit max keys.
+                // What a nifty function!
+                if (extension.filter(obj) < 0) {
+                    break;
+                }
+            }
+            return cb(null, extension.result());
+        });
+    },
+
+    listMultipartUploads(bucketName, listingParams, log, cb) {
+        process.nextTick(() => {
+            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
+                if (bucket === undefined) {
+                    // no on going multipart uploads, return empty listing
+                    return cb(null, {
+                        IsTruncated: false,
+                        NextMarker: undefined,
+                        MaxKeys: 0,
+                    });
+                }
+                return getMultipartUploadListing(bucket, listingParams, cb);
+            });
+        });
+    },
+};
+
+module.exports = metastore;

lib/storage/metadata/mongoclient/DataCounter.js

@@ -0,0 +1,274 @@
+const NEW_OBJ = 0;
+const NEW_VER = 1;
+const UPDATE_VER = 2;
+const UPDATE_MST = 3;
+const RESTORE = 4;
+
+const DEL_VER = 0;
+const DEL_MST = 1;
+
+const CURR = 'curr';
+const PREV = 'prev';
+
+function deepCopyObject(obj) {
+    return JSON.parse(JSON.stringify(obj));
+}
+
+class DataCounter {
+    /**
+     * DataCounter - class for keeping track of the ItemCount metrics
+     * @return {DataCounter} DataCounter object
+     */
+    constructor() {
+        this.objects = 0;
+        this.versions = 0;
+        this.dataManaged = {
+            total: { curr: 0, prev: 0 },
+            byLocation: {},
+        };
+        this.stalled = 0;
+        this.populated = false;
+        this.transientList = {};
+    }
+
+    /**
+     * updateTransientList - update data counter list of transient locations
+     * @param  {Object} newLocations - list of locations constraint details
+     * @return {undefined}
+     */
+    updateTransientList(newLocations) {
+        if (newLocations && Object.keys(newLocations).length > 0) {
+            const tempList = {};
+            Object.keys(newLocations).forEach(loc => {
+                tempList[loc] = newLocations[loc].isTransient;
+            });
+            this.transientList = tempList;
+        }
+    }
+
+    /**
+     * set - set DataCounter values
+     * @param {Object} setVal - object containing values to be used for setting
+     * DataCounter
+     * @param {number} setVal.objects - number of master objects
+     * @param {number} setVal.versions - number of versioned objects
+     * @param {Object} setVal.dataManaged - object containing information about
+     * all the data managed
+     * @param {Object} setVal.total - object containing the total byte count of
+     * data managed
+     * @param {number} setVal.total.curr - the total byte count of master
+     * objects
+     * @param {number} setVal.total.prev - the total byte count of versioned
+     * objects
+     * @param {Object} setVal.byLocaton - object containing the information
+     * about data managed on each location
+     * @return {undefined}
+     */
+    set(setVal) {
+        if (setVal) {
+            this.objects = setVal.objects;
+            this.versions = setVal.versions;
+            this.dataManaged = deepCopyObject(setVal.dataManaged);
+            this.populated = true;
+            this.stalled = setVal.stalled;
+        }
+    }
+
+    /**
+     * results - creates a deep copy of the current DataCounter values
+     * @return {Object} - object containing the current DataCounter values
+     */
+    results() {
+        const obj = {
+            objects: this.objects,
+            versions: this.versions,
+            dataManaged: this.dataManaged,
+            stalled: this.stalled,
+        };
+        return deepCopyObject(obj);
+    }
+
+    /**
+     * addObjectFn - performing add operations
+     * @param {ObjectMD} currMD - new master version metadata
+     * @param {ObjectMD} prevMD - old master version metadata
+     * @param {number} type - index of the current type of add operation
+     * @return {undefined}
+     */
+    addObject(currMD, prevMD, type) {
+        if (type !== undefined && type !== null && this.populated) {
+            switch (type) {
+            case NEW_OBJ: // add new object, replace master if needed
+                if (prevMD) {
+                    this._delValue(prevMD, CURR);
+                    this._addValue(currMD, CURR);
+                } else {
+                    ++this.objects;
+                    this._addValue(currMD, CURR);
+                }
+                break;
+            case NEW_VER: // add new object, archive master
+                ++this.versions;
+                this._delValue(prevMD, CURR);
+                this._addValue(prevMD, PREV);
+                this._addValue(currMD, CURR);
+                break;
+            case UPDATE_VER: // update archived object, replication info
+                this._updateObject(currMD, prevMD, PREV);
+                break;
+            case UPDATE_MST: // update master object, replication info
+                this._updateObject(currMD, prevMD, CURR);
+                break;
+            case RESTORE:
+                --this.versions;
+                this._delValue(currMD, PREV);
+                ++this.objects;
+                this._addValue(currMD, CURR);
+                break;
+            default:
+                // should throw error, noop
+                break;
+            }
+        }
+    }
+
+    /**
+     * delObjectFn - performing del operations
+     * @param {ObjectMD} currMD - object metadata
+     * @param {number} type - index of the current type of delete operation
+     * @return {undefined}
+     */
+    delObject(currMD, type) {
+        if (type !== undefined && type !== null && this.populated) {
+            switch (type) {
+            case DEL_VER:
+                --this.versions;
+                this._delValue(currMD, PREV);
+                break;
+            case DEL_MST:
+                --this.objects;
+                this._delValue(currMD, CURR);
+                break;
+            default:
+                // should throw error, noop
+                break;
+            }
+        }
+    }
+
+    _addLocation(site, size, type) {
+        this.dataManaged.total[type] += size;
+        if (!this.dataManaged.byLocation[site]) {
+            this.dataManaged.byLocation[site] = {
+                curr: 0,
+                prev: 0,
+            };
+        }
+        this.dataManaged.byLocation[site][type] += size;
+    }
+
+    /**
+     * _addValue - helper function for handling put object updates
+     * @param {ObjectMD} objMD - object metadata
+     * @param {string} type - string with value either 'curr' or 'prev'
+     * @return {undefined}
+     */
+    _addValue(objMD, type) {
+        if (objMD) {
+            const { replicationInfo, 'content-length': size } = objMD;
+            const { backends } = replicationInfo || {};
+            this._addLocation(objMD.dataStoreName, size, type);
+            if (backends && Array.isArray(backends)) {
+                backends.forEach(loc => {
+                    const { site, status } = loc;
+                    if (status === 'COMPLETED') {
+                        this._addLocation(site, size, type);
+                    }
+                });
+            }
+        }
+    }
+
+    /**
+     * _updateObject - helper function for handling updates from replication
+     * info changes
+     * @param {ObjectMD} currMD - new object metadata
+     * @param {ObjectMD} prevMD - old object metadata
+     * @param {string} type - string with value either 'curr' or 'prev'
+     * @return {undefined}
+     */
+    _updateObject(currMD, prevMD, type) {
+        const transientList = Object.assign({}, this.transientList);
+        if (currMD && prevMD) {
+            // check for changes in replication
+            const { replicationInfo: currLocs,
+                'content-length': size, dataStoreName } = currMD;
+            const { replicationInfo: prevLocs } = prevMD;
+            const { backends: prevBackends } = prevLocs || {};
+            const { backends: currBackends } = currLocs || {};
+            const oldLocs = {};
+            if (prevBackends && Array.isArray(prevBackends)) {
+                prevBackends.forEach(loc => {
+                    const { site, status } = loc;
+                    oldLocs[site] = status;
+                });
+            }
+            if (currBackends && Array.isArray(currBackends)) {
+                currBackends.forEach(loc => {
+                    const { site, status } = loc;
+                    if (site in oldLocs && status === 'COMPLETED' &&
+                        oldLocs[site] !== status) {
+                        this._addLocation(site, size, type);
+                    }
+                });
+            }
+            if (currLocs.status === 'COMPLETED' &&
+                transientList[dataStoreName]) {
+                this._delLocation(dataStoreName, size, type);
+            }
+        }
+    }
+
+    _delLocation(site, size, type) {
+        if (this.dataManaged.byLocation[site]) {
+            this.dataManaged.total[type] -= size;
+            this.dataManaged.total[type] =
+                Math.max(0, this.dataManaged.total[type]);
+            this.dataManaged.byLocation[site][type] -= size;
+            this.dataManaged.byLocation[site][type] =
+                Math.max(0, this.dataManaged.byLocation[site][type]);
+        }
+    }
+    /**
+     * _delValue - helper function for handling delete object operations
+     * @param {ObjectMD} objMD - object metadata
+     * @param {string} type - string with value either 'curr' or 'prev'
+     * @return {undefined}
+     */
+    _delValue(objMD, type) {
+        if (objMD) {
+            const { replicationInfo, 'content-length': size } = objMD;
+            const { backends } = replicationInfo || {};
+            this._delLocation(objMD.dataStoreName, size, type);
+            if (backends && Array.isArray(backends)) {
+                backends.forEach(loc => {
+                    const { site, status } = loc;
+                    if (status === 'COMPLETED') {
+                        this._delLocation(site, size, type);
+                    }
+                });
+            }
+        }
+    }
+}
+
+module.exports = {
+    NEW_OBJ,
+    NEW_VER,
+    UPDATE_VER,
+    UPDATE_MST,
+    RESTORE,
+    DEL_VER,
+    DEL_MST,
+    DataCounter,
+};

lib/storage/metadata/mongoclient/ListRecordStream.js

@@ -0,0 +1,172 @@
+const stream = require('stream');
+
+/**
+ * @class ListRecordStream
+ * @classdesc Filter and stream records returned from a mongodb query
+ * cursor
+ */
+class ListRecordStream extends stream.Readable {
+    /**
+     * @constructor
+     * @param {mongodb.Cursor} mongoCursor - cursor returned by a
+     *   mongodb query to the oplog (see
+     *   http://mongodb.github.io/node-mongodb-native/2.0/api/Cursor.html)
+     * @param {werelogs.Logger} logger - logger object
+     * @param {string} lastSavedID - unique ID that has been persisted
+     *   of the most recently processed entry in the oplog
+     * @param {string} latestOplogID - unique ID of the most recently
+     *   added entry in the oplog
+     */
+    constructor(mongoCursor, logger, lastSavedID, latestOplogID) {
+        super({ objectMode: true });
+        this._cursor = mongoCursor;
+        this._logger = logger;
+        this._lastSavedID = lastSavedID;
+        this._latestOplogID = latestOplogID;
+        this._lastConsumedID = null;
+        // this._unpublishedListing is true once we pass the oplog
+        // record that has the same uniqID 'h' than last saved. If we
+        // don't find it (e.g. log rolled over before populator could
+        // process its oldest entries), we will restart from the
+        // latest record of the oplog.
+        this._unpublishedListing = false;
+        // cf. this.getSkipCount()
+        this._skipCount = 0;
+    }
+
+    _read() {
+        // MongoDB cursors provide a stream interface. We choose not
+        // to use it though because errors may not be emitted by the
+        // stream when there is an issue with the connection to
+        // MongoDB (especially when pause()/resume() are used).
+        //
+        // Instead we use the async cursor.next() call directly to
+        // fetch records one at a time, errors are then forwarded in
+        // the callback.
+        this._cursor.next((err, item) => {
+            if (err) {
+                this._logger.error('mongodb cursor error', {
+                    method: 'mongoclient.ListRecordStream._read()',
+                    error: err.message,
+                });
+                this.emit('error', err);
+                return undefined;
+            }
+            if (this._processItem(item)) {
+                return process.nextTick(this._read.bind(this));
+            }
+            // wait until _read() gets called again
+            return undefined;
+        });
+    }
+
+    _processItem(itemObj) {
+        // always update to most recent uniqID
+        this._lastConsumedID = itemObj.h.toString();
+
+        // only push to stream unpublished objects
+        if (!this._lastSavedID) {
+            // process from the first entry
+            this._unpublishedListing = true;
+        } else if (!this._unpublishedListing) {
+            // When an oplog with a unique ID that is stored in the
+            // log offset is found, all oplogs AFTER this is unpublished.
+            if (this._lastSavedID === this._lastConsumedID) {
+                this._unpublishedListing = true;
+            } else if (this._latestOplogID === this._lastConsumedID) {
+                this._logger.warn(
+                    'did not encounter the last saved offset in oplog, ' +
+                        'resuming processing right after the latest record ' +
+                        'to date; some entries may have been skipped', {
+                            lastSavedID: this._lastSavedID,
+                            latestRecordID: this._latestOplogID,
+                        });
+                this._unpublishedListing = true;
+            }
+            ++this._skipCount;
+            return true; // read next record
+        }
+
+        const dbName = itemObj.ns.slice(itemObj.ns.indexOf('.') + 1);
+        let entry;
+        if (itemObj.op === 'i' &&
+            itemObj.o && itemObj.o._id) {
+            entry = {
+                type: 'put',
+                key: itemObj.o._id,
+                // value is given as-is for inserts
+                value: JSON.stringify(itemObj.o.value),
+            };
+        } else if (itemObj.op === 'u' &&
+                   itemObj.o && itemObj.o2 && itemObj.o2._id) {
+            entry = {
+                type: 'put', // updates overwrite the whole metadata,
+                             // so they are considered as puts
+                key: itemObj.o2._id,
+                // updated value may be either stored directly in 'o'
+                // attribute or in '$set' attribute (supposedly when
+                // the object pre-exists it will be in '$set')
+                value: JSON.stringify(
+                    (itemObj.o.$set ? itemObj.o.$set : itemObj.o).value),
+            };
+        } else if (itemObj.op === 'd' &&
+                   itemObj.o && itemObj.o._id) {
+            entry = {
+                type: 'delete',
+                key: itemObj.o._id,
+                // deletion yields no value
+            };
+        } else {
+            // skip other entry types as we don't need them for now
+            // ('c', ...?)
+            ++this._skipCount;
+            return true; // read next record
+        }
+        const streamObject = {
+            timestamp: new Date((itemObj.ts ?
+                                 itemObj.ts.toNumber() * 1000 : 0)),
+            db: dbName,
+            entries: [entry],
+        };
+        // push object to the stream, then return false to wait until
+        // _read() is called again (because we are in an asynchronous
+        // context already)
+        this.push(streamObject);
+        return false;
+    }
+
+    /**
+     * Get an opaque JSON blob containing the latest consumed offset
+     * from MongoDB oplog.
+     *
+     * @return {string} opaque JSON blob
+     */
+    getOffset() {
+        return JSON.stringify({
+            uniqID: this._lastConsumedID,
+        });
+    }
+
+    /**
+     * Get the number of entries that have been read and skipped from
+     * MongoDB oplog since the ListRecordStream instance was created.
+     *
+     * @return {integer} number of skipped entries
+     */
+    getSkipCount() {
+        return this._skipCount;
+    }
+
+    /**
+     * Get whether the stream reached yet-unpublished records
+     * (i.e. after we reached either the saved unique ID, or the tip
+     * of the oplog)
+     *
+     * @return {boolean} true if we are now returning unpublished records
+     */
+    reachedUnpublishedListing() {
+        return this._unpublishedListing;
+    }
+}
+
+module.exports = ListRecordStream;

lib/storage/metadata/mongoclient/LogConsumer.js

@@ -0,0 +1,121 @@
+'use strict'; // eslint-disable-line
+
+const MongoClient = require('mongodb').MongoClient;
+const ListRecordStream = require('./ListRecordStream');
+
+/**
+ * @class
+ * @classdesc Class to consume mongo oplog
+ */
+class LogConsumer {
+
+    /**
+     * @constructor
+     *
+     * @param {object} mongoConfig - object with the mongo configuration
+     * @param {string} logger - logger
+     */
+    constructor(mongoConfig, logger) {
+        const { replicaSetHosts, database } = mongoConfig;
+        this._mongoUrl = `mongodb://${replicaSetHosts}/local`;
+        this._logger = logger;
+        this._oplogNsRegExp = new RegExp(`^${database}\\.`);
+        // oplog collection
+        this._coll = null;
+    }
+
+    /**
+     * Connect to MongoClient using Mongo node module to access database and
+     * database oplogs (operation logs)
+     *
+     * @param {function} done - callback function, called with an error object
+     * or null and an object as 2nd parameter
+     * @return {undefined}
+    */
+    connectMongo(done) {
+        MongoClient.connect(this._mongoUrl, {
+            replicaSet: 'rs0',
+            useNewUrlParser: true,
+        },
+        (err, client) => {
+            if (err) {
+                this._logger.error('Unable to connect to MongoDB',
+                { error: err });
+                return done(err);
+            }
+            this._logger.info('connected to mongodb');
+            // 'local' is the database where MongoDB has oplog.rs
+            // capped collection
+            const db = client.db('local', {
+                ignoreUndefined: true,
+            });
+            this._coll = db.collection('oplog.rs');
+            return done();
+        });
+    }
+
+    /**
+     * Open a tailable cursor to mongo oplog and retrieve a stream of
+     * records to read
+     *
+     * @param {Object} [params] - params object
+     * @param {String} [params.startSeq] - fetch starting from this
+     *   opaque offset returned previously by mongo ListRecordStream
+     *   in an 'info' event
+     * @param {function} cb - callback function, called with an error
+     *   object or null and an object as 2nd parameter
+     *
+     * @return {undefined}
+     */
+    readRecords(params, cb) {
+        let startSeq = {};
+        if (params.startSeq) {
+            try {
+                // parse the opaque JSON string passed through from a
+                // previous 'info' event
+                startSeq = JSON.parse(params.startSeq);
+            } catch (err) {
+                this._logger.error('malformed startSeq', {
+                    startSeq: params.startSeq,
+                });
+                // start over if malformed
+            }
+        }
+        this._readLatestOplogID((err, latestOplogID) => {
+            if (err) {
+                return cb(err);
+            }
+            return this._coll.find({
+                ns: this._oplogNsRegExp,
+            }, {
+                tailable: true,
+                awaitData: true,
+                noCursorTimeout: true,
+                numberOfRetries: Number.MAX_VALUE,
+            }, (err, cursor) => {
+                const recordStream = new ListRecordStream(
+                    cursor, this._logger, startSeq.uniqID, latestOplogID);
+                return cb(null, { log: recordStream, tailable: true });
+            });
+        });
+    }
+
+    _readLatestOplogID(cb) {
+        this._coll.find({
+            ns: this._oplogNsRegExp,
+        }, {
+            ts: 1,
+        }).sort({
+            $natural: -1,
+        }).limit(1).toArray((err, data) => {
+            if (err) {
+                return cb(err);
+            }
+            const latestOplogID = data[0].h.toString();
+            this._logger.debug('latest oplog ID read', { latestOplogID });
+            return cb(null, latestOplogID);
+        });
+    }
+}
+
+module.exports = LogConsumer;

lib/storage/metadata/mongoclient/Mongoclient.md

@@ -0,0 +1,173 @@
+# Mongoclient
+
+We introduce a new metadata backend called *mongoclient* for
+[MongoDB](https://www.mongodb.com). This backend takes advantage of
+MongoDB being a document store to store the metadata (bucket and
+object attributes) as JSON objects.
+
+## Overall Design
+
+The mongoclient backend strictly follows the metadata interface that
+stores bucket and object attributes, which consists of the methods
+createBucket(), getBucketAttributes(), getBucketAndObject()
+(attributes), putBucketAttributes(), deleteBucket(), putObject(),
+getObject(), deleteObject(), listObject(), listMultipartUploads() and
+the management methods getUUID(), getDiskUsage() and countItems(). The
+mongoclient backend also knows how to deal with versioning, it is also
+compatible with the various listing algorithms implemented in Arsenal.
+
+FIXME: There should be a document describing the metadata (currently
+duck-typing) interface.
+
+### Why Using MongoDB for Storing Bucket and Object Attributes
+
+We chose MongoDB for various reasons:
+
+- MongoDB supports replication, especially through the Raft protocol.
+
+- MongoDB supports a basic replication scheme called 'Replica Set' and
+  more advanced sharding schemes if required.
+
+- MongoDB is open source and an enterprise standard.
+
+- MongoDB is a document store (natively supports JSON) and supports a
+  very flexible search interface.
+
+### Choice of Mongo Client Library
+
+We chose to use the official MongoDB driver for NodeJS:
+[https://github.com/mongodb/node-mongodb-native](https://github.com/mongodb/node-mongodb-native)
+
+### Granularity for Buckets
+
+We chose to have one collection for one bucket mapping. First because
+in a simple mode of replication called 'replica set' it works from the
+get-go, but if one or many buckets grow to big it is possible to use
+more advanced schemes such as sharding. MongoDB supports a mix of
+sharded and non-sharded collections.
+
+### Storing Database Information
+
+We need a special collection called the *Infostore* (stored under the
+name __infostore which is impossible to create through the S3 bucket
+naming scheme) to store specific database properties such as the
+unique *uuid* for Orbit.
+
+### Storing Bucket Attributes
+
+We need to use a special collection called the *Metastore* (stored
+under the name __metastore which is impossible to create through the
+S3 bucket naming scheme).
+
+### Versioning Format
+
+We chose to keep the same versioning format that we use in some other
+Scality products in order to facilitate the compatibility between the
+different products.
+
+FIXME: Document the versioning internals in the upper layers and
+document the versioning format
+
+### Dealing with Concurrency
+
+We chose not to use transactions (aka
+[https://docs.mongodb.com/manual/tutorial/perform-two-phase-commits/)
+because it is a known fact there is an overhead of using them, and we
+thought there was no real need for them since we could leverage Mongo
+ordered operations guarantees and atomic writes.
+
+Example of corner cases:
+
+#### CreateBucket()
+
+Since it is not possible to create a collection AND at the same time
+register the bucket in the Metastore we chose to only update the
+Metastore. A non-existing collection (NamespaceNotFound error in
+Mongo) is one possible normal state for an empty bucket.
+
+#### DeleteBucket()
+
+In this case the bucket is *locked* by the upper layers (use of a
+transient delete flag) so we don't have to worry about that and by the
+fact the bucket is empty neither (which is also checked by the upper
+layers).
+
+We first drop() the collection and then we asynchronously delete the
+bucket name entry from the metastore (the removal from the metastore
+is atomic which is not absolutely necessary in this case but more
+robust in term of design).
+
+If we fail in between we still have an entry in the metastore which is
+good because we need to manage the delete flag. For the upper layers
+the operation has not completed until this flag is removed. The upper
+layers will restart the deleteBucket() which is fine because we manage
+the case where the collection does not exist.
+
+#### PutObject() with a Version
+
+We need to store the versioned object then update the master object
+(the latest version). For this we use the
+[BulkWrite](http://mongodb.github.io/node-mongodb-native/3.0/api/Collection.html#bulkWrite)
+method. This is not a transaction but guarantees that the 2 operations
+will happen sequentially in the MongoDB oplog. Indeed if the
+BulkWrite() fails in between we would end up creating an orphan (which
+is not critical) but if the operation succeeds then we are sure that
+the master is always pointing to the right object. If there is a
+concurrency between 2 clients then we are sure that the 2 groups of
+operations will be clearly decided in the oplog (the last writer will
+win).
+
+#### DeleteObject()
+
+This is probably the most complex case to manage because it involves a
+lot of different cases:
+
+##### Deleting an Object when Versioning is not Enabled
+
+This case is a straightforward atomic delete. Atomicity is not really
+required because we assume version IDs are random enough but it is
+more robust to do so.
+
+##### Deleting an Object when Versioning is Enabled
+
+This case is more complex since we have to deal with the 2 cases:
+
+Case 1: The caller asks for a deletion of a version which is not a master:
+This case is a straight-forward atomic delete.
+
+Case 2: The caller asks for a deletion of a version which is the master: In
+this case we need to create a special flag called PHD (as PlaceHolDer)
+that indicates the master is no longer valid (with a new unique
+virtual version ID). We force the ordering of operations in a
+bulkWrite() to first replace the master with the PHD flag and then
+physically delete the version. If the call fail in between we will be
+left with a master with a PHD flag. If the call succeeds we try to
+find if the master with the PHD flag is left alone in such case we
+delete it otherwise we trigger an asynchronous repair that will spawn
+after AYNC_REPAIR_TIMEOUT=15s that will reassign the master to the
+latest version.
+
+In all cases the physical deletion or the repair of the master are
+checked against the PHD flag AND the actual unique virtual version
+ID. We do this to check against potential concurrent deletions,
+repairs or updates. Only the last writer/deleter has the right to
+physically perform the operation, otherwise it is superseded by other
+operations.
+
+##### Getting an object with a PHD flag
+
+If the caller is asking for the latest version of an object and the
+PHD flag is set we perform a search on the bucket to find the latest
+version and we return it.
+
+#### Listing Objects
+
+The mongoclient backend implements a readable key/value stream called
+*MongoReadStream* that follows the LevelDB duck typing interface used
+in Arsenal/lib/algos listing algorithms. Note it does not require any
+LevelDB package.
+
+#### Generating the UUID
+
+To avoid race conditions we always (try to) generate a new UUID and we
+condition the insertion to the non-existence of the document.

lib/storage/metadata/mongoclient/readStream.js

@@ -0,0 +1,137 @@
+const Readable = require('stream').Readable;
+const MongoUtils = require('./utils');
+
+class MongoReadStream extends Readable {
+    constructor(c, options, searchOptions) {
+        super({
+            objectMode: true,
+            highWaterMark: 0,
+        });
+
+        if (options.limit === 0) {
+            return;
+        }
+
+        const query = {
+            _id: {},
+        };
+        if (options.reverse) {
+            if (options.start) {
+                query._id.$lte = options.start;
+            }
+            if (options.end) {
+                query._id.$gte = options.end;
+            }
+            if (options.gt) {
+                query._id.$lt = options.gt;
+            }
+            if (options.gte) {
+                query._id.$lte = options.gte;
+            }
+            if (options.lt) {
+                query._id.$gt = options.lt;
+            }
+            if (options.lte) {
+                query._id.$gte = options.lte;
+            }
+        } else {
+            if (options.start) {
+                query._id.$gte = options.start;
+            }
+            if (options.end) {
+                query._id.$lte = options.end;
+            }
+            if (options.gt) {
+                query._id.$gt = options.gt;
+            }
+            if (options.gte) {
+                query._id.$gte = options.gte;
+            }
+            if (options.lt) {
+                query._id.$lt = options.lt;
+            }
+            if (options.lte) {
+                query._id.$lte = options.lte;
+            }
+        }
+
+        if (!Object.keys(query._id).length) {
+            delete query._id;
+        }
+
+        if (searchOptions) {
+            Object.assign(query, searchOptions);
+        }
+
+        this._cursor = c.find(query).sort({
+            _id: options.reverse ? -1 : 1,
+        });
+        if (options.limit && options.limit !== -1) {
+            this._cursor = this._cursor.limit(options.limit);
+        }
+        this._options = options;
+        this._destroyed = false;
+        this.on('end', this._cleanup.bind(this));
+    }
+
+    _read() {
+        if (this._destroyed) {
+            return;
+        }
+
+        this._cursor.next((err, doc) => {
+            if (this._destroyed) {
+                return;
+            }
+            if (err) {
+                this.emit('error', err);
+                return;
+            }
+
+            let key = undefined;
+            let value = undefined;
+
+            if (doc) {
+                key = doc._id;
+                MongoUtils.unserialize(doc.value);
+                value = JSON.stringify(doc.value);
+            }
+
+            if (key === undefined && value === undefined) {
+                this.push(null);
+            } else if (this._options.keys !== false &&
+                       this._options.values === false) {
+                this.push(key);
+            } else if (this._options.keys === false &&
+                       this._options.values !== false) {
+                this.push(value);
+            } else {
+                this.push({
+                    key,
+                    value,
+                });
+            }
+        });
+    }
+
+    _cleanup() {
+        if (this._destroyed) {
+            return;
+        }
+        this._destroyed = true;
+
+        this._cursor.close(err => {
+            if (err) {
+                this.emit('error', err);
+                return;
+            }
+            this.emit('close');
+        });
+    }
+
+    destroy() {
+        return this._cleanup();
+    }
+}
+
+module.exports = MongoReadStream;

lib/storage/metadata/mongoclient/utils.js

@@ -0,0 +1,30 @@
+
+function escape(obj) {
+    return JSON.parse(JSON.stringify(obj).
+                      replace(/\$/g, '\uFF04').
+                      replace(/\./g, '\uFF0E'));
+}
+
+function unescape(obj) {
+    return JSON.parse(JSON.stringify(obj).
+                      replace(/\uFF04/g, '$').
+                      replace(/\uFF0E/g, '.'));
+}
+
+function serialize(objMD) {
+    // Tags require special handling since dot and dollar are accepted
+    if (objMD.tags) {
+        // eslint-disable-next-line
+        objMD.tags = escape(objMD.tags);
+    }
+}
+
+function unserialize(objMD) {
+    // Tags require special handling
+    if (objMD.tags) {
+        // eslint-disable-next-line
+        objMD.tags = unescape(objMD.tags);
+    }
+}
+
+module.exports = { escape, unescape, serialize, unserialize };

lib/storage/metadata/proxy/BucketdRoutes.js

@@ -0,0 +1,375 @@
+'use strict'; // eslint-disable-line strict
+
+const errors = require('../../../errors');
+const BucketInfo = require('../../../models/BucketInfo');
+const { getURIComponents, getRequestBody, sendResponse } = require('./utils');
+
+class BucketdRoutes {
+    /**
+     * Create a new Bucketd routes instance
+     * This class implements the bucketd Metadata protocol and is used in
+     * the Metadata Proxy Server to implement this protocol on top of
+     * various metadata backends.
+     *
+     * Implementation note: the adaptations performed in the methods of
+     * the class MetadataWrapper are not required in this context.
+     * For this reason, the methods of the `client' instance are directly
+     * called from this class, somewhat defeating the encapsulation of the
+     * wrapper.
+     *
+     * @param {Arsenal.storage.metadata.MetadataWrapper} metadataWrapper - to
+     *              be used as a translation target for the bucketd protocol.
+     * @param {werelogs.Logger} logger - werelogs logger object
+     */
+    constructor(metadataWrapper, logger) {
+        this._metadataWrapper = metadataWrapper;
+        this._logger = logger;
+    }
+
+    // Metadata Wrapper's wrapper
+
+    // `attributes' context methods
+
+    _getBucketAttributes(req, res, bucketName, logger) {
+        return this._metadataWrapper.client.getBucketAttributes(
+            bucketName, logger, (err, data) => {
+                if (err) {
+                    logger.error('Failed to get bucket attributes',
+                                 { bucket: bucketName, error: err });
+                    return sendResponse(req, res, logger, err);
+                }
+                if (data === undefined) {
+                    return sendResponse(req, res, logger,
+                                        errors.NoSuchBucket);
+                }
+                return sendResponse(req, res, logger, null,
+                                    BucketInfo.fromObj(data).serialize());
+            });
+    }
+
+    _putBucketAttributes(req, res, bucketName, data, logger) {
+        return this._metadataWrapper.client.putBucketAttributes(
+            bucketName, BucketInfo.deSerialize(data), logger, err =>
+                sendResponse(req, res, logger, err));
+    }
+
+    // `bucket' context methods
+
+    _createBucket(req, res, bucketName, data, logger) {
+        return this._metadataWrapper.client.createBucket(
+            bucketName, BucketInfo.deSerialize(data), logger, err =>
+                sendResponse(req, res, logger, err));
+    }
+
+    _deleteBucket(req, res, bucketName, logger) {
+        return this._metadataWrapper.client.deleteBucket(
+            bucketName, logger, err =>
+                sendResponse(req, res, logger, err));
+    }
+
+    _putObject(req, res, bucketName, objectName, objectValue, params, logger) {
+        let parsedValue;
+        try {
+            parsedValue = JSON.parse(objectValue);
+        } catch (err) {
+            logger.error('Malformed JSON value', { value: objectValue });
+            return sendResponse(req, res, logger, errors.BadRequest);
+        }
+        return this._metadataWrapper.client.putObject(
+            bucketName, objectName, parsedValue,
+            params, logger, (err, data) =>
+                sendResponse(req, res, logger, err, data));
+    }
+
+    _getObject(req, res, bucketName, objectName, params, logger) {
+        return this._metadataWrapper.client.getObject(
+            bucketName, objectName, params, logger, (err, data) =>
+                sendResponse(req, res, logger, err, data));
+    }
+
+    _deleteObject(req, res, bucketName, objectName, params, logger) {
+        return this._metadataWrapper.client.deleteObject(
+            bucketName, objectName, params, logger, (err, data) =>
+                sendResponse(req, res, logger, err, data));
+    }
+
+    _listObject(req, res, bucketName, params, logger) {
+        const listingParameters = params || {};
+        if (listingParameters.listingType === undefined) {
+            listingParameters.listingType = 'Delimiter';
+        }
+        if (listingParameters.maxKeys) {
+            listingParameters.maxKeys = Number.parseInt(params.maxKeys, 10);
+        }
+        return this._metadataWrapper.client.listObject(
+            bucketName, listingParameters, logger, (err, data) =>
+                sendResponse(req, res, logger, err, data));
+    }
+
+    // `admin' context methods
+
+    _checkHealth(req, res, logger) {
+        return this._metadataWrapper.checkHealth(logger, (err, resp) => {
+            if (err) {
+                logger.error('Failed the health check',
+                             { error: err, method: '_checkHealth' });
+                return sendResponse(req, res, logger, err);
+            }
+            return sendResponse(req, res, logger, undefined, resp);
+        });
+    }
+
+    _createRequestLogger(req) {
+        const uids = req.headers['x-scal-request-uids'];
+        const logger = uids === undefined ?
+              this._logger.newRequestLogger() :
+              this._logger.newRequestLoggerFromSerializedUids(uids);
+        logger.trace('new request', { method: req.method, url: req.url });
+        return logger;
+    }
+
+    // `parallel' context methods
+
+    _getBucketAndObjectMD(req, res, bucketName, objectName, params, logger) {
+        return this._metadataWrapper.client.getBucketAndObject(
+            bucketName, objectName, params, logger, (err, data) =>
+                sendResponse(req, res, logger, err, data));
+    }
+
+    // Internal routes
+
+    /**
+     * Handle routes related to operations on bucket attributes
+     *
+     * @param {http.IncomingMessage} req - request being processed
+     * @param {http.OutgoingMessage} res - response associated to the request
+     * @param {object} uriComponents - URI breakdown of the request to process
+     * @param {werelogs.Logger} logger - werelogs logger object
+     * @return {undefined}
+     */
+    _attributesRoutes(req, res, uriComponents, logger) {
+        if (uriComponents.bucketName === undefined) {
+            logger.error('Missing bucket name for attributes route',
+                         { uriComponents });
+            return sendResponse(req, res, logger, errors.BadRequest);
+        }
+        switch (req.method) {
+        case 'GET':
+            return this._getBucketAttributes(
+                req, res,
+                uriComponents.bucketName, logger, (err, attrs) =>
+                    sendResponse(req, res, logger, err, attrs));
+        case 'POST':
+            return getRequestBody(logger, req, (err, body) => {
+                if (err) {
+                    return sendResponse(req, res, logger, err);
+                }
+                return this._putBucketAttributes(
+                    req, res,
+                    uriComponents.bucketName, body, logger, err =>
+                        sendResponse(req, res, logger, err));
+            });
+        default:
+            return sendResponse(req, res, logger, errors.RouteNotFound);
+        }
+    }
+
+    /**
+     * Handle routes related to operations on buckets
+     *
+     * @param {http.IncomingMessage} req - request being processed
+     * @param {http.OutgoingMessage} res - response associated to the request
+     * @param {object} uriComponents - URI breakdown of the request to process
+     * @param {string} uriComponents.namespace - Select the control plane with
+     *                                           `_' or the data plane with
+     *                                           `default'.
+     * @param {string} uriComponents.context - Targets the bucket itself with
+     *                                         `attributes' or the content of
+     *                                         the bucket with `bucket'.
+     * @param {string} uriComponents.bucketName - The name of the bucket
+     * @param {string} uriComponents.objectName - the key of the object in the
+     *                                            bucket
+     * @param {werelogs.Logger} logger - werelogs logger object
+     * @return {undefined}
+     */
+    _bucketRoutes(req, res, uriComponents, logger) {
+        if (uriComponents.bucketName === undefined) {
+            logger.error('Missing bucket name for bucket route',
+                         { uriComponents });
+            return sendResponse(req, res, logger, errors.BadRequest);
+        }
+        switch (req.method) {
+        case 'GET':
+            return this._listObject(req, res,
+                                    uriComponents.bucketName,
+                                    uriComponents.options,
+                                    logger);
+        case 'DELETE':
+            return this._deleteBucket(req, res,
+                                      uriComponents.bucketName, logger);
+        case 'POST':
+            return getRequestBody(logger, req, (err, body) => {
+                if (err) {
+                    return sendResponse(req, res, logger, err);
+                }
+                return this._createBucket(req, res,
+                                          uriComponents.bucketName,
+                                          body, logger);
+            });
+        default:
+            return sendResponse(req, res, logger, errors.RouteNotFound);
+        }
+    }
+
+    /**
+     * Handle routes related to operations on objects
+     *
+     * @param {http.IncomingMessage} req - request being processed
+     * @param {http.OutgoingMessage} res - response associated to the request
+     * @param {object} uriComponents - URI breakdown of the request to process
+     * @param {werelogs.Logger} logger - werelogs logger object
+     * @return {undefined}
+     */
+    _objectRoutes(req, res, uriComponents, logger) {
+        if (uriComponents.bucketName === undefined) {
+            logger.error('Missing bucket name for object route',
+                         { uriComponents });
+            return sendResponse(req, res, logger, errors.BadRequest);
+        }
+        switch (req.method) {
+        case 'GET':
+            return this._getObject(req, res,
+                                   uriComponents.bucketName,
+                                   uriComponents.objectName,
+                                   uriComponents.options,
+                                   logger);
+        case 'DELETE':
+            return this._deleteObject(req, res,
+                                      uriComponents.bucketName,
+                                      uriComponents.objectName,
+                                      uriComponents.options,
+                                      logger);
+        case 'POST':
+            return getRequestBody(logger, req, (err, body) =>
+                                  this._putObject(req, res,
+                                                  uriComponents.bucketName,
+                                                  uriComponents.objectName,
+                                                  body,
+                                                  uriComponents.options,
+                                                  logger));
+        default:
+            return sendResponse(req, res, logger, errors.RouteNotFound);
+        }
+    }
+
+    /**
+     * Handle routes related to operations on both objects and buckets
+     *
+     * @param {http.IncomingMessage} req - request being processed
+     * @param {http.OutgoingMessage} res - response associated to the request
+     * @param {object} uriComponents - URI breakdown of the request to process
+     * @param {werelogs.Logger} logger - werelogs logger object
+     * @return {undefined}
+     */
+    _parallelRoutes(req, res, uriComponents, logger) {
+        if (uriComponents.bucketName === undefined) {
+            logger.error('Missing bucket name for parallel route',
+                         { uriComponents });
+            return sendResponse(req, res, logger, errors.BadRequest);
+        }
+        switch (req.method) {
+        case 'GET':
+            return this._getBucketAndObjectMD(req, res,
+                                              uriComponents.bucketName,
+                                              uriComponents.objectName,
+                                              uriComponents.options,
+                                              logger);
+        default:
+            return sendResponse(req, res, logger, errors.RouteNotFound);
+        }
+    }
+
+    /**
+     * Handle default routes. e.g. URI starting with /default/
+     * (or anything excepted an underscore)
+     *
+     * @param {http.IncomingMessage} req - request being processed
+     * @param {http.OutgoingMessage} res - response associated to the request
+     * @param {object} uriComponents - URI breakdown of the request to process
+     * @param {werelogs.Logger} logger - werelogs logger object
+     * @return {undefined}
+     */
+    _defaultRoutes(req, res, uriComponents, logger) {
+        switch (uriComponents.context) {
+        case 'leader':
+        case 'informations':
+            logger.trace(`${uriComponents.context} operation`);
+            return sendResponse(req, res, logger, errors.NotImplemented);
+        case 'metadataInformation':
+            return sendResponse(req, res, logger, undefined,
+                                '{"metadataVersion":2}');
+        case 'parallel':
+            logger.trace(`${uriComponents.context} operation`);
+            if (uriComponents.objectName) {
+                return this._parallelRoutes(req, res, uriComponents, logger);
+            }
+            return sendResponse(req, res, logger, errors.RouteNotFound);
+        case 'bucket':
+            logger.trace(`${uriComponents.context} operation`);
+            if (uriComponents.objectName) {
+                return this._objectRoutes(req, res, uriComponents, logger);
+            }
+            return this._bucketRoutes(req, res, uriComponents, logger);
+        case 'attributes':
+            logger.trace(`${uriComponents.context} operation`);
+            return this._attributesRoutes(req, res, uriComponents, logger);
+        default:
+            logger.error('invalid URI', { uriComponents });
+            return sendResponse(req, res, logger, errors.RouteNotFound);
+        }
+    }
+
+    /**
+     * Handle admin routes. e.g. URI starting with /_/
+     *
+     * @param {http.IncomingMessage} req - request being processed
+     * @param {http.OutgoingMessage} res - response associated to the request
+     * @param {object} uriComponents - URI breakdown of the request to process
+     * @param {werelogs.Logger} logger - werelogs logger object
+     * @return {undefined}
+     */
+    _adminRoutes(req, res, uriComponents, logger) {
+        switch (uriComponents.context) {
+        case 'healthcheck':
+            return this._checkHealth(req, res, logger);
+        default:
+            return sendResponse(req, res, logger, errors.NotImplemented);
+        }
+    }
+
+    // The route dispatching method
+
+    /**
+     * dispatch the HTTP request to the appropriate handling function.
+     *
+     * @param {http.IncomingMessage} req - request being processed
+     * @param {http.OutgoingMessage} res - response associated to the request
+     * @return {undefined}
+     */
+    dispatch(req, res) {
+        const adminNamespace = '_';
+        const logger = this._createRequestLogger(req);
+        const uriComponents = getURIComponents(req.url, logger);
+        if (!uriComponents) {
+            return sendResponse(req, res, logger, errors.BadRequest);
+        }
+        switch (uriComponents.namespace) {
+        case adminNamespace:
+            return this._adminRoutes(req, res, uriComponents, logger);
+        default: // coincidently matches the `default' literal namespace as well
+            return this._defaultRoutes(req, res, uriComponents, logger);
+        }
+    }
+}
+
+module.exports = BucketdRoutes;

lib/storage/metadata/proxy/README.md

@@ -0,0 +1,33 @@
+# Metatada Proxy Server
+
+## Design goals
+
+## Design choices
+
+## Implementation details
+
+## How to run the proxy server
+
+```js
+const werelogs = require('werelogs');
+const MetadataWrapper = require('arsenal')
+                            .storage.metadata.MetadataWrapper;
+const Server = require('arsenal')
+                            .storage.metadata.proxy.Server;
+
+const logger = new werelogs.Logger('MetadataProxyServer',
+                                   'debug', 'debug');
+const metadataWrapper = new MetadataWrapper('mem', {},
+                                            null, logger);
+const server = new Server(metadataWrapper,
+                          {
+                              port: 9001,
+                              workers: 1,
+                          },
+                          logger);
+server.start(() => {
+    logger.info('Metadata Proxy Server successfully started. ' +
+                `Using the ${metadataWrapper.implName} backend`);
+});
+
+```

lib/storage/metadata/proxy/Server.js

@@ -0,0 +1,105 @@
+'use strict'; // eslint-disable-line strict
+
+const cluster = require('cluster');
+
+const HttpServer = require('../../../network/http/server');
+const BucketdRoutes = require('./BucketdRoutes');
+
+const requiresOneWorker = {
+    // in memory kvs storage is not shared across processes
+    memorybucket: true,
+};
+
+class Server {
+
+    /**
+     * Create a new Metadata Proxy Server instance
+     *
+     * The Metadata Proxy Server is an HTTP server that translates
+     * requests of the bucketd sub-protocol into function calls to
+     * a properly configured MetadataWrapper instance. Such instance
+     * can use any of the available metadata backends available.
+     *
+     * @param {arsenal.storage.metadata.MetadataWrapper} metadataWrapper -
+     * @param {Object} configuration -
+     * @param {number} configuration.port -
+     * @param {number} configuration.workers -
+     * @param {werelogs.Logger} logger -
+     */
+    constructor(metadataWrapper, configuration, logger) {
+        this._configuration = configuration;
+        if (requiresOneWorker[metadataWrapper.implName] &&
+            this._configuration.workers !== 1) {
+            logger.warn('This metadata backend requires only one worker',
+                        { metadataBackend: metadataWrapper.implName });
+            this._configuration.workers = 1;
+        }
+        this._logger = logger;
+        this._metadataWrapper = metadataWrapper;
+
+        this._proxyRoutes = new BucketdRoutes(metadataWrapper, this._logger);
+        this._httpServer = null;
+        this._installSignalHandlers();
+    }
+
+    _cleanup() {
+        if (cluster.isWorker) {
+            this._logger.info('Server worker shutting down...');
+            this._httpServer.stop();
+        } else {
+            this._logger.info('Server shutting down...');
+        }
+        return process.exit(0);
+    }
+
+    _installSignalHandlers() {
+        process.on('SIGINT', () => { this._cleanup(); });
+        process.on('SIGHUP', () => { this._cleanup(); });
+        process.on('SIGQUIT', () => { this._cleanup(); });
+        process.on('SIGTERM', () => { this._cleanup(); });
+        process.on('SIGPIPE', () => {});
+    }
+
+    /**
+     * Start the Metadata Proxy Server instance
+     *
+     * @param {Function} cb - called with no argument when the onListening event
+     *                        is triggered
+     * @return {undefined}
+     */
+    start(cb) {
+        if (cluster.isMaster) {
+            for (let i = 0; i < this._configuration.workers; ++i) {
+                cluster.fork();
+            }
+            cluster.on('disconnect', worker => {
+                this._logger
+                    .info(`worker ${worker.process.pid} exited, respawning.`);
+                cluster.fork();
+            });
+        } else {
+            this._httpServer = new HttpServer(this._configuration.port,
+                                              this._logger);
+            if (this._configuration.bindAddress) {
+                this._httpServer.setBindAddress(
+                    this._configuration.bindAddress);
+            }
+            this._httpServer
+                .onRequest((req, res) => this._proxyRoutes.dispatch(req, res))
+                .onListening(() => {
+                    this._logger.info(
+                        'Metadata Proxy Server now listening on' +
+                        ` port ${this._configuration.port}`);
+                    if (cb) {
+                        return this._metadataWrapper.setup(cb);
+                    }
+                    return this._metadataWrapper.setup(() => {
+                        this._logger.info('MetadataWrapper setup complete.');
+                    });
+                })
+                .start();
+        }
+    }
+}
+
+module.exports = Server;

lib/storage/metadata/proxy/utils.js

@@ -0,0 +1,183 @@
+const url = require('url');
+const querystring = require('querystring');
+const errors = require('../../../errors');
+
+/**
+ * Extracts components from URI.
+ * @param {string} uri - uri part of the received request
+ * @param {werelogs.Logger} logger -
+ * @return {object} ret - URI breakdown of the request to process
+ * @return {string} ret.namespace - targeted plane, control plane is targeted
+ *                                  with `_' and the data plane with `default'.
+ * @return {string} ret.context   - Targets the bucket itself with
+ *                                  `attributes' or the content of
+ *                                  the bucket with `bucket'.
+ * @return {string} ret.bucketName - The name of the bucket
+ * @return {string} ret.objectName - the key of the object in the bucket
+ */
+function getURIComponents(uri, logger) {
+    try {
+        if (uri.charAt(0) !== '/') {
+            return {};
+        }
+
+        const { pathname, query } = url.parse(uri);
+        const options = query ? querystring.parse(query) : {};
+        const typeIndex = pathname.indexOf('/', 1);
+        const bucketIndex = pathname.indexOf('/', typeIndex + 1);
+        const objectIndex = pathname.indexOf('/', bucketIndex + 1);
+
+        if (typeIndex === -1 || typeIndex === pathname.length - 1) {
+            return {};
+        }
+        if (bucketIndex === -1) {
+            return {
+                namespace: pathname.substring(1, typeIndex),
+                context: pathname.substring(typeIndex + 1),
+            };
+        }
+        if (bucketIndex === pathname.length - 1) {
+            return {
+                namespace: pathname.substring(1, typeIndex),
+                context: pathname.substring(typeIndex + 1, bucketIndex),
+            };
+        }
+        if (objectIndex === -1) {
+            return {
+                namespace: pathname.substring(1, typeIndex),
+                context: pathname.substring(typeIndex + 1, bucketIndex),
+                bucketName: pathname.substring(bucketIndex + 1),
+                options,
+            };
+        }
+        if (objectIndex === pathname.length - 1) {
+            return {
+                namespace: pathname.substring(1, typeIndex),
+                context: pathname.substring(typeIndex + 1, bucketIndex),
+                bucketName: pathname.substring(bucketIndex + 1, objectIndex),
+                options,
+            };
+        }
+        return {
+            namespace: pathname.substring(1, typeIndex),
+            context: pathname.substring(typeIndex + 1, bucketIndex),
+            bucketName: pathname.substring(bucketIndex + 1, objectIndex),
+            objectName: decodeURIComponent(pathname.substring(objectIndex + 1)),
+            options,
+        };
+    } catch (ex) {
+        logger.error('Invalid URI: failed to parse',
+                     { uri, error: ex, errorStack: ex.stack,
+                       message: ex.message });
+        return null;
+    }
+}
+
+/**
+ * Extracts the body of the request through a callback
+ * @param {werelogs.Logger} logger - instance of the logger that will emit the
+ *                                   log entry
+ * @param {http.IncomingMessage} request - request received from bucketclient
+ * @param {Function} cb - function which has an interest in the request body.
+ *                        The first parameter is err and may be falsey
+ *                        The second parameter is the body of the request
+ * @return {undefined}
+ */
+function getRequestBody(logger, request, cb) {
+    const body = [];
+    let bodyLen = 0;
+    request.on('data', data => {
+        body.push(data);
+        bodyLen += data.length;
+    }).on('error', cb).on('end', () => {
+        cb(null, Buffer.concat(body, bodyLen).toString());
+    }).on('close', () => {
+        logger.error('Connection closed by remote peer');
+        /* Execution will eventually reach the sendResponse code which will
+         * trigger the proper cleanup as the remote peer already hung up and
+         * nobody is on the line to get the message */
+        cb(errors.BadRequest);
+    });
+}
+
+/**
+ * Emit a log entry corresponding to the end of the request
+ *
+ * @param {werelogs.Logger} logger - instance of the logger that will emit the
+ *                                   log entry
+ * @param {http.IncomingMessage} req - request being processed
+ * @param {object} statusCode - HTTP status code sent back to the client
+ * @param {object} statusMessage - HTTP status message sent back to the client
+ * @return {undefined}
+ */
+function _logRequestEnd(logger, req, statusCode, statusMessage) {
+    const info = {
+        clientIp: req.socket.remoteAddress,
+        clientPort: req.socket.remotePort,
+        httpMethod: req.method,
+        httpURL: req.url,
+        httpCode: statusCode,
+        httpMessage: statusMessage,
+    };
+    logger.end('finished handling request', info);
+}
+
+/**
+ * Request processing exit point, sends back to the client the specified data
+ * and/or error code
+ *
+ * @param {http.IncomingMessage} req - request being processed
+ * @param {http.OutgoingMessage} res - response associated to the request
+ * @param {werelogs.Logger} log - instance of the logger to use
+ * @param {Arsenal.Error} err - if not null, defines the HTTP status
+ *                              code and message
+ * @param {string} data - if not null, used as the response body. If `data'
+ *                        isn't a string, it's considered as a JSON object and
+ *                        it's content get serialized before being sent.
+ * @return {undefined}
+ */
+function sendResponse(req, res, log, err, data) {
+    let statusCode;
+    let statusMessage;
+    if (err) {
+        statusCode = err.code;
+        statusMessage = err.message;
+    } else {
+        statusCode = errors.ok.code;
+        statusMessage = errors.ok.message;
+    }
+
+    if (data) {
+        let resData = data;
+        if (typeof resData === 'object') {
+            resData = JSON.stringify(data);
+        } else if (typeof resData === 'number') {
+            resData = resData.toString();
+        }
+        /*
+        * Encoding data to binary provides a hot path to write data
+        * directly to the socket, without node.js trying to encode the data
+        * over and over again.
+        */
+        const rawData = Buffer.from(resData, 'utf8');
+        /*
+        * Using Buffer.bytelength is not required here because data is binary
+        * encoded, data.length would give us the exact byte length
+        */
+        res.writeHead(statusCode, statusMessage, {
+            'content-length': rawData.length,
+        });
+        res.write(rawData);
+    } else {
+        res.writeHead(statusCode, statusMessage, { 'content-length': 0 });
+    }
+    return res.end(() => {
+        _logRequestEnd(log, req, statusCode, statusMessage);
+    });
+}
+
+module.exports = {
+    getURIComponents,
+    getRequestBody,
+    sendResponse,
+};

package.json

@@ -1,16 +1,16 @@
 {
   "name": "arsenal",
   "engines": {
-    "node": ">=6.9.5"
+    "node": ">=8"
   },
-  "version": "7.4.3",
+  "version": "8.0.6",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/scality/Arsenal.git"
   },
-  "author": "Giorgio Regni",
+  "author": "Scality Inc.",
   "license": "Apache-2.0",
   "bugs": {
     "url": "https://github.com/scality/Arsenal/issues"
@@ -20,13 +20,16 @@
     "JSONStream": "^1.0.0",
     "ajv": "4.10.0",
     "async": "~2.1.5",
+    "bson": "2.0.4",
     "debug": "~2.3.3",
     "diskusage": "^1.1.1",
+    "fcntl": "github:scality/node-fcntl",
     "ioredis": "4.9.5",
     "ipaddr.js": "1.2.0",
     "joi": "^10.6",
     "level": "~5.0.1",
     "level-sublevel": "~6.6.5",
+    "mongodb": "^3.0.1",
     "node-forge": "^0.7.1",
     "simple-glob": "^0.1",
     "socket.io": "~1.7.3",

tests/functional/backbeat/Metrics.js

@@ -0,0 +1,105 @@
+'use strict'; // eslint-disable-line strict
+
+const assert = require('assert');
+
+const RedisClient = require('../../../lib/metrics/RedisClient');
+const { backbeat } = require('../../../');
+
+// expirations
+const EXPIRY = 86400; // 24 hours
+const THROUGHPUT_EXPIRY = 900; // 15 minutes
+
+// setup redis client
+const config = {
+    host: '127.0.0.1',
+    port: 6379,
+    enableOfflineQueue: false,
+};
+const fakeLogger = {
+    trace: () => {},
+    error: () => {},
+};
+const redisClient = new RedisClient(config, fakeLogger);
+
+// setup stats model
+const sites = ['site1', 'site2'];
+const metrics = new backbeat.Metrics({
+    redisConfig: config,
+    validSites: ['site1', 'site2', 'all'],
+    internalStart: Date.now() - (EXPIRY * 1000), // 24 hours ago.
+}, fakeLogger);
+
+// Since many methods were overwritten, these tests should validate the changes
+// made to the original methods
+describe('Metrics class', () => {
+    afterEach(() => redisClient.clear(() => {}));
+
+    it('should not crash on empty results', done => {
+        const redisKeys = {
+            ops: 'bb:crr:ops',
+            bytes: 'bb:crr:bytes',
+            opsDone: 'bb:crr:opsdone',
+            bytesDone: 'bb:crr:bytesdone',
+            bytesFail: 'bb:crr:bytesfail',
+            opsFail: 'bb:crr:opsfail',
+            failedCRR: 'bb:crr:failed',
+            opsPending: 'bb:crr:bytespending',
+            bytesPending: 'bb:crr:opspending',
+        };
+        const routes = backbeat.routes(redisKeys, sites);
+        const details = routes.find(route =>
+            route.category === 'metrics' && route.type === 'all');
+        details.site = 'all';
+        metrics.getAllMetrics(details, (err, res) => {
+            assert.ifError(err);
+            const expected = {
+                pending: {
+                    description: 'Number of pending replication operations ' +
+                        '(count) and bytes (size)',
+                    results: {
+                        count: 0,
+                        size: 0,
+                    },
+                },
+                backlog: {
+                    description: 'Number of incomplete replication operations' +
+                        ' (count) and number of incomplete bytes transferred' +
+                        ' (size)',
+                    results: {
+                        count: 0,
+                        size: 0,
+                    },
+                },
+                completions: {
+                    description: 'Number of completed replication operations' +
+                        ' (count) and number of bytes transferred (size) in ' +
+                        `the last ${EXPIRY} seconds`,
+                    results: {
+                        count: 0,
+                        size: 0,
+                    },
+                },
+                failures: {
+                    description: 'Number of failed replication operations ' +
+                        `(count) and bytes (size) in the last ${EXPIRY} ` +
+                        'seconds',
+                    results: {
+                        count: 0,
+                        size: 0,
+                    },
+                },
+                throughput: {
+                    description: 'Current throughput for replication' +
+                        ' operations in ops/sec (count) and bytes/sec (size) ' +
+                        `in the last ${THROUGHPUT_EXPIRY} seconds`,
+                    results: {
+                        count: '0.00',
+                        size: '0.00',
+                    },
+                },
+            };
+            assert.deepStrictEqual(res, expected);
+            done();
+        });
+    });
+});

tests/functional/kmip/highlevel.js

@@ -0,0 +1,68 @@
+'use strict'; // eslint-disable-line strict
+
+const assert = require('assert');
+const crypto = require('crypto');
+const async = require('async');
+const TTLVCodec = require('../../../lib/network/kmip/codec/ttlv.js');
+const LoopbackServerChannel =
+      require('../../utils/kmip/LoopbackServerChannel.js');
+const TransportTemplate =
+      require('../../../lib/network/kmip/transport/TransportTemplate.js');
+const KMIP = require('../../../lib/network/kmip');
+const KMIPClient = require('../../../lib/network/kmip/Client.js');
+const {
+    logger,
+} = require('../../utils/kmip/ersatz.js');
+
+class LoopbackServerTransport extends TransportTemplate {
+    constructor(options) {
+        super(new LoopbackServerChannel(KMIP, TTLVCodec), options);
+    }
+}
+
+describe('KMIP High Level Driver', () => {
+    [null, 'dummyAttributeName'].forEach(bucketNameAttributeName => {
+        [false, true].forEach(compoundCreateActivate => {
+            const options = {
+                kmip: {
+                    client: {
+                        bucketNameAttributeName,
+                        compoundCreateActivate,
+                    },
+                    codec: {},
+                    transport: {
+                        pipelineDepth: 8,
+                        tls: {
+                            port: 5696,
+                        },
+                    },
+                },
+            };
+            it('should work with' +
+               ` x-name attribute: ${!!bucketNameAttributeName},` +
+               ` compound creation: ${compoundCreateActivate}`,
+               done => {
+                   const kmipClient = new KMIPClient(options, TTLVCodec,
+                                                     LoopbackServerTransport);
+                   const plaintext = Buffer.from(crypto.randomBytes(32));
+                   async.waterfall([
+                       next => kmipClient.createBucketKey('plop', logger, next),
+                       (id, next) =>
+                           kmipClient.cipherDataKey(1, id, plaintext,
+                                                    logger, (err, ciphered) => {
+                                                        next(err, id, ciphered);
+                                                    }),
+                       (id, ciphered, next) =>
+                           kmipClient.decipherDataKey(
+                               1, id, ciphered, logger, (err, deciphered) => {
+                                   assert(plaintext
+                                          .compare(deciphered) === 0);
+                                   next(err, id);
+                               }),
+                       (id, next) =>
+                           kmipClient.destroyBucketKey(id, logger, next),
+                   ], done);
+               });
+        });
+    });
+});

tests/functional/kmip/lowlevel.js

@@ -0,0 +1,52 @@
+'use strict'; // eslint-disable-line strict
+
+const assert = require('assert');
+const TTLVCodec = require('../../../lib/network/kmip/codec/ttlv.js');
+const TransportTemplate =
+      require('../../../lib/network/kmip/transport/TransportTemplate.js');
+const KMIP = require('../../../lib/network/kmip');
+const {
+    logger,
+    MirrorChannel,
+} = require('../../utils/kmip/ersatz.js');
+const lowlevelFixtures = require('../../utils/kmip/lowlevelFixtures.js');
+
+
+class MirrorTransport extends TransportTemplate {
+    constructor(options) {
+        super(new MirrorChannel(KMIP, TTLVCodec), options);
+    }
+}
+
+const options = {
+    kmip: {
+        codec: {},
+        transport: {
+            pipelineDepth: 8,
+            tls: {
+                port: 5696,
+            },
+        },
+    },
+};
+
+describe('KMIP Low Level Driver', () => {
+    lowlevelFixtures.forEach((fixture, n) => {
+        it(`should work with fixture #${n}`, done => {
+            const kmip = new KMIP(TTLVCodec, MirrorTransport, options);
+            const requestPayload = fixture.payload(kmip);
+            kmip.request(logger, fixture.operation,
+                         requestPayload, (err, response) => {
+                             if (err) {
+                                 return done(err);
+                             }
+                             const responsePayload = response.lookup(
+                                 'Response Message/Batch Item/Response Payload'
+                             )[0];
+                             assert.deepStrictEqual(responsePayload,
+                                                    requestPayload);
+                             return done();
+                         });
+        });
+    });
+});

tests/functional/kmip/transport.js

@@ -0,0 +1,82 @@
+'use strict'; // eslint-disable-line
+
+const async = require('async');
+const assert = require('assert');
+const TransportTemplate =
+      require('../../../lib/network/kmip/transport/TransportTemplate.js');
+const { logger, EchoChannel } = require('../../utils/kmip/ersatz.js');
+
+describe('KMIP Transport Template Class', () => {
+    const pipelineDepths = [1, 2, 4, 8, 16, 32];
+    const requestNumbers = [1, 37, 1021, 8191];
+
+    pipelineDepths.forEach(pipelineDepth => {
+        requestNumbers.forEach(iterations => {
+            it(`should survive ${iterations} iterations` +
+               ` with ${pipelineDepth}way pipeline`,
+               done => {
+                   const transport = new TransportTemplate(
+                       new EchoChannel,
+                       {
+                           pipelineDepth,
+                           tls: {
+                               port: 5696,
+                           },
+                       });
+                   const request = Buffer.alloc(10).fill(6);
+                   async.times(iterations, (n, next) => {
+                       transport.send(logger, request,
+                                      (err, conversation, response) => {
+                                          if (err) {
+                                              return next(err);
+                                          }
+                                          if (request.compare(response) !== 0) {
+                                              return next(Error('arg'));
+                                          }
+                                          return next();
+                                      });
+                   }, err => {
+                       transport.end();
+                       done(err);
+                   });
+               });
+
+            [true, false].forEach(doEmit => {
+                it('should report errors to outstanding requests.' +
+                   ` w:${pipelineDepth}, i:${iterations}, e:${doEmit}`,
+                   done => {
+                       const echoChannel = new EchoChannel;
+                       echoChannel.clog();
+                       const transport = new TransportTemplate(
+                           echoChannel,
+                           {
+                               pipelineDepth,
+                               tls: {
+                                   port: 5696,
+                               },
+                           });
+                       const request = Buffer.alloc(10).fill(6);
+                       /* Using a for loop here instead of anything
+                        * asynchronous, the callbacks get stuck in
+                        * the conversation queue and are unwind with
+                        * an error. It is the purpose of this test */
+                       for (let i = 0; i < iterations; ++i) {
+                           transport.send(
+                               logger, request,
+                               (err, conversation, response) => {
+                                   assert(err);
+                                   assert(!response);
+                               });
+                       }
+                       if (doEmit) {
+                           echoChannel.emit('error', new Error('awesome'));
+                       } else {
+                           transport.abortPipeline(echoChannel);
+                       }
+                       transport.end();
+                       done();
+                   });
+            });
+        });
+    });
+});

tests/functional/metadataProxy/routesToMem.js

@@ -0,0 +1,319 @@
+'use strict'; // eslint-disable-line strict
+
+const werelogs = require('werelogs');
+const assert = require('assert');
+const async = require('async');
+
+const logger = new werelogs.Logger('MetadataProxyServer', 'debug', 'debug');
+const MetadataWrapper =
+      require('../../../lib/storage/metadata/MetadataWrapper');
+const BucketRoutes =
+      require('../../../lib/storage/metadata/proxy/BucketdRoutes');
+const metadataWrapper = new MetadataWrapper('mem', {}, null, logger);
+const { RequestDispatcher } = require('../../utils/mdProxyUtils');
+
+const routes = new BucketRoutes(metadataWrapper, logger);
+const dispatcher = new RequestDispatcher(routes);
+
+const Bucket = 'test';
+const bucketInfo = {
+    acl: {
+        Canned: 'private',
+        FULL_CONTROL: [],
+        WRITE: [],
+        WRITE_ACP: [],
+        READ: [],
+        READ_ACP: [],
+    },
+    name: Bucket,
+    owner: '9d8fe19a78974c56dceb2ea4a8f01ed0f5fecb9d29f80e9e3b84104e4a3ea520',
+    ownerDisplayName: 'anonymousCoward',
+    creationDate: '2018-06-04T17:45:42.592Z',
+    mdBucketModelVersion: 8,
+    transient: false,
+    deleted: false,
+    serverSideEncryption: null,
+    versioningConfiguration: null,
+    locationConstraint: 'us-east-1',
+    readLocationConstraint: 'us-east-1',
+    cors: null,
+    replicationConfiguration: null,
+    lifecycleConfiguration: null,
+    uid: 'fea97818-6a9a-11e8-9777-e311618cc5d4',
+    isNFS: null,
+};
+
+const objects = [
+    'aaa',
+    'bbb/xaa',
+    'bbb/xbb',
+    'bbb/xcc',
+    'ccc',
+    'ddd',
+];
+
+function _getExpectedListing(prefix, objects) {
+    const filtered = objects.map(key => {
+        const deprefixed = key.slice(prefix.length);
+        return deprefixed.replace(/[/].*/, '/');
+    });
+    const keySet = {};
+    return filtered.filter(key => {
+        if (keySet[key]) {
+            return false;
+        }
+        if (key === '') {
+            return false;
+        }
+        keySet[key] = true;
+        return true;
+    });
+}
+
+function _listingURL(prefix, marker) {
+    const reSlash = /[/]/g;
+    const escapedPrefix = prefix.replace(reSlash, '%2F');
+    const escapedMarker = marker.replace(reSlash, '%2F');
+    return `/default/bucket/${Bucket}?delimiter=%2F&prefix=` +
+        `${escapedPrefix}&maxKeys=1&marker=${escapedMarker}`;
+}
+
+function _listObjects(prefix, objects, cb) {
+    const keys = _getExpectedListing(prefix, objects);
+    const markers = keys.slice(0);
+    markers.unshift(undefined);
+    const lastKey = keys[keys.length - 1];
+    const listing = keys.map((key, index) => ({
+        key,
+        marker: markers[index],
+        NextMarker: markers[index + 1],
+        IsTruncated: key !== lastKey,
+        isPrefix: key.endsWith('/'),
+    }));
+    async.mapLimit(listing, 5, (obj, next) => {
+        const currentMarker = obj.marker === undefined ? '' : obj.marker;
+        dispatcher.get(_listingURL(prefix, prefix + currentMarker),
+                       (err, response, body) => {
+                           if (err) {
+                               return next(err);
+                           }
+                           if (obj.isPrefix) {
+                               assert.strictEqual(body.Contents.length, 0);
+                               assert.strictEqual(body.CommonPrefixes.length,
+                                                  1);
+                               assert.strictEqual(body.CommonPrefixes[0],
+                                                  prefix + obj.key);
+                           } else {
+                               assert.strictEqual(body.Contents.length, 1);
+                               assert.strictEqual(body.CommonPrefixes.length,
+                                                  0);
+                               assert.strictEqual(body.Contents[0].key,
+                                                  prefix + obj.key);
+                           }
+                           assert.strictEqual(body.IsTruncated,
+                                              obj.IsTruncated);
+                           if (body.IsTruncated) {
+                               assert.strictEqual(body.NextMarker,
+                                                  prefix + obj.NextMarker);
+                           }
+                           return next();
+                       });
+    }, err => cb(err));
+}
+
+function _createObjects(objects, cb) {
+    async.mapLimit(objects, 5, (key, next) => {
+        dispatcher.post(`/default/bucket/${Bucket}/${key}`,
+                        { key }, next);
+    }, err => {
+        cb(err);
+    });
+}
+
+function _readObjects(objects, cb) {
+    async.mapLimit(objects, 5, (key, next) => {
+        dispatcher.get(`/default/bucket/${Bucket}/${key}`,
+                       (err, response, body) => {
+                           assert.deepStrictEqual(body.key, key);
+                           next(err);
+                       });
+    }, err => {
+        cb(err);
+    });
+}
+
+function _deleteObjects(objects, cb) {
+    async.mapLimit(objects, 5, (key, next) => {
+        dispatcher.delete(`/default/bucket/${Bucket}/${key}`,
+                          err => next(err));
+    }, err => {
+        cb(err);
+    });
+}
+
+describe('Basic Metadata Proxy Server test',
+         function bindToThis() {
+             this.timeout(10000);
+             it('Shoud get the metadataInformation', done => {
+                 dispatcher.get('/default/metadataInformation',
+                                (err, response, body) => {
+                                    if (err) {
+                                        return done(err);
+                                    }
+                                    assert.deepStrictEqual(
+                                        body, { metadataVersion: 2 });
+                                    return done();
+                                });
+             });
+         });
+
+describe('Basic Metadata Proxy Server CRUD test', function bindToThis() {
+    this.timeout(10000);
+
+    beforeEach(done => {
+        dispatcher.post(`/default/bucket/${Bucket}`, bucketInfo,
+                        done);
+    });
+
+    afterEach(done => {
+        dispatcher.delete(`/default/bucket/${Bucket}`, done);
+    });
+
+    it('Should get the bucket attributes', done => {
+        dispatcher.get(`/default/attributes/${Bucket}`,
+                       (err, response, body) => {
+                           if (err) {
+                               return done(err);
+                           }
+                           assert.deepStrictEqual(body.name,
+                                                  bucketInfo.name);
+                           return done();
+                       });
+    });
+
+    it('Should crud an object', done => {
+        async.waterfall([
+            next => dispatcher.post(`/default/bucket/${Bucket}/test1`,
+                                    { foo: 'gabu' }, err => next(err)),
+            next => dispatcher.get(`/default/bucket/${Bucket}/test1`,
+                                   (err, response, body) => {
+                                       if (!err) {
+                                           assert.deepStrictEqual(body.foo,
+                                                                  'gabu');
+                                           next(err);
+                                       }
+                                   }),
+            next => dispatcher.post(`/default/bucket/${Bucket}/test1`,
+                                    { foo: 'zome' }, err => next(err)),
+            next => dispatcher.get(`/default/bucket/${Bucket}/test1`,
+                                   (err, response, body) => {
+                                       if (!err) {
+                                           assert.deepStrictEqual(body.foo,
+                                                                  'zome');
+                                           next(err);
+                                       }
+                                   }),
+            next => dispatcher.delete(`/default/bucket/${Bucket}/test1`,
+                                      err => next(err)),
+        ], err => done(err));
+    });
+
+    it('Should list objects', done => {
+        async.waterfall([
+            next => _createObjects(objects, next),
+            next => _readObjects(objects, next),
+            next => _listObjects('', objects, next),
+            next => _listObjects('bbb/', objects, next),
+            next => _deleteObjects(objects, next),
+        ], err => {
+            done(err);
+        });
+    });
+
+    it('Should update bucket properties', done => {
+        dispatcher.get(
+            `/default/attributes/${Bucket}`, (err, response, body) => {
+                assert.strictEqual(err, null);
+                const bucketInfo = body;
+                const newOwnerDisplayName = 'divertedfrom';
+                bucketInfo.ownerDisplayName = newOwnerDisplayName;
+                dispatcher.post(
+                    `/default/attributes/${Bucket}`, bucketInfo, err => {
+                        assert.strictEqual(err, null);
+                        dispatcher.get(
+                            `/default/attributes/${Bucket}`,
+                            (err, response, body) => {
+                                assert.strictEqual(err, null);
+                                const newBucketInfo = body;
+                                assert.strictEqual(
+                                    newBucketInfo.ownerDisplayName,
+                                    newOwnerDisplayName);
+                                done(null);
+                            });
+                    });
+            });
+    });
+
+    it('Should fail to list a non existing bucket', done => {
+        dispatcher.get('/default/bucket/nonexisting',
+                       (err, response) => {
+                           assert.strictEqual(
+                               response.responseHead.statusCode,
+                               404);
+                           done(err);
+                       });
+    });
+
+    it('Should fail to get attributes from a non existing bucket', done => {
+        dispatcher.get('/default/attributes/nonexisting',
+                       (err, response) => {
+                           assert.strictEqual(
+                               response.responseHead.statusCode,
+                               404);
+                           done(err);
+                       });
+    });
+
+    it('should succeed a health check', done => {
+        dispatcher.get('/_/healthcheck', (err, response, body) => {
+            if (err) {
+                return done(err);
+            }
+            const expectedResponse = {
+                memorybucket: {
+                    code: 200,
+                    message: 'OK',
+                },
+            };
+            assert.strictEqual(response.responseHead.statusCode, 200);
+            assert.deepStrictEqual(body, expectedResponse);
+            return done(err);
+        });
+    });
+
+    it('should work with parallel route', done => {
+        const objectName = 'theObj';
+        async.waterfall([
+            next => _createObjects([objectName], next),
+            next => {
+                dispatcher.get(
+                    `/default/parallel/${Bucket}/${objectName}`,
+                    (err, response, body) => {
+                        if (err) {
+                            return next(err);
+                        }
+                        assert.strictEqual(response.responseHead.statusCode,
+                                           200);
+                        const bucketMD = JSON.parse(body.bucket);
+                        const objectMD = JSON.parse(body.obj);
+                        const expectedObjectMD = { key: objectName };
+                        assert.deepStrictEqual(bucketMD.name,
+                                               bucketInfo.name);
+                        assert.deepStrictEqual(objectMD, expectedObjectMD);
+                        return next(err);
+                    });
+            },
+            next => _deleteObjects([objectName], next),
+        ], done);
+    });
+});

tests/functional/metrics/StatsModel.js

@@ -0,0 +1,318 @@
+'use strict'; // eslint-disable-line strict
+
+const assert = require('assert');
+const async = require('async');
+
+const RedisClient = require('../../../lib/metrics/RedisClient');
+const StatsModel = require('../../../lib/metrics/StatsModel');
+
+// setup redis client
+const config = {
+    host: '127.0.0.1',
+    port: 6379,
+    enableOfflineQueue: false,
+};
+const fakeLogger = {
+    trace: () => {},
+    error: () => {},
+};
+const redisClient = new RedisClient(config, fakeLogger);
+
+// setup stats model
+const STATS_INTERVAL = 300; // 5 minutes
+const STATS_EXPIRY = 86400; // 24 hours
+const statsModel = new StatsModel(redisClient, STATS_INTERVAL, STATS_EXPIRY);
+
+function setExpectedStats(expected) {
+    return expected.concat(
+        Array((STATS_EXPIRY / STATS_INTERVAL) - expected.length).fill(0));
+}
+
+// Since many methods were overwritten, these tests should validate the changes
+// made to the original methods
+describe('StatsModel class', () => {
+    const id = 'arsenal-test';
+    const id2 = 'test-2';
+    const id3 = 'test-3';
+
+    afterEach(() => redisClient.clear(() => {}));
+
+    it('should convert a 2d array columns into rows and vice versa using _zip',
+    () => {
+        const arrays = [
+            [1, 2, 3],
+            [4, 5, 6],
+            [7, 8, 9],
+        ];
+
+        const res = statsModel._zip(arrays);
+        const expected = [
+            [1, 4, 7],
+            [2, 5, 8],
+            [3, 6, 9],
+        ];
+
+        assert.deepStrictEqual(res, expected);
+    });
+
+    it('_zip should return an empty array if given an invalid array', () => {
+        const arrays = [];
+
+        const res = statsModel._zip(arrays);
+
+        assert.deepStrictEqual(res, []);
+    });
+
+    it('_getCount should return a an array of all valid integer values',
+    () => {
+        const res = statsModel._getCount([
+            [null, '1'],
+            [null, '2'],
+            [null, null],
+        ]);
+        assert.deepStrictEqual(res, setExpectedStats([1, 2, 0]));
+    });
+
+    it('should correctly record a new request by default one increment',
+    done => {
+        async.series([
+            next => {
+                statsModel.reportNewRequest(id, (err, res) => {
+                    assert.ifError(err);
+
+                    const expected = [[null, 1], [null, 1]];
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+            next => {
+                statsModel.reportNewRequest(id, (err, res) => {
+                    assert.ifError(err);
+
+                    const expected = [[null, 2], [null, 1]];
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should record new requests by defined amount increments', done => {
+        function noop() {}
+
+        async.series([
+            next => {
+                statsModel.reportNewRequest(id, 9);
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepStrictEqual(res.requests, setExpectedStats([9]));
+                    next();
+                });
+            },
+            next => {
+                statsModel.reportNewRequest(id);
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepStrictEqual(res.requests,
+                        setExpectedStats([10]));
+                    next();
+                });
+            },
+            next => {
+                statsModel.reportNewRequest(id, noop);
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepStrictEqual(res.requests,
+                        setExpectedStats([11]));
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should correctly record a 500 on the server', done => {
+        statsModel.report500(id, (err, res) => {
+            assert.ifError(err);
+
+            const expected = [[null, 1], [null, 1]];
+            assert.deepStrictEqual(res, expected);
+            done();
+        });
+    });
+
+    it('should respond back with total requests as an array', done => {
+        async.series([
+            next => {
+                statsModel.reportNewRequest(id, err => {
+                    assert.ifError(err);
+                    next();
+                });
+            },
+            next => {
+                statsModel.report500(id, err => {
+                    assert.ifError(err);
+                    next();
+                });
+            },
+            next => {
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+
+                    const expected = {
+                        'requests': setExpectedStats([1]),
+                        '500s': setExpectedStats([1]),
+                        'sampleDuration': STATS_EXPIRY,
+                    };
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should not crash on empty results', done => {
+        async.series([
+            next => {
+                statsModel.getStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+                    const expected = {
+                        'requests': setExpectedStats([]),
+                        '500s': setExpectedStats([]),
+                        'sampleDuration': STATS_EXPIRY,
+                    };
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+            next => {
+                statsModel.getAllStats(fakeLogger, id, (err, res) => {
+                    assert.ifError(err);
+                    const expected = {
+                        'requests': setExpectedStats([]),
+                        '500s': setExpectedStats([]),
+                        'sampleDuration': STATS_EXPIRY,
+                    };
+                    assert.deepStrictEqual(res, expected);
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should return a zero-filled array if no ids are passed to getAllStats',
+    done => {
+        statsModel.getAllStats(fakeLogger, [], (err, res) => {
+            assert.ifError(err);
+
+            assert.deepStrictEqual(res.requests, setExpectedStats([]));
+            assert.deepStrictEqual(res['500s'], setExpectedStats([]));
+            done();
+        });
+    });
+
+    it('should get accurately reported data for given id from getAllStats',
+    done => {
+        statsModel.reportNewRequest(id, 9);
+        statsModel.reportNewRequest(id2, 2);
+        statsModel.reportNewRequest(id3, 3);
+        statsModel.report500(id);
+
+        async.series([
+            next => {
+                statsModel.getAllStats(fakeLogger, [id], (err, res) => {
+                    assert.ifError(err);
+
+                    assert.equal(res.requests[0], 9);
+                    assert.equal(res['500s'][0], 1);
+                    next();
+                });
+            },
+            next => {
+                statsModel.getAllStats(fakeLogger, [id, id2, id3],
+                (err, res) => {
+                    assert.ifError(err);
+
+                    assert.equal(res.requests[0], 14);
+                    assert.deepStrictEqual(res.requests,
+                        setExpectedStats([14]));
+                    next();
+                });
+            },
+        ], done);
+    });
+
+    it('should normalize to the nearest hour using normalizeTimestampByHour',
+    () => {
+        const date = new Date('2018-09-13T23:30:59.195Z');
+        const newDate = new Date(statsModel.normalizeTimestampByHour(date));
+
+        assert.strictEqual(date.getHours(), newDate.getHours());
+        assert.strictEqual(newDate.getMinutes(), 0);
+        assert.strictEqual(newDate.getSeconds(), 0);
+        assert.strictEqual(newDate.getMilliseconds(), 0);
+    });
+
+    it('should get previous hour using _getDatePreviousHour', () => {
+        const date = new Date('2018-09-13T23:30:59.195Z');
+        const newDate = statsModel._getDatePreviousHour(new Date(date));
+
+        const millisecondsInOneHour = 3600000;
+        assert.strictEqual(date - newDate, millisecondsInOneHour);
+    });
+
+    it('should get an array of hourly timestamps using getSortedSetHours',
+    () => {
+        const epoch = 1536882476501;
+        const millisecondsInOneHour = 3600000;
+
+        const expected = [];
+        let dateInMilliseconds = statsModel.normalizeTimestampByHour(
+            new Date(epoch));
+
+        for (let i = 0; i < 24; i++) {
+            expected.push(dateInMilliseconds);
+            dateInMilliseconds -= millisecondsInOneHour;
+        }
+        const res = statsModel.getSortedSetHours(epoch);
+
+        assert.deepStrictEqual(res, expected);
+    });
+
+    it('should apply TTL on a new sorted set using addToSortedSet', done => {
+        const key = 'a-test-key';
+        const score = 100;
+        const value = 'a-value';
+
+        const now = Date.now();
+        const nearestHour = statsModel.normalizeTimestampByHour(new Date(now));
+
+        statsModel.addToSortedSet(key, score, value, (err, res) => {
+            assert.ifError(err);
+            // check both a "zadd" and "expire" occurred
+            assert.equal(res, 1);
+            redisClient.ttl(key, (err, res) => {
+                assert.ifError(err);
+                // assert this new set has a ttl applied
+                assert(res > 0);
+
+                const adjustmentSecs = now - nearestHour;
+                const msInADay = 24 * 60 * 60 * 1000;
+                const msInAnHour = 60 * 60 * 1000;
+                const upperLimitSecs =
+                    Math.ceil((msInADay - adjustmentSecs) / 1000);
+                const lowerLimitSecs =
+                    Math.floor((msInADay - adjustmentSecs - msInAnHour) / 1000);
+
+                // assert new ttl is between 23 and 24 hours adjusted by time
+                // elapsed since normalized hourly time
+                assert(res >= lowerLimitSecs);
+                assert(res <= upperLimitSecs);
+
+                done();
+            });
+        });
+    });
+});

tests/unit/auth/v4/headerAuthCheck.js

@@ -269,4 +269,33 @@ describe('v4 headerAuthCheck', () => {
         assert.strictEqual(res.params.version, 4);
         done();
     });
+
+    it('should not return error if proxy_path header is added', done => {
+        // Freezes time so date created within function will be Feb 8, 2016
+        const clock = lolex.install(1454962445000);
+        /* eslint-disable camelcase */
+        const alteredRequest = createAlteredRequest({
+            proxy_path: 'proxy/1234' }, 'headers', request, headers);
+        /* eslint-enable camelcase */
+        const res = headerAuthCheck(alteredRequest, log);
+        clock.uninstall();
+        assert.strictEqual(res.err, null);
+        done();
+    });
+
+    it('should return InvalidRequest error if proxy_path header is invalid',
+    done => {
+        // Freezes time so date created within function will be Feb 8, 2016
+        const clock = lolex.install(1454962445000);
+        /* eslint-disable camelcase */
+        const alteredRequest = createAlteredRequest({
+            proxy_path: 'absc%2proxy/1234' }, 'headers', request, headers);
+        /* eslint-enable camelcase */
+        const res = headerAuthCheck(alteredRequest, log);
+        clock.uninstall();
+        assert.deepStrictEqual(res.err,
+            errors.InvalidArgument.customizeDescription(
+            'invalid proxy_path header'));
+        done();
+    });
 });

tests/unit/auth/v4/queryAuthCheck.js

@@ -225,4 +225,34 @@ describe('v4 queryAuthCheck', () => {
         assert.strictEqual(res.params.version, 4);
         done();
     });
+
+    it('should successfully return no error if proxy_path header is added',
+    done => {
+        // Freezes time so date created within function will be Feb 8, 2016
+        const clock = lolex.install(1454974984001);
+        /* eslint-disable camelcase */
+        const alteredRequest = createAlteredRequest({ proxy_path:
+        'proxy/1234' }, 'headers', request, query);
+        /* eslint-enable camelcase */
+        const res = queryAuthCheck(alteredRequest, log, alteredRequest.query);
+        clock.uninstall();
+        assert.deepStrictEqual(res.err, null);
+        done();
+    });
+
+    it('should return InvalidRequest error if proxy_path header is invalid',
+    done => {
+        // Freezes time so date created within function will be Feb 8, 2016
+        const clock = lolex.install(1454974984001);
+        /* eslint-disable camelcase */
+        const alteredRequest = createAlteredRequest({ proxy_path:
+        'absc%2proxy/1234' }, 'headers', request, query);
+        /* eslint-enable camelcase */
+        const res = queryAuthCheck(alteredRequest, log, alteredRequest.query);
+        clock.uninstall();
+        assert.deepStrictEqual(res.err,
+            errors.InvalidArgument.customizeDescription(
+            'invalid proxy_path header'));
+        done();
+    });
 });

tests/unit/kmip/ttlvCodec.js

@@ -0,0 +1,172 @@
+'use strict'; // eslint-disable-line strict
+/* eslint new-cap: "off" */
+
+const assert = require('assert');
+
+const TTLVCodec = require('../../../lib/network/kmip/codec/ttlv.js');
+const KMIP = require('../../../lib/network/kmip');
+const ttlvFixtures = require('../../utils/kmip/ttlvFixtures');
+const badTtlvFixtures = require('../../utils/kmip/badTtlvFixtures');
+const messageFixtures = require('../../utils/kmip/messageFixtures');
+const { logger } = require('../../utils/kmip/ersatz.js');
+
+function newKMIP() {
+    return new KMIP(TTLVCodec,
+                    class DummyTransport {},
+                    { kmip: {} }, () => {});
+}
+
+describe('KMIP TTLV Codec', () => {
+    it('should map, encode and decode an extension', done => {
+        const kmip = newKMIP();
+        kmip.mapExtension('Dummy Extension', 0x54a000);
+        const msg = KMIP.Message([
+            KMIP.TextString('Dummy Extension', 'beautiful'),
+        ]);
+        const encodedMsg = kmip._encodeMessage(msg);
+        const decodedMsg = kmip._decodeMessage(logger, encodedMsg);
+        assert.deepStrictEqual(msg, decodedMsg);
+        done();
+    });
+
+    ttlvFixtures.forEach((item, idx) => {
+        ['request', 'response'].forEach(fixture => {
+            it(`should decode the TTLV ${fixture} fixture[${idx}]`, done => {
+                const kmip = newKMIP();
+                const msg = kmip._decodeMessage(logger, item[fixture]);
+                if (!item.degenerated) {
+                    const encodedMsg = kmip._encodeMessage(msg);
+                    assert(encodedMsg.compare(item[fixture]) === 0);
+                }
+                done();
+            });
+        });
+    });
+
+    it('should validate supported operations', done => {
+        const kmip = newKMIP();
+        const msg = kmip._decodeMessage(logger, ttlvFixtures[1].response);
+
+        const supportedOperations =
+              msg.lookup('Response Message/Batch Item/' +
+                         'Response Payload/Operation');
+        const supportedObjectTypes =
+              msg.lookup('Response Message/Batch Item/' +
+                         'Response Payload/Object Type');
+        const protocolVersionMajor =
+              msg.lookup('Response Message/Response Header/' +
+                         'Protocol Version/Protocol Version Major');
+        const protocolVersionMinor =
+              msg.lookup('Response Message/Response Header/' +
+                         'Protocol Version/Protocol Version Minor');
+
+        assert(supportedOperations.includes('Encrypt'));
+        assert(supportedOperations.includes('Decrypt'));
+        assert(supportedOperations.includes('Create'));
+        assert(supportedOperations.includes('Destroy'));
+        assert(supportedOperations.includes('Query'));
+        assert(supportedObjectTypes.includes('Symmetric Key'));
+        assert(protocolVersionMajor[0] >= 2 ||
+               (protocolVersionMajor[0] === 1 &&
+                protocolVersionMinor[0] >= 2));
+        done();
+    });
+
+    it('should detect unsupported operations', done => {
+        const kmip = newKMIP();
+        const msg = kmip._decodeMessage(logger, ttlvFixtures[2].response);
+
+        const supportedOperations =
+              msg.lookup('Response Message/Batch Item/' +
+                         'Response Payload/Operation');
+
+        assert(!supportedOperations.includes('Encrypt'));
+        assert(!supportedOperations.includes('Decrypt'));
+        done();
+    });
+
+    it('should support non canonical search path', done => {
+        const kmip = newKMIP();
+        const msg = kmip._decodeMessage(logger, ttlvFixtures[1].response);
+
+        const supportedOperations =
+              msg.lookup('/Response Message/Batch Item/' +
+                         'Response Payload/Operation');
+        const supportedObjectTypes =
+              msg.lookup('Response Message/Batch Item/' +
+                         'Response Payload/Object Type/');
+        const protocolVersionMajor =
+              msg.lookup('Response Message//Response Header///' +
+                         'Protocol Version////Protocol Version Major');
+        const protocolVersionMinor =
+              msg.lookup('/Response Message////Response Header///' +
+                         'Protocol Version//Protocol Version Minor/');
+
+        assert(supportedOperations.includes('Encrypt'));
+        assert(supportedOperations.includes('Decrypt'));
+        assert(supportedOperations.includes('Create'));
+        assert(supportedOperations.includes('Destroy'));
+        assert(supportedOperations.includes('Query'));
+        assert(supportedObjectTypes.includes('Symmetric Key'));
+        assert(protocolVersionMajor[0] >= 2 ||
+               (protocolVersionMajor[0] === 1 &&
+                protocolVersionMinor[0] >= 2));
+        done();
+    });
+
+    it('should return nothing with an empty search path', done => {
+        const kmip = newKMIP();
+        const msg = kmip._decodeMessage(logger, ttlvFixtures[2].response);
+
+        const empty1 = msg.lookup('');
+        const empty2 = msg.lookup('////');
+        assert(empty1.length === 0);
+        assert(empty2.length === 0);
+        done();
+    });
+
+    it('should encode/decode a bit mask', done => {
+        const kmip = newKMIP();
+        const usageMask = ['Encrypt', 'Decrypt', 'Export'];
+        const decodedMask =
+              kmip.decodeMask('Cryptographic Usage Mask',
+                              kmip.encodeMask('Cryptographic Usage Mask',
+                                              usageMask));
+        assert.deepStrictEqual(usageMask.sort(), decodedMask.sort());
+        done();
+    });
+
+    it('should detect invalid bit name', done => {
+        const kmip = newKMIP();
+        const usageMask = ['Encrypt', 'Decrypt', 'Exprot'];
+        try {
+            kmip.encodeMask('Cryptographic Usage Mask', usageMask);
+            done(Error('Must not succeed'));
+        } catch (e) {
+            done();
+        }
+    });
+
+    messageFixtures.forEach((item, idx) => {
+        it(`should encode the KMIP message fixture[${idx}]`, done => {
+            const kmip = newKMIP();
+            const encodedMessage = kmip._encodeMessage(item);
+            const decodedMessage = kmip._decodeMessage(logger, encodedMessage);
+            assert.deepStrictEqual(item.content, decodedMessage.content);
+            done();
+        });
+    });
+
+    badTtlvFixtures.forEach((rawMessage, idx) => {
+        it(`should fail to parse invalid TTLV message fixture[${idx}]`,
+           done => {
+               const kmip = newKMIP();
+               try {
+                   kmip._decodeMessage(logger, rawMessage);
+                   done(Error('Must not succeed'));
+               } catch (e) {
+                   done();
+               }
+           });
+    });
+});

tests/unit/models/BucketInfo.js

@@ -59,6 +59,7 @@ const testWebsiteConfiguration = new WebsiteConfiguration({
 });
 
 const testLocationConstraint = 'us-west-1';
+const testReadLocationConstraint = 'us-west-2';
 
 const testCorsConfiguration = [
     { id: 'test',
@@ -115,6 +116,8 @@ const testLifecycleConfiguration = {
         },
     ],
 };
+
+const testUid = '99ae3446-7082-4c17-ac97-52965dc004ec';
 // create a dummy bucket to test getters and setters
 
 Object.keys(acl).forEach(
@@ -132,7 +135,8 @@ Object.keys(acl).forEach(
             testWebsiteConfiguration,
             testCorsConfiguration,
             testReplicationConfiguration,
-            testLifecycleConfiguration);
+            testLifecycleConfiguration,
+            testUid, undefined, true);
 
         describe('serialize/deSerialize on BucketInfo class', () => {
             const serialized = dummyBucket.serialize();
@@ -151,6 +155,7 @@ Object.keys(acl).forEach(
                     versioningConfiguration:
                         dummyBucket._versioningConfiguration,
                     locationConstraint: dummyBucket._locationConstraint,
+                    readLocationConstraint: dummyBucket._readLocationConstraint,
                     websiteConfiguration: dummyBucket._websiteConfiguration
                         .getConfig(),
                     cors: dummyBucket._cors,
@@ -158,6 +163,8 @@ Object.keys(acl).forEach(
                         dummyBucket._replicationConfiguration,
                     lifecycleConfiguration:
                         dummyBucket._lifecycleConfiguration,
+                    uid: dummyBucket._uid,
+                    isNFS: dummyBucket._isNFS,
                 };
                 assert.strictEqual(serialized, JSON.stringify(bucketInfos));
                 done();
@@ -182,6 +189,7 @@ Object.keys(acl).forEach(
                                       'string');
                    assert.strictEqual(typeof dummyBucket.getCreationDate(),
                                       'string');
+                   assert.strictEqual(typeof dummyBucket.getUid(), 'string');
                });
             it('this should have the right acl\'s types', () => {
                 assert.strictEqual(typeof dummyBucket.getAcl(), 'object');
@@ -249,6 +257,18 @@ Object.keys(acl).forEach(
                 assert.deepStrictEqual(dummyBucket.getLocationConstraint(),
                 testLocationConstraint);
             });
+            it('getReadLocationConstraint should return locationConstraint ' +
+            'if readLocationConstraint hasn\'t been set', () => {
+                assert.deepStrictEqual(dummyBucket.getReadLocationConstraint(),
+                testLocationConstraint);
+            });
+            it('getReadLocationConstraint should return readLocationConstraint',
+            () => {
+                dummyBucket._readLocationConstraint =
+                    testReadLocationConstraint;
+                assert.deepStrictEqual(dummyBucket.getReadLocationConstraint(),
+                testReadLocationConstraint);
+            });
             it('getCors should return CORS configuration', () => {
                 assert.deepStrictEqual(dummyBucket.getCors(),
                         testCorsConfiguration);
@@ -257,6 +277,16 @@ Object.keys(acl).forEach(
                 assert.deepStrictEqual(dummyBucket.getLifecycleConfiguration(),
                     testLifecycleConfiguration);
             });
+            it('getUid should return unique id of bucket', () => {
+                assert.deepStrictEqual(dummyBucket.getUid(), testUid);
+            });
+            it('isNFS should return whether bucket is on NFS', () => {
+                assert.deepStrictEqual(dummyBucket.isNFS(), true);
+            });
+            it('setIsNFS should set whether bucket is on NFS', () => {
+                dummyBucket.setIsNFS(false);
+                assert.deepStrictEqual(dummyBucket.isNFS(), false);
+            });
         });
 
         describe('setters on BucketInfo class', () => {
@@ -328,8 +358,7 @@ Object.keys(acl).forEach(
                         protocol: 'https',
                     },
                 };
-                dummyBucket
-                    .setWebsiteConfiguration(newWebsiteConfiguration);
+                dummyBucket.setWebsiteConfiguration(newWebsiteConfiguration);
                 assert.deepStrictEqual(dummyBucket.getWebsiteConfiguration(),
                     newWebsiteConfiguration);
             });
@@ -381,3 +410,26 @@ Object.keys(acl).forEach(
         });
     })
 );
+
+describe('uid default', () => {
+    it('should set uid if none is specified by constructor params', () => {
+        const dummyBucket = new BucketInfo(
+            bucketName, owner, ownerDisplayName, testDate,
+            BucketInfo.currentModelVersion(), acl[emptyAcl],
+            false, false, {
+                cryptoScheme: 1,
+                algorithm: 'sha1',
+                masterKeyId: 'somekey',
+                mandatory: true,
+            }, testVersioningConfiguration,
+            testLocationConstraint,
+            testWebsiteConfiguration,
+            testCorsConfiguration,
+            testReplicationConfiguration,
+            testLifecycleConfiguration);
+
+        const defaultUid = dummyBucket.getUid();
+        assert(defaultUid);
+        assert.strictEqual(defaultUid.length, 36);
+    });
+});

tests/unit/models/LifecycleConfiguration.js

@@ -175,6 +175,10 @@ function generateFilter(errorTag, tagObj) {
             middleTags = '<Prefix>foo</Prefix><Prefix>bar</Prefix>' +
                 `<Prefix>${tagObj.lastPrefix}</Prefix>`;
         }
+        if (tagObj.label === 'mult-tags') {
+            middleTags = '<And><Tag><Key>color</Key><Value>blue</Value></Tag>' +
+                '<Tag><Key>shape</Key><Value>circle</Value></Tag></And>';
+        }
         Filter = `<Filter>${middleTags}</Filter>`;
         if (tagObj.label === 'also-prefix') {
             Filter = '<Filter></Filter><Prefix></Prefix>';
@@ -349,4 +353,16 @@ describe('LifecycleConfiguration class getLifecycleConfiguration', () => {
             done();
         });
     });
+
+    it('should apply all unique Key tags if multiple tags included', done => {
+        tagObj.label = 'mult-tags';
+        generateParsedXml('Filter', tagObj, parsedXml => {
+            const lcConfig = new LifecycleConfiguration(parsedXml).
+                getLifecycleConfiguration();
+            const expected = [{ key: 'color', val: 'blue' },
+                { key: 'shape', val: 'circle' }];
+            assert.deepStrictEqual(expected, lcConfig.rules[0].filter.tags);
+            done();
+        });
+    });
 });

tests/unit/models/ReplicationConfiguration.spec.js

@@ -0,0 +1,74 @@
+const assert = require('assert');
+const { parseString } = require('xml2js');
+
+const werelogs = require('werelogs');
+
+const ReplicationConfiguration =
+      require('../../../lib/models/ReplicationConfiguration');
+
+const logger = new werelogs.Logger('test:ReplicationConfiguration');
+
+const mockedConfig = {
+    replicationEndpoints: [{
+        type: 'scality',
+        site: 'ring',
+        default: true,
+    }, {
+        type: 'aws_s3',
+        site: 'awsbackend',
+    }, {
+        type: 'gcp',
+        site: 'gcpbackend',
+    }, {
+        type: 'azure',
+        site: 'azurebackend',
+    }],
+};
+
+
+function getXMLConfig(hasPreferredRead) {
+    return `
+    <ReplicationConfiguration>
+        <Role>arn:aws:iam::root:role/s3-replication-role</Role>
+        <Rule>
+            <ID>Replication-Rule-1</ID>
+            <Status>Enabled</Status>
+            <Prefix>someprefix/</Prefix>
+            <Destination>
+                <Bucket>arn:aws:s3:::destbucket</Bucket>
+                <StorageClass>awsbackend,` +
+        `gcpbackend${hasPreferredRead ? ':preferred_read' : ''},azurebackend` +
+        `</StorageClass>
+            </Destination>
+        </Rule>
+    </ReplicationConfiguration>
+`;
+}
+
+describe('ReplicationConfiguration class', () => {
+    it('should parse replication config XML without preferred read', done => {
+        const repConfigXML = getXMLConfig(false);
+        parseString(repConfigXML, (err, parsedXml) => {
+            assert.ifError(err);
+            const repConf = new ReplicationConfiguration(
+                parsedXml, logger, mockedConfig);
+            const repConfErr = repConf.parseConfiguration();
+            assert.ifError(repConfErr);
+            assert.strictEqual(repConf.getPreferredReadLocation(), null);
+            done();
+        });
+    });
+    it('should parse replication config XML with preferred read', done => {
+        const repConfigXML = getXMLConfig(true);
+        parseString(repConfigXML, (err, parsedXml) => {
+            assert.ifError(err);
+            const repConf = new ReplicationConfiguration(
+                parsedXml, logger, mockedConfig);
+            const repConfErr = repConf.parseConfiguration();
+            assert.ifError(repConfErr);
+            assert.strictEqual(repConf.getPreferredReadLocation(),
+                               'gcpbackend');
+            done();
+        });
+    });
+});

tests/unit/models/object.js

@@ -82,6 +82,7 @@ describe('ObjectMD class setters/getters', () => {
             role: '',
             storageType: '',
             dataStoreVersionId: '',
+            isNFS: null,
         }],
         ['ReplicationInfo', {
             status: 'PENDING',
@@ -97,8 +98,11 @@ describe('ObjectMD class setters/getters', () => {
                 'arn:aws:iam::account-id:role/dest-resource',
             storageType: 'aws_s3',
             dataStoreVersionId: '',
+            isNFS: null,
         }],
         ['DataStoreName', null, ''],
+        ['ReplicationIsNFS', null, null],
+        ['ReplicationIsNFS', true],
     ].forEach(test => {
         const property = test[0];
         const testValue = test[1];
@@ -192,6 +196,15 @@ describe('ObjectMD class setters/getters', () => {
         assert.strictEqual(
             md.getReplicationSiteDataStoreVersionId('zenko'), 'a');
     });
+
+    it('ObjectMd::isMultipartUpload', () => {
+        md.setContentMd5('68b329da9893e34099c7d8ad5cb9c940');
+        assert.strictEqual(md.isMultipartUpload(), false);
+        md.setContentMd5('741e0f4bad5b093044dc54a74d911094-1');
+        assert.strictEqual(md.isMultipartUpload(), true);
+        md.setContentMd5('bda0c0bed89c8bdb9e409df7ae7073c5-9876');
+        assert.strictEqual(md.isMultipartUpload(), true);
+    });
 });
 
 describe('ObjectMD import from stored blob', () => {

tests/unit/network/probe/HealthProbeServer.spec.js

@@ -0,0 +1,170 @@
+const assert = require('assert');
+const HealthProbeServer =
+    require('../../../../lib/network/probe/HealthProbeServer');
+const http = require('http');
+
+function makeRequest(meth, uri) {
+    const params = {
+        hostname: 'localhost',
+        port: 4042,
+        method: meth,
+        path: uri,
+    };
+    const req = http.request(params);
+    req.setNoDelay(true);
+    return req;
+}
+
+const endpoints = [
+    '/_/health/liveness',
+    '/_/health/readiness',
+];
+
+const badEndpoints = [
+    '/_/health/liveness_thisiswrong',
+    '/_/health/readiness_thisiswrong',
+];
+
+describe('network.probe.HealthProbeServer', () => {
+    describe('service is "up"', () => {
+        let server;
+        function setup(done) {
+            server = new HealthProbeServer({ port: 4042 });
+            server.start();
+            done();
+        }
+
+        before(done => {
+            setup(done);
+        });
+
+        after(done => {
+            server.stop();
+            done();
+        });
+        endpoints.forEach(ep => {
+            it('should perform a GET and ' +
+                'return 200 OK', done => {
+                makeRequest('GET', ep)
+                    .on('response', res => {
+                        assert(res.statusCode === 200);
+                        done();
+                    })
+                    .on('error', err => {
+                        assert.ifError(err);
+                        done();
+                    }).end();
+            });
+        });
+    });
+
+    describe('service is "down"', () => {
+        let server;
+        function setup(done) {
+            function falseStub() {
+                return false;
+            }
+            server = new HealthProbeServer({
+                port: 4042,
+                livenessCheck: falseStub,
+                readinessCheck: falseStub,
+            });
+            server.start();
+            done();
+        }
+
+        before(done => {
+            setup(done);
+        });
+
+        after(done => {
+            server.stop();
+            done();
+        });
+
+        endpoints.forEach(ep => {
+            it('should perform a GET and ' +
+                'return 503 ServiceUnavailable', done => {
+                makeRequest('GET', ep)
+                    .on('response', res => {
+                        assert(res.statusCode === 503);
+                        done();
+                    })
+                    .on('error', err => {
+                        assert.ifError(err);
+                        done();
+                    }).end();
+            });
+        });
+    });
+
+    describe('Invalid Methods', () => {
+        let server;
+        function setup(done) {
+            server = new HealthProbeServer({
+                port: 4042,
+            });
+            server.start();
+            done();
+        }
+
+        before(done => {
+            setup(done);
+        });
+
+        after(done => {
+            server.stop();
+            done();
+        });
+
+        endpoints.forEach(ep => {
+            it('should perform a POST and ' +
+                'return 405 MethodNotAllowed', done => {
+                makeRequest('POST', ep)
+                    .on('response', res => {
+                        assert(res.statusCode === 405);
+                        done();
+                    })
+                    .on('error', err => {
+                        assert.ifError(err);
+                        done();
+                    }).end();
+            });
+        });
+    });
+
+    describe('Invalid URI', () => {
+        let server;
+        function setup(done) {
+            server = new HealthProbeServer({
+                port: 4042,
+            });
+            server.start();
+            done();
+        }
+
+        before(done => {
+            setup(done);
+        });
+
+        after(done => {
+            server.stop();
+            done();
+        });
+
+        badEndpoints.forEach(ep => {
+            it('should perform a GET and ' +
+                'return 400 InvalidURI', done => {
+                makeRequest('GET', ep)
+                    .on('response', res => {
+                        assert(res.statusCode === 400);
+                        done();
+                    })
+                    .on('error', err => {
+                        assert.ifError(err);
+                        done();
+                    }).end();
+            });
+        });
+    });
+});

tests/unit/s3middleware/validateConditionalHeaders.js

@@ -7,6 +7,7 @@ const {
     _checkEtagNoneMatch,
     _checkModifiedSince,
     _checkUnmodifiedSince,
+    checkDateModifiedHeaders,
     validateConditionalHeaders,
 } = require('../../../lib/s3middleware/validateConditionalHeaders');
 
@@ -172,6 +173,59 @@ describe('validateConditionalHeaders util function ::', () => {
     });
 });
 
+describe('checkDateModifiedHeaders util function: ', () => {
+    const expectedSuccess = {
+        present: true,
+        error: null,
+    };
+
+    const expectedAbsense = {
+        present: false,
+        error: null,
+    };
+
+    it('should return NotModified error for \'if-modified-since\' header',
+    () => {
+        const header = {};
+        header['if-modified-since'] = afterLastModified;
+        const { modifiedSinceRes, unmodifiedSinceRes } =
+            checkDateModifiedHeaders(header, lastModified);
+        assert.deepStrictEqual(modifiedSinceRes.error, errors.NotModified);
+        assert.deepStrictEqual(unmodifiedSinceRes, expectedAbsense);
+    });
+
+    it('should return PreconditionFailed error for \'if-unmodified-since\' ' +
+    'header', () => {
+        const header = {};
+        header['if-unmodified-since'] = beforeLastModified;
+        const { modifiedSinceRes, unmodifiedSinceRes } =
+            checkDateModifiedHeaders(header, lastModified);
+        assert.deepStrictEqual(unmodifiedSinceRes.error,
+            errors.PreconditionFailed);
+        assert.deepStrictEqual(modifiedSinceRes, expectedAbsense);
+    });
+
+    it('should succeed if \'if-modified-since\' header value is earlier ' +
+    'than last modified', () => {
+        const header = {};
+        header['if-modified-since'] = beforeLastModified;
+        const { modifiedSinceRes, unmodifiedSinceRes } =
+            checkDateModifiedHeaders(header, lastModified);
+        assert.deepStrictEqual(modifiedSinceRes, expectedSuccess);
+        assert.deepStrictEqual(unmodifiedSinceRes, expectedAbsense);
+    });
+
+    it('should succeed if \'if-unmodified-since\' header value is later ' +
+    'than last modified', () => {
+        const header = {};
+        header['if-unmodified-since'] = afterLastModified;
+        const { modifiedSinceRes, unmodifiedSinceRes } =
+            checkDateModifiedHeaders(header, lastModified);
+        assert.deepStrictEqual(unmodifiedSinceRes, expectedSuccess);
+        assert.deepStrictEqual(modifiedSinceRes, expectedAbsense);
+    });
+});
+
 describe('_checkEtagMatch function :', () => {
     const expectedSuccess = {
         present: true,

tests/unit/s3routes/routesUtils/isValidObjectKey.js

@@ -0,0 +1,36 @@
+const assert = require('assert');
+const routesUtils = require('../../../../lib/s3routes/routesUtils.js');
+
+const bannedStr = 'banned';
+const prefixBlacklist = [];
+
+// byte size of 915
+const keyutf8 = '%EA%9D%8B崰㈌㒈保轖䳷䀰⺩ቆ楪秲ⴝ㿅鼎ꓜ퇬枅࿷염곞召㸾⌙ꪊᆐ庍뉆䌗幐鸆䛃➟녩' +
+    'ˍ뙪臅⠙≼绒벊냂詴 끴鹲萯⇂㭢䈊퉉楝舳㷖족痴䧫㾵᏷ำꎆ꼵껪멷㄀誕㳓腜쒃컹㑻鳃삚舿췈孨੦⮀NJ곓⵪꺼꜈' +
+    '嗼뫘悕錸瑺⁤륒㜓垻ㆩꝿ詀펉ᆙ舑䜾힑藪碙ꀎꂰ췊Ᏻ  㘺幽醛잯ද汧Ꟑꛒⶨ쪸숞헹㭔ꡔᘼ뺓ᡆ᡾ᑟ䅅퀭耓弧⢠⇙' +
+    '폪ް蛧⃪Ἔ돫ꕢ븥ヲ캂䝄쟐颺ᓾ둾Ұ껗礞ᾰ瘹蒯硳풛瞋襎奺熝妒컚쉴⿂㽝㝳駵鈚䄖戭䌸᫲ᇁ䙪鸮ᐴ稫ⶭ뀟ھ⦿' +
+    '䴳稉ꉕ捈袿놾띐✯伤䃫⸧ꠏ瘌틳藔ˋ㫣敀䔩㭘식↴⧵佶痊牌ꪌ搒꾛æᤈべ쉴挜敗羥誜嘳ֶꫜ걵ࣀ묟ኋ拃秷膤䨸菥' +
+    '䟆곘縧멀煣卲챸⧃⏶혣ਔ뙞밺㊑ک씌촃Ȅ頰ᖅ懚ホῐ꠷㯢먈㝹୥밷㮇䘖桲阥黾噘烻ᓧ鈠ᴥ徰穆ꘛ蹕綻表鯍裊' +
+    '鮕漨踒ꠍ픸Ä☶莒浏钸목탬툖氭ˠٸ൪㤌ᶟ訧ᜒೳ揪Ⴛ摖㸣᳑⹞걀ꢢ䏹ῖ"';
+
+describe('routesUtils.isValidObjectKey', () => {
+    it('should return isValid false if object key name starts with a ' +
+    'blacklisted prefix', () => {
+        const result = routesUtils.isValidObjectKey('bannedkey', [bannedStr]);
+        // return { isValid: false, invalidPrefix };
+        assert.strictEqual(result.isValid, false);
+        assert.strictEqual(result.invalidPrefix, bannedStr);
+    });
+
+    it('should return isValid false if object key name exceeds length of 915',
+    () => {
+        const key = 'a'.repeat(916);
+        const result = routesUtils.isValidObjectKey(key, prefixBlacklist);
+        assert.strictEqual(result.isValid, false);
+    });
+
+    it('should return isValid true for a utf8 string of byte size 915', () => {
+        const result = routesUtils.isValidObjectKey(keyutf8, prefixBlacklist);
+        assert.strictEqual(result.isValid, true);
+    });
+});

tests/unit/storage/metadata/in_memory/bucket_utilities.js

@@ -0,0 +1,76 @@
+'use strict'; //eslint-disable-line
+
+const assert = require('assert');
+
+const { markerFilterMPU } =
+    require('../../../../../lib/storage/metadata/in_memory/bucket_utilities');
+
+function dupeArray(arr) {
+    const dupe = [];
+
+    arr.forEach(i => {
+        dupe.push(Object.assign({}, i));
+    });
+
+    return dupe;
+}
+
+describe('bucket utility methods for in_memory backend', () => {
+    it('should return an array of multipart uploads starting with the item ' +
+    'right after the specified keyMarker and uploadIdMarker', () => {
+        const mpus = [
+            {
+                key: 'key-1',
+                uploadId: '2624ca6080c841d48a2481941df868a9',
+            },
+            {
+                key: 'key-1',
+                uploadId: '4ffeca96b0c24ea9b538b8f0b60cede3',
+            },
+            {
+                key: 'key-1',
+                uploadId: '52e5b94474894990a2b94330bb3c8fa9',
+            },
+            {
+                key: 'key-1',
+                uploadId: '54e530c5d4c741898c8e161d426591cb',
+            },
+            {
+                key: 'key-1',
+                uploadId: '6cc59f9d29254e81ab6cb6332fb46314',
+            },
+            {
+                key: 'key-1',
+                uploadId: 'fe9dd10776c9476697632d0b55960a05',
+            },
+            {
+                key: 'key-2',
+                uploadId: '68e24ccb96c14beea79bf01fc130fdf5',
+            },
+        ];
+
+        [
+            {
+                keyMarker: 'key-1',
+                uploadIdMarker: '54e530c5d4c741898c8e161d426591cb',
+                expected: 3,
+            },
+            {
+                keyMarker: 'key-2',
+                uploadIdMarker: '68e24ccb96c14beea79bf01fc130fdf5',
+                expected: 0,
+            },
+            {
+                keyMarker: 'key-1',
+                uploadIdMarker: '2624ca6080c841d48a2481941df868a9',
+                expected: 6,
+            },
+        ].forEach(item => {
+            const res = markerFilterMPU(item, dupeArray(mpus));
+            assert.equal(res.length, item.expected);
+
+            const expected = mpus.slice(mpus.length - res.length);
+            assert.deepStrictEqual(res, expected);
+        });
+    });
+});

tests/unit/storage/metadata/mongoclient/DataCounter.js

@@ -0,0 +1,507 @@
+const assert = require('assert');
+
+const {
+    NEW_OBJ,
+    NEW_VER,
+    UPDATE_VER,
+    UPDATE_MST,
+    RESTORE,
+    DEL_VER,
+    DEL_MST,
+    DataCounter,
+} = require('../../../../../lib/storage/metadata/mongoclient/DataCounter');
+
+const refZeroObj = {
+    objects: 0,
+    versions: 0,
+    dataManaged: {
+        total: { curr: 0, prev: 0 },
+        byLocation: {},
+    },
+    stalled: 0,
+};
+
+const refSingleObj = {
+    objects: 2,
+    versions: 0,
+    dataManaged: {
+        total: { curr: 200, prev: 0 },
+        byLocation: {
+            locationOne: { curr: 200, prev: 0 },
+        },
+    },
+    stalled: 0,
+};
+
+const refSingleObjVer = {
+    objects: 1,
+    versions: 1,
+    dataManaged: {
+        total: { curr: 100, prev: 100 },
+        byLocation: {
+            locationOne: { curr: 100, prev: 100 },
+        },
+    },
+    stalled: 0,
+};
+
+const refMultiObjVer = {
+    objects: 1,
+    versions: 1,
+    dataManaged: {
+        total: { curr: 200, prev: 200 },
+        byLocation: {
+            locationOne: { curr: 100, prev: 100 },
+            locationTwo: { curr: 100, prev: 100 },
+        },
+    },
+    stalled: 0,
+};
+
+const refMultiObj = {
+    objects: 2,
+    versions: 0,
+    dataManaged: {
+        total: { curr: 400, prev: 0 },
+        byLocation: {
+            locationOne: { curr: 200, prev: 0 },
+            locationTwo: { curr: 200, prev: 0 },
+        },
+    },
+    stalled: 0,
+};
+
+const singleSite = size => ({
+    'content-length': size,
+    'dataStoreName': 'locationOne',
+    'replicationInfo': {
+        backends: [],
+    },
+});
+
+const multiSite = (size, isComplete) => ({
+    'content-length': size,
+    'dataStoreName': 'locationOne',
+    'replicationInfo': {
+        backends: [{
+            site: 'locationTwo',
+            status: isComplete ? 'COMPLETED' : 'PENDING',
+        }],
+    },
+});
+
+const transientSite = (size, status, backends) => ({
+    'content-length': size,
+    'dataStoreName': 'locationOne',
+    'replicationInfo': { status, backends },
+});
+
+const locationConstraints = {
+    locationOne: { isTransient: true },
+    locationTwo: { isTransient: false },
+};
+
+const dataCounter = new DataCounter();
+
+describe('DataCounter Class', () => {
+    it('should create a zero object', () => {
+        dataCounter.set(refZeroObj);
+        assert.deepStrictEqual(dataCounter.results(), refZeroObj);
+    });
+
+    it('should skip dataCounter methods if initial values are not set', () => {
+        const testCounter = new DataCounter();
+        testCounter.addObject(singleSite(100), null, NEW_OBJ);
+        assert.deepStrictEqual(testCounter.results(), refZeroObj);
+    });
+});
+
+describe('DateCounter::updateTransientList', () => {
+    afterEach(() => dataCounter.updateTransientList({}));
+    it('should set transient list', () => {
+        assert.deepStrictEqual(dataCounter.transientList, {});
+        dataCounter.updateTransientList(locationConstraints);
+        const expectedRes = { locationOne: true, locationTwo: false };
+        assert.deepStrictEqual(dataCounter.transientList, expectedRes);
+    });
+});
+
+describe('DataCounter::addObject', () => {
+    const tests = [
+        {
+            it: 'should correctly update DataCounter, new object one site',
+            init: refZeroObj,
+            input: [singleSite(100), null, NEW_OBJ],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 100, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, new object multi site',
+            init: refZeroObj,
+            input: [multiSite(100, true), null, NEW_OBJ],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 200, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                        locationTwo: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, overwrite single site',
+            init: refSingleObj,
+            input: [singleSite(100), singleSite(50), NEW_OBJ],
+            expectedRes: {
+                objects: 2, versions: 0,
+                dataManaged: {
+                    total: { curr: 250, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 250, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, overwrite multi site',
+            init: refMultiObj,
+            input: [multiSite(100, true), multiSite(50, true), NEW_OBJ],
+            expectedRes: {
+                objects: 2, versions: 0,
+                dataManaged: {
+                    total: { curr: 500, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 250, prev: 0 },
+                        locationTwo: { curr: 250, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, new version single site',
+            init: refSingleObj,
+            input: [singleSite(100), singleSite(50), NEW_VER],
+            expectedRes: {
+                objects: 2, versions: 1,
+                dataManaged: {
+                    total: { curr: 250, prev: 50 },
+                    byLocation: {
+                        locationOne: { curr: 250, prev: 50 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, new version multi site',
+            init: refMultiObj,
+            input: [multiSite(100, true), multiSite(50, true), NEW_VER],
+            expectedRes: {
+                objects: 2, versions: 1,
+                dataManaged: {
+                    total: { curr: 500, prev: 100 },
+                    byLocation: {
+                        locationOne: { curr: 250, prev: 50 },
+                        locationTwo: { curr: 250, prev: 50 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly ignore pending status, multi site',
+            init: refZeroObj,
+            input: [multiSite(100, false), null, NEW_OBJ],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 100, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'replication completion update in master object',
+            init: refSingleObj,
+            input: [multiSite(100, true), multiSite(100, false), UPDATE_MST],
+            expectedRes: {
+                objects: 2, versions: 0,
+                dataManaged: {
+                    total: { curr: 300, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 200, prev: 0 },
+                        locationTwo: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'replication completion update in versioned object',
+            init: refSingleObjVer,
+            input: [multiSite(100, true), multiSite(100, false), UPDATE_VER],
+            expectedRes: {
+                objects: 1, versions: 1,
+                dataManaged: {
+                    total: { curr: 100, prev: 200 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 100 },
+                        locationTwo: { curr: 0, prev: 100 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'restoring versioned object as master',
+            init: refMultiObjVer,
+            input: [multiSite(100, true), multiSite(100, true), RESTORE],
+            expectedRes: {
+                objects: 2, versions: 0,
+                dataManaged: {
+                    total: { curr: 400, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 200, prev: 0 },
+                        locationTwo: { curr: 200, prev: 0 },
+                    },
+                },
+            },
+        },
+    ];
+    tests.forEach(test => it(test.it, () => {
+        const { expectedRes, input, init } = test;
+        dataCounter.set(init);
+        dataCounter.addObject(...input);
+        const testResults = dataCounter.results();
+        Object.keys(expectedRes).forEach(key => {
+            if (typeof expectedRes[key] === 'object') {
+                assert.deepStrictEqual(testResults[key], expectedRes[key]);
+            } else {
+                assert.strictEqual(testResults[key], expectedRes[key]);
+            }
+        });
+    }));
+});
+
+describe('DataCounter, update with transient location', () => {
+    before(() => dataCounter.updateTransientList(locationConstraints));
+    after(() => dataCounter.updateTransientList({}));
+
+    const pCurrMD = transientSite(100, 'PENDING', [
+        { site: 'site1', status: 'PENDING' },
+        { site: 'site2', status: 'COMPLETED' },
+    ]);
+    const cCurrMD = transientSite(100, 'COMPLETED', [
+        { site: 'site1', status: 'COMPLETED' },
+        { site: 'site2', status: 'COMPLETED' },
+    ]);
+    const prevMD = transientSite(100, 'PENDING', [
+        { site: 'site1', status: 'PENDING' },
+        { site: 'site2', status: 'PENDING' },
+    ]);
+    const transientTest = [
+        {
+            it: 'should correctly update DataCounter, ' +
+            'version object, replication status = PENDING',
+            init: refSingleObjVer,
+            input: [pCurrMD, prevMD, UPDATE_VER],
+            expectedRes: {
+                objects: 1, versions: 1,
+                dataManaged: {
+                    total: { curr: 100, prev: 200 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 100 },
+                        site2: { curr: 0, prev: 100 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'version object, replication status = COMPLETED',
+            init: refSingleObjVer,
+            input: [cCurrMD, prevMD, UPDATE_VER],
+            expectedRes: {
+                objects: 1, versions: 1,
+                dataManaged: {
+                    total: { curr: 100, prev: 200 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                        site1: { curr: 0, prev: 100 },
+                        site2: { curr: 0, prev: 100 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'master object, replication status = PENDING',
+            init: refSingleObjVer,
+            input: [pCurrMD, prevMD, UPDATE_MST],
+            expectedRes: {
+                objects: 1, versions: 1,
+                dataManaged: {
+                    total: { curr: 200, prev: 100 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 100 },
+                        site2: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'master object, replication status = COMPLETED',
+            init: refSingleObjVer,
+            input: [cCurrMD, prevMD, UPDATE_MST],
+            expectedRes: {
+                objects: 1, versions: 1,
+                dataManaged: {
+                    total: { curr: 200, prev: 100 },
+                    byLocation: {
+                        locationOne: { curr: 0, prev: 100 },
+                        site1: { curr: 100, prev: 0 },
+                        site2: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+    ];
+
+    transientTest.forEach(test => it(test.it, () => {
+        const { expectedRes, input, init } = test;
+        dataCounter.set(init);
+        dataCounter.addObject(...input);
+        const testResults = dataCounter.results();
+        Object.keys(expectedRes).forEach(key => {
+            if (typeof expectedRes[key] === 'object') {
+                assert.deepStrictEqual(testResults[key], expectedRes[key]);
+            } else {
+                assert.strictEqual(testResults[key], expectedRes[key]);
+            }
+        });
+    }));
+});
+
+describe('DataCounter::delObject', () => {
+    const tests = [
+        {
+            it: 'should correctly update DataCounter, ' +
+            'delete master object single site',
+            init: refMultiObj,
+            input: [singleSite(100), DEL_MST],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 300, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                        locationTwo: { curr: 200, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'delete master object multi site',
+            init: refMultiObj,
+            input: [multiSite(100, true), DEL_MST],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 200, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                        locationTwo: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'delete versioned object single site',
+            init: refMultiObjVer,
+            input: [singleSite(100), DEL_VER],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 200, prev: 100 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                        locationTwo: { curr: 100, prev: 100 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should correctly update DataCounter, ' +
+            'delete versioned object multi site',
+            init: refMultiObjVer,
+            input: [multiSite(100, true), DEL_VER],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 200, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                        locationTwo: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should clamp negative values to 0, master object',
+            init: refMultiObjVer,
+            input: [multiSite(300, true), DEL_MST],
+            expectedRes: {
+                objects: 0, versions: 1,
+                dataManaged: {
+                    total: { curr: 0, prev: 200 },
+                    byLocation: {
+                        locationOne: { curr: 0, prev: 100 },
+                        locationTwo: { curr: 0, prev: 100 },
+                    },
+                },
+            },
+        },
+        {
+            it: 'should clamp negative values to 0, versioned object',
+            init: refMultiObjVer,
+            input: [multiSite(300, true), DEL_VER],
+            expectedRes: {
+                objects: 1, versions: 0,
+                dataManaged: {
+                    total: { curr: 200, prev: 0 },
+                    byLocation: {
+                        locationOne: { curr: 100, prev: 0 },
+                        locationTwo: { curr: 100, prev: 0 },
+                    },
+                },
+            },
+        },
+    ];
+
+    tests.forEach(test => it(test.it, () => {
+        const { expectedRes, input, init } = test;
+        dataCounter.set(init);
+        dataCounter.delObject(...input);
+        const testResults = dataCounter.results();
+        Object.keys(expectedRes).forEach(key => {
+            if (typeof expectedRes[key] === 'object') {
+                assert.deepStrictEqual(testResults[key], expectedRes[key]);
+            } else {
+                assert.strictEqual(testResults[key], expectedRes[key]);
+            }
+        });
+    }));
+});

tests/unit/storage/metadata/mongoclient/ListRecordStream.spec.js

@@ -0,0 +1,367 @@
+const assert = require('assert');
+const { Timestamp } = require('bson');
+
+const ListRecordStream = require(
+    '../../../../../lib/storage/metadata/mongoclient/ListRecordStream');
+const DummyRequestLogger = require('./utils/DummyRequestLogger');
+
+const logger = new DummyRequestLogger();
+
+const mongoProcessedLogEntries = {
+    insert: {
+        h: -42,
+        ts: Timestamp.fromNumber(42),
+        op: 'i',
+        ns: 'metadata.replicated-bucket',
+        o: {
+            _id: 'replicated-key\u000098467518084696999999RG001  19.3',
+            value: {
+                someField: 'someValue',
+            },
+        },
+    },
+    updateObject: {
+        h: -42,
+        ts: Timestamp.fromNumber(42),
+        op: 'u',
+        ns: 'metadata.replicated-bucket',
+        o2: {
+            _id: 'replicated-key\u000098467518084696999999RG001  19.3',
+        },
+        o: {
+            $set: {
+                value: {
+                    someField: 'someUpdatedValue',
+                },
+            },
+        },
+    },
+    deleteObject: {
+        h: -42,
+        ts: Timestamp.fromNumber(42),
+        op: 'd',
+        ns: 'metadata.replicated-bucket',
+        o: {
+            _id: 'replicated-key\u000098467518084696999999RG001  19.3',
+        },
+    },
+    putBucketAttributes: {
+        h: -42,
+        ts: Timestamp.fromNumber(42),
+        op: 'u',
+        ns: 'metadata.__metastore',
+        o2: {
+            _id: 'new-bucket',
+        }, o: {
+            _id: 'new-bucket',
+            value: {
+                someField: 'someValue',
+            },
+        },
+    },
+    deleteBucket: {
+        h: -42,
+        ts: Timestamp.fromNumber(42),
+        op: 'd',
+        ns: 'metadata.__metastore',
+        o: {
+            _id: 'new-bucket',
+        },
+    },
+};
+
+const mongoIgnoredLogEntries = {
+    createBucket: {
+        h: -42,
+        ts: Timestamp.fromNumber(42),
+        op: 'c',
+        ns: 'metadata.$cmd',
+        o: {
+            create: 'new-bucket',
+            idIndex: {
+                v: 2,
+                key: { _id: 1 },
+                name: '_id_',
+                ns: 'metadata.new-bucket',
+            },
+        },
+    },
+    dropBucketDb: {
+        h: -42,
+        ts: Timestamp.fromNumber(42),
+        op: 'c',
+        ns: 'metadata.$cmd',
+        o: {
+            drop: 'new-bucket',
+        },
+    },
+};
+
+const expectedStreamEntries = {
+    insert: {
+        db: 'replicated-bucket',
+        entries: [
+            {
+                key: 'replicated-key\u000098467518084696999999RG001  19.3',
+                type: 'put',
+                value: '{"someField":"someValue"}',
+            },
+        ],
+        timestamp: new Date(42000),
+    },
+    updateObject: {
+        db: 'replicated-bucket',
+        entries: [
+            {
+                key: 'replicated-key\u000098467518084696999999RG001  19.3',
+                type: 'put',
+                value: '{"someField":"someUpdatedValue"}',
+            },
+        ],
+        timestamp: new Date(42000),
+    },
+    deleteObject: {
+        db: 'replicated-bucket',
+        entries: [
+            {
+                key: 'replicated-key\u000098467518084696999999RG001  19.3',
+                type: 'delete',
+            },
+        ],
+        timestamp: new Date(42000),
+    },
+    putBucketAttributes: {
+        db: '__metastore',
+        entries: [
+            {
+                key: 'new-bucket',
+                type: 'put',
+                value: '{"someField":"someValue"}',
+            },
+        ],
+        timestamp: new Date(42000),
+    },
+    deleteBucket: {
+        db: '__metastore',
+        entries: [
+            {
+                key: 'new-bucket',
+                type: 'delete',
+            },
+        ],
+        timestamp: new Date(42000),
+    },
+    dropBucketDb: {
+        h: -42,
+        op: 'c',
+        ns: 'metadata.$cmd',
+        o: {
+            drop: 'new-bucket',
+        },
+    },
+};
+
+class MongoCursorMock {
+    constructor(itemsToYield, errorAtPos) {
+        this.itemsToYield = itemsToYield;
+        this.pos = 0;
+        this.errorAtPos = errorAtPos;
+    }
+
+    next(cb) {
+        // if there's no more item, just hang out there waiting for
+        // items that will never come (this is how the real mongo
+        // tailable cursor would behave)
+        if (this.pos === this.errorAtPos) {
+            return process.nextTick(() => cb(new Error('boo')));
+        }
+        if (!this.hasSentAllItems()) {
+            const pos = this.pos;
+            this.pos += 1;
+            return process.nextTick(() => cb(null, this.itemsToYield[pos]));
+        }
+        return undefined;
+    }
+    hasSentAllItems() {
+        return this.pos === this.itemsToYield.length;
+    }
+}
+
+describe('mongoclient.ListRecordStream', () => {
+    const lastEndIDEntry = {
+        h: -43,
+        ts: Timestamp.fromNumber(42),
+    };
+    Object.keys(mongoProcessedLogEntries).forEach(entryType => {
+        it(`should transform ${entryType}`, done => {
+            // first write will be ignored by ListRecordStream because
+            // of the last end ID (-42), it's needed though to bootstrap it
+            const cursor = new MongoCursorMock([
+                lastEndIDEntry,
+                mongoProcessedLogEntries[entryType],
+            ]);
+            const lrs = new ListRecordStream(cursor, logger,
+                                             lastEndIDEntry.h.toString());
+            let hasReceivedData = false;
+            lrs.on('data', entry => {
+                assert.strictEqual(hasReceivedData, false);
+                hasReceivedData = true;
+                assert.deepStrictEqual(entry, expectedStreamEntries[entryType]);
+                if (cursor.hasSentAllItems()) {
+                    assert.strictEqual(hasReceivedData, true);
+                    assert.deepStrictEqual(JSON.parse(lrs.getOffset()),
+                                           { uniqID: '-42' });
+                    done();
+                }
+            });
+        });
+    });
+    it('should ignore other entry types', done => {
+        // first write will be ignored by ListRecordStream because
+        // of the last end ID (-43), it's needed though to bootstrap it
+        const logEntries = [lastEndIDEntry];
+        Object.keys(mongoIgnoredLogEntries).forEach(entryType => {
+            logEntries.push(mongoIgnoredLogEntries[entryType]);
+        });
+        const cursor = new MongoCursorMock(logEntries);
+        const lrs = new ListRecordStream(cursor, logger,
+                                         lastEndIDEntry.h.toString());
+        lrs.on('data', entry => {
+            assert(false, `ListRecordStream did not ignore entry ${entry}`);
+        });
+        setTimeout(() => {
+            assert.strictEqual(cursor.hasSentAllItems(), true);
+            assert.deepStrictEqual(JSON.parse(lrs.getOffset()),
+                                   { uniqID: '-42' });
+            done();
+        }, 200);
+    });
+    it('should skip entries until uniqID is encountered', done => {
+        const logEntries = [
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 1234, ts: Timestamp.fromNumber(45) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 5678, ts: Timestamp.fromNumber(44) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: -1234, ts: Timestamp.fromNumber(42) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 2345, ts: Timestamp.fromNumber(42) }),
+        ];
+        const cursor = new MongoCursorMock(logEntries);
+        const lrs = new ListRecordStream(cursor, logger, '5678');
+        assert.strictEqual(lrs.reachedUnpublishedListing(), false);
+        let nbReceivedEntries = 0;
+        lrs.on('data', entry => {
+            assert.deepStrictEqual(entry, expectedStreamEntries.insert);
+            assert.strictEqual(lrs.reachedUnpublishedListing(), true);
+            ++nbReceivedEntries;
+            if (cursor.hasSentAllItems()) {
+                assert.strictEqual(nbReceivedEntries, 2);
+                assert.deepStrictEqual(JSON.parse(lrs.getOffset()),
+                                       { uniqID: '2345' });
+                assert.strictEqual(lrs.getSkipCount(), 2);
+                assert.strictEqual(lrs.reachedUnpublishedListing(), true);
+                done();
+            }
+        });
+    });
+
+    it('should start after latest entry if uniqID is not encountered', done => {
+        const logEntries = [
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 1234, ts: Timestamp.fromNumber(45) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 5678, ts: Timestamp.fromNumber(44) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: -1234, ts: Timestamp.fromNumber(42) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 2345, ts: Timestamp.fromNumber(42) }),
+        ];
+        const cursor = new MongoCursorMock(logEntries);
+        const lrs = new ListRecordStream(cursor, logger, '4242', '-1234');
+        let nbReceivedEntries = 0;
+        lrs.on('data', entry => {
+            assert.deepStrictEqual(entry, expectedStreamEntries.insert);
+            ++nbReceivedEntries;
+            if (cursor.hasSentAllItems()) {
+                assert.strictEqual(nbReceivedEntries, 1);
+                assert.deepStrictEqual(JSON.parse(lrs.getOffset()),
+                                       { uniqID: '2345' });
+                assert.strictEqual(lrs.getSkipCount(), 3);
+                assert.strictEqual(lrs.reachedUnpublishedListing(), true);
+                done();
+            }
+        });
+    });
+    it('should consume from the first entry if there is no saved ID', done => {
+        const logEntries = [
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 1234, ts: Timestamp.fromNumber(42) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 5678, ts: Timestamp.fromNumber(42) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: -1234, ts: Timestamp.fromNumber(42) }),
+            Object.assign({}, mongoProcessedLogEntries.insert,
+                          { h: 2345, ts: Timestamp.fromNumber(42) }),
+        ];
+        const cursor = new MongoCursorMock(logEntries);
+        const lrs = new ListRecordStream(cursor, logger, undefined, '-1234');
+        let nbReceivedEntries = 0;
+        lrs.on('data', entry => {
+            assert.deepStrictEqual(entry, expectedStreamEntries.insert);
+            ++nbReceivedEntries;
+            if (cursor.hasSentAllItems()) {
+                assert.strictEqual(nbReceivedEntries, 4);
+                assert.deepStrictEqual(JSON.parse(lrs.getOffset()),
+                                       { uniqID: '2345' });
+                assert.strictEqual(lrs.getSkipCount(), 0);
+                assert.strictEqual(lrs.reachedUnpublishedListing(), true);
+                done();
+            }
+        });
+    });
+    it('should emit an error event when cursor returns an error', done => {
+        const cursor = new MongoCursorMock([], 0);
+        const lrs = new ListRecordStream(cursor, logger, '4242', '-1234');
+        lrs.on('data', () => {
+            assert(false, 'did not expect data');
+        });
+        lrs.on('error', () => done());
+    });
+
+    it('should support bucket names with dots', done => {
+        const logEntry = {
+            h: -42,
+            ts: Timestamp.fromNumber(42),
+            op: 'i',
+            ns: 'metadata.some.bucket.with.dots',
+            o: {
+                _id: 'replicated-key\u000098467518084696999999RG001  19.3',
+                value: {
+                    someField: 'someValue',
+                },
+            },
+        };
+        const expectedLogEntry = {
+            db: 'some.bucket.with.dots',
+            entries: [
+                {
+                    key: 'replicated-key\u000098467518084696999999RG001  19.3',
+                    type: 'put',
+                    value: '{"someField":"someValue"}',
+                },
+            ],
+            timestamp: new Date(42000),
+        };
+        const cursor = new MongoCursorMock([
+            lastEndIDEntry,
+            logEntry,
+        ]);
+        const lrs = new ListRecordStream(cursor, logger,
+                                         lastEndIDEntry.h.toString());
+        lrs.on('data', entry => {
+            assert.deepStrictEqual(entry, expectedLogEntry);
+            done();
+        });
+    });
+});

tests/unit/storage/metadata/mongoclient/MongoClientInterface.js

@@ -0,0 +1,238 @@
+const assert = require('assert');
+
+const MongoClientInterface = require(
+    '../../../../../lib/storage/metadata/mongoclient/MongoClientInterface');
+const DummyMongoDB = require('./utils/DummyMongoDB');
+const DummyConfigObject = require('./utils/DummyConfigObject');
+const DummyRequestLogger = require('./utils/DummyRequestLogger');
+
+const log = new DummyRequestLogger();
+const mongoTestClient = new MongoClientInterface({});
+mongoTestClient.db = new DummyMongoDB();
+
+describe('MongoClientInterface, init behavior', () => {
+    let s3ConfigObj;
+    const locationConstraints = {
+        locationOne: { isTransient: true },
+        locationTwo: { isTransient: false },
+    };
+
+    beforeEach(() => {
+        s3ConfigObj = new DummyConfigObject();
+    });
+
+    it('should set DataCounter transientList when declaring a ' +
+    'new MongoClientInterface object', () => {
+        s3ConfigObj.setLocationConstraints(locationConstraints);
+        const mongoClient = new MongoClientInterface({ config: s3ConfigObj });
+        const expectedRes = { locationOne: true, locationTwo: false };
+        assert.deepStrictEqual(
+            mongoClient.dataCount.transientList, expectedRes);
+    });
+
+    it('should update DataCounter transientList if location constraints ' +
+    'are updated', done => {
+        const mongoClient = new MongoClientInterface({ config: s3ConfigObj });
+        assert.deepStrictEqual(mongoClient.dataCount.transientList, {});
+        const expectedRes = { locationOne: true, locationTwo: false };
+        s3ConfigObj.once('MongoClientTestDone', () => {
+            assert.deepStrictEqual(
+                mongoClient.dataCount.transientList, expectedRes);
+            return done();
+        });
+        s3ConfigObj.setLocationConstraints(locationConstraints);
+    });
+});
+
+describe('MongoClientInterface::_handleResults', () => {
+    it('should return zero-result', () => {
+        const testInput = {
+            masterCount: 0, masterData: {},
+            nullCount: 0, nullData: {},
+            versionCount: 0, versionData: {},
+        };
+        const testResults = mongoTestClient._handleResults(testInput, true);
+        const expectedRes = {
+            versions: 0, objects: 0,
+            dataManaged: {
+                total: { curr: 0, prev: 0 },
+                locations: {},
+            },
+        };
+        assert.deepStrictEqual(testResults, expectedRes);
+    });
+
+    it('should return correct value if isVer is false', () => {
+        const testInput = {
+            masterCount: 2, masterData: { test1: 10, test2: 10 },
+            nullCount: 2, nullData: { test1: 10, test2: 10 },
+            versionCount: 2, versionData: { test1: 20, test2: 20 },
+        };
+        const testResults = mongoTestClient._handleResults(testInput, false);
+        const expectedRes = {
+            versions: 0, objects: 4,
+            dataManaged: {
+                total: { curr: 40, prev: 0 },
+                locations: {
+                    test1: { curr: 20, prev: 0 },
+                    test2: { curr: 20, prev: 0 },
+                },
+            },
+        };
+        assert.deepStrictEqual(testResults, expectedRes);
+    });
+
+    it('should return correct value if isVer is true', () => {
+        const testInput = {
+            masterCount: 2, masterData: { test1: 10, test2: 10 },
+            nullCount: 2, nullData: { test1: 10, test2: 10 },
+            versionCount: 4, versionData: { test1: 20, test2: 20 },
+        };
+        const testResults = mongoTestClient._handleResults(testInput, true);
+        const expectedRes = {
+            versions: 2, objects: 4,
+            dataManaged: {
+                total: { curr: 40, prev: 20 },
+                locations: {
+                    test1: { curr: 20, prev: 10 },
+                    test2: { curr: 20, prev: 10 },
+                },
+            },
+        };
+        assert.deepStrictEqual(testResults, expectedRes);
+    });
+});
+
+describe('MongoClientInterface::_handleMongo', () => {
+    beforeEach(() => mongoTestClient.db.reset());
+
+    it('should return error if mongo aggregate fails', done => {
+        const retValues = [new Error('testError')];
+        mongoTestClient.db.setReturnValues(retValues);
+        const testCollection = mongoTestClient.db.collection('test');
+        mongoTestClient._handleMongo(testCollection, {}, false, log, err => {
+            assert(err, 'Expected error, but got success');
+            return done();
+        });
+    });
+
+    it('should return empty object if mongo aggregate has no results', done => {
+        const testCollection = mongoTestClient.db.collection('test');
+        mongoTestClient._handleMongo(testCollection, {}, false, log,
+        (err, res) => {
+            assert.ifError(err, `Expected success, but got error ${err}`);
+            assert.deepStrictEqual(res, {});
+            return done();
+        });
+    });
+
+    it('should return empty object if mongo aggregate has missing results',
+    done => {
+        const retValues = [[{
+            count: undefined,
+            data: undefined,
+            repData: undefined,
+        }]];
+        mongoTestClient.db.setReturnValues(retValues);
+        const testCollection = mongoTestClient.db.collection('test');
+        mongoTestClient._handleMongo(testCollection, {}, false, log,
+        (err, res) => {
+            assert.ifError(err, `Expected success, but got error ${err}`);
+            assert.deepStrictEqual(res, {});
+            return done();
+        });
+    });
+
+    const testRetValues = [[{
+        count: [{ _id: null, count: 100 }],
+        data: [
+            { _id: 'locationone', bytes: 1000 },
+            { _id: 'locationtwo', bytes: 1000 },
+        ],
+        repData: [
+            { _id: 'awsbackend', bytes: 500 },
+            { _id: 'azurebackend', bytes: 500 },
+            { _id: 'gcpbackend', bytes: 500 },
+        ],
+        compData: [
+            { _id: 'locationone', bytes: 500 },
+            { _id: 'locationtwo', bytes: 500 },
+        ],
+    }]];
+
+    it('should return correct results, transient false', done => {
+        mongoTestClient.db.setReturnValues(testRetValues);
+        const testCollection = mongoTestClient.db.collection('test');
+        mongoTestClient._handleMongo(testCollection, {}, false, log,
+        (err, res) => {
+            assert.ifError(err, `Expected success, but got error ${err}`);
+            assert.deepStrictEqual(res, {
+                count: 100,
+                data: {
+                    locationone: 1000,
+                    locationtwo: 1000,
+                    awsbackend: 500,
+                    azurebackend: 500,
+                    gcpbackend: 500,
+                },
+            });
+            return done();
+        });
+    });
+
+    it('should return correct results, transient true', done => {
+        mongoTestClient.db.setReturnValues(testRetValues);
+        const testCollection = mongoTestClient.db.collection('test');
+        mongoTestClient._handleMongo(testCollection, {}, true, log,
+        (err, res) => {
+            assert.ifError(err, `Expected success, but got error ${err}`);
+            assert.deepStrictEqual(res, {
+                count: 100,
+                data: {
+                    locationone: 500,
+                    locationtwo: 500,
+                    awsbackend: 500,
+                    azurebackend: 500,
+                    gcpbackend: 500,
+                },
+            });
+            return done();
+        });
+    });
+
+    const testRetValuesNeg = [[{
+        count: [{ _id: null, count: 100 }],
+        data: [
+            { _id: 'locationone', bytes: 100 },
+            { _id: 'locationtwo', bytes: 100 },
+        ],
+        repData: [
+            { _id: 'awsbackend', bytes: 500 },
+            { _id: 'azurebackend', bytes: 500 },
+            { _id: 'gcpbackend', bytes: 500 },
+        ],
+        compData: [
+            { _id: 'locationone', bytes: 500 },
+            { _id: 'locationtwo', bytes: 500 },
+        ],
+    }]];
+    it('should return clamp negative values to 0', done => {
+        mongoTestClient.db.setReturnValues(testRetValuesNeg);
+        const testCollection = mongoTestClient.db.collection('test');
+        mongoTestClient._handleMongo(testCollection, {}, true, log,
+        (err, res) => {
+            assert.ifError(err, `Expected success, but got error ${err}`);
+            assert.deepStrictEqual(res, {
+                count: 100,
+                data: {
+                    locationone: 0,
+                    locationtwo: 0,
+                    awsbackend: 500,
+                    azurebackend: 500,
+                    gcpbackend: 500,
+                },
+            });
+            return done();
+        });
+    });
+});

tests/unit/storage/metadata/mongoclient/utils/DummyConfigObject.js

@@ -0,0 +1,16 @@
+const { EventEmitter } = require('events');
+
+class DummyConfigObject extends EventEmitter {
+    constructor() {
+        super();
+        this.locationConstraints = null;
+        this.isTest = true;
+    }
+
+    setLocationConstraints(locationConstraints) {
+        this.locationConstraints = locationConstraints;
+        this.emit('location-constraints-update');
+    }
+}
+
+module.exports = DummyConfigObject;

tests/unit/storage/metadata/mongoclient/utils/DummyMongoDB.js

@@ -0,0 +1,103 @@
+const testError = new Error('test error');
+
+class DummyCollection {
+    constructor(name, isFail) {
+        this.s = {
+            name,
+        };
+        this.fail = isFail;
+        this.retQueue = [];
+    }
+
+    setReturnValues(retArray) {
+        this.retQueue.push(...retArray);
+    }
+
+    aggregate() {
+        return {
+            toArray: cb => {
+                if (this.retQueue.length <= 0) {
+                    return cb(null, []);
+                }
+                const retVal = this.retQueue[0];
+                this.retQueue = this.retQueue.slice(1);
+                if (retVal instanceof Error) {
+                    return cb(retVal);
+                }
+                return cb(null, retVal);
+            },
+        };
+    }
+
+    bulkWrite(cmds, opt, cb) {
+        process.stdout.write('mock mongodb.bulkWrite call\n');
+        if (this.fail) {
+            return cb(testError);
+        }
+        return cb();
+    }
+
+    update(target, doc, opt, cb) {
+        process.stdout.write('mock mongodb.update call\n');
+        if (this.fail) {
+            return cb(testError);
+        }
+        return cb();
+    }
+
+    find() {
+        return {
+            toArray: cb => {
+                if (this.retQueue.length <= 0) {
+                    return cb(null, []);
+                }
+                const retVal = this.retQueue[0];
+                this.retQueue = this.retQueue.slice(1);
+                if (retVal instanceof Error) {
+                    return cb(retVal);
+                }
+                return cb(null, retVal);
+            },
+        };
+    }
+
+    findOne(query, opt, cb) {
+        if (typeof opt === 'function' && cb === undefined) {
+            // eslint-disable-next-line no-param-reassign
+            cb = opt;
+        }
+        if (this.retQueue.length <= 0) {
+            return cb(null);
+        }
+        const retVal = this.retQueue[0];
+        this.retQueue = this.retQueue.slice(1);
+        if (retVal instanceof Error) {
+            return cb(retVal);
+        }
+        return cb(null, retVal);
+    }
+}
+
+class DummyMongoDB {
+    contructor() {
+        this.fail = false;
+        this.returnQueue = [];
+    }
+
+    reset() {
+        this.fail = false;
+        this.returnQueue = [];
+    }
+
+    setReturnValues(retValues) {
+        this.returnQueue.push(...retValues);
+    }
+
+    collection(name) {
+        const c = new DummyCollection(name, this.fail);
+        c.setReturnValues(this.returnQueue);
+        return c;
+    }
+}
+
+module.exports = DummyMongoDB;

tests/unit/storage/metadata/mongoclient/utils/DummyRequestLogger.js

@@ -0,0 +1,58 @@
+class DummyRequestLogger {
+    constructor() {
+        this.ops = [];
+        this.counts = {
+            trace: 0,
+            debug: 0,
+            info: 0,
+            warn: 0,
+            error: 0,
+            fatal: 0,
+        };
+        this.defaultFields = {};
+    }
+
+    trace(msg) {
+        this.ops.push(['trace', [msg]]);
+        this.counts.trace += 1;
+    }
+
+    debug(msg) {
+        this.ops.push(['debug', [msg]]);
+        this.counts.debug += 1;
+    }
+
+    info(msg) {
+        this.ops.push(['info', [msg]]);
+        this.counts.info += 1;
+    }
+
+    warn(msg) {
+        this.ops.push(['warn', [msg]]);
+        this.counts.warn += 1;
+    }
+
+    error(msg) {
+        this.ops.push(['error', [msg]]);
+        this.counts.error += 1;
+    }
+
+    fatal(msg) {
+        this.ops.push(['fatal', [msg]]);
+        this.counts.fatal += 1;
+    }
+
+    getSerializedUids() { // eslint-disable-line class-methods-use-this
+        return 'dummy:Serialized:Uids';
+    }
+
+    addDefaultFields(fields) {
+        Object.assign(this.defaultFields, fields);
+    }
+
+    end() {
+        return this;
+    }
+}
+
+module.exports = DummyRequestLogger;

tests/unit/storage/metadata/mongoclient/utils/helper.js

@@ -0,0 +1,24 @@
+const basicMD = {
+    'content-length': 0,
+    'key': '',
+    'versionId': '',
+    'replicationInfo': {
+        backends: [], // site, status
+    },
+    'dataStoreName': 'mongotest',
+};
+
+function generateMD(objKey, size, versionId, repBackends) {
+    const retMD = JSON.parse(JSON.stringify(basicMD));
+    retMD.key = objKey;
+    retMD['content-length'] = size;
+    retMD.versionId = versionId;
+    if (repBackends && Array.isArray(repBackends)) {
+        retMD.replicationInfo.backends.push(...repBackends);
+    }
+    return retMD;
+}
+
+module.exports = {
+    generateMD,
+};

tests/unit/versioning/VersionID.js

@@ -28,9 +28,7 @@ describe('test generating versionIds', () => {
 
     // nodejs 10 no longer returns error for non-hex string versionIds
     it.skip('should return error decoding non-hex string versionIds', () => {
-        const encoded = vids.map(vid => VID.encode(vid));
-        const decoded = encoded.map(vid => VID.decode(`${vid}foo`));
-        decoded.forEach(result => assert(result instanceof Error));
+        assert(VID.decode('foo') instanceof Error);
     });
 
     it('should encode and decode versionIds', () => {

tests/utils/kmip/LoopbackServerChannel.js

@@ -0,0 +1,372 @@
+'use strict'; // eslint-disable-line strict
+/* eslint new-cap: "off" */
+/* eslint dot-notation: "off" */
+
+const assert = require('assert');
+const crypto = require('crypto');
+const uuidv4 = require('uuid/v4');
+
+const {
+    EchoChannel,
+    logger,
+} = require('./ersatz.js');
+
+const expectedObjectType = 'Symmetric Key';
+const expectedAlgorithm = 'AES';
+const expectedLength = 256;
+const expectedBlockCipherMode = 'CBC';
+const expectedPaddingMethod = 'PKCS5';
+const expectedIV = Buffer.alloc(16).fill(0);
+const versionMajor = 1;
+const versionMinor = 4;
+const vendorIdentification = 'Scality Loopback KMIP Server';
+const serverExtensions = [
+    {
+        name: 'Security Level',
+        tag: 0x541976,
+        type: 7,
+        value: 'Gangsta Grade',
+    },
+    {
+        name: 'Prefered Algorithm',
+        tag: 0x542008,
+        type: 7,
+        value: 'Kevin Bacon',
+    },
+    {
+        name: 'Yo Dawg',
+        tag: 0x542011,
+        type: 7,
+        value: 'I heard you like kmip, so I put a server in your client ' +
+            'so you can do both ends of the conversation while you are ' +
+            'talking about server side encryption',
+    },
+];
+
+
+class DummyServerTransport {
+    registerHandshakeFunction() {
+        throw new Error('DummyServerTransport::registerHandshakeFunction: ' +
+                        'Client side operations not implemented');
+    }
+    send() {
+        throw new Error('DummyServerTransport::send: ' +
+                        'Client side operations not implemented');
+    }
+}
+
+
+class LoopbackServerChannel extends EchoChannel {
+    constructor(KMIPClass, Codec, options) {
+        super();
+        this.options = options || {
+            kmip: {
+                codec: {},
+                transport: {},
+            },
+        };
+        this.KMIP = KMIPClass;
+        this.kmip = new KMIPClass(Codec, DummyServerTransport,
+                                  this.options);
+        serverExtensions.forEach(extension => {
+            this.kmip.mapExtension(extension.name, extension.tag);
+        });
+        this.managedKeys = {};
+    }
+
+    write(data) {
+        const request = this.kmip._decodeMessage(logger, data);
+        const requestOperation = request.lookup(
+            'Request Message/Batch Item/Operation')[0];
+        this.routeRequest(
+            requestOperation, request, (err, responsePayload) => {
+                const uniqueBatchItemID = request.lookup(
+                    'Request Message/Batch Item/Unique Batch Item ID')[0];
+                const requestProtocolVersionMinor = request.lookup(
+                    'Request Message/Request Header/Protocol Version/' +
+                        'Protocol Version Minor')[0];
+                const requestProtocolVersionMajor = request.lookup(
+                    'Request Message/Request Header/Protocol Version/' +
+                        'Protocol Version Major')[0];
+
+                let result;
+                if (err) {
+                    logger.error('Request processing failed', { error: err });
+                    result = err;
+                } else {
+                    result = [
+                        this.KMIP.Enumeration('Result Status', 'Success'),
+                        this.KMIP.Structure('Response Payload',
+                                            responsePayload),
+                    ];
+                }
+
+                const response = this.KMIP.Message([
+                    this.KMIP.Structure('Response Message', [
+                        this.KMIP.Structure('Response Header', [
+                            this.KMIP.Structure('Protocol Version', [
+                                this.KMIP.Integer('Protocol Version Major',
+                                                  requestProtocolVersionMajor),
+                                this.KMIP.Integer('Protocol Version Minor',
+                                                  requestProtocolVersionMinor),
+                            ]),
+                            this.KMIP.DateTime('Time Stamp', new Date),
+                            this.KMIP.Integer('Batch Count', 1),
+                        ]),
+                        this.KMIP.Structure('Batch Item', [
+                            this.KMIP.Enumeration('Operation',
+                                                  requestOperation),
+                            this.KMIP.ByteString('Unique Batch Item ID',
+                                                 uniqueBatchItemID),
+                            ...result,
+                        ]),
+                    ]),
+                ]);
+                super.write(this.kmip._encodeMessage(response));
+            });
+        return this;
+    }
+
+    errorResponse(reason, message) {
+        return [
+            this.KMIP.Enumeration('Result Status', 'Operation Failed'),
+            this.KMIP.Enumeration('Result Reason', reason),
+            this.KMIP.Enumeration('Result Message', message),
+        ];
+    }
+
+    routeRequest(operation, request, cb) {
+        switch (operation) {
+        case 'Query': return this.routeQuery(request, cb);
+        case 'Discover Versions':
+            return this.routeDiscoverVersions(request, cb);
+        case 'Create': return this.routeCreate(request, cb);
+        case 'Activate': return this.routeActivate(request, cb);
+        case 'Encrypt': return this.routeEncrypt(request, cb);
+        case 'Decrypt': return this.routeDecrypt(request, cb);
+        case 'Revoke': return this.routeRevoke(request, cb);
+        case 'Destroy': return this.routeDestroy(request, cb);
+        default: return cb(new Error(`Unknown Operation: ${operation}`));
+        }
+    }
+
+    routeQuery(request, cb) {
+        const queryFunctions = request.lookup(
+            'Request Message/Batch Item/Request Payload/Query Function');
+        const response = [];
+        if (queryFunctions.includes('Query Operations')) {
+            response.push(
+                this.KMIP.Enumeration('Operation', 'Query'),
+                this.KMIP.Enumeration('Operation', 'Discover Versions'),
+                this.KMIP.Enumeration('Operation', 'Create'),
+                this.KMIP.Enumeration('Operation', 'Activate'),
+                this.KMIP.Enumeration('Operation', 'Encrypt'),
+                this.KMIP.Enumeration('Operation', 'Decrypt'),
+                this.KMIP.Enumeration('Operation', 'Revoke'),
+                this.KMIP.Enumeration('Operation', 'Destroy'));
+        }
+        if (queryFunctions.includes('Query Objects')) {
+            response.push(
+                this.KMIP.Enumeration('Object Type', 'Symmetric Key'));
+        }
+        if (queryFunctions.includes('Query Server Information')) {
+            response.push(
+                this.KMIP.TextString('Vendor Identification',
+                                     vendorIdentification),
+                this.KMIP.Structure('Server Information',
+                                    serverExtensions.map(extension =>
+                                                         this.KMIP.TextString(
+                                                             extension.name,
+                                                             extension.value)
+                                                        )));
+        }
+        if (queryFunctions.includes('Query Extension Map')) {
+            serverExtensions.forEach(extension => {
+                response.push(
+                    this.KMIP.Structure('Extension Information', [
+                        this.KMIP.TextString('Extension Name', extension.name),
+                        this.KMIP.Integer('Extension Tag', extension.tag),
+                        /* 7 is KMIP TextString, not really used anyway in
+                         * this implenetation, it could be anything
+                         * without changing the behavior of the client code. */
+                        this.KMIP.Integer('Extension Type', 7),
+                    ]));
+            });
+        }
+        return cb(null, response);
+    }
+
+    routeDiscoverVersions(request, cb) {
+        const response = [
+            this.KMIP.Structure('Protocol Version', [
+                this.KMIP.Integer('Protocol Version Major', versionMajor),
+                this.KMIP.Integer('Protocol Version Minor', versionMinor),
+            ]),
+        ];
+        return cb(null, response);
+    }
+
+    routeCreate(request, cb) {
+        let cryptographicAlgorithm;
+        let cryptographicLength;
+        let cryptographicUsageMask;
+        let activationDate;
+        const attributes = request.lookup(
+            'Request Message/Batch Item/Request Payload/Template-Attribute')[0];
+        attributes.forEach(item => {
+            const attribute = item['Attribute'];
+            const attributeValue = attribute.value[1]['Attribute Value'];
+            const diversion = attributeValue.diversion;
+            const value = attributeValue.value;
+            switch (diversion) {
+            case 'Cryptographic Algorithm':
+                assert(!cryptographicAlgorithm);
+                cryptographicAlgorithm = value;
+                break;
+            case 'Cryptographic Length':
+                assert(!cryptographicLength);
+                cryptographicLength = value;
+                break;
+            case 'Cryptographic Usage Mask':
+                assert(!cryptographicUsageMask);
+                cryptographicUsageMask = value;
+                break;
+            case 'Activation Date':
+                assert(!activationDate);
+                activationDate = value;
+                break;
+            default:
+            }
+        });
+        const decodedUsageMask =
+              this.kmip.decodeMask('Cryptographic Usage Mask',
+                                   cryptographicUsageMask);
+        assert(cryptographicAlgorithm === expectedAlgorithm);
+        assert(cryptographicLength === expectedLength);
+        assert(decodedUsageMask.includes('Encrypt'));
+        assert(decodedUsageMask.includes('Decrypt'));
+        const key = Buffer.from(crypto.randomBytes(cryptographicLength / 8));
+        const keyIdentifier = uuidv4();
+        this.managedKeys[keyIdentifier] = {
+            key,
+            activationDate,
+        };
+        const response = [
+            this.KMIP.Enumeration('Object Type', expectedObjectType),
+            this.KMIP.TextString('Unique Identifier', keyIdentifier),
+        ];
+        return cb(null, response);
+    }
+
+    routeActivate(request, cb) {
+        const keyIdentifier = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Unique Identifier') || [undefined])[0];
+        this.managedKeys[keyIdentifier].activationDate =
+            new Date;
+        const response = [
+            this.KMIP.TextString('Unique Identifier', keyIdentifier),
+        ];
+        return cb(null, response);
+    }
+
+    _getIvCounterNonce(request) {
+        /* Having / in the path is not a good idea for the server side.
+         * Because of this, Message::lookup() cannot be directly used to
+         * extract the IV, hence this function */
+        const requestPayload = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload')
+                || [undefined])[0];
+        let ivCounterNonce;
+        requestPayload.forEach(attr => {
+            const ivCounterNonceAttr = attr['IV/Counter/Nonce'];
+            if (ivCounterNonceAttr) {
+                ivCounterNonce = ivCounterNonceAttr.value;
+            }
+        });
+        return ivCounterNonce;
+    }
+
+    _transform(cipherFunc, request, cb) {
+        const keyIdentifier = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Unique Identifier') || [undefined])[0];
+        const blockCipherMode = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Cryptographic Parameters/Block Cipher Mode')
+                || [undefined])[0];
+        const paddingMethod = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Cryptographic Parameters/Padding Method')
+                || [undefined])[0];
+        const cryptographicAlgorithm = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Cryptographic Parameters/Cryptographic Algorithm')
+                || [undefined])[0];
+        const ivCounterNonce = this._getIvCounterNonce(request);
+        const data = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Data')
+                || [undefined])[0];
+        assert(blockCipherMode === expectedBlockCipherMode);
+        assert(paddingMethod === expectedPaddingMethod);
+        assert(cryptographicAlgorithm === expectedAlgorithm);
+        assert(expectedIV.compare(ivCounterNonce) === 0);
+        const keySpec = this.managedKeys[keyIdentifier];
+        const now = new Date;
+        assert(keySpec.activationDate && keySpec.activationDate <= now);
+        const cipher = cipherFunc('aes-256-cbc', keySpec.key, ivCounterNonce);
+        let cipheredData = cipher.update(data);
+        const final = cipher.final();
+        if (final.length !== 0) {
+            cipheredData = Buffer.concat([cipheredData, final]);
+        }
+        const response = [
+            this.KMIP.TextString('Unique Identifier', keyIdentifier),
+            this.KMIP.ByteString('Data', cipheredData),
+        ];
+        return cb(null, response);
+    }
+
+    routeEncrypt(request, cb) {
+        return this._transform(crypto.createCipheriv, request, cb);
+    }
+
+    routeDecrypt(request, cb) {
+        return this._transform(crypto.createDecipheriv, request, cb);
+    }
+
+    routeRevoke(request, cb) {
+        const keyIdentifier = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Unique Identifier') || [undefined])[0];
+        this.managedKeys[keyIdentifier].activationDate = null;
+        const response = [
+            this.KMIP.TextString('Unique Identifier', keyIdentifier),
+        ];
+        return cb(null, response);
+    }
+
+    routeDestroy(request, cb) {
+        const keyIdentifier = (
+            request.lookup(
+                'Request Message/Batch Item/Request Payload/' +
+                    'Unique Identifier') || [undefined])[0];
+        assert(!this.managedKeys[keyIdentifier].activationDate);
+        this.managedKeys[keyIdentifier] = null;
+        const response = [
+            this.KMIP.TextString('Unique Identifier', keyIdentifier),
+        ];
+        return cb(null, response);
+    }
+}
+
+module.exports = LoopbackServerChannel;

tests/utils/kmip/badTtlvFixtures.js

@@ -0,0 +1,127 @@
+'use strict'; // eslint-disable-line strict
+
+module.exports = [
+
+    /* Invalid type */
+
+    Buffer.from('2100000000000000', 'hex'),
+    Buffer.from('2100000b00000000', 'hex'),
+
+    /* Structure */
+    // too short
+    Buffer.from('42', 'hex'),
+    Buffer.from('4200', 'hex'),
+    Buffer.from('420078', 'hex'),
+    Buffer.from('42007801', 'hex'),
+    Buffer.from('4200780100', 'hex'),
+    Buffer.from('420078010000', 'hex'),
+    Buffer.from('4200780100000001', 'hex'),
+    Buffer.from('420078010000000100', 'hex'),
+    Buffer.from('4200780100000008', 'hex'),
+    Buffer.from('420078010000000800', 'hex'),
+    Buffer.from('42007801000000080000', 'hex'),
+    Buffer.from('4200780100000008000000', 'hex'),
+    Buffer.from('420078010000000800000000', 'hex'),
+    Buffer.from('4200780100000010', 'hex'),
+
+    /* Integer */
+    // too short
+    Buffer.from('4200780200000004', 'hex'),
+    Buffer.from('420078020000000400', 'hex'),
+    Buffer.from('42007802000000040000', 'hex'),
+    Buffer.from('4200780200000004000000', 'hex'),
+    // invalid length for the type
+    Buffer.from('42007802000000080000000000000000', 'hex'),
+    Buffer.from('42007802000000020000000000000000', 'hex'),
+    Buffer.from('42007802000000000000000000000000', 'hex'),
+
+    /* Long Integer */
+    // too short
+    Buffer.from('4200780300000008', 'hex'),
+    Buffer.from('420078030000000810', 'hex'),
+    Buffer.from('42007803000000081000', 'hex'),
+    Buffer.from('4200780300000008100000', 'hex'),
+    Buffer.from('420078030000000810000000', 'hex'),
+    Buffer.from('42007803000000081000000000', 'hex'),
+    Buffer.from('4200780300000008100000000000', 'hex'),
+    Buffer.from('420078030000000810000000000000', 'hex'),
+    // 53bit overflow
+    Buffer.from('42007803000000081000000000000000', 'hex'),
+    Buffer.from('4200780300000008ffffffffffffffff', 'hex'),
+    // invalid length for the type
+    Buffer.from('420078030000000400000001', 'hex'),
+    Buffer.from('42007803000000100000000000000000100000000000000000', 'hex'),
+
+    /* Big Integer */
+    // too short
+    Buffer.from('4200780400000001', 'hex'),
+    Buffer.from('420078040000000200', 'hex'),
+
+    /* Enumeration */
+    // too short
+    Buffer.from('4200740500000004', 'hex'),
+    Buffer.from('4200740500000004000000', 'hex'),
+    // invalid length for the type
+    Buffer.from('42007405000000020000', 'hex'),
+    Buffer.from('4200740500000006000000000000', 'hex'),
+    // non existing tag and enum value with invalid length
+    Buffer.from('45007405000000020000', 'hex'),
+    Buffer.from('4500740500000006000000000000', 'hex'),
+
+    /* Boolean */
+    // too short
+    Buffer.from('4200740600000008', 'hex'),
+    Buffer.from('420074060000000800', 'hex'),
+    Buffer.from('42007406000000080000', 'hex'),
+    Buffer.from('4200740600000008000000', 'hex'),
+    Buffer.from('420074060000000800000000', 'hex'),
+    Buffer.from('42007406000000080000000000', 'hex'),
+    Buffer.from('4200740600000008000000000000', 'hex'),
+    // invalid length
+    Buffer.from('420074060000000400000000', 'hex'),
+    Buffer.from('420074060000001000000000000000000000000000000000', 'hex'),
+
+    /* TextString */
+    // too short
+    Buffer.from('4200740700000008', 'hex'),
+    Buffer.from('420074070000000800', 'hex'),
+    Buffer.from('42007407000000080000', 'hex'),
+    Buffer.from('4200740700000008000000', 'hex'),
+    Buffer.from('420074070000000800000000', 'hex'),
+    Buffer.from('42007407000000080000000000', 'hex'),
+    Buffer.from('4200740700000008000000000000', 'hex'),
+
+    /* ByteString */
+    // too short
+    Buffer.from('4200740800000008', 'hex'),
+    Buffer.from('420074080000000800', 'hex'),
+    Buffer.from('42007408000000080000', 'hex'),
+    Buffer.from('4200740800000008000000', 'hex'),
+    Buffer.from('420074080000000800000000', 'hex'),
+    Buffer.from('42007408000000080000000000', 'hex'),
+    Buffer.from('4200740800000008000000000000', 'hex'),
+
+    /* Date-Time */
+    // too short
+    Buffer.from('4200740900000008', 'hex'),
+    Buffer.from('420074090000000800', 'hex'),
+    Buffer.from('42007409000000080000', 'hex'),
+    Buffer.from('4200740900000008000000', 'hex'),
+    Buffer.from('420074090000000800000000', 'hex'),
+    Buffer.from('42007409000000080000000000', 'hex'),
+    Buffer.from('4200740900000008000000000000', 'hex'),
+    // invalid length
+    Buffer.from('420074090000000400000000', 'hex'),
+    Buffer.from('420074090000001000000000000000000000000000000000', 'hex'),
+
+    /* Interval */
+    // too short
+    Buffer.from('4200780a00000004', 'hex'),
+    Buffer.from('4200780a0000000400', 'hex'),
+    Buffer.from('4200780a000000040000', 'hex'),
+    Buffer.from('4200780a00000004000000', 'hex'),
+    // invalid length for the type
+    Buffer.from('4200780a000000080000000000000000', 'hex'),
+    Buffer.from('4200780a000000020000000000000000', 'hex'),
+    Buffer.from('4200780a000000000000000000000000', 'hex'),
+];

tests/utils/kmip/ersatz.js

@@ -0,0 +1,125 @@
+'use strict'; // eslint-disable-line
+/* eslint new-cap: "off" */
+
+
+const logger = {
+    info: () => {},
+    debug: () => {},
+    warn: () => {},
+    error: () => {},
+};
+
+/* Fake tls AND socket objects, duck type */
+class EchoChannel {
+    constructor() {
+        this.clogged = false;
+        this.eventHandler = {};
+        this.deferedSignal = {};
+    }
+
+    /* tls object members substitutes */
+
+    connect(port, options, cb) {
+        process.nextTick(cb);
+        return this;
+    }
+
+    on(event, cb) {
+        this.eventHandler[event] = cb;
+        if (this.deferedSignal[event] &&
+            this.deferedSignal[event].length > 0) {
+            this.deferedSignal[event].forEach(this.eventHandler[event]);
+            this.deferedSignal[event] = undefined;
+        }
+        return this;
+    }
+
+    /* socket object members substitutes */
+
+    cork() {
+        return this;
+    }
+
+    uncork() {
+        return this;
+    }
+
+    write(data) {
+        if (!this.clogged) {
+            return this.emit('data', data);
+        }
+        return this;
+    }
+
+    end() {
+        return this.emit('end');
+    }
+
+    /* Instrumentation member functions */
+
+    emit(event, data) {
+        if (this.eventHandler[event]) {
+            this.eventHandler[event](data);
+        } else {
+            if (!this.deferedSignal[event]) {
+                this.deferedSignal[event] = [];
+            }
+            this.deferedSignal[event].push(data);
+        }
+        return this;
+    }
+
+    clog() {
+        this.clogged = true;
+        return this;
+    }
+
+}
+
+class MirrorChannel extends EchoChannel {
+    constructor(KMIPClass, Codec) {
+        super();
+        this.codec = new Codec({});
+        this.KMIP = KMIPClass;
+    }
+    write(data) {
+        const request = this.codec.decode(logger, data);
+        const uniqueBatchItemID = request.lookup(
+            'Request Message/Batch Item/Unique Batch Item ID')[0];
+        const requestPayload = request.lookup(
+            'Request Message/Batch Item/Request Payload')[0];
+        const requestProtocolVersionMinor = request.lookup(
+            'Request Message/Request Header/Protocol Version/' +
+                'Protocol Version Minor')[0];
+        const requestProtocolVersionMajor = request.lookup(
+            'Request Message/Request Header/Protocol Version/' +
+                'Protocol Version Major')[0];
+        const requestOperation = request.lookup(
+            'Request Message/Batch Item/Operation')[0];
+        const response = this.KMIP.Message([
+            this.KMIP.Structure('Response Message', [
+                this.KMIP.Structure('Response Header', [
+                    this.KMIP.Structure('Protocol Version', [
+                        this.KMIP.Integer('Protocol Version Major',
+                                          requestProtocolVersionMajor),
+                        this.KMIP.Integer('Protocol Version Minor',
+                                          requestProtocolVersionMinor),
+                    ]),
+                    this.KMIP.DateTime('Time Stamp', new Date),
+                    this.KMIP.Integer('Batch Count', 1),
+                ]),
+                this.KMIP.Structure('Batch Item', [
+                    this.KMIP.Enumeration('Operation', requestOperation),
+                    this.KMIP.ByteString('Unique Batch Item ID',
+                                         uniqueBatchItemID),
+                    this.KMIP.Enumeration('Result Status', 'Success'),
+                    this.KMIP.Structure('Response Payload', requestPayload),
+                ]),
+            ]),
+        ]);
+        super.write(this.codec.encode(response));
+        return this;
+    }
+}
+
+module.exports = { logger, EchoChannel, MirrorChannel };

tests/utils/kmip/lowlevelFixtures.js

@@ -0,0 +1,82 @@
+'use strict'; // eslint-disable-line strict
+/* eslint new-cap: "off" */
+
+const KMIP = require('../../../lib/network/kmip');
+
+module.exports = [
+    {
+        operation: 'Query',
+        payload: () => [
+            KMIP.Enumeration('Query Function', 'Query Operations'),
+            KMIP.Enumeration('Query Function', 'Query Objects'),
+        ],
+    },
+    {
+        operation: 'Query',
+        payload: () => [
+            KMIP.Enumeration('Query Function', 'Query Operations'),
+            KMIP.Enumeration('Query Function', 'Query Objects'),
+            KMIP.Enumeration('Query Function',
+                             'Query Server Information'),
+            KMIP.Enumeration('Query Function', 'Query Profiles'),
+            KMIP.Enumeration('Query Function', 'Query Capabilities'),
+            KMIP.Enumeration('Query Function',
+                             'Query Application Namespaces'),
+            KMIP.Enumeration('Query Function', 'Query Extension List'),
+            KMIP.Enumeration('Query Function', 'Query Extension Map'),
+            KMIP.Enumeration('Query Function',
+                             'Query Attestation Types'),
+            KMIP.Enumeration('Query Function', 'Query RNGs'),
+            KMIP.Enumeration('Query Function', 'Query Validations'),
+            KMIP.Enumeration('Query Function',
+                             'Query Client Registration Methods'),
+        ],
+    },
+    {
+        operation: 'Discover Versions',
+        payload: () => [
+            KMIP.Structure('Protocol Version', [
+                KMIP.Integer('Protocol Version Major', 2),
+                KMIP.Integer('Protocol Version Minor', 0),
+            ]),
+            KMIP.Structure('Protocol Version', [
+                KMIP.Integer('Protocol Version Major', 1),
+                KMIP.Integer('Protocol Version Minor', 4),
+            ]),
+            KMIP.Structure('Protocol Version', [
+                KMIP.Integer('Protocol Version Major', 1),
+                KMIP.Integer('Protocol Version Minor', 3),
+            ]),
+            KMIP.Structure('Protocol Version', [
+                KMIP.Integer('Protocol Version Major', 1),
+                KMIP.Integer('Protocol Version Minor', 2),
+            ]),
+            KMIP.Structure('Protocol Version', [
+                KMIP.Integer('Protocol Version Major', 1),
+                KMIP.Integer('Protocol Version Minor', 1),
+            ]),
+            KMIP.Structure('Protocol Version', [
+                KMIP.Integer('Protocol Version Major', 1),
+                KMIP.Integer('Protocol Version Minor', 0),
+            ]),
+        ],
+    },
+    {
+        operation: 'Create',
+        payload: kmip => [
+            KMIP.Enumeration('Object Type', 'Symmetric Key'),
+            KMIP.Structure('Template-Attribute', [
+                KMIP.Attribute('TextString', 'x-Name', 's3-thekey'),
+                KMIP.Attribute('Enumeration', 'Cryptographic Algorithm',
+                               'AES'),
+                KMIP.Attribute('Integer', 'Cryptographic Length', 256),
+                KMIP.Attribute('Integer', 'Cryptographic Usage Mask',
+                               kmip.encodeMask(
+                                   'Cryptographic Usage Mask',
+                                   ['Encrypt', 'Decrypt'])),
+                KMIP.Attribute('Date-Time', 'Activation Date',
+                               new Date),
+            ]),
+        ],
+    },
+];

tests/utils/kmip/messageFixtures.js

@@ -0,0 +1,120 @@
+'use strict'; // eslint-disable-line strict
+/* eslint new-cap: "off" */
+
+const TTLVCodec = require('../../../lib/network/kmip/codec/ttlv.js');
+const KMIP = require('../../../lib/network/kmip');
+
+const kmip = new KMIP(TTLVCodec,
+                      class DummyTransport {},
+                      { kmip: {} }, () => {});
+
+module.exports = [
+    KMIP.Message([
+        KMIP.Structure('Request Message', [
+            KMIP.Structure('Request Header', [
+                KMIP.Structure('Protocol Version', [
+                    KMIP.Integer('Protocol Version Major', 1),
+                    KMIP.Integer('Protocol Version Minor', 3),
+                ]),
+                KMIP.Integer('Maximum Response Size', 256),
+                KMIP.Integer('Batch Count', 1),
+            ]),
+            KMIP.Structure('Batch Item', [
+                KMIP.Enumeration('Operation', 'Query'),
+                KMIP.Structure('Request Payload', [
+                    KMIP.Enumeration('Query Function', 'Query Operations'),
+                    KMIP.Enumeration('Query Function', 'Query Objects'),
+                ]),
+            ]),
+        ]),
+    ]),
+    KMIP.Message([
+        KMIP.Structure('Request Message', [
+            KMIP.Structure('Request Header', [
+                KMIP.Structure('Protocol Version', [
+                    KMIP.Integer('Protocol Version Major', 1),
+                    KMIP.Integer('Protocol Version Minor', 2),
+                ]),
+                KMIP.Integer('Maximum Response Size', 2048),
+                KMIP.Boolean('Asynchronous Indicator', false),
+                KMIP.Integer('Batch Count', 3),
+                KMIP.ByteString('Asynchronous Correlation Value',
+                                Buffer.from('Arrggg...', 'utf8')),
+            ]),
+            KMIP.Structure('Batch Item', [
+                KMIP.Enumeration('Operation', 'Query'),
+                KMIP.ByteString('Unique Batch Item ID',
+                                Buffer.from('foo', 'utf8')),
+                KMIP.Structure('Request Payload', [
+                    KMIP.Enumeration('Query Function', 'Query Operations'),
+                    KMIP.Enumeration('Query Function', 'Query Objects'),
+                    KMIP.Enumeration('Query Function',
+                                     'Query Server Information'),
+                    KMIP.Enumeration('Query Function', 'Query Profiles'),
+                    KMIP.Enumeration('Query Function', 'Query Capabilities'),
+                    KMIP.Enumeration('Query Function',
+                                     'Query Application Namespaces'),
+                    KMIP.Enumeration('Query Function', 'Query Extension List'),
+                    KMIP.Enumeration('Query Function', 'Query Extension Map'),
+                    KMIP.Enumeration('Query Function',
+                                     'Query Attestation Types'),
+                    KMIP.Enumeration('Query Function', 'Query RNGs'),
+                    KMIP.Enumeration('Query Function', 'Query Validations'),
+                    KMIP.Enumeration('Query Function',
+                                     'Query Client Registration Methods'),
+                ]),
+            ]),
+            KMIP.Structure('Batch Item', [
+                KMIP.Enumeration('Operation', 'Discover Versions'),
+                KMIP.ByteString('Unique Batch Item ID',
+                                Buffer.from('bar', 'utf8')),
+                KMIP.Structure('Request Payload', [
+                    KMIP.Structure('Protocol Version', [
+                        KMIP.Integer('Protocol Version Major', 2),
+                        KMIP.Integer('Protocol Version Minor', 0),
+                    ]),
+                    KMIP.Structure('Protocol Version', [
+                        KMIP.Integer('Protocol Version Major', 1),
+                        KMIP.Integer('Protocol Version Minor', 4),
+                    ]),
+                    KMIP.Structure('Protocol Version', [
+                        KMIP.Integer('Protocol Version Major', 1),
+                        KMIP.Integer('Protocol Version Minor', 3),
+                    ]),
+                    KMIP.Structure('Protocol Version', [
+                        KMIP.Integer('Protocol Version Major', 1),
+                        KMIP.Integer('Protocol Version Minor', 2),
+                    ]),
+                    KMIP.Structure('Protocol Version', [
+                        KMIP.Integer('Protocol Version Major', 1),
+                        KMIP.Integer('Protocol Version Minor', 1),
+                    ]),
+                    KMIP.Structure('Protocol Version', [
+                        KMIP.Integer('Protocol Version Major', 1),
+                        KMIP.Integer('Protocol Version Minor', 0),
+                    ]),
+                ]),
+            ]),
+            KMIP.Structure('Batch Item', [
+                KMIP.Enumeration('Operation', 'Create'),
+                KMIP.ByteString('Unique Batch Item ID',
+                                Buffer.from('baz', 'utf8')),
+                KMIP.Structure('Request Payload', [
+                    KMIP.Enumeration('Object Type', 'Symmetric Key'),
+                    KMIP.Structure('Template-Attribute', [
+                        KMIP.Attribute('TextString', 'x-Name', 's3-thekey'),
+                        KMIP.Attribute('Enumeration', 'Cryptographic Algorithm',
+                                       'AES'),
+                        KMIP.Attribute('Integer', 'Cryptographic Length', 256),
+                        KMIP.Attribute('Integer', 'Cryptographic Usage Mask',
+                                       kmip.encodeMask(
+                                           'Cryptographic Usage Mask',
+                                           ['Encrypt', 'Decrypt'])),
+                        KMIP.Attribute('Date-Time', 'Activation Date',
+                                       new Date),
+                    ]),
+                ]),
+            ]),
+        ]),
+    ]),
+];

tests/utils/kmip/ttlvFixtures.js

@@ -0,0 +1,247 @@
+'use strict'; // eslint-disable-line strict
+
+module.exports = [
+    {
+        request: Buffer.from('42007801000000904200770100000048' +
+                             '420069010000002042006a0200000004' +
+                             '000000010000000042006b0200000004' +
+                             '00000003000000004200500200000004' +
+                             '000001000000000042000d0200000004' +
+                             '000000010000000042000f0100000038' +
+                             '42005c05000000040000001800000000' +
+                             '42007901000000204200740500000004' +
+                             '00000001000000004200740500000004' +
+                             '0000000200000000', 'hex'),
+        response: Buffer.from('42007b01000000a042007a0100000048' +
+                              '420069010000002042006a0200000004' +
+                              '000000010000000042006b0200000004' +
+                              '00000003000000004200920900000008' +
+                              '00000000568a5be242000d0200000004' +
+                              '000000010000000042000f0100000048' +
+                              '42005c05000000040000001800000000' +
+                              '42007f05000000040000000100000000' +
+                              '42007e05000000040000000200000000' +
+                              '42007d0700000009544f4f5f4c415247' +
+                              '4500000000000000', 'hex'),
+    },
+    {
+        request: Buffer.from('42007801000000904200770100000048' +
+                             '420069010000002042006a0200000004' +
+                             '000000010000000042006b0200000004' +
+                             '00000003000000004200500200000004' +
+                             '000008000000000042000d0200000004' +
+                             '000000010000000042000f0100000038' +
+                             '42005c05000000040000001800000000' +
+                             '42007901000000204200740500000004' +
+                             '00000001000000004200740500000004' +
+                             '0000000200000000', 'hex'),
+        response: Buffer.from('42007b010000038042007a0100000048' +
+                              '420069010000002042006a0200000004' +
+                              '000000010000000042006b0200000004' +
+                              '00000003000000004200920900000008' +
+                              '00000000568a5be242000d0200000004' +
+                              '000000010000000042000f0100000328' +
+                              '42005c05000000040000001800000000' +
+                              '42007f05000000040000000000000000' +
+                              '42007c010000030042005c0500000004' +
+                              '000000180000000042005c0500000004' +
+                              '000000080000000042005c0500000004' +
+                              '000000140000000042005c0500000004' +
+                              '0000000a0000000042005c0500000004' +
+                              '000000010000000042005c0500000004' +
+                              '000000030000000042005c0500000004' +
+                              '0000000b0000000042005c0500000004' +
+                              '0000000c0000000042005c0500000004' +
+                              '0000000d0000000042005c0500000004' +
+                              '0000000e0000000042005c0500000004' +
+                              '0000000f0000000042005c0500000004' +
+                              '000000120000000042005c0500000004' +
+                              '000000130000000042005c0500000004' +
+                              '0000001a0000000042005c0500000004' +
+                              '000000190000000042005c0500000004' +
+                              '000000090000000042005c0500000004' +
+                              '000000110000000042005c0500000004' +
+                              '000000020000000042005c0500000004' +
+                              '000000040000000042005c0500000004' +
+                              '000000150000000042005c0500000004' +
+                              '000000160000000042005c0500000004' +
+                              '000000100000000042005c0500000004' +
+                              '0000001d0000000042005c0500000004' +
+                              '000000060000000042005c0500000004' +
+                              '000000070000000042005c0500000004' +
+                              '0000001e0000000042005c0500000004' +
+                              '0000001b0000000042005c0500000004' +
+                              '0000001c0000000042005c0500000004' +
+                              '000000250000000042005c0500000004' +
+                              '000000260000000042005c0500000004' +
+                              '0000001f0000000042005c0500000004' +
+                              '000000200000000042005c0500000004' +
+                              '000000210000000042005c0500000004' +
+                              '000000220000000042005c0500000004' +
+                              '000000230000000042005c0500000004' +
+                              '000000240000000042005c0500000004' +
+                              '000000270000000042005c0500000004' +
+                              '000000280000000042005c0500000004' +
+                              '00000029000000004200570500000004' +
+                              '00000001000000004200570500000004' +
+                              '00000002000000004200570500000004' +
+                              '00000007000000004200570500000004' +
+                              '00000003000000004200570500000004' +
+                              '00000004000000004200570500000004' +
+                              '00000006000000004200570500000004' +
+                              '00000008000000004200570500000004' +
+                              '00000005000000004200570500000004' +
+                              '0000000900000000', 'hex'),
+    },
+    {
+        request: Buffer.from('42007801000000d04200770100000048' +
+                             '420069010000002042006a0200000004' +
+                             '000000010000000042006b0200000004' +
+                             '00000003000000004200500200000004' +
+                             '000008000000000042000d0200000004' +
+                             '000000020000000042000f0100000048' +
+                             '4200930800000003666f6f0000000000' +
+                             '42005c05000000040000001800000000' +
+                             '42007901000000204200740500000004' +
+                             '00000001000000004200740500000004' +
+                             '000000020000000042000f0100000028' +
+                             '42005c05000000040000001800000000' +
+                             '42007901000000104200740500000004' +
+                             '0000000300000000', 'hex'),
+        response: Buffer.from('42007b010000028042007a0100000048' +
+                              '420069010000002042006a0200000004' +
+                              '000000010000000042006b0200000004' +
+                              '00000003000000004200920900000008' +
+                              '000000005c2d3df242000d0200000004' +
+                              '000000020000000042000f0100000188' +
+                              '42005c05000000040000001800000000' +
+                              '4200930800000003666f6f0000000000' +
+                              '42007f05000000040000000000000000' +
+                              '42007c010000015042005c0500000004' +
+                              '000000010000000042005c0500000004' +
+                              '000000020000000042005c0500000004' +
+                              '000000030000000042005c0500000004' +
+                              '000000080000000042005c0500000004' +
+                              '0000000a0000000042005c0500000004' +
+                              '0000000b0000000042005c0500000004' +
+                              '0000000c0000000042005c0500000004' +
+                              '0000000d0000000042005c0500000004' +
+                              '0000000e0000000042005c0500000004' +
+                              '0000000f0000000042005c0500000004' +
+                              '000000120000000042005c0500000004' +
+                              '000000130000000042005c0500000004' +
+                              '000000140000000042005c0500000004' +
+                              '000000180000000042005c0500000004' +
+                              '0000001e000000004200570500000004' +
+                              '00000001000000004200570500000004' +
+                              '00000002000000004200570500000004' +
+                              '00000003000000004200570500000004' +
+                              '00000004000000004200570500000004' +
+                              '00000006000000004200570500000004' +
+                              '000000070000000042000f0100000098' +
+                              '42005c05000000040000001800000000' +
+                              '42009308000000036261720000000000' +
+                              '42007f05000000040000000000000000' +
+                              '42007c010000006042009d070000000c' +
+                              '4b4b4b4b4b4b4b4b4b4b4b4b00000000' +
+                              '4200880100000040420088070000000d' +
+                              '4b4b4b4b4b4b4b4b4b4b4b4b4b000000' +
+                              '420088070000000c4b4b4b4b4b4b4b4b' +
+                              '4b4b4b4b000000004200880700000005' +
+                              '4b4b4b4b4b000000', 'hex'),
+    },
+    {
+        request: Buffer.from('', 'hex'),
+        response: Buffer.from('', 'hex'),
+    },
+    {
+        request: Buffer.from('4200780100000000', 'hex'),
+        response: Buffer.from('4200780100000000', 'hex'),
+    },
+    {
+        /* Valid message with unknown tag */
+        request: Buffer.from('56000002000000040000000100000000', 'hex'),
+        response: Buffer.from('56000006000000080000000000000001', 'hex'),
+        degenerated: true,
+    },
+    {
+        /* Valid message with unknown enum */
+        /* on a non-enumeration tag */
+        request: Buffer.from('42007805000000040000000100000000', 'hex'),
+        /* on an enumeration tag */
+        response: Buffer.from('42007405000000040000000000000000', 'hex'),
+        degenerated: true,
+    },
+    {
+        /* padding is missing at the end of the message */
+        request: Buffer.from('42000f080000000400000001', 'hex'),
+        response: Buffer.from('42000f0a0000000400000001', 'hex'),
+        degenerated: true,
+    },
+    {
+        request: Buffer.from('42000f08000000040000000100000000', 'hex'),
+        response: Buffer.from('42000f0a000000040000000100000000', 'hex'),
+        degenerated: false,
+    },
+    {
+        /* chained message, shouldn't happen but validate the structure loop */
+        request: Buffer.from('42000f08000000040000000100000000' +
+                             '42000f08000000040000000100000000' +
+                             '42000f08000000040000000100000000' +
+                             '42000f08000000040000000100000000'
+                             , 'hex'),
+        response: Buffer.from('42000f0a000000040000000100000000' +
+                              '42000f0a000000040000000100000000' +
+                              '42000f0a000000040000000100000000' +
+                              '42000f0a000000040000000100000000'
+                              , 'hex'),
+        degenerated: false,
+    },
+    {
+        /* zero-length payload */
+        request: Buffer.from('4200780400000000', 'hex'),
+        response: Buffer.from('4200780400000000', 'hex'),
+        degenerated: false,
+    },
+    {
+        /* no padding */
+        request: Buffer.from('420078040000000100', 'hex'),
+        response: Buffer.from('420078040000000100', 'hex'),
+        degenerated: true,
+    },
+    {
+        request: Buffer.from('42007406000000080000000000000001', 'hex'),
+        response: Buffer.from('42007406000000080000000000000000', 'hex'),
+        degenerated: false,
+    },
+    {
+        request: Buffer.from('42007406000000081111111111111111', 'hex'),
+        response: Buffer.from('42007406000000080101010101010101', 'hex'),
+        degenerated: true,
+    },
+    {
+        request: Buffer.from('4200740700000000', 'hex'),
+        response: Buffer.from('42007407000000010100000000000000', 'hex'),
+        degenerated: false,
+    },
+    {
+        request: Buffer.from('42007407000000010000000000000000', 'hex'),
+        response: Buffer.from('42007407000000020100000000000000', 'hex'),
+        degenerated: false,
+    },
+    {
+        request: Buffer.from('4200740800000000', 'hex'),
+        response: Buffer.from('42007408000000010100000000000000', 'hex'),
+        degenerated: false,
+    },
+    {
+        request: Buffer.from('42007408000000010000000000000000', 'hex'),
+        response: Buffer.from('42007408000000020100000000000000', 'hex'),
+        degenerated: false,
+    },
+    {
+        request: Buffer.from('42007409000000080000000000000001', 'hex'),
+        response: Buffer.from('42007409000000080000000000000000', 'hex'),
+        degenerated: false,
+    },
+];

tests/utils/mdProxyUtils.js

@@ -0,0 +1,185 @@
+'use strict'; // eslint-disable-line strict
+
+
+class DummyProxyResponse {
+    /**
+     * Create a new instance of this dummy class
+     *
+     * This dummy class implements the minimum feature set
+     * of the class http.OutgoingMessage suitable for the
+     * arsenal.storage.metadata.proxy.BucketdRoutes test
+     * without using an actuall http server.
+     *
+     * @param {function} doneCB - function called once the response is
+     *                            ready to be consummed. (err, response, body)
+     */
+    constructor(doneCB) {
+        this.headers = {};
+        this.body = null;
+        this.endCalled = false;
+        this.responseHead = null;
+        this.doneCB = doneCB;
+    }
+    writeHead(statusCode, statusMessage, header) {
+        this.responseHead = {
+            statusCode,
+            statusMessage,
+            header,
+        };
+    }
+    write(data) {
+        this.body = data;
+    }
+    end(cb) {
+        if (this.endCalled) {
+            return;
+        }
+        this.endCalled = true;
+        process.nextTick(() => {
+            cb(null);
+            this.doneCB(null, this, JSON.parse(this.body));
+        });
+    }
+}
+
+class DummyProxyRequest {
+    /**
+     * Create a new instance of this dummy class
+     *
+     * This dummy class implements the minimum feature set
+     * of the class http.IncomingMessage suitable for the
+     * arsenal.storage.metadata.proxy.BucketdRoutes test
+     * without using an actuall http server.
+     *
+     * @param {object} params - parameter set describing the intended request
+     * @param {string} params.method - http method to fake
+     * @param {string} params.url - url to fake
+     * @param {string} params.body - request body to fake
+     * @param {boolean} params.json - if set, assume the body to be a JSON
+     *                                value to be serialized
+     * @param {object} params.headers - request headers to fake
+     */
+    constructor(params) {
+        this.method = params.method;
+        this.url = params.url;
+        this.json = params.json;
+        this.body = new Buffer(
+            this.json ? JSON.stringify(params.body) : (params.body || ''));
+        this.headers = params.headers;
+        this.socket = {
+            remoteAddress: '127.0.0.1',
+            remotePort: 32769,
+        };
+        this.dataConsummed = false;
+        this.endCB = null;
+    }
+
+    /**
+     * bind a callback to a particular event on the request processing
+     *
+     * @param {string} event - one of 'data', 'end' or 'error'
+     * @param {function} callback - a function suitable for the associated event
+     * @returns {object} this
+     */
+    on(event, callback) {
+        switch (event) {
+        case 'data':
+            process.nextTick(() => {
+                callback(this.body);
+                this.dataConsummed = true;
+                if (this.endCB) {
+                    this.endCB();
+                }
+            });
+            break;
+        case 'end':
+            if (!this.dataConsummed) {
+                this.endCB = callback;
+            } else {
+                process.nextTick(() => {
+                    callback();
+                });
+            }
+            break;
+        case 'error':
+            // never happen with this mockup class
+            break;
+        default:
+            process.nextTick(() => callback(new Error(
+                `Unsupported DummyProxyRequest.on event '${event}'`)));
+        }
+        return this;
+    }
+}
+
+class RequestDispatcher {
+    /**
+     * Construct a new RequestDispatcher object.
+     *
+     * This class connects the provided Routes class to a dummy interface
+     * that enables tests to perform requests without using an actual http
+     * server.
+     *
+     * @param {object} routes - an instance of a Routes dispatcher class
+     */
+    constructor(routes) {
+        this.routes = routes;
+    }
+
+    /**
+     * fake a POST request on the associated Routes dispatcher
+     *
+     * @param {string} path - the path of the object to be posted
+     * @param {object} objectMD - the metadata to post for this object
+     * @param {function} callback - called once the request has been processed
+     *                              with these parameters (err)
+     * @returns {undefined}
+     */
+    post(path, objectMD, callback) {
+        this.routes.dispatch(new DummyProxyRequest({
+            method: 'POST',
+            url: path,
+            json: true,
+            body: objectMD,
+            headers: {},
+        }), new DummyProxyResponse(callback));
+    }
+
+    /**
+     * fake a GET request on the associated Routes dispatcher
+     *
+     * @param {string} path - the path of the object to be retrieved
+     * @param {function} callback - called once the request has been processed
+     *                              with these parameters (err, response, body)
+     * @returns {undefined}
+     */
+    get(path, callback) {
+        this.routes.dispatch(new DummyProxyRequest({
+            method: 'GET',
+            url: path,
+            json: true,
+            body: '',
+            headers: {},
+        }), new DummyProxyResponse(callback));
+    }
+
+    /**
+     * fake a DELETE request on the associated Routes dispatcher
+     *
+     * @param {string} path - the path of the object to be deleted
+     * @param {function} callback - called once the request has been processed
+     *                              with these parameters (err)
+     * @returns {undefined}
+     */
+    delete(path, callback) {
+        this.routes.dispatch(new DummyProxyRequest({
+            method: 'DELETE',
+            url: path,
+            json: true,
+            body: '',
+            headers: {},
+        }), new DummyProxyResponse(callback));
+    }
+}
+
+module.exports = { RequestDispatcher };