Merge remote-tracking branch 'origin/development/7.5' into improvement/ZENKO-2062-port

Conflicts:
	package-lock.json
	package.json

.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+localData/*
+localMetadata/*

.github/ISSUE_TEMPLATE.md

@@ -1,19 +1,32 @@
-# Issue template
+# General support information
 
-If you are reporting a new issue, make sure that we do not have any
-duplicates already open. You can ensure this by searching the issue list for
-this repository. If there is a duplicate, please close your issue and add a
-comment to the existing issue instead.
+GitHub Issues are **reserved** for actionable bug reports (including
+documentation inaccuracies), and feature requests.
+**All questions** (regarding configuration, use cases, performance, community,
+events, setup and usage recommendations, among other things) should be asked on
+the **[Zenko Forum](http://forum.zenko.io/)**.
 
-## General support information
+> Questions opened as GitHub issues will systematically be closed, and moved to
+> the [Zenko Forum](http://forum.zenko.io/).
 
-GitHub Issues are reserved for actionable bug reports and feature requests.
-General questions should be sent to the
-[S3 scality server Forum](http://forum.scality.com/).
+--------------------------------------------------------------------------------
+
+## Avoiding duplicates
+
+When reporting a new issue/requesting a feature, make sure that we do not have
+any duplicates already open:
+
+- search the issue list for this repository (use the search bar, select
+  "Issues" on the left pane after searching);
+- if there is a duplicate, please do not open your issue, and add a comment
+  to the existing issue instead.
+
+--------------------------------------------------------------------------------
 
 ## Bug report information
 
-(delete this section if not applicable)
+(delete this section (everything between the lines) if you're not reporting a bug
+but requesting a feature)
 
 ### Description
 
@@ -29,13 +42,22 @@ Describe the results you received
 
 ### Expected result
 
-Describe the results you expecteds
+Describe the results you expected
 
-### Additional information: (Node.js version, Docker version, etc)
+### Additional information
+
+- Node.js version,
+- Docker version,
+- npm version,
+- distribution/OS,
+- optional: anything else you deem helpful to us.
+
+--------------------------------------------------------------------------------
 
 ## Feature Request
 
-(delete this section if not applicable)
+(delete this section (everything between the lines) if you're not requesting
+a feature but reporting a bug)
 
 ### Proposal
 
@@ -52,3 +74,14 @@ What you would like to happen
 ### Use case
 
 Please provide use cases for changing the current behavior
+
+### Additional information
+
+- Is this request for your company? Y/N
+  - If Y: Company name:
+  - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N
+- Are you willing to contribute this feature yourself?
+- Position/Title:
+- How did you hear about us?
+
+--------------------------------------------------------------------------------

.gitignore

@@ -22,9 +22,14 @@ coverage
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
 
+# Sphinx build dir
+_build
+
 # Dependency directory
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
+yarn.lock
+.tox
 
 # Junit directory
 junit

Dockerfile

@@ -19,6 +19,9 @@ COPY ./ ./
 
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
+ENV NO_PROXY localhost,127.0.0.1
+ENV no_proxy localhost,127.0.0.1
+
 ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
 CMD [ "npm", "start" ]
 

README.md

@@ -121,8 +121,9 @@ where the data is saved. If the bucket has no location
 constraint, the endpoint of the PUT request will be
 used to determine location.
 
-See the Configuration section below to learn how to set
-location constraints.
+See the Configuration section in our documentation
+[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
+to learn how to set location constraints.
 
 ## Run it with an in-memory backend
 

bin/create_bucket_with_NFS_enabled.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict'; // eslint-disable-line strict
+
+require('../lib/nfs/utilities.js').createBucketWithNFSEnabled();

bin/search_bucket.js

@@ -0,0 +1,100 @@
+#!/bin/sh
+// 2>/dev/null ; exec "$(which nodejs 2>/dev/null || which node)" "$0" "$@"
+'use strict'; // eslint-disable-line strict
+
+const { auth } = require('arsenal');
+const commander = require('commander');
+
+const http = require('http');
+const https = require('https');
+const logger = require('../lib/utilities/logger');
+
+function _performSearch(host,
+                        port,
+                        bucketName,
+                        query,
+                        accessKey,
+                        secretKey,
+                        verbose, ssl) {
+    const escapedSearch = encodeURIComponent(query);
+    const options = {
+        host,
+        port,
+        method: 'GET',
+        path: `/${bucketName}/?search=${escapedSearch}`,
+        headers: {
+            'Content-Length': 0,
+        },
+        rejectUnauthorized: false,
+    };
+    const transport = ssl ? https : http;
+    const request = transport.request(options, response => {
+        if (verbose) {
+            logger.info('response status code', {
+                statusCode: response.statusCode,
+            });
+            logger.info('response headers', { headers: response.headers });
+        }
+        const body = [];
+        response.setEncoding('utf8');
+        response.on('data', chunk => body.push(chunk));
+        response.on('end', () => {
+            if (response.statusCode >= 200 && response.statusCode < 300) {
+                logger.info('Success');
+                process.stdout.write(body.join(''));
+                process.exit(0);
+            } else {
+                logger.error('request failed with HTTP Status ', {
+                    statusCode: response.statusCode,
+                    body: body.join(''),
+                });
+                process.exit(1);
+            }
+        });
+    });
+    // generateV4Headers exepects request object with path that does not
+    // include query
+    request.path = `/${bucketName}`;
+    auth.client.generateV4Headers(request, { search: query },
+        accessKey, secretKey, 's3');
+    request.path = `/${bucketName}?search=${escapedSearch}`;
+    if (verbose) {
+        logger.info('request headers', { headers: request._headers });
+    }
+    request.end();
+}
+
+/**
+ * This function is used as a binary to send a request to S3 to perform a
+ * search on the objects in a bucket
+ *
+ * @return {undefined}
+ */
+function searchBucket() {
+    // TODO: Include other bucket listing possible query params?
+    commander
+        .version('0.0.1')
+        .option('-a, --access-key <accessKey>', 'Access key id')
+        .option('-k, --secret-key <secretKey>', 'Secret access key')
+        .option('-b, --bucket <bucket>', 'Name of the bucket')
+        .option('-q, --query <query>', 'Search query')
+        .option('-h, --host <host>', 'Host of the server')
+        .option('-p, --port <port>', 'Port of the server')
+        .option('-s', '--ssl', 'Enable ssl')
+        .option('-v, --verbose')
+        .parse(process.argv);
+
+    const { host, port, accessKey, secretKey, bucket, query, verbose, ssl } =
+        commander;
+
+    if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
+        logger.error('missing parameter');
+        commander.outputHelp();
+        process.exit(1);
+    }
+
+    _performSearch(host, port, bucket, query, accessKey, secretKey, verbose,
+        ssl);
+}
+
+searchBucket();

circle.yml

@@ -1,45 +0,0 @@
----
-general:
-  branches:
-    ignore:
-      - /^ultron\/.*/   # Ignore ultron/* branches
-  artifacts:
-    - coverage/
-
-machine:
-  node:
-    version: 6.13.1
-  services:
-    - redis
-    - docker
-  ruby:
-    version: "2.4.1"
-  environment:
-    CXX: g++-4.9
-    ENABLE_LOCAL_CACHE: true
-    REPORT_TOKEN: report-token-1
-  hosts:
-    bucketwebsitetester.s3-website-us-east-1.amazonaws.com: 127.0.0.1
-
-dependencies:
-  override:
-    - rm -rf node_modules
-    - npm install
-  post:
-    - sudo pip install flake8 yamllint
-    - sudo pip install s3cmd==1.6.1
-    # fog and ruby testing dependencies
-    - gem install fog-aws -v 1.3.0
-    - gem install mime-types -v 3.1
-    - gem install rspec -v 3.5
-    - gem install json
-    # java sdk dependencies
-    - sudo apt-get install -y -q default-jdk
-
-
-test:
-  override:
-    - docker run --name squid-proxy -d --net=host
-      --publish 3128:3128 sameersbn/squid:3.3.8-23
-    - bash tests.bash:
-        parallel: true

conf/authdata.json

@@ -19,5 +19,38 @@
             "access": "accessKey2",
             "secret": "verySecretKey2"
         }]
+    },
+    {
+        "name": "Clueso",
+        "email": "inspector@clueso.info",
+        "arn": "arn:aws:iam::123456789014:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/clueso",
+        "shortid": "123456789014",
+        "keys": [{
+            "access": "cluesoKey1",
+            "secret": "cluesoSecretKey1"
+        }]
+    },
+    {
+        "name": "Replication",
+        "email": "inspector@replication.info",
+        "arn": "arn:aws:iam::123456789015:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/replication",
+        "shortid": "123456789015",
+        "keys": [{
+            "access": "replicationKey1",
+            "secret": "replicationSecretKey1"
+        }]
+    },
+    {
+        "name": "Lifecycle",
+        "email": "inspector@lifecycle.info",
+        "arn": "arn:aws:iam::123456789016:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle",
+        "shortid": "123456789016",
+        "keys": [{
+            "access": "lifecycleKey1",
+            "secret": "lifecycleSecretKey1"
+        }]
     }]
 }

config.json

@@ -8,7 +8,9 @@
         "cloudserver-front": "us-east-1",
         "s3.docker.test": "us-east-1",
         "127.0.0.2": "us-east-1",
-        "s3.amazonaws.com": "us-east-1"
+        "s3.amazonaws.com": "us-east-1",
+        "zenko-cloudserver-replicator": "us-east-1",
+        "lb": "us-east-1"
     },
     "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
                         "s3-website.us-east-2.amazonaws.com",
@@ -23,7 +25,8 @@
                         "s3-website-eu-west-1.amazonaws.com",
                         "s3-website-sa-east-1.amazonaws.com",
                         "s3-website.localhost",
-                        "s3-website.scality.test"],
+                        "s3-website.scality.test",
+                        "zenkoazuretest.blob.core.windows.net"],
     "replicationEndpoints": [{
         "site": "zenko",
         "servers": ["127.0.0.1:8000"],
@@ -32,6 +35,10 @@
         "site": "us-east-2",
         "type": "aws_s3"
     }],
+    "backbeat": {
+        "host": "localhost",
+        "port": 8900
+    },
     "cdmi": {
         "host": "localhost",
         "port": 81,
@@ -45,7 +52,7 @@
         "host": "localhost",
         "port": 8500
     },
-    "clusters": 10,
+    "clusters": 1,
     "log": {
         "logLevel": "info",
         "dumpLevel": "error"
@@ -70,12 +77,32 @@
         "port": 9991
     },
     "recordLog": {
-        "enabled": false,
+        "enabled": true,
         "recordLogName": "s3-recordlog"
     },
     "mongodb": {
-	"host": "localhost",
-	"port": 27018,
+       "replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
+       "writeConcern": "majority",
+       "replicaSet": "rs0",
+       "readPreference": "primary",
        "database": "metadata"
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
     }
 }

constants.js

@@ -39,6 +39,8 @@ const constants = {
     // once the multipart upload is complete.
     mpuBucketPrefix: 'mpuShadowBucket',
     blacklistedPrefixes: { bucket: [], object: [] },
+    // GCP Object Tagging Prefix
+    gcpTaggingPrefix: 'aws-tag-',
     // PublicId is used as the canonicalID for a request that contains
     // no authentication information.  Requestor can access
     // only public resources
@@ -64,11 +66,21 @@ const constants = {
     // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
     minimumAllowedPartSize: 5242880,
 
+    // AWS sets a maximum total parts limit
+    // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
+    maximumAllowedPartCount: 10000,
+
+    gcpMaximumAllowedPartCount: 1024,
+
     // Max size on put part or copy part is 5GB. For functional
     // testing use 110 MB as max
     maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :
         5368709120,
 
+    // Max size allowed in a single put object request is 5GB
+    // https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
+    maximumAllowedUploadSize: 5368709120,
+
     // AWS states max size for user-defined metadata (x-amz-meta- headers) is
     // 2 KB: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
     // In testing, AWS seems to allow up to 88 more bytes, so we do the same.
@@ -105,6 +117,7 @@ const constants = {
     legacyLocations: ['sproxyd', 'legacy'],
     /* eslint-disable camelcase */
     externalBackends: { aws_s3: true, azure: true, gcp: true },
+    replicationBackends: { aws_s3: true, azure: true, gcp: true },
     // some of the available data backends  (if called directly rather
     // than through the multiple backend gateway) need a key provided
     // as a string as first parameter of the get/delete methods.
@@ -113,13 +126,17 @@ const constants = {
     // for external backends, don't call unless at least 1 minute
     // (60,000 milliseconds) since last call
     externalBackendHealthCheckInterval: 60000,
-    versioningNotImplBackends: { azure: true },
-    mpuMDStoredExternallyBackend: { aws_s3: true },
+    versioningNotImplBackends: { azure: true, gcp: true },
+    mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
+    skipBatchDeleteBackends: { azure: true, gcp: true },
+    s3HandledBackends: { azure: true, gcp: true },
+    hasCopyPartBackends: { aws_s3: true, gcp: true },
     /* eslint-enable camelcase */
     mpuMDStoredOnS3Backend: { azure: true },
     azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
     base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
         '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
+    productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
     // user metadata applied on zenko objects
     zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
     bucketOwnerActions: [

dataserver.js

@@ -17,13 +17,17 @@ process.on('uncaughtException', err => {
 if (config.backends.data === 'file' ||
     (config.backends.data === 'multiple' &&
      config.backends.metadata !== 'scality')) {
-    const dataServer = new arsenal.network.rest.RESTServer(
-        { bindAddress: config.dataDaemon.bindAddress,
+    const dataServer = new arsenal.network.rest.RESTServer({
+        bindAddress: config.dataDaemon.bindAddress,
         port: config.dataDaemon.port,
-            dataStore: new arsenal.storage.data.file.DataFileStore(
-                { dataPath: config.dataDaemon.dataPath,
-                    log: config.log }),
-            log: config.log });
+        dataStore: new arsenal.storage.data.file.DataFileStore({
+            dataPath: config.dataDaemon.dataPath,
+            log: config.log,
+            noSync: config.dataDaemon.noSync,
+            noCache: config.dataDaemon.noCache,
+        }),
+        log: config.log,
+    });
     dataServer.setup(err => {
         if (err) {
             logger.error('Error initializing REST data server',

docker-entrypoint.sh

@@ -6,14 +6,15 @@ set -e
 # modifying config.json
 JQ_FILTERS_CONFIG="."
 
+# ENDPOINT var can accept comma separated values
+# for multiple endpoint locations
 if [[ "$ENDPOINT" ]]; then
-    HOST_NAME="$ENDPOINT"
-fi
-
-if [[ "$HOST_NAME" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$HOST_NAME\"]=\"us-east-1\""
-    echo "Host name has been modified to $HOST_NAME"
-    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with $HOST_NAME"
+    IFS="," read -ra HOST_NAMES <<< "$ENDPOINT"
+    for host in "${HOST_NAMES[@]}"; do
+        JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$host\"]=\"us-east-1\""
+    done
+    echo "Host name has been modified to ${HOST_NAMES[@]}"
+    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with ${HOST_NAMES[@]}"
 fi
 
 if [[ "$LOG_LEVEL" ]]; then
@@ -25,7 +26,7 @@ if [[ "$LOG_LEVEL" ]]; then
     fi
 fi
 
-if [[ "$SSL" && "$HOST_NAME" ]]; then
+if [[ "$SSL" && "$HOST_NAMES" ]]; then
 # This condition makes sure that the certificates are not generated twice. (for docker restart)
 if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
 # Compute config for utapi tests
@@ -36,15 +37,15 @@ prompt = no
 req_extensions = s3_req
 
 [req_distinguished_name]
-CN = ${HOST_NAME}
+CN = ${HOST_NAMES[0]}
 
 [s3_req]
 subjectAltName = @alt_names
 extendedKeyUsage = serverAuth, clientAuth
 
 [alt_names]
-DNS.1 = *.${HOST_NAME}
-DNS.2 = ${HOST_NAME}
+DNS.1 = *.${HOST_NAMES[0]}
+DNS.2 = ${HOST_NAMES[0]}
 
 EOF
 
@@ -81,6 +82,18 @@ if [[ "$METADATA_HOST" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
 fi
 
+if [[ "$MONGODB_HOSTS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
+fi
+
+if [[ "$MONGODB_RS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSet=\"$MONGODB_RS\""
+fi
+
+if [[ "$MONGODB_DATABASE" ]]; then
+   JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.database=\"$MONGODB_DATABASE\""
+fi
+
 if [ -z "$REDIS_HA_NAME" ]; then
     REDIS_HA_NAME='mymaster'
 fi
@@ -113,11 +126,67 @@ if [[ "$RECORDLOG_ENABLED" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true"
 fi
 
+if [[ "$STORAGE_LIMIT_ENABLED" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.metrics[.utapi.metrics | length]=\"location\""
+fi
+
+if [[ "$CRR_METRICS_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\""
+fi
+
+if [[ "$CRR_METRICS_PORT" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
+fi
+
+if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
+fi
+
+# external backends http(s) agent config
+
+# AWS
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
+#GCP
+if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
 if [[ $JQ_FILTERS_CONFIG != "." ]]; then
     jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
     mv config.json.tmp config.json
 fi
 
+if test -v INITIAL_INSTANCE_ID && test -v S3METADATAPATH && ! test -f ${S3METADATAPATH}/uuid ; then
+    echo -n ${INITIAL_INSTANCE_ID} > ${S3METADATAPATH}/uuid
+fi
+
 # s3 secret credentials for Zenko
 if [ -r /run/secrets/s3-credentials ] ; then
     . /run/secrets/s3-credentials

docs/ARCHITECTURE.rst

@@ -27,7 +27,7 @@ including null versions and delete markers, described in the above
 links.
 
 Implementation of Bucket Versioning in Zenko CloudServer
------------------------------------------
+--------------------------------------------------------
 
 Overview of Metadata and API Component Roles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -179,12 +179,13 @@ PUT
    the master version with this version.
 -  ``versionId: <versionId>`` create or update a specific version (for updating
    version's ACL or tags, or remote updates in geo-replication)
-   -  if the version identified by ``versionId`` happens to be the latest
+
+   *  if the version identified by ``versionId`` happens to be the latest
       version, the master version will be updated as well
-   -  if the master version is not as recent as the version identified by
+   *  if the master version is not as recent as the version identified by
       ``versionId``, as may happen with cross-region replication, the master
       will be updated as well
-   -  note that with ``versionId`` set to an empty string ``''``, it will
+   *  note that with ``versionId`` set to an empty string ``''``, it will
       overwrite the master version only (same as no options, but the master
       version will have a ``versionId`` property set in its metadata like
       any other version). The ``versionId`` will never be exposed to an
@@ -208,10 +209,13 @@ A deletion targeting the latest version of an object has to:
 -  delete the specified version identified by ``versionId``
 -  replace the master version with a version that is a placeholder for
    deletion
+
     -  this version contains a special keyword, 'isPHD', to indicate the
          master version was deleted and needs to be updated
+
 -  initiate a repair operation to update the value of the master
    version:
+
     -  involves listing the versions of the object and get the latest
        version to replace the placeholder delete version
     -  if no more versions exist, metadata deletes the master version,
@@ -764,7 +768,7 @@ This will open two ports:
 Then, one or more instances of Zenko CloudServer without the dmd can be started
 elsewhere with:
 
-::
+.. code:: sh
 
    npm run start_s3server
 

docs/CLIENTS.rst

@@ -178,7 +178,7 @@ Ruby
 ~~~~
 
 `AWS SDK for Ruby - Version 2 <http://docs.aws.amazon.com/sdkforruby/api/>`__
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code:: ruby
 
@@ -239,6 +239,7 @@ Python
 Client integration
 
 .. code:: python
+
     import boto3
 
     client = boto3.client(
@@ -253,6 +254,7 @@ Client integration
 Full integration (with object mapping)
 
 .. code:: python
+
     import os
 
     from botocore.utils import fix_s3_host

docs/DOCKER.rst

@@ -71,15 +71,15 @@ to get in the source bucket. ACL's would have to be updated
 on AWS directly to enable this.
 
 S3BACKEND
-~~~~~~
+~~~~~~~~~
 
 S3BACKEND=file
-^^^^^^^^^^^
+^^^^^^^^^^^^^^
 When storing file data, for it to be persistent you must mount docker volumes
 for both data and metadata. See `this section <#using-docker-volumes-in-production>`__
 
 S3BACKEND=mem
-^^^^^^^^^^
+^^^^^^^^^^^^^
 This is ideal for testing - no data will remain after container is shutdown.
 
 ENDPOINT
@@ -89,9 +89,9 @@ This variable specifies your endpoint. If you have a domain such as
 new.host.com, by specifying that here, you and your users can direct s3
 server requests to new.host.com.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e ENDPOINT=new.host.com scality/s3server
+    $ docker run -d --name s3server -p 8000:8000 -e ENDPOINT=new.host.com scality/s3server
 
 Note: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
 permissions, make sure to associate 127.0.0.1 with ``new.host.com``
@@ -107,7 +107,7 @@ You can set credentials for many accounts by editing
 want to specify one set of your own, you can use these environment
 variables.
 
-.. code:: shell
+.. code-block:: shell
 
     docker run -d --name s3server -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey
     -e SCALITY_SECRET_ACCESS_KEY=newSecretKey scality/s3server
@@ -123,9 +123,9 @@ This variable allows you to change the log level: info, debug or trace.
 The default is info. Debug will give you more detailed logs and trace
 will give you the most detailed.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
+    $ docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
 
 SSL
 ~~~
@@ -144,9 +144,9 @@ extra container to do SSL/TLS termination such as haproxy/nginx/stunnel
 to limit what an exploit on either component could expose, as well as
 certificates in a mounted volume
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT>
+    $ docker run -d --name s3server -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT>
     scality/s3server
 
 More information about how to use S3 server with SSL
@@ -159,9 +159,9 @@ This variable instructs the Zenko CloudServer, and its data and metadata
 components to listen on the specified address. This allows starting the data
 or metadata servers as standalone services, for example.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0
+    $ docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0
     scality/s3server npm run start_dataserver
 
 
@@ -172,9 +172,9 @@ These variables configure the data and metadata servers to use,
 usually when they are running on another host and only starting the stateless
 Zenko CloudServer.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -e DATA_HOST=s3server-data
+    $ docker run -d --name s3server -e DATA_HOST=s3server-data
     -e METADATA_HOST=s3server-metadata scality/s3server npm run start_s3server
 
 REDIS\_HOST
@@ -183,9 +183,9 @@ REDIS\_HOST
 Use this variable to connect to the redis cache server on another host than
 localhost.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000
+    $ docker run -d --name s3server -p 8000:8000
     -e REDIS_HOST=my-redis-server.example.com scality/s3server
 
 REDIS\_PORT
@@ -194,9 +194,9 @@ REDIS\_PORT
 Use this variable to connect to the redis cache server on another port than
 the default 6379.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000
+    $ docker run -d --name s3server -p 8000:8000
     -e REDIS_PORT=6379 scality/s3server
 
 Tunables and Setup Tips
@@ -215,9 +215,9 @@ Docker volumes to host your data and metadata outside your Zenko CloudServer
 Docker container. Otherwise, the data and metadata will be destroyed
 when you erase the container.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata
+    $ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata
     -p 8000:8000 ­-d scality/s3server
 
 This command mounts the host directory, ``./data``, into the container
@@ -233,11 +233,12 @@ Adding modifying or deleting accounts or users credentials
 2. Use `Docker
    Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__
    to override the default ``authdata.json`` through a docker file mapping.
+
 For example:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
+    $ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
     scality/s3server
 
 Specifying your own host name
@@ -272,9 +273,9 @@ More information about location configuration
 Then, run your Scality S3 Server using `Docker
 Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
+    $ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
 
 Your local ``config.json`` file will override the default one through a
 docker file mapping.
@@ -292,7 +293,7 @@ The user needs to exist within the container, and own the folder
 
 For instance, you can modify these lines in the dockerfile:
 
-.. code:: shell
+.. code-block:: shell
 
     ...
     && groupadd -r -g 1001 scality \
@@ -316,7 +317,7 @@ Sample ways to run it for CI are:
 - With custom locations (one in-memory, one hosted on AWS), and custom
   credentials mounted:
 
-.. code:: shell
+.. code-block:: shell
 
     docker run --name CloudServer -p 8000:8000
     -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
@@ -328,7 +329,7 @@ Sample ways to run it for CI are:
   and custom credentials set as environment variables
   (see `this section <#scality-access-key-id-and-scality-secret-access-key>`__):
 
-.. code:: shell
+.. code-block:: shell
 
     docker run --name CloudServer -p 8000:8000
     -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
@@ -346,7 +347,7 @@ multiple backends capabilities of Zenko CloudServer, and that you will have a
 custom endpoint for your local storage, and custom credentials for your local
 storage:
 
-.. code:: shell
+.. code-block:: shell
 
     docker run -d --name CloudServer
     -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata

docs/GETTING_STARTED.rst

@@ -1,5 +1,5 @@
 Getting Started
-=================
+===============
 
 .. figure:: ../res/scality-cloudserver-logo.png
    :alt: Zenko CloudServer logo
@@ -19,7 +19,7 @@ npm v3 . Up-to-date versions can be found at
 Clone source code
 ~~~~~~~~~~~~~~~~~
 
-.. code:: shell
+.. code-block:: shell
 
     git clone https://github.com/scality/S3.git
 
@@ -28,14 +28,14 @@ Install js dependencies
 
 Go to the ./S3 folder,
 
-.. code:: shell
+.. code-block:: shell
 
     npm install
 
 Run it with a file backend
 --------------------------
 
-.. code:: shell
+.. code-block:: shell
 
     npm start
 
@@ -53,7 +53,7 @@ pre-created within the repository. If you would like to save the data or
 metadata in different locations of your choice, you must specify them
 with absolute paths. So, when starting the server:
 
-.. code:: shell
+.. code-block:: shell
 
     mkdir -m 700 $(pwd)/myFavoriteDataPath
     mkdir -m 700 $(pwd)/myFavoriteMetadataPath
@@ -64,7 +64,7 @@ with absolute paths. So, when starting the server:
 Run it with multiple data backends
 ----------------------------------
 
-.. code:: shell
+.. code-block:: shell
 
     export S3DATA='multiple'
     npm start
@@ -76,7 +76,7 @@ With multiple backends, you have the ability to choose where each object
 will be saved by setting the following header with a locationConstraint
 on a PUT request:
 
-.. code:: shell
+.. code-block:: shell
 
     'x-amz-meta-scal-location-constraint':'myLocationConstraint'
 
@@ -91,7 +91,7 @@ constraints.
 Run it with an in-memory backend
 --------------------------------
 
-.. code:: shell
+.. code-block:: shell
 
     npm run mem_backend
 
@@ -108,19 +108,20 @@ Testing
 
 You can run the unit tests with the following command:
 
-.. code:: shell
+.. code-block:: shell
 
     npm test
 
 You can run the multiple backend unit tests with:
 
-.. code:: shell
+.. code-block:: shell
+
     CI=true S3DATA=multiple npm start
     npm run multiple_backend_test
 
 You can run the linter with:
 
-.. code:: shell
+.. code-block:: shell
 
     npm run lint
 
@@ -152,13 +153,13 @@ instance port (``6379`` by default)
 
 -  Add the following to the etc/hosts file on your machine:
 
-.. code:: shell
+.. code-block:: shell
 
     127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
 
 -  Start the Zenko CloudServer in memory and run the functional tests:
 
-.. code:: shell
+.. code-block:: shell
 
     CI=true npm run mem_backend
     CI=true npm run ft_test
@@ -263,7 +264,7 @@ These variables specify authentication credentials for an account named
 
 Note: Anything in the ``authdata.json`` file will be ignored.
 
-.. code:: shell
+.. code-block:: shell
 
     SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey npm start
 
@@ -275,7 +276,7 @@ If you wish to use https with your local Zenko CloudServer, you need to set up
 SSL certificates. Here is a simple guide of how to do it.
 
 Deploying Zenko CloudServer
-^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 First, you need to deploy **Zenko CloudServer**. This can be done very easily
 via `our **DockerHub**
@@ -287,7 +288,7 @@ with a file backend).
     distribution <https://docs.docker.com/engine/installation/>`__*
 
 Updating your Zenko CloudServer container's config
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You're going to add your certificates to your container. In order to do
 so, you need to exec inside your Zenko CloudServer container. Run a
@@ -295,7 +296,7 @@ so, you need to exec inside your Zenko CloudServer container. Run a
 name should be ``scality/s3server``. Copy the corresponding container id
 (here we'll use ``894aee038c5e``, and run:
 
-.. code:: sh
+.. code-block:: shell
 
     $> docker exec -it 894aee038c5e bash
 
@@ -307,7 +308,7 @@ Generate SSL key and certificates
 There are 5 steps to this generation. The paths where the different
 files are stored are defined after the ``-out`` option in each command
 
-.. code:: sh
+.. code-block:: shell
 
     # Generate a private key for your CSR
     $> openssl genrsa -out ca.key 2048
@@ -322,7 +323,7 @@ files are stored are defined after the ``-out`` option in each command
     $> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
 
 Update Zenko CloudServer ``config.json``
-**********************************
+****************************************
 
 Add a ``certFilePaths`` section to ``./config.json`` with the
 appropriate paths:
@@ -336,7 +337,7 @@ appropriate paths:
         }
 
 Run your container with the new config
-****************************************
+**************************************
 
 First, you need to exit your container. Simply run ``$> exit``. Then,
 you need to restart your container. Normally, a simple
@@ -356,13 +357,13 @@ permissions), edit the line of localhost so it looks like this:
     127.0.0.1      localhost s3.scality.test
 
 Copy the local certificate authority from your container
-*********************************************************
+********************************************************
 
 In the above commands, it's the file named ``ca.crt``. Choose the path
 you want to save this file at (here we chose ``/root/ca.crt``), and run
 something like:
 
-.. code:: sh
+.. code-block:: shell
 
     $> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
 

docs/INTEGRATIONS.rst

@@ -337,7 +337,7 @@ tutorial, we used an Ubuntu 14.04 host to deploy and use s3fs over
 Scality's Zenko CloudServer.
 
 Deploying Zenko CloudServer with SSL
-----------------------------
+------------------------------------
 
 First, you need to deploy **Zenko CloudServer**. This can be done very easily
 via `our DockerHub
@@ -401,7 +401,7 @@ s3fs expects you to provide it with a password file. Our file is
     $> chmod 600 /etc/passwd-s3fs
 
 Using Zenko CloudServer with s3fs
-------------------------
+---------------------------------
 
 First, you're going to need a mountpoint; we chose ``/mnt/tests3fs``:
 
@@ -634,7 +634,7 @@ backing up is a folder I modify permanently during my workday, so I want
 incremental backups every 5mn from 8AM to 9PM monday to friday. Here is
 the line I will paste in my crontab:
 
-.. code:: cron
+.. code:: sh
 
     */5 8-20 * * 1-5 /usr/local/sbin/backup.sh
 

docs/MD_SEARCH.md

@@ -0,0 +1,229 @@
+# Metadata Search Documenation
+
+## Description
+
+This feature enables metadata search to be performed on the metadata of objects
+stored in Zenko.
+
+## Requirements
+
++ MongoDB
+
+## Design
+
+The MD Search feature expands on the existing `GET Bucket` S3 API. It allows
+users to conduct metadata searches by adding the custom Zenko querystring
+parameter, `search`. The `search` parameter is of a pseudo
+SQL WHERE clause structure and supports basic SQL operators:
+ex. `"A=1 AND B=2 OR C=3"` (more complex queries can also be achieved with the
+use of nesting operators, `(` and `)`).
+
+The search process is as follows:
+
++ Zenko receives a `GET` request.
+
+    ```
+    # regular getBucket request
+    GET /bucketname HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # getBucket versions request
+    GET /bucketname?versions HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # search getBucket request
+    GET /bucketname?search=key%3Dsearch-item HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+    ```
+
++ If the request does not contain the query param `search`, a normal bucket
+  listing is performed and a XML result containing the list of objects will be
+  returned as the response.
++ If the request does contain the query parameter `search`, the search string is
+  parsed and validated.
+
+    + If the search string is invalid, an `InvalidArgument` error will be
+      returned as response.
+    + If the search string is valid, it will be parsed and an abstract syntax
+      tree (AST) is generated.
+
++ The AST is then passed to the MongoDB backend to be used as the query filter
+  for retrieving objects in a bucket that satisfies the requested search
+  conditions.
++ The filtered results are then parsed and returned as the response.
+
+The results from MD search is of the same structure as the `GET Bucket`
+results:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+    <Name>bucketname</Name>
+    <Prefix/>
+    <Marker/>
+    <MaxKeys>1000</MaxKeys>
+    <IsTruncated>false</IsTruncated>
+    <Contents>
+        <Key>objectKey</Key>
+        <LastModified>2018-04-19T18:31:49.426Z</LastModified>
+        <ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
+        <Size>0</Size>
+        <Owner>
+            <ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
+            <DisplayName>Bart</DisplayName>
+        </Owner>
+        <StorageClass>STANDARD</StorageClass>
+    </Contents>
+    <Contents>
+        ...
+    </Contents>
+</ListBucketResult>
+```
+
+## Performing MD Search with Zenko
+
+To make a successful request to Zenko, you would need
+
++ Zenko Credentials
++ Sign request with Auth V4
+
+With requirements, you can peform metadata searches by:
+
++ using the `search_bucket` tool in the
+  [Scality/S3](https://github.com/scality/S3) GitHub repository.
++ creating an AuthV4 signed HTTP request to Zenko in the programming language of
+  choice
+
+### Using the S3 Tool
+
+After cloning the [Scality/S3](https://github.com/scality/S3) GitHub repository
+and installing the necessary dependencies, you can run the following command
+in the S3 project root directory to access the search tool.
+
+```
+node bin/search_bucket
+```
+
+This will generate the following output
+
+```
+Usage: search_bucket [options]
+
+Options:
+
+  -V, --version                 output the version number
+  -a, --access-key <accessKey>  Access key id
+  -k, --secret-key <secretKey>  Secret access key
+  -b, --bucket <bucket>         Name of the bucket
+  -q, --query <query>           Search query
+  -h, --host <host>             Host of the server
+  -p, --port <port>             Port of the server
+  -s                            --ssl
+  -v, --verbose
+  -h, --help                    output usage information
+```
+
+In the following examples, our Zenko Server is accessible on endpoint
+`http://127.0.0.1:8000` and contains the bucket `zenkobucket`.
+
+```
+# search for objects with metadata "blue"
+node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+    -q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
+
+# search for objects tagged with "type=color"
+node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+    -q "tags.type=color" -h 127.0.0.1 -p 8000
+```
+
+### Coding Examples
+
+Search requests can be also performed by making HTTP requests authenticated
+with the `AWS Signature version 4` scheme.\
+See the following urls for more information about the V4 authentication scheme.
+
++ http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
++ http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+
+You can also view examples for making requests with Auth V4 in various
+languages [here](../exmaples).
+
+### Specifying Metadata Fields
+
+To search common metadata headers:
+
+```
+    {metadata-key}{supported SQL op}{search value}
+    # example
+    key = blueObject
+    size > 0
+    key LIKE "blue.*"
+```
+
+To search custom user metadata:
+
+```
+    # metadata must be prefixed with "x-amz-meta-"
+    x-amz-meta-{usermetadata-key}{supported SQL op}{search value}
+    # example
+    x-amz-meta-color = blue
+    x-amz-meta-color != red
+    x-amz-meta-color LIKE "b.*"
+```
+
+To search tags:
+
+```
+    # tag searches must be prefixed with "tags."
+    tags.{tag-key}{supported SQL op}{search value}
+    # example
+    tags.type = color
+```
+
+### Differences from SQL
+
+The MD search queries are similar to the `WHERE` clauses of SQL queries, but
+they differ in that:
+
++ MD search queries follow the `PCRE` format
++ Search queries do not require values with hyphens to be enclosed in
+  backticks, ``(`)``
+
+    ```
+        # SQL query
+        `x-amz-meta-search-item` = `ice-cream-cone`
+
+        # MD Search query
+        x-amz-meta-search-item = ice-cream-cone
+    ```
+
++ The search queries do not support all of the SQL operators.
+
+  + Supported SQL Operators: `=`, `<`, `>`, `<=`, `>=`, `!=`,
+    `AND`, `OR`, `LIKE`, `<>`
+  + Unsupported SQL Operators: `NOT`, `BETWEEN`, `IN`, `IS`, `+`,
+    `-`, `%`, `^`, `/`, `*`, `!`
+
+#### Using Regular Expressions in MD Search
+
++ Regular expressions used in MD search differs from SQL in that wildcards are
+  represented with `.*` instead of `%`.
++ Regex patterns must be wrapped in quotes as not doing so can lead to
+  misinterpretation of patterns.
++ Regex patterns can be written in form of the `/pattern/` syntax or
+  just the pattern if one does not require regex options, similar to `PCRE`.
+
+Example regular expressions:
+
+    ```
+    # search for strings containing word substring "helloworld"
+        ".*helloworld.*"
+        "/.*helloworld.*/"
+        "/.*helloworld.*/i"
+    ```

docs/Makefile

@@ -0,0 +1,21 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Zenko
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+

docs/USING_PUBLIC_CLOUDS.rst

@@ -1,3 +1,5 @@
+.. _use-public-cloud:
+
 Using Public Clouds as data backends
 ====================================
 
@@ -243,7 +245,7 @@ There are a few configurable options here:
   this region should behave like any other AWS S3 region (in the case of MS Azure
   hosted data, this is mostly relevant for the format of errors);
 - :code:`azureStorageEndpoint` : set to your storage account's endpoint, usually
-  :code:`https://{{storageAccountName}}.blob.core.windows.name`;
+  :code:`https://{{storageAccountName}}.blob.core.windows.net`;
 - :code:`azureContainerName` : set to an *existing container* in your MS Azure
   storage account; this is the container in which your data will be stored for
   this location constraint;

docs/developers/getting-started.rst

@@ -0,0 +1,79 @@
+============================================
+Add New Backend Storage To Zenko Cloudserver
+============================================
+
+This set of documents aims at bootstrapping developers with Zenko's Cloudserver
+module, so they can then go on and contribute features.
+
+.. toctree::
+    :maxdepth: 2
+
+    non-s3-compatible-backend
+    s3-compatible-backend
+
+We always encourage our community to offer new extensions to Zenko,
+and new backend support is paramount to meeting more community needs.
+If that is something you want to contribute (or just do on your own
+version of the cloudserver image), this is the guid to read. Please
+make sure you follow our `Contributing Guidelines`_/.
+
+If you need help with anything, please search our `forum`_ for more
+information.
+
+Add support for a new backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently the main public cloud protocols are supported and more can
+be added. There are two main types of backend: those compatible with
+Amazon's S3 protocol and those not compatible.
+
+================= ========== ============ ===========
+Backend type      Supported  Active WIP   Not started
+================= ========== ============ ===========
+Private disk/fs       x
+AWS S3                x
+Microsoft Azure       x
+Backblaze B2                    x
+Google Cloud          x
+Openstack Swift                               x
+================= ========== ============ ===========
+
+.. important:: Should you want to request for a new backend to be
+               supported, please do so by opening a `Github issue`_,
+               and filling out the "Feature Request" section of our
+               template.
+
+To add support for a new backend support to Cloudserver official
+repository, please follow these steps:
+
+- familiarize yourself with our `Contributing Guidelines`_
+- open a `Github issue`_ and fill out Feature Request form, and
+  specify you would like to contribute it yourself;
+- wait for our core team to get back to you with an answer on whether
+  we are interested in taking that contribution in (and hence
+  committing to maintaining it over time);
+- once approved, fork the repository and start your development;
+- use the `forum`_ with any question you may have during the
+  development process;
+- when you think it's ready, let us know so that we create a feature
+  branch against which we'll compare and review your code;
+- open a pull request with your changes against that dedicated feature
+  branch;
+- once that pull request gets merged, you're done.
+
+.. tip::
+
+    While we do take care of the final rebase (when we merge your feature
+    branch on the latest default branch), we do ask that you keep up to date with our latest default branch
+    until then.
+
+.. important::
+
+    If we do not approve your feature request, you may of course still
+    work on supporting a new backend: all our "no" means is that we do not
+    have the resources, as part of our core development team, to maintain
+    this feature for the moment.
+
+.. _GitHub issue: https://github.com/scality/S3/issues
+.. _Contributing Guidelines: https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md
+.. _forum: https://forum.zenko.io

docs/developers/new-backend.rst

@@ -0,0 +1,53 @@
+=================
+Add A New Backend
+=================
+
+Supporting all possible public cloud storage APIs is Cloudserver's
+ultimate goal. As an open source project, contributions are welcome.
+
+The first step is to get familiar with building a custom Docker image
+for Cloudserver.
+
+Build a Custom Docker Image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clone Zenko's Cloudserver, install all dependencies and start the
+service:
+
+.. code-block:: shell
+
+  $ git clone https://github.com/scality/cloudserver
+  $ cd cloudserver
+  $ npm install
+  $ npm start
+
+.. tip::
+
+    Some optional dependencies may fail, resulting in you seeing `NPM
+    WARN` messages; these can safely be ignored.  Refer to the User
+    documentation for all available options.
+
+Build the Docker image:
+
+.. code-block:: shell
+
+     # docker build . -t
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+Push the newly created Docker image to your own hub:
+
+.. code-block:: shell
+
+     # docker push
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+.. note::
+
+    To perform this last operation, you need to be authenticated with DockerHub
+
+There are two main types of backend you could want Zenko to support:
+
+== link:S3_COMPATIBLE_BACKENDS.adoc[S3 compatible data backends]
+
+== link:NON_S3_COMPATIBLE_BACKENDS.adoc[Data backends using another protocol than the S3 protocol]
+

docs/developers/non-s3-compatible-backend.rst

@@ -0,0 +1,530 @@
+==========================================================
+Adding support for data backends not supporting the S3 API
+==========================================================
+
+These backends abstract the complexity of multiple APIs to let users
+work on a single common namespace across multiple clouds.
+
+This documents aims at introducing you to the right files in
+Cloudserver (the Zenko stack's subcomponent in charge of API
+translation, among other things) to add support to your own backend of
+choice.
+
+General configuration
+~~~~~~~~~~~~~~~~~~~~~
+
+There are a number of constants and environment variables to define to support a
+new data backend; here is a list and where to find them:
+
+:file:`/constants.js`
+---------------------
+
+ * give your backend type a name, as part of the `externalBackends` object;
+ * specify whether versioning is implemented, as part of the
+   `versioningNotImplemented` object;
+
+:file:`/lib/Config.js`
+----------------------
+
+ * this is where you should put common utility functions, like the ones to parse
+   the location object from `locationConfig.json`;
+ * make sure you define environment variables (like `GCP_SERVICE_EMAIL` as we'll
+   use those internally for the CI to test against the real remote backend;
+
+:file:`/lib/data/external/{{backendName}}Client.js`
+---------------------------------------------------
+
+ * this file is where you'll instantiate your backend client; this should be a
+   class with a constructor taking the config object built in `/lib/Config.js` as
+   parameter;
+ * over time, you may need some utility functions which we've defined in the
+   folder `/api/apiUtils`, and in the file `/lib/data/external/utils`;
+
+:file:`/lib/data/external/utils.js`
+-----------------------------------
+
+ * make sure to add options for `sourceLocationConstraintType` to be equal to
+   the name you gave your backend in :file:`/constants.js`;
+
+:file:`/lib/data/external/{{BackendName}}_lib/`
+-----------------------------------------------
+
+ * this folder is where you'll put the functions needed for supporting your
+   backend; keep your files as atomic as possible;
+
+:file:`/tests/locationConfig/locationConfigTests.json`
+------------------------------------------------------
+
+ * this file is where you'll create location profiles to be used by your
+   functional tests;
+
+:file:`/lib/data/locationConstraintParser.js`
+---------------------------------------------
+
+ * this is where you'll instantiate your client if the operation the end user
+   sent effectively writes to your backend; everything happens inside the
+   function `parseLC()`; you should add a condition that executes if
+   `locationObj.type` is the name of your backend (that you defined in
+   `constants.js`), and instantiates a client of yours. See pseudocode below,
+   assuming location type name is `ztore`:
+
+
+.. code-block:: js
+   :linenos:
+   :emphasize-lines: 12
+
+    (...) //<1>
+    const ZtoreClient = require('./external/ZtoreClient');
+    const { config } = require('../Config'); //<1>
+
+    function parseLC(){ //<1>
+    (...) //<1>
+        Object.keys(config.locationConstraints).forEach(location => { //<1>
+            const locationObj = config.locationConstraints[location]; //<1>
+            (...) //<1>
+            if (locationObj.type === 'ztore' {
+                const ztoreEndpoint = config.getZtoreEndpoint(location);
+                const ztoreCredentials = config.getZtoreCredentials(location); //<2>
+                clients[location] = new ZtoreClient({
+                    ztoreEndpoint,
+                    ztoreCredentials,
+                    ztoreBucketname: locationObj.details.ztoreBucketName,
+                    bucketMatch:  locationObj.details.BucketMatch,
+                    dataStoreName: location,
+                }); //<3>
+                clients[location].clientType = 'ztore';
+            });
+            (...) //<1>
+        });
+    }
+
+
+1. Code that is already there
+2. You may need more utility functions depending on your backend specs
+3. You may have more fields required in your constructor object depending on
+   your backend specs
+
+Operation of type PUT
+~~~~~~~~~~~~~~~~~~~~~
+
+PUT routes are usually where people get started, as it's the easiest to check!
+Simply go on your remote backend console and you'll be able to see whether your
+object actually went up in the cloud...
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `put()` function is also called
+  `put()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `put(stream, size, keyContext, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+  //TODO: generate this from jsdoc
+
+ - `stream`: the stream of data you want to put in the cloud; if you're
+   unfamiliar with node.js streams, we suggest you start training, as we use
+   them a lot !
+
+ - `size`: the size of the object you're trying to put;
+
+ - `keyContext`: an object with metadata about the operation; common entries are
+   `namespace`, `buckerName`, `owner`, `cipherBundle`, and `tagging`; if these
+   are not sufficient for your integration, contact us to get architecture
+   validation before adding new entries;
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your PUT operation, and
+  then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/put/put{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+  function returning ajusted `keys[]` to your tests.
+
+Operation of type GET
+~~~~~~~~~~~~~~~~~~~~~
+
+GET routes are easy to test after PUT routes are implemented, hence why we're
+covering them second.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `get()` function is also called
+  `get()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `get(objectGetInfo, range, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+
+ - `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+
+ - `range`: the range of bytes you will get, for "get-by-range" operations (we
+   recommend you do simple GETs first, and then look at this);
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your GET operation, and
+  then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type DELETE
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+DELETE routes are easy to test after PUT routes are implemented, and they are
+similar to GET routes in our implementation, hence why we're covering them
+third.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `delete()` function is also called
+  `delete()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `delete(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your DELETE operation,
+  and then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/delete/delete{{BackendName}}js`
+-----------------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the
+          tutorial in order (that is, if you have covered the PUT operation
+          already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type HEAD
+~~~~~~~~~~~~~~~~~~~~~~
+
+HEAD routes are very similar to DELETE routes in our implementation, hence why
+we're covering them fourth.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `head()` function is also called
+  `head()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `head(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+// TODO:: generate this from jsdoc
+
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your HEAD operation,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Healthcheck
+~~~~~~~~~~~
+
+Healtchecks are used to make sure failure to write to a remote cloud is due to
+a problem on that remote cloud, an not on Zenko's side.
+This is usually done by trying to create a bucket that already exists, and
+making sure you get the expected answer.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `healthcheck()` function is called
+  `checkExternalBackend()` and it's defined in
+  :file:`/lib/data/multipleBackendGateway.js`; you will need to add your own;
+- your healtcheck function should get `location` as a parameter, which is an
+  object comprising:`
+
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/{{backendName}}_create_bucket.js`
+-------------------------------------------------------------------------------
+
+- this is where you should write the function performing the actual bucket
+  creation;
+
+:file:`/lib/data/external/{{backendName}}_lib/utils.js`
+-------------------------------------------------------
+
+- add an object named per your backend's name to the `backendHealth` dictionary,
+  with proper `response` and `time` entries;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `healthcheck` function to add your location's array, and call your
+  healthcheck; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+
+.. code-block:: js
+   :linenos:
+
+    (...) //<1>
+
+        healthcheck: (flightCheckOnStartUp, log, callback) => { //<1>
+            (...) //<1>
+            const ztoreArray = []; //<2>
+            async.each(Object.keys(clients), (location, cb) => { //<1>
+                (...) //<1>
+                } else if (client.clientType === 'ztore' {
+                    ztoreArray.push(location); //<3>
+                    return cb();
+                }
+            (...) //<1>
+            multBackendResp[location] = { code: 200, message: 'OK' }; //<1>
+            return cb();
+        }, () => { //<1>
+            async.parallel([
+                (...) //<1>
+                next => checkExternalBackend( //<4>
+                    clients, ztoreArray, 'ztore', flightCheckOnStartUp,
+                    externalBackendHealthCheckInterval, next),
+            ] (...) //<1>
+            });
+            (...) //<1>
+        });
+    }
+
+1. Code that is already there
+2. The array that will store all locations of type 'ztore'
+3. Where you add locations of type 'ztore' to the array
+4. Where you actually call the healthcheck function on all 'ztore' locations
+
+Multipart upload (MPU)
+~~~~~~~~~~~~~~~~~~~~~~
+
+This is the final part to supporting a new backend! MPU is far from
+the easiest subject, but you've come so far it shouldn't be a problem.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+You'll be creating four functions with template signatures:
+
+- `createMPU(Key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
+  cacheControl, contentDisposition, contentEncoding, log, callback)` will
+  initiate the multi part upload process; now, here, all parameters are
+  metadata headers except for:
+
+ * `Key`, the key id for the final object (collection of all parts);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `uploadPart(request, streamingV4Params, stream, size, key, uploadId, partNumber, bucketName, log, callback)`
+   will be called for each part; the parameters can be explicited as follow:
+
+ * `request`, the request object for putting the part;
+ * `streamingV4Params`, parameters for auth V4 parameters against S3;
+ * `stream`, the node.js readable stream used to put the part;
+ * `size`, the size of the part;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `partNumber`, the number of the part in this MPU (ordered);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback)` will
+   end the MPU process once all parts are uploaded; parameters can be explicited
+   as follows:
+
+ * `jsonList`, user-sent list of parts to include in final mpu object;
+ * `mdInfo`, object containing 3 keys: storedParts, mpuOverviewKey, and
+   splitter;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance:
+
+- `abortMPU(key, uploadId, bucketName, log, callback)` will handle errors, and
+  make sure that all parts that may have been uploaded will be deleted if the
+  MPU ultimately fails; the parameters are:
+
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance.
+
+:file:`/lib/api/objectPutPart.js`
+---------------------------------
+
+- you'll need to add your backend type in appropriate sections (simply look for
+  other backends already implemented).
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your MPU operations,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `createMPU` function to add your location type, and call your
+  `createMPU()`; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+.. code-block:: javascript
+   :linenos:
+
+    (...) //<1>
+        createMPU:(key, metaHeaders, bucketName, websiteRedirectHeader, //<1>
+         location, contentType, cacheControl, contentDisposition,
+         contentEncoding, log, cb) => {
+            const client = clients[location]; //<1>
+            if (client.clientType === 'aws_s3') { //<1>
+                return client.createMPU(key, metaHeaders, bucketName,
+                websiteRedirectHeader, contentType, cacheControl,
+                contentDisposition, contentEncoding, log, cb);
+            } else if (client.clientType === 'ztore') { //<2>
+                return client.createMPU(key, metaHeaders, bucketName,
+                  websiteRedirectHeader, contentType, cacheControl,
+                  contentDisposition, contentEncoding, log, cb);
+            }
+            return cb();
+        };
+    (...) //<1>
+
+1. Code that is already there
+2. Where the `createMPU()` of your client is actually called
+
+Add functional tests
+~~~~~~~~~~~~~~~~~~~~
+
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/initMPU/{{BackendName}}InitMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/listParts/{{BackendName}}ListPart.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/{{BackendName}}AbortMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/{{BackendName}}CompleteMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/{{BackendName}}UploadPart.js`
+
+Adding support in Orbit, Zenko's UI for simplified Multi Cloud Management
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can only be done by our core developers' team. Once your backend
+integration is merged, you may open a feature request on the
+`Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/developers/s3-compatible-backend.rst

@@ -0,0 +1,43 @@
+======================
+S3 compatible backends
+======================
+
+
+Adding support in Zenko’s Cloudserver
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is the easiest case for backend support integration: there is nothing to do
+but configuration!  Follow the steps described in our
+:ref:`use-public-cloud` and make sure you:
+
+- set ``details.awsEndpoint`` to your storage provider endpoint;
+
+- use ``details.credentials`` and *not* ``details.credentialsProfile`` to set your
+  credentials for that S3-compatible backend.
+
+For example, if you’re using a Wasabi bucket as a backend, then your region
+definition for that backend will look something like:
+::
+
+    "wasabi-bucket-zenkobucket": {
+    "type": "aws_s3",
+    "legacyAwsBehavior": true,
+    "details": {
+    "awsEndpoint": "s3.wasabisys.com",
+    "bucketName": "zenkobucket",
+    "bucketMatch": true,
+    "credentials": {
+    "accessKey": "\\{YOUR_WASABI_ACCESS_KEY}",
+    "secretKey": "\\{YOUR_WASABI_SECRET_KEY}"
+    }
+    }
+    },
+
+Adding support in Zenko Orbit
+#############################
+
+This can only be done by our core developpers' team. If that’s what you’re
+after, open a feature request on the `Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/index.rst

@@ -1,11 +1,12 @@
 Scality Zenko CloudServer
-==================
+=========================
 
 .. _user-docs:
 
 .. toctree::
    :maxdepth: 2
    :caption: Documentation
+   :glob:
 
    CONTRIBUTING
    GETTING_STARTED
@@ -14,3 +15,4 @@ Scality Zenko CloudServer
    DOCKER
    INTEGRATIONS
    ARCHITECTURE
+   developers/*

docs/requirements.in

@@ -0,0 +1,2 @@
+Sphinx >= 1.7.5
+recommonmark >= 0.4.0

docs/requirements.txt

@@ -0,0 +1,119 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    tox -e pip-compile
+#
+alabaster==0.7.12 \
+    --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
+    --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 \
+    # via sphinx
+babel==2.6.0 \
+    --hash=sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669 \
+    --hash=sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23 \
+    # via sphinx
+certifi==2018.10.15 \
+    --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
+    --hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+commonmark==0.5.4 \
+    --hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5 \
+    # via recommonmark
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
+    # via recommonmark, sphinx
+idna==2.7 \
+    --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
+    --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
+    # via requests
+imagesize==1.1.0 \
+    --hash=sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8 \
+    --hash=sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5 \
+    # via sphinx
+jinja2==2.10 \
+    --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
+    --hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
+    # via sphinx
+markupsafe==1.1.0 \
+    --hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \
+    --hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \
+    --hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \
+    --hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \
+    --hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \
+    --hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \
+    --hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \
+    --hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \
+    --hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \
+    --hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \
+    --hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \
+    --hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \
+    --hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \
+    --hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \
+    --hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \
+    --hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \
+    --hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \
+    --hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \
+    --hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \
+    --hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \
+    --hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \
+    --hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \
+    --hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \
+    --hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \
+    --hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \
+    --hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \
+    --hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \
+    --hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 \
+    # via jinja2
+packaging==18.0 \
+    --hash=sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807 \
+    --hash=sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9 \
+    # via sphinx
+pygments==2.2.0 \
+    --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
+    --hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc \
+    # via sphinx
+pyparsing==2.3.0 \
+    --hash=sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b \
+    --hash=sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592 \
+    # via packaging
+pytz==2018.7 \
+    --hash=sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca \
+    --hash=sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6 \
+    # via babel
+recommonmark==0.4.0 \
+    --hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d \
+    --hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7
+requests==2.20.1 \
+    --hash=sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54 \
+    --hash=sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263 \
+    # via sphinx
+six==1.11.0 \
+    --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
+    --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \
+    # via packaging, sphinx
+snowballstemmer==1.2.1 \
+    --hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 \
+    --hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \
+    # via sphinx
+sphinx==1.8.2 \
+    --hash=sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea \
+    --hash=sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64
+sphinxcontrib-websupport==1.1.0 \
+    --hash=sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd \
+    --hash=sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9 \
+    # via sphinx
+typing==3.6.6 \
+    --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \
+    --hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
+    --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
+    # via sphinx
+urllib3==1.24.1 \
+    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
+    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via requests

eve/main.yml

@@ -96,6 +96,14 @@ models:
         s3:
           filename: /artifacts/s3.log
           follow: true
+  - ShellCommand: &add-hostname
+      name: add hostname
+      command: |
+        echo "127.0.0.1 testrequestbucket.localhost" >> /etc/hosts
+        echo \
+        "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" \
+        >> /etc/hosts
+      haltOnFailure: True
   - ShellCommand: &setup-junit-upload
       name: preparing junit files for upload
       command: |
@@ -133,6 +141,7 @@ stages:
     steps:
       - Git: *clone
       - ShellCommand: *npm-install
+      - ShellCommand: *add-hostname
       - ShellCommand: *credentials
       - ShellCommand:
           name: Linting
@@ -146,6 +155,8 @@ stages:
           name: Unit Coverage
           command: |
             set -ex
+            unset HTTP_PROXY HTTPS_PROXY NO_PROXY
+            unset http_proxy https_proxy no_proxy
             mkdir -p $CIRCLE_TEST_REPORTS/unit
             npm test
             npm run test_legacy_location

eve/workers/pod.yaml

@@ -10,6 +10,7 @@ spec:
   - ip: "127.0.0.1"
     hostnames:
     - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com"
+    - "testrequestbucket.localhost"
     - "pykmip.local"
   {% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
   initContainers:

examples/go-md-search.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+    "fmt"
+    "time"
+    "bytes"
+    "net/http"
+    "net/url"
+    "io/ioutil"
+    "github.com/aws/aws-sdk-go/aws/credentials"
+    "github.com/aws/aws-sdk-go/aws/signer/v4"
+)
+
+func main() {
+    // Input AWS access key, secret key
+    aws_access_key_id := "accessKey1"
+    aws_secret_access_key := "verySecretKey1"
+    endpoint := "http://localhost:8000"
+    bucket_name := "bucketname"
+    searchQuery := url.QueryEscape("x-amz-meta-color=blue")
+    buf := bytes.NewBuffer([]byte{})
+
+    requestUrl := fmt.Sprintf("%s/%s?search=%s",
+        endpoint, bucket_name, searchQuery)
+
+    request, err := http.NewRequest("GET", requestUrl, buf)
+    if err != nil {
+        panic(err)
+    }
+    reader := bytes.NewReader(buf.Bytes())
+    credentials := credentials.NewStaticCredentials(aws_access_key_id,
+        aws_secret_access_key, "")
+    signer := v4.NewSigner(credentials)
+    signer.Sign(request, reader, "s3", "us-east-1", time.Now())
+    client := &http.Client{}
+    resp, err := client.Do(request)
+    if err != nil {
+        panic(err)
+    }
+    defer resp.Body.Close()
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println(string(body))
+}

examples/node-md-search.js

@@ -0,0 +1,28 @@
+const { S3 } = require('aws-sdk');
+const config = {
+    sslEnabled: false,
+    endpoint: 'http://127.0.0.1:8000',
+    signatureCache: false,
+    signatureVersion: 'v4',
+    region: 'us-east-1',
+    s3ForcePathStyle: true,
+    accessKeyId: 'accessKey1',
+    secretAccessKey: 'verySecretKey1',
+};
+const s3Client = new S3(config);
+
+const encodedSearch =
+    encodeURIComponent('x-amz-meta-color="blue"');
+const req = s3Client.listObjects({ Bucket: 'bucketname' });
+
+// the build event
+req.on('build', () => {
+    req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
+});
+req.on('success', res => {
+    process.stdout.write(`Result ${res.data}`);
+});
+req.on('error', err => {
+    process.stdout.write(`Error ${err}`);
+});
+req.send();

examples/python-md-search.py

@@ -0,0 +1,79 @@
+import datetime
+import hashlib
+import hmac
+import urllib
+# pip install requests
+import requests
+
+access_key = 'accessKey1'
+secret_key = 'verySecretKey1'
+
+method = 'GET'
+service = 's3'
+host = 'localhost:8000'
+region = 'us-east-1'
+canonical_uri = '/bucketname'
+query = 'x-amz-meta-color=blue'
+canonical_querystring = 'search=%s' % (urllib.quote(query))
+algorithm = 'AWS4-HMAC-SHA256'
+
+t = datetime.datetime.utcnow()
+amz_date = t.strftime('%Y%m%dT%H%M%SZ')
+date_stamp = t.strftime('%Y%m%d')
+
+# Key derivation functions. See:
+# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
+
+
+def sign(key, msg):
+    return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
+
+
+def getSignatureKey(key, date_stamp, regionName, serviceName):
+    kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
+    kRegion = sign(kDate, regionName)
+    kService = sign(kRegion, serviceName)
+    kSigning = sign(kService, 'aws4_request')
+    return kSigning
+
+
+payload_hash = hashlib.sha256('').hexdigest()
+
+canonical_headers = \
+    'host:{0}\nx-amz-content-sha256:{1}\nx-amz-date:{2}\n' \
+    .format(host, payload_hash, amz_date)
+
+signed_headers = 'host;x-amz-content-sha256;x-amz-date'
+
+canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
+    .format(method, canonical_uri, canonical_querystring, canonical_headers,
+            signed_headers, payload_hash)
+print canonical_request
+
+credential_scope = '{0}/{1}/{2}/aws4_request' \
+    .format(date_stamp, region, service)
+
+string_to_sign = '{0}\n{1}\n{2}\n{3}' \
+    .format(algorithm, amz_date, credential_scope,
+            hashlib.sha256(canonical_request).hexdigest())
+
+signing_key = getSignatureKey(secret_key, date_stamp, region, service)
+
+signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
+                     hashlib.sha256).hexdigest()
+
+authorization_header = \
+    '{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}' \
+    .format(algorithm, access_key, credential_scope, signed_headers, signature)
+
+# The 'host' header is added automatically by the Python 'requests' library.
+headers = {
+    'X-Amz-Content-Sha256': payload_hash,
+    'X-Amz-Date': amz_date,
+    'Authorization': authorization_header
+}
+
+endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
+
+r = requests.get(endpoint, headers=headers)
+print (r.text)

lib/Config.js

@@ -10,15 +10,64 @@ const cronParser = require('cron-parser');
 const { isValidBucketName } = require('arsenal').s3routes.routesUtils;
 const validateAuthConfig = require('arsenal').auth.inMemory.validateAuthConfig;
 const { buildAuthDataAccount } = require('./auth/in_memory/builder');
-const externalBackends = require('../constants').externalBackends;
+const validExternalBackends = require('../constants').externalBackends;
 const { azureAccountNameRegex, base64Regex } = require('../constants');
 
 // whitelist IP, CIDR for health checks
 const defaultHealthChecks = { allowFrom: ['127.0.0.1/8', '::1'] };
 
 const defaultLocalCache = { host: '127.0.0.1', port: 6379 };
+const defaultExternalBackendsConfig = {
+    // eslint-disable-next-line camelcase
+    aws_s3: {
+        httpAgent: {
+            keepAlive: false,
+            keepAliveMsecs: 1000,
+            maxFreeSockets: 256,
+            maxSockets: null,
+        },
+    },
+    gcp: {
+        httpAgent: {
+            keepAlive: true,
+            keepAliveMsecs: 1000,
+            maxFreeSockets: 256,
+            maxSockets: null,
+        },
+    },
+};
 
-const gcpScope = 'https://www.googleapis.com/auth/cloud-platform';
+function assertCertPaths(key, cert, ca, basePath) {
+    const certObj = {};
+    certObj.paths = {};
+    certObj.certs = {};
+    if (key) {
+        const keypath = key.startsWith('/') ? key : `${basePath}/${key}`;
+        assert.doesNotThrow(() =>
+            fs.accessSync(keypath, fs.F_OK | fs.R_OK),
+            `File not found or unreachable: ${keypath}`);
+        certObj.paths.key = keypath;
+        certObj.certs.key = fs.readFileSync(keypath, 'ascii');
+    }
+    if (cert) {
+        const certpath = cert.startsWith('/') ? cert : `${basePath}/${cert}`;
+        assert.doesNotThrow(() =>
+            fs.accessSync(certpath, fs.F_OK | fs.R_OK),
+            `File not found or unreachable: ${certpath}`);
+        certObj.paths.cert = certpath;
+        certObj.certs.cert = fs.readFileSync(certpath, 'ascii');
+    }
+
+    if (ca) {
+        const capath = ca.startsWith('/') ? ca : `${basePath}/${ca}`;
+        assert.doesNotThrow(() =>
+            fs.accessSync(capath, fs.F_OK | fs.R_OK),
+            `File not found or unreachable: ${capath}`);
+        certObj.paths.ca = capath;
+        certObj.certs.ca = fs.readFileSync(capath, 'ascii');
+    }
+    return certObj;
+}
 
 function sproxydAssert(configSproxyd) {
     const sproxydFields = [];
@@ -62,57 +111,18 @@ function gcpLocationConstraintAssert(location, locationObj) {
         gcpEndpoint,
         bucketName,
         mpuBucketName,
-        overflowBucketName,
-        serviceCredentials,
     } = locationObj.details;
-    const serviceKeyFileFromEnv =
-        process.env[`${location}_GCP_SERVICE_KEYFILE`];
-    const serviceEmailFromEnv =
-        process.env[`${location}_GCP_SERVICE_EMAIL`];
-    const serviceKeyFromEnv =
-        process.env[`${location}_GCP_SERVICE_KEY`];
-    const serviceScopeFromEnv =
-        process.env[`${location}_GCP_SERVICE_SCOPE`];
-    const scopes = serviceScopeFromEnv || serviceCredentials &&
-        serviceCredentials.scopes || gcpScope;
-    const keyFilename = serviceKeyFileFromEnv || serviceCredentials &&
-        serviceCredentials.keyFilename;
-    const serviceEmail = serviceEmailFromEnv || serviceCredentials &&
-        serviceCredentials.serviceEmail;
-    const serviceKey = serviceKeyFromEnv || serviceCredentials &&
-        serviceCredentials.serviceKey;
     const stringFields = [
         gcpEndpoint,
         bucketName,
         mpuBucketName,
-        overflowBucketName,
     ];
-
-    assert(typeof scopes === 'string', `bad config: ${location} ` +
-        'serviceCredentials scopes must be a string');
     stringFields.forEach(field => {
         if (field !== undefined) {
             assert(typeof field === 'string',
                 `bad config: ${field} must be a string`);
         }
     });
-    assert.strictEqual(
-        [keyFilename, (serviceEmail && serviceKey)].some(param => param),
-        true, `bad location constriant: "${location}" ` +
-        'serviceCredentials keyFilename and/or both serviceEmail and ' +
-        'serviceKey must be set in locationConfig or environment variable');
-    if (keyFilename) {
-        assert.strictEqual(typeof keyFilename, 'string',
-        `bad location constriant: "${location}" serviceCredentials ` +
-        `keyFilename "${keyFilename}" must be a string`);
-    } else {
-        assert.strictEqual(typeof serviceEmail, 'string',
-        `bad location constriant: "${location}" serviceCredentials ` +
-        `serviceEmail "${serviceEmail}" must be a string`);
-        assert.strictEqual(typeof serviceKey, 'string',
-        `bad location constriant: "${location}"" serviceCredentials ` +
-        `serviceKey "${serviceKey}" must be a string`);
-    }
 }
 
 function azureLocationConstraintAssert(location, locationObj) {
@@ -158,7 +168,7 @@ function azureLocationConstraintAssert(location, locationObj) {
 function locationConstraintAssert(locationConstraints) {
     const supportedBackends =
           ['mem', 'file', 'scality',
-           'mongodb'].concat(Object.keys(externalBackends));
+           'mongodb'].concat(Object.keys(validExternalBackends));
     assert(typeof locationConstraints === 'object',
         'bad config: locationConstraints must be an object');
     Object.keys(locationConstraints).forEach(l => {
@@ -170,25 +180,45 @@ function locationConstraintAssert(locationConstraints) {
         assert(supportedBackends.indexOf(locationConstraints[l].type) > -1,
             'bad config: locationConstraints[region].type must ' +
             `be one of ${supportedBackends}`);
+        assert(typeof locationConstraints[l].objectId === 'string',
+               'bad config: locationConstraints[region].objectId is ' +
+               'mandatory and must be a unique string across locations');
+        assert(Object.keys(locationConstraints)
+               .filter(loc => (locationConstraints[loc].objectId ===
+                               locationConstraints[l].objectId))
+               .length === 1,
+               'bad config: location constraint objectId ' +
+               `"${locationConstraints[l].objectId}" is not unique across ` +
+               'configured locations');
         assert(typeof locationConstraints[l].legacyAwsBehavior
             === 'boolean',
             'bad config: locationConstraints[region]' +
             '.legacyAwsBehavior is mandatory and must be a boolean');
-        if (locationConstraints[l].details.serverSideEncryption !== undefined) {
-            assert(typeof locationConstraints[l].details.serverSideEncryption
-              === 'boolean',
+        assert(['undefined', 'boolean'].includes(
+            typeof locationConstraints[l].isTransient),
+               'bad config: locationConstraints[region]' +
+               '.isTransient must be a boolean');
+        if (locationConstraints[l].sizeLimitGB !== undefined) {
+            assert(typeof locationConstraints[l].sizeLimitGB === 'number' ||
+                locationConstraints[l].sizeLimitGB === null,
+                'bad config: locationConstraints[region].sizeLimitGB ' +
+                'must be a number (in gigabytes)');
+        }
+
+        const details = locationConstraints[l].details;
+        assert(typeof details === 'object',
+            'bad config: locationConstraints[region].details is ' +
+            'mandatory and must be an object');
+        if (details.serverSideEncryption !== undefined) {
+            assert(typeof details.serverSideEncryption === 'boolean',
               'bad config: locationConstraints[region]' +
               '.details.serverSideEncryption must be a boolean');
         }
-        assert(typeof locationConstraints[l].details
-            === 'object',
-            'bad config: locationConstraints[region].details is ' +
-            'mandatory and must be an object');
-        const details = locationConstraints[l].details;
         const stringFields = [
             'awsEndpoint',
             'bucketName',
             'credentialsProfile',
+            'region',
         ];
         stringFields.forEach(field => {
             if (details[field] !== undefined) {
@@ -208,28 +238,6 @@ function locationConstraintAssert(locationConstraints) {
             assert(typeof details.credentials.secretKey === 'string',
                 'bad config: credentials must include secretKey as string');
         }
-        if (details.proxy !== undefined) {
-            const { protocol, hostname, port, auth } = url.parse(details.proxy);
-            assert(protocol === 'http:' || protocol === 'https:',
-                'bad config: protocol must be http or https in ' +
-                'locationConstraints[region].details');
-            assert(typeof hostname === 'string' && hostname !== '',
-                'bad config: hostname must be a non-empty string');
-            if (port) {
-                const portInt = Number.parseInt(port, 10);
-                assert(!Number.isNaN(portInt) && portInt > 0, 'bad config: ' +
-                    'locationConstraints[region].details port must be a ' +
-                    'number greater than 0');
-            }
-            if (auth) {
-                assert(typeof auth === 'string',
-                    'bad config: proxy auth must be string');
-                const authArray = auth.split(':');
-                assert(authArray.length === 2 && authArray[0].length > 0
-                    && authArray[1].length > 0, 'bad config: proxy auth ' +
-                    'must be of format username:password');
-            }
-        }
         if (details.https !== undefined) {
             assert(typeof details.https === 'boolean', 'bad config: ' +
                 'locationConstraints[region].details https must be a boolean');
@@ -237,6 +245,24 @@ function locationConstraintAssert(locationConstraints) {
             // eslint-disable-next-line no-param-reassign
             locationConstraints[l].details.https = true;
         }
+        if (details.pathStyle !== undefined) {
+            assert(typeof details.pathStyle === 'boolean', 'bad config: ' +
+                'locationConstraints[region].pathStyle must be a boolean');
+        } else {
+            // eslint-disable-next-line no-param-reassign
+            locationConstraints[l].details.pathStyle = false;
+        }
+
+        if (details.supportsVersioning !== undefined) {
+            assert(typeof details.supportsVersioning === 'boolean',
+                'bad config: locationConstraints[region].supportsVersioning' +
+                'must be a boolean');
+        } else {
+            // default to true
+            // eslint-disable-next-line no-param-reassign
+            locationConstraints[l].details.supportsVersioning = true;
+        }
+
         if (locationConstraints[l].type === 'azure') {
             azureLocationConstraintAssert(l, locationConstraints[l]);
         }
@@ -305,6 +331,9 @@ class Config extends EventEmitter {
         if (process.env.S3_LOCATION_FILE !== undefined) {
             this.locationConfigPath = process.env.S3_LOCATION_FILE;
         }
+        if (process.env.S3_REPLICATION_FILE !== undefined) {
+            this.replicationConfigPath = process.env.S3_REPLICATION_FILE;
+        }
 
         // Read config automatically
         this._getLocationConfig();
@@ -397,6 +426,17 @@ class Config extends EventEmitter {
         } catch (err) {
             throw new Error(`could not parse config file: ${err.message}`);
         }
+        if (this.replicationConfigPath) {
+            try {
+                const repData = fs.readFileSync(this.replicationConfigPath,
+                  { encoding: 'utf-8' });
+                const replicationEndpoints = JSON.parse(repData);
+                config.replicationEndpoints.push(...replicationEndpoints);
+            } catch (err) {
+                throw new Error(
+                    `could not parse replication file: ${err.message}`);
+            }
+        }
 
         this.port = 8000;
         if (config.port !== undefined) {
@@ -458,10 +498,10 @@ class Config extends EventEmitter {
                 assert.notStrictEqual(site, '', 'bad config: `site` property ' +
                     "of object in `replicationEndpoints` must not be ''");
                 if (type !== undefined) {
-                    assert(externalBackends[type], 'bad config: `type` ' +
+                    assert(validExternalBackends[type], 'bad config: `type` ' +
                         'property of `replicationEndpoints` object must be ' +
                         'a valid external backend (one of: "' +
-                        `${Object.keys(externalBackends).join('", "')}")`);
+                        `${Object.keys(validExternalBackends).join('", "')})`);
                 } else {
                     assert.notStrictEqual(servers, undefined, 'bad config: ' +
                         'each object of `replicationEndpoints` array that is ' +
@@ -480,6 +520,15 @@ class Config extends EventEmitter {
             this.replicationEndpoints = replicationEndpoints;
         }
 
+        if (config.backbeat) {
+            const { backbeat } = config;
+            assert.strictEqual(typeof backbeat.host, 'string',
+                'bad config: backbeat host must be a string');
+            assert(Number.isInteger(backbeat.port) && backbeat.port > 0,
+                'bad config: backbeat port must be a positive integer');
+            this.backbeat = backbeat;
+        }
+
         // legacy
         if (config.regions !== undefined) {
             throw new Error('bad config: regions key is deprecated. ' +
@@ -626,6 +675,8 @@ class Config extends EventEmitter {
             this.dataDaemon.dataPath =
                 process.env.S3DATAPATH ?
                 process.env.S3DATAPATH : `${__dirname}/../localData`;
+            this.dataDaemon.noSync = process.env.S3DATA_NOSYNC === 'true';
+            this.dataDaemon.noCache = process.env.S3DATA_NOCACHE === 'true';
         }
 
         if (config.metadataDaemon) {
@@ -669,14 +720,45 @@ class Config extends EventEmitter {
             assert(typeof config.localCache === 'object',
                 'config: invalid local cache configuration. localCache must ' +
                 'be an object');
+            if (config.localCache.sentinels) {
+                this.localCache = { sentinels: [], name: null };
+
+                assert(typeof config.localCache.name === 'string',
+                    'bad config: localCache sentinel name must be a string');
+                this.localCache.name = config.localCache.name;
+
+                assert(Array.isArray(config.localCache.sentinels) ||
+                    typeof config.localCache.sentinels === 'string',
+                    'bad config: localCache sentinels' +
+                    'must be an array or string');
+
+                if (typeof config.localCache.sentinels === 'string') {
+                    config.localCache.sentinels.split(',').forEach(item => {
+                        const [host, port] = item.split(':');
+                        this.localCache.sentinels.push({ host,
+                            port: Number.parseInt(port, 10) });
+                    });
+                } else if (Array.isArray(config.localCache.sentinels)) {
+                    config.localCache.sentinels.forEach(item => {
+                        const { host, port } = item;
+                        assert(typeof host === 'string',
+                            'bad config: localCache' +
+                            'sentinel host must be a string');
+                        assert(typeof port === 'number',
+                            'bad config: localCache' +
+                            'sentinel port must be a number');
+                        this.localCache.sentinels.push({ host, port });
+                    });
+                }
+            } else {
                 assert(typeof config.localCache.host === 'string',
-                'config: invalid host for localCache. host must be a string');
+                      'config: bad host for localCache. host must be a string');
                 assert(typeof config.localCache.port === 'number',
-                'config: invalid port for localCache. port must be a number');
+                      'config: bad port for localCache. port must be a number');
                 if (config.localCache.password !== undefined) {
                     assert(
                         this._verifyRedisPassword(config.localCache.password),
-                    'config: invalid password for localCache. password must' +
+                        'config: vad password for localCache. password must' +
                     ' be a string');
                 }
                 this.localCache = {
@@ -685,6 +767,7 @@ class Config extends EventEmitter {
                     password: config.localCache.password,
                 };
             }
+        }
 
         if (config.mongodb) {
             this.mongodb = config.mongodb;
@@ -762,8 +845,9 @@ class Config extends EventEmitter {
             assert(config.localCache, 'missing required property of utapi ' +
                 'configuration: localCache');
             this.utapi.localCache = config.localCache;
-            assert(config.utapi.redis, 'missing required property of utapi ' +
+            assert(config.redis, 'missing required property of utapi ' +
                 'configuration: redis');
+            if (config.utapi.redis) {
                 if (config.utapi.redis.sentinels) {
                     this.utapi.redis = { sentinels: [], name: null };
 
@@ -800,12 +884,14 @@ class Config extends EventEmitter {
                 }
                 if (config.utapi.redis.sentinelPassword !== undefined) {
                     assert(
-                this._verifyRedisPassword(config.utapi.redis.sentinelPassword),
+                    this._verifyRedisPassword(
+                        config.utapi.redis.sentinelPassword),
                         'config: invalid password for utapi redis. password' +
                         ' must be a string');
                     this.utapi.redis.sentinelPassword =
                         config.utapi.redis.sentinelPassword;
                 }
+            }
             if (config.utapi.metrics) {
                 this.utapi.metrics = config.utapi.metrics;
             }
@@ -878,6 +964,13 @@ class Config extends EventEmitter {
                 this.utapi.reindex = config.utapi.reindex;
             }
         }
+        if (Object.keys(this.locationConstraints).some(
+        loc => this.locationConstraints[loc].sizeLimitGB)) {
+            assert(this.utapi && this.utapi.metrics &&
+                this.utapi.metrics.includes('location'),
+                'bad config: if storage size limit set on a location ' +
+                'constraint, Utapi must also be configured correctly');
+        }
 
         this.log = { logLevel: 'debug', dumpLevel: 'error' };
         if (config.log !== undefined) {
@@ -1040,38 +1133,77 @@ class Config extends EventEmitter {
                     !config.certFilePaths.ca)
                );
         }
+
         const { key, cert, ca } = config.certFilePaths ?
             config.certFilePaths : {};
+        let certObj = undefined;
         if (key && cert) {
-            const keypath = (key[0] === '/') ? key : `${this._basePath}/${key}`;
-            const certpath = (cert[0] === '/') ?
-                cert : `${this._basePath}/${cert}`;
-            let capath;
-            if (ca) {
-                capath = (ca[0] === '/') ? ca : `${this._basePath}/${ca}`;
-                assert.doesNotThrow(() =>
-                    fs.accessSync(capath, fs.F_OK | fs.R_OK),
-                    `File not found or unreachable: ${capath}`);
-            }
-            assert.doesNotThrow(() =>
-                fs.accessSync(keypath, fs.F_OK | fs.R_OK),
-                `File not found or unreachable: ${keypath}`);
-            assert.doesNotThrow(() =>
-                fs.accessSync(certpath, fs.F_OK | fs.R_OK),
-                `File not found or unreachable: ${certpath}`);
-            this.https = {
-                cert: fs.readFileSync(certpath, 'ascii'),
-                key: fs.readFileSync(keypath, 'ascii'),
-                ca: ca ? fs.readFileSync(capath, 'ascii') : undefined,
-            };
-            this.httpsPath = {
-                ca: capath,
-                cert: certpath,
-            };
+            certObj = assertCertPaths(key, cert, ca, this._basePath);
         } else if (key || cert) {
             throw new Error('bad config: both certFilePaths.key and ' +
                 'certFilePaths.cert must be defined');
         }
+        if (certObj) {
+            if (Object.keys(certObj.certs).length > 0) {
+                this.https = certObj.certs;
+            }
+            if (Object.keys(certObj.paths).length > 0) {
+                this.httpsPath = certObj.paths;
+            }
+        }
+
+        this.outboundProxy = {};
+        const envProxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY
+            || process.env.http_proxy || process.env.https_proxy;
+        const p = config.outboundProxy;
+        const proxyUrl = envProxy || (p ? p.url : '');
+        if (proxyUrl) {
+            assert(typeof proxyUrl === 'string',
+                'bad proxy config: url must be a string');
+            const { protocol, hostname, port, auth } = url.parse(proxyUrl);
+            assert(protocol === 'http:' || protocol === 'https:',
+                'bad proxy config: protocol must be http or https');
+            assert(typeof hostname === 'string' && hostname !== '',
+                'bad proxy config: hostname must be a non-empty string');
+            if (port) {
+                const portInt = Number.parseInt(port, 10);
+                assert(!Number.isNaN(portInt) && portInt > 0,
+                    'bad proxy config: port must be a number greater than 0');
+            }
+            if (auth) {
+                assert(typeof auth === 'string',
+                    'bad proxy config: auth must be string');
+                const authArray = auth.split(':');
+                assert(authArray.length === 2 && authArray[0].length > 0
+                    && authArray[1].length > 0, 'bad proxy config: ' +
+                    'auth must be of format username:password');
+            }
+            this.outboundProxy.url = proxyUrl;
+            this.outboundProxy.certs = {};
+            const envCert = process.env.HTTPS_PROXY_CERTIFICATE;
+            const key = p ? p.key : '';
+            const cert = p ? p.cert : '';
+            const caBundle = envCert || (p ? p.caBundle : '');
+            if (p) {
+                assert(typeof p === 'object',
+                    'bad config: "proxy" should be an object');
+            }
+            if (key) {
+                assert(typeof key === 'string',
+                    'bad config: proxy.key should be a string');
+            }
+            if (cert) {
+                assert(typeof cert === 'string',
+                    'bad config: proxy.cert should be a string');
+            }
+            if (caBundle) {
+                assert(typeof caBundle === 'string',
+                    'bad config: proxy.caBundle should be a string');
+            }
+            const certObj =
+                assertCertPaths(key, cert, caBundle, this._basePath);
+            this.outboundProxy.certs = certObj.certs;
+        }
 
         // Ephemeral token to protect the reporting endpoint:
         // try inherited from parent first, then hardcoded in conf file,
@@ -1080,6 +1212,41 @@ class Config extends EventEmitter {
             process.env.REPORT_TOKEN ||
             config.reportToken ||
             uuid.v4().toString();
+        this.reportEndpoint = process.env.REPORT_ENDPOINT;
+
+        // External backends
+        // Currently supports configuring httpAgent(s) for keepAlive
+        this.externalBackends = defaultExternalBackendsConfig;
+        if (config.externalBackends) {
+            const extBackendsConfig = Object.keys(config.externalBackends);
+            extBackendsConfig.forEach(b => {
+                // assert that it's a valid backend
+                assert(validExternalBackends[b] !== undefined,
+                    `bad config: ${b} is not one of valid external backends: ` +
+                    `${Object.keys(validExternalBackends).join(', ')}`);
+
+                const { httpAgent } = config.externalBackends[b];
+                assert(typeof httpAgent === 'object',
+                    `bad config: ${b} must have httpAgent object defined`);
+                const { keepAlive, keepAliveMsecs, maxFreeSockets, maxSockets }
+                    = httpAgent;
+                assert(typeof keepAlive === 'boolean',
+                    `bad config: ${b}.httpAgent.keepAlive must be a boolean`);
+                assert(typeof keepAliveMsecs === 'number' &&
+                    httpAgent.keepAliveMsecs > 0,
+                    `bad config: ${b}.httpAgent.keepAliveMsecs must be` +
+                    ' a number > 0');
+                assert(typeof maxFreeSockets === 'number' &&
+                    httpAgent.maxFreeSockets >= 0,
+                    `bad config: ${b}.httpAgent.maxFreeSockets must be ` +
+                    'a number >= 0');
+                assert((typeof maxSockets === 'number' && maxSockets >= 0) ||
+                    maxSockets === null,
+                    `bad config: ${b}.httpAgent.maxFreeSockets must be ` +
+                    'null or a number >= 0');
+                Object.assign(this.externalBackends[b].httpAgent, httpAgent);
+            });
+        }
     }
 
     _configureBackends() {
@@ -1087,7 +1254,7 @@ class Config extends EventEmitter {
          * Configure the backends for Authentication, Data and Metadata.
          */
         let auth = 'mem';
-        let data = 'file';
+        let data = 'multiple';
         let metadata = 'file';
         let kms = 'file';
         if (process.env.S3BACKEND) {
@@ -1172,14 +1339,17 @@ class Config extends EventEmitter {
         const {
             bucketName,
             mpuBucketName,
-            overflowBucketName,
         } = this.locationConstraints[locationConstraint].details;
-        return { bucketName, mpuBucketName, overflowBucketName };
+        return { bucketName, mpuBucketName };
     }
 
     getLocationConstraintType(locationConstraint) {
-        return this.locationConstraints[locationConstraint] &&
-            this.locationConstraints[locationConstraint].type;
+        const dataStoreName = this.locationConstraints[locationConstraint];
+        return dataStoreName && dataStoreName.type;
+    }
+
+    getLocationConstraint(locationConstraintName) {
+        return this.locationConstraints[locationConstraintName];
     }
 
     setRestEndpoints(restEndpoints) {
@@ -1194,6 +1364,12 @@ class Config extends EventEmitter {
         this.emit('location-constraints-update');
     }
 
+    setReplicationEndpoints(locationConstraints) {
+        this.replicationEndpoints =
+        Object.keys(locationConstraints)
+        .map(key => ({ site: key, type: locationConstraints[key].type }));
+    }
+
     getAzureEndpoint(locationConstraint) {
         let azureStorageEndpoint =
         process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] ||
@@ -1241,33 +1417,6 @@ class Config extends EventEmitter {
         return this.locationConstraints[locationConstraint].details
         .serverSideEncryption === true;
     }
-
-    getGcpServiceParams(locationConstraint) {
-        const { serviceCredentials } =
-            this.locationConstraints[locationConstraint].details;
-        const serviceKeyFileFromEnv =
-            process.env[`${locationConstraint}_GCP_SERVICE_KEYFILE`];
-        const serviceEmailFromEnv =
-            process.env[`${locationConstraint}_GCP_SERVICE_EMAIL`];
-        const serviceKeyFromEnv =
-            process.env[`${locationConstraint}_GCP_SERVICE_KEY`];
-        const serviceScopeFromEnv =
-            process.env[`${locationConstraint}_GCP_SERVICE_SCOPE`];
-        return {
-            scopes: serviceScopeFromEnv || serviceCredentials &&
-                serviceCredentials.scopes || gcpScope,
-            keyFilename: serviceKeyFileFromEnv || serviceCredentials &&
-                serviceCredentials.keyFilename,
-            /* eslint-disable camelcase */
-            credentials: {
-                client_email: serviceEmailFromEnv || serviceCredentials &&
-                    serviceCredentials.serviceEmail,
-                private_key: serviceKeyFromEnv || serviceCredentials &&
-                    serviceCredentials.serviceKey,
-            },
-            /* eslint-enable camelcase */
-        };
-    }
 }
 
 module.exports = {

lib/api/apiUtils/bucket/bucketCreation.js

@@ -9,6 +9,7 @@ const createKeyForUserBucket = require('./createKeyForUserBucket');
 const metadata = require('../../../metadata/wrapper');
 const kms = require('../../../kms/wrapper');
 const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
+const { isBackbeatUser } = require('../authorization/aclChecks');
 
 const usersBucket = constants.usersBucket;
 const oldUsersBucket = constants.oldUsersBucket;
@@ -165,9 +166,12 @@ function createBucket(authInfo, bucketName, headers,
     const ownerDisplayName =
         authInfo.getAccountDisplayName();
     const creationDate = new Date().toJSON();
+    const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
     const bucket = new BucketInfo(bucketName,
         canonicalID, ownerDisplayName, creationDate,
-        BucketInfo.currentModelVersion());
+        BucketInfo.currentModelVersion(), null, null, null,
+        null, null, null, null, null, null, null, null,
+        null, isNFSEnabled);
 
     if (locationConstraint !== undefined) {
         bucket.setLocationConstraint(locationConstraint);
@@ -210,7 +214,8 @@ function createBucket(authInfo, bucketName, headers,
         }
         const existingBucketMD = results.getAnyExistingBucketInfo;
         if (existingBucketMD instanceof BucketInfo &&
-            existingBucketMD.getOwner() !== canonicalID) {
+            existingBucketMD.getOwner() !== canonicalID &&
+            !isBackbeatUser(canonicalID)) {
             // return existingBucketMD to collect cors headers
             return cb(errors.BucketAlreadyExists, existingBucketMD);
         }

lib/api/apiUtils/bucket/checkPreferredLocations.js

@@ -0,0 +1,26 @@
+const { errors } = require('arsenal');
+
+function checkPreferredLocations(location, locationConstraints, log) {
+    const retError = loc => {
+        const errMsg = 'value of the location you are attempting to set - ' +
+        `${loc} - is not listed in the locationConstraint config`;
+        log.trace(`locationConstraint is invalid - ${errMsg}`,
+          { locationConstraint: loc });
+        return errors.InvalidLocationConstraint.customizeDescription(errMsg);
+    };
+    if (typeof location === 'string' && !locationConstraints[location]) {
+        return retError(location);
+    }
+    if (typeof location === 'object') {
+        const { read, write } = location;
+        if (!locationConstraints[read]) {
+            return retError(read);
+        }
+        if (!locationConstraints[write]) {
+            return retError(write);
+        }
+    }
+    return null;
+}
+
+module.exports = checkPreferredLocations;

lib/api/apiUtils/bucket/parseLikeExpression.js

@@ -0,0 +1,19 @@
+/**
+ * parse LIKE expressions
+ * @param {string} regex - regex pattern
+ * @return {object} MongoDB search object
+ */
+function parseLikeExpression(regex) {
+    if (typeof regex !== 'string') {
+        return null;
+    }
+    const split = regex.split('/');
+    if (split.length < 3 || split[0] !== '') {
+        return { $regex: regex };
+    }
+    const pattern = split.slice(1, split.length - 1).join('/');
+    const regexOpt = split[split.length - 1];
+    return { $regex: new RegExp(pattern), $options: regexOpt };
+}
+
+module.exports = parseLikeExpression;

lib/api/apiUtils/bucket/parseWhere.js

@@ -0,0 +1,85 @@
+const parseLikeExpression = require('./parseLikeExpression');
+
+/*
+  This code is based on code from https://github.com/olehch/sqltomongo
+  with the following license:
+
+  The MIT License (MIT)
+
+  Copyright (c) 2016 Oleh
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+/**
+ * A helper object to map SQL-like naming to MongoDB query syntax
+ */
+const exprMapper = {
+    '=': '$eq',
+    '!=': '$ne',
+    '<>': '$ne',
+    '>': '$gt',
+    '<': '$lt',
+    '>=': '$gte',
+    '<=': '$lte',
+    'LIKE': '$regex',
+};
+
+/*
+ * Parses object with WHERE clause recursively
+ * and generates MongoDB `find` query object
+ */
+function parseWhere(root) {
+    const operator = Object.keys(root)[0];
+
+    // extract leaf binary expressions
+    if (operator === 'AND') {
+        const e1 = parseWhere(root[operator][0]);
+        const e2 = parseWhere(root[operator][1]);
+
+        // eslint-disable-next-line
+        return { '$and' : [
+            e1,
+            e2,
+        ] };
+    } else if (operator === 'OR') {
+        const e1 = parseWhere(root[operator][0]);
+        const e2 = parseWhere(root[operator][1]);
+
+        // eslint-disable-next-line
+        return  { '$or' : [
+            e1,
+            e2,
+        ] };
+    }
+    const field = root[operator][0];
+    const value = root[operator][1];
+    const expr = exprMapper[operator];
+    const obj = {};
+
+    if (operator === 'LIKE') {
+        obj[`value.${field}`] = parseLikeExpression(value);
+    } else {
+        obj[`value.${field}`] = { [expr]: value };
+    }
+
+    return obj;
+}
+
+module.exports = parseWhere;

lib/api/apiUtils/bucket/validateReplicationConfig.js

@@ -0,0 +1,32 @@
+const config = require('../../../Config').config;
+
+/**
+ * Validates that the replication configuration contains a preferred
+ * read location if the bucket location is a transient source
+ *
+ * @param {object} repConfig - replication configuration
+ * @param {object} bucket - bucket metadata
+ *
+ * @return {boolean} validity of replication configuration with
+ * transient source
+ */
+function validateReplicationConfig(repConfig, bucket) {
+    const bucketLocationName = bucket.getLocationConstraint();
+    if (!repConfig || !repConfig.rules) {
+        return false;
+    }
+    const bucketLocation = config.locationConstraints[bucketLocationName];
+    if (!bucketLocation.isTransient) {
+        return true;
+    }
+    return repConfig.rules.every(rule => {
+        if (!rule.storageClass) {
+            return true;
+        }
+        const storageClasses = rule.storageClass.split(',');
+        return storageClasses.some(
+            site => site.endsWith(':preferred_read'));
+    });
+}
+
+module.exports = validateReplicationConfig;

lib/api/apiUtils/bucket/validateSearch.js

@@ -0,0 +1,96 @@
+const Parser = require('sql-where-parser');
+const { errors } = require('arsenal');
+const objModel = require('arsenal').models.ObjectMD;
+
+const BINARY_OP = 2;
+const sqlConfig = {
+    operators: [
+        {
+            '=': BINARY_OP,
+            '<': BINARY_OP,
+            '>': BINARY_OP,
+            '<>': BINARY_OP,
+            '<=': BINARY_OP,
+            '>=': BINARY_OP,
+            '!=': BINARY_OP,
+        },
+        { LIKE: BINARY_OP },
+        { AND: BINARY_OP },
+        { OR: BINARY_OP },
+    ],
+    tokenizer: {
+        shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'],
+        shouldMatch: ['"', '\'', '`'],
+        shouldDelimitBy: [' ', '\n', '\r', '\t'],
+    },
+};
+const parser = new Parser(sqlConfig);
+
+function _validateTree(whereClause, possibleAttributes) {
+    let invalidAttribute;
+
+    function _searchTree(node) {
+        if (typeof node !== 'object') {
+            invalidAttribute = node;
+        } else {
+            const operator = Object.keys(node)[0];
+            if (operator === 'AND' || operator === 'OR') {
+                _searchTree(node[operator][0]);
+                _searchTree(node[operator][1]);
+            } else {
+                const field = node[operator][0];
+                if (!field.startsWith('tags.') &&
+                    !possibleAttributes[field] &&
+                    !field.startsWith('replicationInfo.') &&
+                    !field.startsWith('x-amz-meta-')) {
+                    invalidAttribute = field;
+                }
+            }
+        }
+    }
+    _searchTree(whereClause);
+    return invalidAttribute;
+}
+
+/**
+ * validateSearchParams - validate value of ?search= in request
+ * @param  {string} searchParams - value of search params in request
+ *  which should be jsu sql where clause
+ * For metadata: x-amz-meta-color=\"blue\"
+ * For tags: tags.x-amz-meta-color=\"blue\"
+ * For replication status : replication-status=\"PENDING\"
+ * For any other attribute: `content-length`=5
+ * @return {undefined | error} undefined if validates or arsenal error if not
+ */
+function validateSearchParams(searchParams) {
+    let ast;
+    try {
+        // allow using 'replicationStatus' as search param to increase
+        // ease of use, pending metadata search rework
+        // eslint-disable-next-line no-param-reassign
+        searchParams = searchParams.replace(
+            'replication-status', 'replicationInfo.status');
+        ast = parser.parse(searchParams);
+    } catch (e) {
+        if (e) {
+            return {
+                error: errors.InvalidArgument
+                .customizeDescription('Invalid sql where clause ' +
+                'sent as search query'),
+            };
+        }
+    }
+    const possibleAttributes = objModel.getAttributes();
+    const invalidAttribute = _validateTree(ast, possibleAttributes);
+    if (invalidAttribute) {
+        return {
+            error: errors.InvalidArgument
+                .customizeDescription('Search param ' +
+                `contains unknown attribute: ${invalidAttribute}`) };
+    }
+    return {
+        ast,
+    };
+}
+
+module.exports = validateSearchParams;

lib/api/apiUtils/object/checkReadLocation.js

@@ -0,0 +1,27 @@
+/**
+ * checkReadLocation - verify that a bucket's default read location exists
+ * for a specified read data locator
+ * @param {Config} config - Config object
+ * @param {string} locationName - location constraint
+ * @param {string} objectKey - object key
+ * @param {string} bucketName - bucket name
+ * @return {Object | null} return object containing location information
+ * if location exists; otherwise, null
+ */
+function checkReadLocation(config, locationName, objectKey, bucketName) {
+    const readLocation = config.getLocationConstraint(locationName);
+    if (readLocation) {
+        const bucketMatch = readLocation.details &&
+            readLocation.details.bucketMatch;
+        const backendKey = bucketMatch ? objectKey :
+            `${bucketName}/${objectKey}`;
+        return {
+            location: locationName,
+            key: backendKey,
+            locationType: readLocation.type,
+        };
+    }
+    return null;
+}
+
+module.exports = checkReadLocation;

lib/api/apiUtils/object/createAndStoreObject.js

@@ -19,7 +19,7 @@ const {
 } = constants;
 
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
-'a versioned object to a location-constraint of type Azure.';
+'a versioned object to a location-constraint of type Azure or GCP.';
 
 /**
  * Retro-propagation is where S3C ingestion will re-ingest an object whose
@@ -136,7 +136,8 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         size,
         headers,
         isDeleteMarker,
-        replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size),
+        replicationInfo: getReplicationInfo(
+            objectKey, bucketMD, false, size, null, null, authInfo),
         log,
     };
     if (!isDeleteMarker) {

lib/api/apiUtils/object/getReplicationBackendDataLocator.js

@@ -0,0 +1,51 @@
+const { errors } = require('arsenal');
+
+/**
+ * getReplicationBackendDataLocator - compares given location constraint to
+ * replication backends
+ * @param {object} locationObj - object containing location information
+ * @param {string} locationObj.location - name of location constraint
+ * @param {string} locationObj.key - keyname of object in location constraint
+ * @param {string} locationObj.locationType - type of location constraint
+ * @param {object} replicationInfo - information about object replication
+ * @param {array} replicationInfo.backends - array containing information about
+ * each replication location
+ * @param {string} replicationInfo.backends[].site - name of replication
+ * location
+ * @param {string} replicationInfo.backends[].status - status of replication
+ * @param {string} replicationInfo.backends[].dataStoreVersionId - version id
+ * of object at replication location
+ * @return {object} res - response object
+ *     {array} [res.dataLocator] - if COMPLETED status: array
+ *                                 containing the cloud location,
+ *                                 undefined otherwise
+ *     {string} [res.status] - replication status if no error
+ *     {string} [res.reason] - reason message if PENDING/FAILED
+ *     {Error} [res.error] - defined if object is not replicated to
+ *                           location passed in locationObj
+ */
+function getReplicationBackendDataLocator(locationObj, replicationInfo) {
+    const repBackendResult = {};
+    const locMatch = replicationInfo.backends.find(
+        backend => backend.site === locationObj.location);
+    if (!locMatch) {
+        repBackendResult.error = errors.InvalidLocationConstraint.
+            customizeDescription('Object is not replicated to location ' +
+            'passed in location header');
+        return repBackendResult;
+    }
+    repBackendResult.status = locMatch.status;
+    if (['PENDING', 'FAILED'].includes(locMatch.status)) {
+        repBackendResult.reason =
+            `Object replication to specified backend is ${locMatch.status}`;
+        return repBackendResult;
+    }
+    repBackendResult.dataLocator = [{
+        key: locationObj.key,
+        dataStoreName: locationObj.location,
+        dataStoreType: locationObj.locationType,
+        dataStoreVersionId: locMatch.dataStoreVersionId }];
+    return repBackendResult;
+}
+
+module.exports = getReplicationBackendDataLocator;

lib/api/apiUtils/object/getReplicationInfo.js

@@ -1,4 +1,6 @@
 const s3config = require('../../../Config').config;
+const constants = require('../../../../constants');
+const { isBackbeatUser } = require('../authorization/aclChecks');
 
 function _getBackend(objectMD, site) {
     const backends = objectMD ? objectMD.replicationInfo.backends : [];
@@ -29,16 +31,19 @@ function _getStorageClasses(rule) {
 }
 
 function _getReplicationInfo(rule, replicationConfig, content, operationType,
-    objectMD) {
+    objectMD, bucketMD) {
     const storageTypes = [];
     const backends = [];
     const storageClasses = _getStorageClasses(rule);
     storageClasses.forEach(storageClass => {
-        const location = s3config.locationConstraints[storageClass];
-        if (location && ['aws_s3', 'azure'].includes(location.type)) {
+        const storageClassName =
+              storageClass.endsWith(':preferred_read') ?
+              storageClass.split(':')[0] : storageClass;
+        const location = s3config.locationConstraints[storageClassName];
+        if (location && constants.replicationBackends[location.type]) {
             storageTypes.push(location.type);
         }
-        backends.push(_getBackend(objectMD, storageClass));
+        backends.push(_getBackend(objectMD, storageClassName));
     });
     if (storageTypes.length > 0 && operationType) {
         content.push(operationType);
@@ -51,6 +56,7 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
         storageClass: storageClasses.join(','),
         role: replicationConfig.role,
         storageType: storageTypes.join(','),
+        isNFS: bucketMD.isNFS(),
     };
 }
 
@@ -63,19 +69,34 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
  * @param {boolean} objSize - The size, in bytes, of the object being PUT
  * @param {string} operationType - The type of operation to replicate
  * @param {object} objectMD - The object metadata
+ * @param {AuthInfo} [authInfo] - authentication info of object owner
  * @return {undefined}
  */
 function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
-    objectMD) {
+    objectMD, authInfo) {
     const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
     const config = bucketMD.getReplicationConfiguration();
-    // If bucket does not have a replication configuration, do not replicate.
-    if (config) {
+
+    // Do not replicate object in the following cases:
+    //
+    // - bucket does not have a replication configuration
+    //
+    // - replication configuration does not apply to the object
+    //   (i.e. no rule matches object prefix)
+    //
+    // - replication configuration applies to the object (i.e. a rule matches
+    //   object prefix) but the status is disabled
+    //
+    // - object owner is an internal service account like Lifecycle
+    //   (because we do not want to replicate objects created from
+    //   actions triggered by internal services, by design)
+
+    if (config && (!authInfo || !isBackbeatUser(authInfo.getCanonicalID()))) {
         const rule = config.rules.find(rule =>
             (objKey.startsWith(rule.prefix) && rule.enabled));
         if (rule) {
             return _getReplicationInfo(rule, config, content, operationType,
-                objectMD);
+                objectMD, bucketMD);
         }
     }
     return undefined;

lib/api/apiUtils/object/locationHeaderCheck.js

@@ -0,0 +1,37 @@
+const { errors } = require('arsenal');
+
+const { config } = require('../../../Config');
+
+/**
+ * locationHeaderCheck - compares 'x-amz-location-constraint' header
+ * to location constraints in config
+ * @param {object} headers - request headers
+ * @param {string} objectKey - key name of object
+ * @param {string} bucketName - name of bucket
+ * @return {undefined|Object} returns error, object, or undefined
+ * @return {string} return.location - name of location constraint
+ * @return {string} return.key - name of object at location constraint
+ * @return {string} - return.locationType - type of location constraint
+ */
+function locationHeaderCheck(headers, objectKey, bucketName) {
+    const location = headers['x-amz-location-constraint'];
+    if (location) {
+        const validLocation = config.locationConstraints[location];
+        if (!validLocation) {
+            return errors.InvalidLocationConstraint.customizeDescription(
+                'Invalid location constraint specified in header');
+        }
+        const bucketMatch = validLocation.details.bucketMatch;
+        const backendKey = bucketMatch ? objectKey :
+            `${bucketName}/${objectKey}`;
+        return {
+            location,
+            key: backendKey,
+            locationType: validLocation.type,
+        };
+    }
+    // no location header was passed
+    return undefined;
+}
+
+module.exports = locationHeaderCheck;

lib/api/apiUtils/object/locationStorageCheck.js

@@ -0,0 +1,48 @@
+const { errors } = require('arsenal');
+
+const { config } = require('../../../Config');
+const { getLocationMetric, pushLocationMetric } =
+    require('../../../utapi/utilities');
+
+function _gbToBytes(gb) {
+    return gb * 1024 * 1024 * 1024;
+}
+
+/**
+ * locationStorageCheck - will ensure there is enough space left for object on
+ * PUT operations, or will update metric on DELETE
+ * NOTE: storage limit may not be exactly enforced in the case of concurrent
+ * requests when near limit
+ * @param {string} location - name of location to check quota
+ * @param {number} updateSize - new size to check against quota in bytes
+ * @param {object} log - werelogs logger
+ * @param {function} cb - callback function
+ * @return {undefined}
+ */
+function locationStorageCheck(location, updateSize, log, cb) {
+    const lc = config.locationConstraints;
+    const sizeLimitGB = lc[location] ? lc[location].sizeLimitGB : undefined;
+    if (updateSize === 0 || sizeLimitGB === undefined || sizeLimitGB === null) {
+        return cb();
+    }
+    // no need to list location metric, since it should be decreased
+    if (updateSize < 0) {
+        return pushLocationMetric(location, updateSize, log, cb);
+    }
+    return getLocationMetric(location, log, (err, bytesStored) => {
+        if (err) {
+            log.error(`Error listing metrics from Utapi: ${err.message}`);
+            return cb(err);
+        }
+        const newStorageSize = parseInt(bytesStored, 10) + updateSize;
+        const sizeLimitBytes = _gbToBytes(sizeLimitGB);
+        if (sizeLimitBytes < newStorageSize) {
+            return cb(errors.AccessDenied.customizeDescription(
+                `The assigned storage space limit for location ${location} ` +
+                'will be exceeded'));
+        }
+        return pushLocationMetric(location, updateSize, log, cb);
+    });
+}
+
+module.exports = locationStorageCheck;

lib/api/apiUtils/object/processMpuParts.js

@@ -238,4 +238,5 @@ splitter, log) {
 module.exports = {
     generateMpuPartStorageInfo,
     validateAndFilterMpuParts,
+    createAggregateETag,
 };

lib/api/bucketDelete.js

@@ -4,6 +4,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const deleteBucket = require('./apiUtils/bucket/bucketDeletion');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketDelete - DELETE bucket (currently supports only non-versioned buckets)
@@ -20,6 +21,8 @@ function bucketDelete(authInfo, request, log, cb) {
 
     if (authInfo.isRequesterPublicUser()) {
         log.debug('operation not available for public user');
+        monitoring.promMetrics(
+            'DELETE', request.bucketName, 403, 'deleteBucket');
         return cb(errors.AccessDenied);
     }
     const bucketName = request.bucketName;
@@ -37,6 +40,8 @@ function bucketDelete(authInfo, request, log, cb) {
             if (err) {
                 log.debug('error processing request',
                     { method: 'metadataValidateBucket', error: err });
+                monitoring.promMetrics(
+                    'DELETE', bucketName, err.code, 'deleteBucket');
                 return cb(err, corsHeaders);
             }
             log.trace('passed checks',
@@ -44,12 +49,16 @@ function bucketDelete(authInfo, request, log, cb) {
             return deleteBucket(authInfo, bucketMD, bucketName,
                 authInfo.getCanonicalID(), log, err => {
                     if (err) {
+                        monitoring.promMetrics(
+                            'DELETE', bucketName, err.code, 'deleteBucket');
                         return cb(err, corsHeaders);
                     }
                     pushMetric('deleteBucket', log, {
                         authInfo,
                         bucket: bucketName,
                     });
+                    monitoring.promMetrics(
+                        'DELETE', bucketName, '204', 'deleteBucket');
                     return cb(null, corsHeaders);
                 });
         });

lib/api/bucketDeleteCors.js

@@ -6,6 +6,7 @@ const { isBucketAuthorized } =
     require('./apiUtils/authorization/permissionChecks');
 const metadata = require('../metadata/wrapper');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketDeleteCors';
 
@@ -27,9 +28,13 @@ function bucketDeleteCors(authInfo, request, log, callback) {
             request.method, bucket);
         if (err) {
             log.debug('metadata getbucket failed', { error: err });
+            monitoring.promMetrics('DELETE', bucketName, 400,
+                'deleteBucketCors');
             return callback(err);
         }
         if (bucketShield(bucket, requestType)) {
+            monitoring.promMetrics('DELETE', bucketName, 400,
+                'deleteBucketCors');
             return callback(errors.NoSuchBucket);
         }
         log.trace('found bucket in metadata');
@@ -39,6 +44,8 @@ function bucketDeleteCors(authInfo, request, log, callback) {
                 requestType,
                 method: 'bucketDeleteCors',
             });
+            monitoring.promMetrics('DELETE', bucketName, 403,
+                'deleteBucketCors');
             return callback(errors.AccessDenied, corsHeaders);
         }
 
@@ -58,12 +65,16 @@ function bucketDeleteCors(authInfo, request, log, callback) {
         bucket.setCors(null);
         return metadata.updateBucket(bucketName, bucket, log, err => {
             if (err) {
+                monitoring.promMetrics('DELETE', bucketName, 400,
+                    'deleteBucketCors');
                 return callback(err, corsHeaders);
             }
             pushMetric('deleteBucketCors', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, '204', 'deleteBucketCors');
             return callback(err, corsHeaders);
         });
     });

lib/api/bucketDeleteLifecycle.js

@@ -2,6 +2,7 @@ const metadata = require('../metadata/wrapper');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketDeleteLifecycle - Delete the bucket Lifecycle configuration
@@ -26,6 +27,8 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) {
                 error: err,
                 method: 'bucketDeleteLifecycle',
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, err.code, 'deleteBucketLifecycle');
             return callback(err, corsHeaders);
         }
         if (!bucket.getLifecycleConfiguration()) {
@@ -42,12 +45,16 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) {
         bucket.setLifecycleConfiguration(null);
         return metadata.updateBucket(bucketName, bucket, log, err => {
             if (err) {
+                monitoring.promMetrics(
+                    'DELETE', bucketName, err.code, 'deleteBucketLifecycle');
                 return callback(err, corsHeaders);
             }
             pushMetric('deleteBucketLifecycle', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, '200', 'deleteBucketLifecycle');
             return callback(null, corsHeaders);
         });
     });

lib/api/bucketDeleteReplication.js

@@ -2,6 +2,7 @@ const metadata = require('../metadata/wrapper');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketDeleteReplication - Delete the bucket replication configuration
@@ -26,6 +27,8 @@ function bucketDeleteReplication(authInfo, request, log, callback) {
                 error: err,
                 method: 'bucketDeleteReplication',
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, err.code, 'deleteBucketReplication');
             return callback(err, corsHeaders);
         }
         if (!bucket.getReplicationConfiguration()) {
@@ -42,12 +45,16 @@ function bucketDeleteReplication(authInfo, request, log, callback) {
         bucket.setReplicationConfiguration(null);
         return metadata.updateBucket(bucketName, bucket, log, err => {
             if (err) {
+                monitoring.promMetrics(
+                    'DELETE', bucketName, err.code, 'deleteBucketReplication');
                 return callback(err, corsHeaders);
             }
             pushMetric('deleteBucketReplication', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, '200', 'deleteBucketReplication');
             return callback(null, corsHeaders);
         });
     });

lib/api/bucketDeleteWebsite.js

@@ -6,6 +6,7 @@ const { isBucketAuthorized } =
     require('./apiUtils/authorization/permissionChecks');
 const metadata = require('../metadata/wrapper');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketDeleteWebsite';
 
@@ -19,9 +20,13 @@ function bucketDeleteWebsite(authInfo, request, log, callback) {
             request.method, bucket);
         if (err) {
             log.debug('metadata getbucket failed', { error: err });
+            monitoring.promMetrics(
+                'DELETE', bucketName, err.code, 'deleteBucketWebsite');
             return callback(err);
         }
         if (bucketShield(bucket, requestType)) {
+            monitoring.promMetrics(
+                'DELETE', bucketName, 404, 'deleteBucketWebsite');
             return callback(errors.NoSuchBucket);
         }
         log.trace('found bucket in metadata');
@@ -31,6 +36,8 @@ function bucketDeleteWebsite(authInfo, request, log, callback) {
                 requestType,
                 method: 'bucketDeleteWebsite',
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, 403, 'deleteBucketWebsite');
             return callback(errors.AccessDenied, corsHeaders);
         }
 
@@ -50,12 +57,16 @@ function bucketDeleteWebsite(authInfo, request, log, callback) {
         bucket.setWebsiteConfiguration(null);
         return metadata.updateBucket(bucketName, bucket, log, err => {
             if (err) {
+                monitoring.promMetrics(
+                    'DELETE', bucketName, err.code, 'deleteBucketWebsite');
                 return callback(err, corsHeaders);
             }
             pushMetric('deleteBucketWebsite', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, '200', 'deleteBucketWebsite');
             return callback(null, corsHeaders);
         });
     });

lib/api/bucketGet.js

@@ -1,14 +1,15 @@
 const querystring = require('querystring');
 const { errors, versioning, s3middleware } = require('arsenal');
-
 const constants = require('../../constants');
 const services = require('../services');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const escapeForXml = s3middleware.escapeForXml;
 const { pushMetric } = require('../utapi/utilities');
-
+const validateSearchParams = require('../api/apiUtils/bucket/validateSearch');
+const parseWhere = require('../api/apiUtils/bucket/parseWhere');
 const versionIdUtils = versioning.VersionID;
+const monitoring = require('../utilities/monitoringHandler');
 const { generateToken, decryptToken }
     = require('../api/apiUtils/object/continueToken');
 
@@ -29,8 +30,7 @@ const { generateToken, decryptToken }
 </ListBucketResult>
 */
 
-//  Sample XML response for GET bucket objects:
-/*
+/*	Sample XML response for GET bucket objects:
 <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
   <Name>example-bucket</Name>
   <Prefix></Prefix>
@@ -60,7 +60,6 @@ const { generateToken, decryptToken }
 // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETVersion.html#RESTBucketGET_Examples
 /*
 <?xml version="1.0" encoding="UTF-8"?>
-
 <ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
     <Name>bucket</Name>
     <Prefix>my</Prefix>
@@ -259,6 +258,7 @@ function handleResult(listParams, requestMaxKeys, encoding, authInfo,
         res = processMasterVersions(bucketName, listParams, list);
     }
     pushMetric('listBucket', log, { authInfo, bucket: bucketName });
+    monitoring.promMetrics('GET', bucketName, '200', 'listBucket');
     return callback(null, res, corsHeaders);
 }
 
@@ -283,14 +283,26 @@ function bucketGet(authInfo, request, log, callback) {
     }
     const encoding = params['encoding-type'];
     if (encoding !== undefined && encoding !== 'url') {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listBucket');
         return callback(errors.InvalidArgument.customizeDescription('Invalid ' +
             'Encoding Method specified in Request'));
     }
     const requestMaxKeys = params['max-keys'] ?
         Number.parseInt(params['max-keys'], 10) : 1000;
     if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listBucket');
         return callback(errors.InvalidArgument);
     }
+    let validatedAst;
+    if (params.search !== undefined) {
+        const astOrError = validateSearchParams(params.search);
+        if (astOrError.error) {
+            return callback(astOrError.error);
+        }
+        validatedAst = astOrError.ast;
+    }
     // AWS only returns 1000 keys even if max keys are greater.
     // Max keys stated in response xml can be greater than actual
     // keys returned.
@@ -322,6 +334,8 @@ function bucketGet(authInfo, request, log, callback) {
             request.method, bucket);
         if (err) {
             log.debug('error processing request', { error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'listBucket');
             return callback(err, null, corsHeaders);
         }
         if (params.versions !== undefined) {
@@ -341,10 +355,27 @@ function bucketGet(authInfo, request, log, callback) {
             return handleResult(listParams, requestMaxKeys, encoding, authInfo,
                 bucketName, emptyList, corsHeaders, log, callback);
         }
+        if (params.search !== undefined) {
+            log.info('performing search listing', { search: params.search });
+            try {
+                listParams.mongifiedSearch = parseWhere(validatedAst);
+            } catch (err) {
+                log.debug(err.message, {
+                    stack: err.stack,
+                });
+                monitoring.promMetrics(
+                    'GET', bucketName, 400, 'listBucket');
+                return callback(errors.InvalidArgument
+                .customizeDescription('Invalid sql where clause ' +
+                'sent as search query'));
+            }
+        }
         return services.getObjectListing(bucketName, listParams, log,
         (err, list) => {
             if (err) {
                 log.debug('error processing request', { error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'listBucket');
                 return callback(err, null, corsHeaders);
             }
             return handleResult(listParams, requestMaxKeys, encoding, authInfo,

lib/api/bucketGetACL.js

@@ -4,6 +4,7 @@ const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const vault = require('../auth/vault');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 //  Sample XML response:
 /*
@@ -65,6 +66,8 @@ function bucketGetACL(authInfo, request, log, callback) {
         if (err) {
             log.debug('error processing request',
                 { method: 'bucketGetACL', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getBucketAcl');
             return callback(err, null, corsHeaders);
         }
         const bucketACL = bucket.getAcl();
@@ -136,6 +139,8 @@ function bucketGetACL(authInfo, request, log, callback) {
             if (err) {
                 log.debug('error processing request',
                     { method: 'vault.getEmailAddresses', error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'getBucketAcl');
                 return callback(err, null, corsHeaders);
             }
             const individualGrants = canonicalIDs.map(canonicalID => {
@@ -166,6 +171,7 @@ function bucketGetACL(authInfo, request, log, callback) {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics('GET', bucketName, '200', 'getBucketAcl');
             return callback(null, xml, corsHeaders);
         });
     });

lib/api/bucketGetCors.js

@@ -7,6 +7,7 @@ const { isBucketAuthorized } =
     require('./apiUtils/authorization/permissionChecks');
 const metadata = require('../metadata/wrapper');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketGetCors';
 
@@ -26,9 +27,13 @@ function bucketGetCors(authInfo, request, log, callback) {
     metadata.getBucket(bucketName, log, (err, bucket) => {
         if (err) {
             log.debug('metadata getbucket failed', { error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getBucketCors');
             return callback(err);
         }
         if (bucketShield(bucket, requestType)) {
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getBucketCors');
             return callback(errors.NoSuchBucket);
         }
         log.trace('found bucket in metadata');
@@ -40,6 +45,8 @@ function bucketGetCors(authInfo, request, log, callback) {
                 requestType,
                 method: 'bucketGetCors',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, 403, 'getBucketCors');
             return callback(errors.AccessDenied, null, corsHeaders);
         }
 
@@ -48,6 +55,8 @@ function bucketGetCors(authInfo, request, log, callback) {
             log.debug('cors configuration does not exist', {
                 method: 'bucketGetCors',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getBucketCors');
             return callback(errors.NoSuchCORSConfiguration, null, corsHeaders);
         }
         log.trace('converting cors configuration to xml');
@@ -57,6 +66,7 @@ function bucketGetCors(authInfo, request, log, callback) {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics('GET', bucketName, '200', 'getBucketCors');
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/bucketGetLifecycle.js

@@ -5,6 +5,7 @@ const LifecycleConfiguration =
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketGetLifecycle - Get the bucket lifecycle configuration
@@ -29,6 +30,8 @@ function bucketGetLifecycle(authInfo, request, log, callback) {
                 error: err,
                 method: 'bucketGetLifecycle',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getBucketLifecycle');
             return callback(err, null, corsHeaders);
         }
         const lifecycleConfig = bucket.getLifecycleConfiguration();
@@ -37,6 +40,8 @@ function bucketGetLifecycle(authInfo, request, log, callback) {
                 error: errors.NoSuchLifecycleConfiguration,
                 method: 'bucketGetLifecycle',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getBucketLifecycle');
             return callback(errors.NoSuchLifecycleConfiguration, null,
                 corsHeaders);
         }
@@ -45,6 +50,7 @@ function bucketGetLifecycle(authInfo, request, log, callback) {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics('GET', bucketName, '200', 'getBucketLifecycle');
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/bucketGetLocation.js

@@ -7,6 +7,7 @@ const metadata = require('../metadata/wrapper');
 const { pushMetric } = require('../utapi/utilities');
 const escapeForXml = s3middleware.escapeForXml;
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketGetLocation';
 
@@ -27,9 +28,13 @@ function bucketGetLocation(authInfo, request, log, callback) {
     return metadata.getBucket(bucketName, log, (err, bucket) => {
         if (err) {
             log.debug('metadata getbucket failed', { error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getBucketLocation');
             return callback(err);
         }
         if (bucketShield(bucket, requestType)) {
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getBucketLocation');
             return callback(errors.NoSuchBucket);
         }
         log.trace('found bucket in metadata');
@@ -42,6 +47,8 @@ function bucketGetLocation(authInfo, request, log, callback) {
                 requestType,
                 method: 'bucketGetLocation',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, 403, 'getBucketLocation');
             return callback(errors.AccessDenied, null, corsHeaders);
         }
 
@@ -60,7 +67,8 @@ function bucketGetLocation(authInfo, request, log, callback) {
             authInfo,
             bucket: bucketName,
         });
-
+        monitoring.promMetrics(
+            'GET', bucketName, '200', 'getBucketLocation');
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/bucketGetReplication.js

@@ -5,6 +5,7 @@ const { pushMetric } = require('../utapi/utilities');
 const { getReplicationConfigurationXML } =
     require('./apiUtils/bucket/getReplicationConfiguration');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketGetReplication - Get the bucket replication configuration
@@ -29,6 +30,8 @@ function bucketGetReplication(authInfo, request, log, callback) {
                 error: err,
                 method: 'bucketGetReplication',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getBucketReplication');
             return callback(err, null, corsHeaders);
         }
         const replicationConfig = bucket.getReplicationConfiguration();
@@ -37,6 +40,8 @@ function bucketGetReplication(authInfo, request, log, callback) {
                 error: errors.ReplicationConfigurationNotFoundError,
                 method: 'bucketGetReplication',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getBucketReplication');
             return callback(errors.ReplicationConfigurationNotFoundError, null,
                 corsHeaders);
         }
@@ -45,6 +50,8 @@ function bucketGetReplication(authInfo, request, log, callback) {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics(
+            'GET', bucketName, '200', 'getBucketReplication');
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/bucketGetVersioning.js

@@ -1,6 +1,7 @@
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 //  Sample XML response:
 /*
@@ -62,6 +63,8 @@ function bucketGetVersioning(authInfo, request, log, callback) {
         if (err) {
             log.debug('error processing request',
                 { method: 'bucketGetVersioning', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getBucketVersioning');
             return callback(err, null, corsHeaders);
         }
         const versioningConfiguration = bucket.getVersioningConfiguration();
@@ -70,6 +73,8 @@ function bucketGetVersioning(authInfo, request, log, callback) {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics(
+            'GET', bucketName, '200', 'getBucketVersioning');
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/bucketGetWebsite.js

@@ -7,6 +7,7 @@ const { isBucketAuthorized } =
     require('./apiUtils/authorization/permissionChecks');
 const metadata = require('../metadata/wrapper');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketGetWebsite';
 
@@ -26,9 +27,13 @@ function bucketGetWebsite(authInfo, request, log, callback) {
     metadata.getBucket(bucketName, log, (err, bucket) => {
         if (err) {
             log.debug('metadata getbucket failed', { error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getBucketWebsite');
             return callback(err);
         }
         if (bucketShield(bucket, requestType)) {
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getBucketWebsite');
             return callback(errors.NoSuchBucket);
         }
         log.trace('found bucket in metadata');
@@ -40,6 +45,8 @@ function bucketGetWebsite(authInfo, request, log, callback) {
                 requestType,
                 method: 'bucketGetWebsite',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, 403, 'getBucketWebsite');
             return callback(errors.AccessDenied, null, corsHeaders);
         }
 
@@ -48,6 +55,8 @@ function bucketGetWebsite(authInfo, request, log, callback) {
             log.debug('bucket website configuration does not exist', {
                 method: 'bucketGetWebsite',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getBucketWebsite');
             return callback(errors.NoSuchWebsiteConfiguration, null,
                 corsHeaders);
         }
@@ -58,6 +67,8 @@ function bucketGetWebsite(authInfo, request, log, callback) {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics(
+            'GET', bucketName, '200', 'getBucketWebsite');
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/bucketHead.js

@@ -2,6 +2,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * Determine if bucket exists and if user has permission to access it
@@ -24,12 +25,16 @@ function bucketHead(authInfo, request, log, callback) {
         const corsHeaders = collectCorsHeaders(request.headers.origin,
             request.method, bucket);
         if (err) {
+            monitoring.promMetrics(
+                        'HEAD', bucketName, err.code, 'headBucket');
             return callback(err, corsHeaders);
         }
         pushMetric('headBucket', log, {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics(
+                    'HEAD', bucketName, '200', 'headBucket');
         return callback(null, corsHeaders);
     });
 }

lib/api/bucketPut.js

@@ -8,6 +8,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { config } = require('../Config');
 const aclUtils = require('../utilities/aclUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 let { restEndpoints, locationConstraints } = config;
 config.on('rest-endpoints-update', () => {
@@ -109,15 +110,25 @@ function bucketPut(authInfo, request, log, callback) {
 
     if (authInfo.isRequesterPublicUser()) {
         log.debug('operation not available for public user');
+        monitoring.promMetrics(
+            'PUT', request.bucketName, 403, 'createBucket');
         return callback(errors.AccessDenied);
     }
     if (!aclUtils.checkGrantHeaderValidity(request.headers)) {
         log.trace('invalid acl header');
+        monitoring.promMetrics(
+            'PUT', request.bucketName, 400, 'createBucket');
         return callback(errors.InvalidArgument);
     }
     const { bucketName } = request;
 
-    if (request.bucketName === 'METADATA') {
+
+    if (request.bucketName === 'METADATA'
+    // Note: for this to work with Vault, would need way to set
+    // canonical ID to http://acs.zenko.io/accounts/service/clueso
+        && !authInfo.isRequesterThisServiceAccount('clueso')) {
+        monitoring.promMetrics(
+            'PUT', bucketName, 403, 'createBucket');
         return callback(errors.AccessDenied
             .customizeDescription('The bucket METADATA is used ' +
             'for internal purposes'));
@@ -179,6 +190,7 @@ function bucketPut(authInfo, request, log, callback) {
                   authInfo,
                   bucket: bucketName,
               });
+              monitoring.promMetrics('PUT', bucketName, '200', 'createBucket');
               return next(null, corsHeaders);
           }),
     ], callback);

lib/api/bucketPutACL.js

@@ -9,6 +9,7 @@ const constants = require('../../constants');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const vault = require('../auth/vault');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 /*
    Format of xml request:
@@ -58,10 +59,12 @@ function bucketPutACL(authInfo, request, log, callback) {
             acl: newCannedACL,
             method: 'bucketPutACL',
         });
+        monitoring.promMetrics('PUT', bucketName, 400, 'bucketPutACL');
         return callback(errors.InvalidArgument);
     }
     if (!aclUtils.checkGrantHeaderValidity(request.headers)) {
         log.trace('invalid acl header');
+        monitoring.promMetrics('PUT', bucketName, 400, 'bucketPutACL');
         return callback(errors.InvalidArgument);
     }
     const possibleGroups = [constants.allAuthedUsersId,
@@ -229,6 +232,8 @@ function bucketPutACL(authInfo, request, log, callback) {
                         id,
                         method: 'bucketPutACL',
                     });
+                    monitoring.promMetrics('PUT', bucketName, 400,
+                        'bucketPutACL');
                     return callback(errors.InvalidArgument, bucket);
                 }
             }
@@ -284,11 +289,13 @@ function bucketPutACL(authInfo, request, log, callback) {
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'bucketPutACL' });
+            monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutACL');
         } else {
             pushMetric('putBucketAcl', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutACL');
         }
         return callback(err, corsHeaders);
     });

lib/api/bucketPutCors.js

@@ -9,6 +9,7 @@ const { isBucketAuthorized } =
 const metadata = require('../metadata/wrapper');
 const { parseCorsXml } = require('./apiUtils/bucket/bucketCors');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketPutCors';
 
@@ -29,6 +30,7 @@ function bucketPutCors(authInfo, request, log, callback) {
     if (!request.post) {
         log.debug('CORS xml body is missing',
         { error: errors.MissingRequestBodyError });
+        monitoring.promMetrics('PUT', bucketName, 400, 'putBucketCors');
         return callback(errors.MissingRequestBodyError);
     }
 
@@ -36,12 +38,14 @@ function bucketPutCors(authInfo, request, log, callback) {
         .update(request.post, 'utf8').digest('base64');
     if (md5 !== request.headers['content-md5']) {
         log.debug('bad md5 digest', { error: errors.BadDigest });
+        monitoring.promMetrics('PUT', bucketName, 400, 'putBucketCors');
         return callback(errors.BadDigest);
     }
 
     if (parseInt(request.headers['content-length'], 10) > 65536) {
         const errMsg = 'The CORS XML document is limited to 64 KB in size.';
         log.debug(errMsg, { error: errors.MalformedXML });
+        monitoring.promMetrics('PUT', bucketName, 400, 'putBucketCors');
         return callback(errors.MalformedXML.customizeDescription(errMsg));
     }
 
@@ -86,11 +90,14 @@ function bucketPutCors(authInfo, request, log, callback) {
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'bucketPutCors' });
+            monitoring.promMetrics('PUT', bucketName, err.code,
+                'putBucketCors');
         }
         pushMetric('putBucketCors', log, {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics('PUT', bucketName, '200', 'putBucketCors');
         return callback(err, corsHeaders);
     });
 }

lib/api/bucketPutLifecycle.js

@@ -7,6 +7,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const metadata = require('../metadata/wrapper');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * Bucket Put Lifecycle - Create or update bucket lifecycle configuration
@@ -55,12 +56,15 @@ function bucketPutLifecycle(authInfo, request, log, callback) {
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'bucketPutLifecycle' });
+            monitoring.promMetrics(
+                'PUT', bucketName, err.code, 'putBucketLifecycle');
             return callback(err, corsHeaders);
         }
         pushMetric('putBucketLifecycle', log, {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics('PUT', bucketName, '200', 'putBucketLifecycle');
         return callback(null, corsHeaders);
     });
 }

lib/api/bucketPutReplication.js

@@ -6,7 +6,10 @@ const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const { getReplicationConfiguration } =
     require('./apiUtils/bucket/getReplicationConfiguration');
+const validateConfiguration =
+    require('./apiUtils/bucket/validateReplicationConfig');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const monitoring = require('../utilities/monitoringHandler');
 
 // The error response when a bucket does not have versioning 'Enabled'.
 const versioningNotEnabledError = errors.InvalidRequest.customizeDescription(
@@ -39,16 +42,26 @@ function bucketPutReplication(authInfo, request, log, callback) {
                 if (err) {
                     return next(err);
                 }
-                // Replication requires that versioning is 'Enabled'.
-                if (!bucket.isVersioningEnabled(bucket)) {
+                // Replication requires that versioning is 'Enabled' unless it
+                // is an NFS bucket.
+                if (!bucket.isNFS() && !bucket.isVersioningEnabled(bucket)) {
                     return next(versioningNotEnabledError);
                 }
                 return next(null, config, bucket);
             }),
         // Set the replication configuration and update the bucket metadata.
         (config, bucket, next) => {
+            // validate there's a preferred read location in case the
+            // bucket location is a transient source
+            if (!validateConfiguration(config, bucket)) {
+                const msg = 'Replication configuration lacks a preferred ' +
+                      'read location';
+                log.error(msg, { bucketName: bucket.getName() });
+                return next(errors.ValidationError
+                            .customizeDescription(msg));
+            }
             bucket.setReplicationConfiguration(config);
-            metadata.updateBucket(bucket.getName(), bucket, log, err =>
+            return metadata.updateBucket(bucket.getName(), bucket, log, err =>
                 next(err, bucket));
         },
     ], (err, bucket) => {
@@ -58,12 +71,16 @@ function bucketPutReplication(authInfo, request, log, callback) {
                 error: err,
                 method: 'bucketPutReplication',
             });
+            monitoring.promMetrics(
+                'PUT', bucketName, err.code, 'putBucketReplication');
             return callback(err, corsHeaders);
         }
         pushMetric('putBucketReplication', log, {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics(
+            'PUT', bucketName, '200', 'putBucketReplication');
         return callback(null, corsHeaders);
     });
 }

lib/api/bucketPutVersioning.js

@@ -9,9 +9,10 @@ const { pushMetric } = require('../utapi/utilities');
 const versioningNotImplBackends =
     require('../../constants').versioningNotImplBackends;
 const { config } = require('../Config');
+const monitoring = require('../utilities/monitoringHandler');
 
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
-'a versioned object to a location-constraint of type Azure.';
+'a versioned object to a location-constraint of type Azure or GCP.';
 
 /**
  * Format of xml request:
@@ -60,9 +61,17 @@ function _checkBackendVersioningImplemented(bucket) {
     const bucketLocation = bucket.getLocationConstraint();
     const bucketLocationType = config.getLocationConstraintType(bucketLocation);
 
+    // backend types known not to support versioning
     if (versioningNotImplBackends[bucketLocationType]) {
         return false;
     }
+
+    // versioning disabled per-location constraint
+    const lc = config.getLocationConstraint(bucketLocation);
+    if (lc.details && !lc.details.supportsVersioning) {
+        return false;
+    }
+
     return true;
 }
 
@@ -83,7 +92,6 @@ function bucketPutVersioning(authInfo, request, log, callback) {
         bucketName,
         requestType: 'bucketPutVersioning',
     };
-
     return waterfall([
         next => _parseXML(request, log, next),
         next => metadataValidateBucket(metadataValParams, log,
@@ -93,6 +101,12 @@ function bucketPutVersioning(authInfo, request, log, callback) {
             if (err) {
                 return next(err, bucket);
             }
+            // prevent enabling versioning on an nfs exported bucket
+            if (bucket.isNFS()) {
+                const error = new Error();
+                error.code = 'NFSBUCKET';
+                return next(error);
+            }
             // _checkBackendVersioningImplemented returns false if versioning
             // is not implemented on the bucket backend
             if (!_checkBackendVersioningImplemented(bucket)) {
@@ -124,14 +138,22 @@ function bucketPutVersioning(authInfo, request, log, callback) {
     ], (err, bucket) => {
         const corsHeaders = collectCorsHeaders(request.headers.origin,
             request.method, bucket);
+        if (err && err.code === 'NFSBUCKET') {
+            log.trace('skipping versioning for nfs exported bucket');
+            return callback(null, corsHeaders);
+        }
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'bucketPutVersioning' });
+            monitoring.promMetrics(
+                'PUT', bucketName, err.code, 'putBucketVersioning');
         } else {
             pushMetric('putBucketVersioning', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                'PUT', bucketName, '200', 'putBucketVersioning');
         }
         return callback(err, corsHeaders);
     });

lib/api/bucketPutWebsite.js

@@ -8,6 +8,7 @@ const { isBucketAuthorized } =
 const metadata = require('../metadata/wrapper');
 const { parseWebsiteConfigXml } = require('./apiUtils/bucket/bucketWebsite');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketPutWebsite';
 
@@ -26,6 +27,8 @@ function bucketPutWebsite(authInfo, request, log, callback) {
     const requestArn = authInfo.getArn();
 
     if (!request.post) {
+        monitoring.promMetrics(
+            'PUT', bucketName, 400, 'putBucketWebsite');
         return callback(errors.MissingRequestBodyError);
     }
     return async.waterfall([
@@ -70,11 +73,15 @@ function bucketPutWebsite(authInfo, request, log, callback) {
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'bucketPutWebsite' });
+            monitoring.promMetrics(
+                'PUT', bucketName, err.code, 'putBucketWebsite');
         } else {
             pushMetric('putBucketWebsite', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                'PUT', bucketName, '200', 'putBucketWebsite');
         }
         return callback(err, corsHeaders);
     });

lib/api/initiateMultipartUpload.js

@@ -14,9 +14,10 @@ const validateWebsiteHeader = require('./apiUtils/object/websiteServing')
     .validateWebsiteHeader;
 const { config } = require('../Config');
 const multipleBackendGateway = require('../data/multipleBackendGateway');
+const monitoring = require('../utilities/monitoringHandler');
 
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
-'a versioned object to a location-constraint of type Azure.';
+'a versioned object to a location-constraint of type Azure or GCP.';
 
 /*
 Sample xml response:
@@ -142,6 +143,8 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
                             log.trace('error storing multipart object', {
                                 error: err,
                             });
+                            monitoring.promMetrics('PUT', bucketName, err.code,
+                                'initiateMultipartUpload');
                             return callback(err, null, corsHeaders);
                         }
                         log.addDefaultFields({ uploadId });
@@ -151,6 +154,8 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
                             bucket: bucketName,
                             keys: [objectKey],
                         });
+                        monitoring.promMetrics('PUT', bucketName, '200',
+                            'initiateMultipartUpload');
                         return callback(null, xml, corsHeaders);
                     });
             });
@@ -184,6 +189,8 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
             undefined, undefined, undefined, log,
             (err, dataBackendResObj) => {
                 if (err) {
+                    monitoring.promMetrics('PUT', bucketName, err.code,
+                                'initiateMultipartUpload');
                     return callback(err);
                 }
                 if (locConstraint &&
@@ -198,6 +205,8 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
                         log.debug(externalVersioningErrorMessage,
                             { method: 'initiateMultipartUpload',
                                 error: errors.NotImplemented });
+                        monitoring.promMetrics('PUT', bucketName, 501,
+                            'initiateMultipartUpload');
                         return callback(errors.NotImplemented
                         .customizeDescription(externalVersioningErrorMessage));
                     }
@@ -229,12 +238,16 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
                     error: err,
                     method: 'metadataValidateBucketAndObj',
                 });
+                monitoring.promMetrics('PUT', bucketName, err.code,
+                    'initiateMultipartUpload');
                 return callback(err, null, corsHeaders);
             }
             if (destinationBucket.hasDeletedFlag() &&
                 accountCanonicalID !== destinationBucket.getOwner()) {
                 log.trace('deleted flag on bucket and request ' +
                     'from non-owner account');
+                monitoring.promMetrics('PUT', bucketName, 404,
+                    'initiateMultipartUpload');
                 return callback(errors.NoSuchBucket);
             }
             if (destinationBucket.hasTransientFlag() ||
@@ -255,6 +268,8 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
                                 // To avoid confusing user with error
                                 // from cleaning up
                                 // bucket return InternalError
+                                monitoring.promMetrics('PUT', bucketName, 500,
+                                    'initiateMultipartUpload');
                                 return callback(errors.InternalError,
                                     null, corsHeaders);
                             }

lib/api/listMultipartUploads.js

@@ -8,6 +8,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const services = require('../services');
 const { metadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 //  Sample XML response:
 /*
@@ -118,6 +119,8 @@ function listMultipartUploads(authInfo, request, log, callback) {
             let maxUploads = query['max-uploads'] !== undefined ?
                 Number.parseInt(query['max-uploads'], 10) : 1000;
             if (maxUploads < 0) {
+                monitoring.promMetrics('GET', bucketName, 400,
+                    'listMultipartUploads');
                 return callback(errors.InvalidArgument, bucket);
             }
             if (maxUploads > constants.listingHardLimit) {
@@ -141,6 +144,8 @@ function listMultipartUploads(authInfo, request, log, callback) {
         const corsHeaders = collectCorsHeaders(request.headers.origin,
             request.method, bucket);
         if (err) {
+            monitoring.promMetrics('GET', bucketName, err.code,
+                'listMultipartUploads');
             return callback(err, null, corsHeaders);
         }
         const xmlParams = {
@@ -156,6 +161,8 @@ function listMultipartUploads(authInfo, request, log, callback) {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics(
+            'GET', bucketName, '200', 'listMultipartUploads');
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/listParts.js

@@ -11,6 +11,7 @@ const services = require('../services');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const escapeForXml = s3middleware.escapeForXml;
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 const { config } = require('../../lib/Config');
 const multipleBackendGateway = require('../data/multipleBackendGateway');
@@ -82,6 +83,8 @@ function listParts(authInfo, request, log, callback) {
     let maxParts = Number.parseInt(request.query['max-parts'], 10) ?
         Number.parseInt(request.query['max-parts'], 10) : 1000;
     if (maxParts < 0) {
+        monitoring.promMetrics('GET', bucketName, 400,
+            'listMultipartUploadParts');
         return callback(errors.InvalidArgument);
     }
     if (maxParts > constants.listingHardLimit) {
@@ -280,11 +283,15 @@ function listParts(authInfo, request, log, callback) {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                    'GET', bucketName, '200', 'listMultipartUploadParts');
             next(null, destBucket, xml.join(''));
         },
     ], (err, destinationBucket, xml) => {
         const corsHeaders = collectCorsHeaders(request.headers.origin,
             request.method, destinationBucket);
+        monitoring.promMetrics('GET', bucketName, 400,
+            'listMultipartUploadParts');
         return callback(err, xml, corsHeaders);
     });
     return undefined;

lib/api/multiObjectDelete.js

@@ -17,6 +17,7 @@ const { preprocessingVersioningDelete }
     = require('./apiUtils/object/versioning');
 const createAndStoreObject = require('./apiUtils/object/createAndStoreObject');
 const { metadataGetObject } = require('../metadata/metadataUtils');
+const monitoring = require('../utilities/monitoringHandler');
 
 const versionIdUtils = versioning.VersionID;
 
@@ -196,6 +197,8 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
                         'null' : versionIdUtils.decode(entry.versionId);
                 }
                 if (decodedVersionId instanceof Error) {
+                    monitoring.promMetrics('DELETE', bucketName, 404,
+                        'multiObjectDelete');
                     return callback(errors.NoSuchVersion);
                 }
                 return callback(null, decodedVersionId);
@@ -206,6 +209,8 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
                 versionId, log, (err, objMD) => {
                     // if general error from metadata return error
                     if (err && !err.NoSuchKey) {
+                        monitoring.promMetrics('DELETE', bucketName, err.code,
+                            'multiObjectDelete');
                         return callback(err);
                     }
                     if (err && err.NoSuchKey) {
@@ -307,11 +312,15 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
 function multiObjectDelete(authInfo, request, log, callback) {
     log.debug('processing request', { method: 'multiObjectDelete' });
     if (!request.post) {
+        monitoring.promMetrics('DELETE', request.bucketName, 400,
+            'multiObjectDelete');
         return callback(errors.MissingRequestBodyError);
     }
     const md5 = crypto.createHash('md5')
         .update(request.post, 'utf8').digest('base64');
     if (md5 !== request.headers['content-md5']) {
+        monitoring.promMetrics('DELETE', request.bucketName, 400,
+            'multiObjectDelete');
         return callback(errors.BadDigest);
     }
 
@@ -480,6 +489,8 @@ function multiObjectDelete(authInfo, request, log, callback) {
         const corsHeaders = collectCorsHeaders(request.headers.origin,
             request.method, bucket);
         if (err) {
+            monitoring.promMetrics('DELETE', bucketName, err.code,
+                'multiObjectDelete');
             return callback(err, null, corsHeaders);
         }
         const xml = _formatXML(quietSetting, errorResults,
@@ -492,6 +503,10 @@ function multiObjectDelete(authInfo, request, log, callback) {
             byteLength: Number.parseInt(totalContentLengthDeleted, 10),
             numberOfObjects: numOfObjectsRemoved,
         });
+        monitoring.promMetrics('DELETE', bucketName, '200',
+        'multiObjectDelete',
+        Number.parseInt(totalContentLengthDeleted, 10), null, null,
+        numOfObjectsRemoved);
         return callback(null, xml, corsHeaders);
     });
 }

lib/api/multipartDelete.js

@@ -3,6 +3,7 @@ const { errors } = require('arsenal');
 const abortMultipartUpload = require('./apiUtils/object/abortMultipartUpload');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const isLegacyAWSBehavior = require('../utilities/legacyAWSBehavior');
+const monitoring = require('../utilities/monitoringHandler');
 const { pushMetric } = require('../utapi/utilities');
 
 /**
@@ -37,13 +38,13 @@ function multipartDelete(authInfo, request, log, callback) {
                     method: 'multipartDelete',
                     uploadId,
                 });
-                // if legacy behavior is enabled for 'us-east-1' and
-                // request is from 'us-east-1', return 404 instead of
-                // 204
+                monitoring.promMetrics('DELETE', bucketName, 400,
+                    'abortMultipartUpload');
                 return callback(err, corsHeaders);
             }
+            monitoring.promMetrics('DELETE', bucketName, 400,
+                'abortMultipartUpload');
             if (!err) {
-                // NoSuchUpload should not be recorded by Utapi
                 pushMetric('abortMultipartUpload', log, {
                     authInfo,
                     bucket: bucketName,

lib/api/objectCopy.js

@@ -20,12 +20,13 @@ const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const validateWebsiteHeader = require('./apiUtils/object/websiteServing')
     .validateWebsiteHeader;
 const { config } = require('../Config');
+const monitoring = require('../utilities/monitoringHandler');
 
 const versionIdUtils = versioning.VersionID;
 const locationHeader = constants.objectLocationConstraintHeader;
 const versioningNotImplBackends = constants.versioningNotImplBackends;
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
-'a versioned object to a location-constraint of type AWS or Azure.';
+'a versioned object to a location-constraint of type AWS or Azure or GCP.';
 
 /**
  * Preps metadata to be saved (based on copy or replace request header)
@@ -218,6 +219,8 @@ function objectCopy(authInfo, request, sourceBucket,
         const err = errors.InvalidRedirectLocation;
         log.debug('invalid x-amz-website-redirect-location' +
             `value ${websiteRedirectHeader}`, { error: err });
+        monitoring.promMetrics(
+            'PUT', destBucketName, err.code, 'copyObject');
         return callback(err);
     }
     const queryContainsVersionId = checkQueryVersionId(request.query);
@@ -467,6 +470,8 @@ function objectCopy(authInfo, request, sourceBucket,
             request.method, destBucketMD);
 
         if (err) {
+            monitoring.promMetrics(
+                'PUT', destBucketName, err.code, 'copyObject');
             return callback(err, null, corsHeaders);
         }
         const xml = [
@@ -503,6 +508,8 @@ function objectCopy(authInfo, request, sourceBucket,
             newByteLength: sourceObjSize,
             oldByteLength: isVersioned ? null : destObjPrevSize,
         });
+        monitoring.promMetrics('PUT', destBucketName, '200',
+                'copyObject', sourceObjSize, destObjPrevSize, isVersioned);
         // Add expiration header if lifecycle enabled
         return callback(null, xml, additionalHeaders);
     });

lib/api/objectDelete.js

@@ -1,6 +1,6 @@
 const async = require('async');
-const { errors, versioning } = require('arsenal');
-
+const { errors, versioning, s3middleware } = require('arsenal');
+const checkDateModifiedHeaders = s3middleware.checkDateModifiedHeaders;
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const services = require('../services');
 const { pushMetric } = require('../utapi/utilities');
@@ -8,6 +8,7 @@ const createAndStoreObject = require('./apiUtils/object/createAndStoreObject');
 const { decodeVersionId, preprocessingVersioningDelete }
     = require('./apiUtils/object/versioning');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
+const monitoring = require('../utilities/monitoringHandler');
 
 const versionIdUtils = versioning.VersionID;
 
@@ -24,6 +25,8 @@ function objectDelete(authInfo, request, log, cb) {
     log.debug('processing request', { method: 'objectDelete' });
     if (authInfo.isRequesterPublicUser()) {
         log.debug('operation not available for public user');
+        monitoring.promMetrics(
+            'DELETE', request.bucketName, 403, 'deleteObject');
         return cb(errors.AccessDenied);
     }
     const bucketName = request.bucketName;
@@ -99,11 +102,46 @@ function objectDelete(authInfo, request, log, cb) {
                     return next(null, bucketMD, objectMD, options);
                 });
         },
+        function validateHeaders(bucketMD, objectMD, options, next) {
+            if (objectMD) {
+                const lastModified = objectMD['last-modified'];
+                const { modifiedSinceRes, unmodifiedSinceRes } =
+                    checkDateModifiedHeaders(request.headers, lastModified);
+                const err = modifiedSinceRes.error || unmodifiedSinceRes.error;
+                if (err) {
+                    return process.nextTick(() => next(err, bucketMD));
+                }
+            }
+            return process.nextTick(() =>
+                next(null, bucketMD, objectMD, options));
+        },
         function deleteOperation(bucketMD, objectMD, delOptions, next) {
             const deleteInfo = {
                 removeDeleteMarker: false,
                 newDeleteMarker: false,
             };
+            if (delOptions && delOptions.deleteData && bucketMD.isNFS() &&
+                bucketMD.getReplicationConfiguration()) {
+                // If an NFS bucket that has replication configured, we want
+                // to put a delete marker on the destination even though the
+                // source does not have versioning.
+                return createAndStoreObject(bucketName, bucketMD, objectKey,
+                    objectMD, authInfo, canonicalID, null, request, true, null,
+                    log, err => {
+                        if (err) {
+                            return next(err);
+                        }
+                        if (objectMD.isDeleteMarker) {
+                            // record that we deleted a delete marker to set
+                            // response headers accordingly
+                            deleteInfo.removeDeleteMarker = true;
+                        }
+                        return services.deleteObject(bucketName, objectMD,
+                            objectKey, delOptions, log, (err, delResult) =>
+                            next(err, bucketMD, objectMD, delResult,
+                                deleteInfo));
+                    });
+            }
             if (delOptions && delOptions.deleteData) {
                 if (objectMD.isDeleteMarker) {
                     // record that we deleted a delete marker to set
@@ -136,6 +174,8 @@ function objectDelete(authInfo, request, log, cb) {
         if (err) {
             log.debug('error processing request', { error: err,
                 method: 'objectDelete' });
+            monitoring.promMetrics(
+                'DELETE', bucketName, err.code, 'deleteObject');
             return cb(err, resHeaders);
         }
         if (deleteInfo.newDeleteMarker) {
@@ -158,6 +198,8 @@ function objectDelete(authInfo, request, log, cb) {
                 keys: [objectKey],
                 byteLength: Number.parseInt(objectMD['content-length'], 10),
                 numberOfObjects: 1 });
+            monitoring.promMetrics('DELETE', bucketName, '200', 'deleteObject',
+                Number.parseInt(objectMD['content-length'], 10));
         }
         return cb(err, resHeaders);
     });

lib/api/objectDeleteTagging.js

@@ -6,6 +6,7 @@ const { decodeVersionId, getVersionIdResHeader }
 
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const metadata = require('../metadata/wrapper');
 const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
@@ -96,12 +97,16 @@ function objectDeleteTagging(authInfo, request, log, callback) {
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'objectDeleteTagging' });
+            monitoring.promMetrics(
+                'DELETE', bucketName, err.code, 'deleteObjectTagging');
         } else {
             pushMetric('deleteObjectTagging', log, {
                 authInfo,
                 bucket: bucketName,
                 keys: [objectKey],
             });
+            monitoring.promMetrics(
+                'DELETE', bucketName, '200', 'deleteObjectTagging');
             const verCfg = bucket.getVersioningConfiguration();
             additionalResHeaders['x-amz-version-id'] =
                 getVersionIdResHeader(verCfg, objectMD);

lib/api/objectGet.js

@@ -9,7 +9,15 @@ const collectResponseHeaders = require('../utilities/collectResponseHeaders');
 const { pushMetric } = require('../utapi/utilities');
 const { getVersionIdResHeader } = require('./apiUtils/object/versioning');
 const setPartRanges = require('./apiUtils/object/setPartRanges');
+const locationHeaderCheck =
+    require('./apiUtils/object/locationHeaderCheck');
+const getReplicationBackendDataLocator =
+    require('./apiUtils/object/getReplicationBackendDataLocator');
+const checkReadLocation = require('./apiUtils/object/checkReadLocation');
+
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
+const { config } = require('../Config');
+const monitoring = require('../utilities/monitoringHandler');
 
 const validateHeaders = s3middleware.validateConditionalHeaders;
 
@@ -27,6 +35,17 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
     const bucketName = request.bucketName;
     const objectKey = request.objectKey;
 
+    // returns name of location to get from and key if successful
+    const locCheckResult =
+        locationHeaderCheck(request.headers, objectKey, bucketName);
+    if (locCheckResult instanceof Error) {
+        log.trace('invalid location constraint to get from', {
+            location: request.headers['x-amz-location-constraint'],
+            error: locCheckResult,
+        });
+        return callback(locCheckResult);
+    }
+
     const decodedVidResult = decodeVersionId(request.query);
     if (decodedVidResult instanceof Error) {
         log.trace('invalid versionId query', {
@@ -54,10 +73,14 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                 error: err,
                 method: 'metadataValidateBucketAndObj',
             });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getObject');
             return callback(err, null, corsHeaders);
         }
         if (!objMD) {
             const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getObject');
             return callback(err, null, corsHeaders);
         }
         const verCfg = bucket.getVersioningConfiguration();
@@ -65,12 +88,16 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
             const responseMetaHeaders = Object.assign({},
                 { 'x-amz-delete-marker': true }, corsHeaders);
             if (!versionId) {
+                monitoring.promMetrics(
+                    'GET', bucketName, 404, 'getObject');
                 return callback(errors.NoSuchKey, null, responseMetaHeaders);
             }
             // return MethodNotAllowed if requesting a specific
             // version that has a delete marker
             responseMetaHeaders['x-amz-version-id'] =
                 getVersionIdResHeader(verCfg, objMD);
+            monitoring.promMetrics(
+                'GET', bucketName, 405, 'getObject');
             return callback(errors.MethodNotAllowed, null,
                 responseMetaHeaders);
         }
@@ -90,6 +117,8 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
             const { range, error } = parseRange(request.headers.range,
                                                 objLength);
             if (error) {
+                monitoring.promMetrics(
+                    'GET', bucketName, 400, 'getObject');
                 return callback(error, null, corsHeaders);
             }
             responseMetaHeaders['Accept-Ranges'] = 'bytes';
@@ -113,6 +142,37 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
             // objMD.location is just a string
             dataLocator = Array.isArray(objMD.location) ?
                 objMD.location : [{ key: objMD.location }];
+
+            const repConf = bucket.getReplicationConfiguration();
+            const prefReadLocation = repConf && repConf.preferredReadLocation;
+            const prefReadDataLocator = checkReadLocation(config,
+                prefReadLocation, objectKey, bucketName);
+            const targetLocation = locCheckResult || prefReadDataLocator ||
+                null;
+
+            if (targetLocation &&
+                targetLocation.location !== objMD.dataStoreName) {
+                const repBackendResult = getReplicationBackendDataLocator(
+                    targetLocation, objMD.replicationInfo);
+                if (repBackendResult.error) {
+                    log.error('Error with location constraint header', {
+                        bucketName, objectKey, versionId,
+                        error: repBackendResult.error,
+                        status: repBackendResult.status,
+                    });
+                    return callback(repBackendResult.error, null, corsHeaders);
+                }
+                const targetDataLocator = repBackendResult.dataLocator;
+                if (targetDataLocator) {
+                    dataLocator = targetDataLocator;
+                } else {
+                    log.debug('using source location as preferred read ' +
+                              'is unavailable', {
+                                  bucketName, objectKey, versionId,
+                                  reason: repBackendResult.reason,
+                              });
+                }
+            }
             // if the data backend is azure, there will only ever be at
             // most one item in the dataLocator array
             if (dataLocator[0] && dataLocator[0].dataStoreType === 'azure') {
@@ -125,18 +185,24 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                     const error = errors.InvalidRequest
                         .customizeDescription('Cannot specify both Range ' +
                             'header and partNumber query parameter.');
+                    monitoring.promMetrics(
+                        'GET', bucketName, 400, 'getObject');
                     return callback(error, null, corsHeaders);
                 }
                 partNumber = Number.parseInt(request.query.partNumber, 10);
                 if (Number.isNaN(partNumber)) {
                     const error = errors.InvalidArgument
                         .customizeDescription('Part number must be a number.');
+                    monitoring.promMetrics(
+                        'GET', bucketName, 400, 'getObject');
                     return callback(error, null, corsHeaders);
                 }
                 if (partNumber < 1 || partNumber > 10000) {
                     const error = errors.InvalidArgument
                         .customizeDescription('Part number must be an ' +
                             'integer between 1 and 10000, inclusive.');
+                    monitoring.promMetrics(
+                        'GET', bucketName, 400, 'getObject');
                     return callback(error, null, corsHeaders);
                 }
             }
@@ -144,6 +210,8 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
             // get range for objects with multiple parts
             if (byteRange && dataLocator.length > 1 &&
                 dataLocator[0].start === undefined) {
+                monitoring.promMetrics(
+                    'GET', bucketName, 501, 'getObject');
                 return callback(errors.NotImplemented, null, corsHeaders);
             }
             if (objMD['x-amz-server-side-encryption']) {
@@ -161,6 +229,8 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                     // Location objects prior to GA7.1 do not include the
                     // dataStoreETag field so we cannot find the part range
                     if (dataStoreETag === undefined) {
+                        monitoring.promMetrics(
+                            'GET', bucketName, 400, 'getObject');
                         return callback(errors.NotImplemented
                             .customizeDescription('PartNumber parameter for ' +
                             'this object is not supported'));
@@ -175,6 +245,8 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                     }
                 }
                 if (locations.length === 0) {
+                    monitoring.promMetrics(
+                        'GET', bucketName, 400, 'getObject');
                     return callback(errors.InvalidPartNumber, null,
                         corsHeaders);
                 }
@@ -192,6 +264,8 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
             if (err) {
                 log.error('error from external backend checking for ' +
                 'object existence', { error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'getObject');
                 return callback(err);
             }
             pushMetric('getObject', log, {
@@ -200,6 +274,8 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                 newByteLength:
                 Number.parseInt(responseMetaHeaders['Content-Length'], 10),
             });
+            monitoring.promMetrics('GET', bucketName, '200', 'getObject',
+                Number.parseInt(responseMetaHeaders['Content-Length'], 10));
             return callback(null, dataLocator, responseMetaHeaders,
                 byteRange);
         });

lib/api/objectGetACL.js

@@ -9,6 +9,7 @@ const { decodeVersionId, getVersionIdResHeader }
     = require('./apiUtils/object/versioning');
 const vault = require('../auth/vault');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
+const monitoring = require('../utilities/monitoringHandler');
 
 //  Sample XML response:
 /*
@@ -210,12 +211,15 @@ function objectGetACL(authInfo, request, log, callback) {
         const resHeaders = collectCorsHeaders(request.headers.origin,
             request.method, bucket);
         if (err) {
+            monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'getObjectAcl');
             return callback(err, null, resHeaders);
         }
         pushMetric('getObjectAcl', log, {
             authInfo,
             bucket: bucketName,
         });
+        monitoring.promMetrics('GET', bucketName, '200', 'getObjectAcl');
         resHeaders['x-amz-version-id'] = resVersionId;
         return callback(null, xml, resHeaders);
     });

lib/api/objectGetTagging.js

@@ -8,6 +8,7 @@ const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { convertToXml } = s3middleware.tagging;
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * Object Get Tagging - Return tag for object
@@ -79,11 +80,15 @@ function objectGetTagging(authInfo, request, log, callback) {
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'objectGetTagging' });
+            monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'getObjectTagging');
         } else {
             pushMetric('getObjectTagging', log, {
                 authInfo,
                 bucket: bucketName,
             });
+            monitoring.promMetrics(
+                'GET', bucketName, '200', 'getObjectTagging');
             const verCfg = bucket.getVersioningConfiguration();
             additionalResHeaders['x-amz-version-id'] =
                 getVersionIdResHeader(verCfg, objectMD);

lib/api/objectHead.js

@@ -6,6 +6,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const collectResponseHeaders = require('../utilities/collectResponseHeaders');
 const { pushMetric } = require('../utapi/utilities');
 const { getVersionIdResHeader } = require('./apiUtils/object/versioning');
+const monitoring = require('../utilities/monitoringHandler');
 
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 
@@ -51,10 +52,14 @@ function objectHead(authInfo, request, log, callback) {
                     error: err,
                     method: 'objectHead',
                 });
+                monitoring.promMetrics(
+                        'HEAD', bucketName, err.code, 'headObject');
                 return callback(err, corsHeaders);
             }
             if (!objMD) {
                 const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
+                monitoring.promMetrics(
+                    'HEAD', bucketName, err.code, 'headObject');
                 return callback(err, corsHeaders);
             }
             const verCfg = bucket.getVersioningConfiguration();
@@ -62,12 +67,16 @@ function objectHead(authInfo, request, log, callback) {
                 const responseHeaders = Object.assign({},
                     { 'x-amz-delete-marker': true }, corsHeaders);
                 if (!versionId) {
+                    monitoring.promMetrics(
+                        'HEAD', bucketName, 404, 'headObject');
                     return callback(errors.NoSuchKey, responseHeaders);
                 }
                 // return MethodNotAllowed if requesting a specific
                 // version that has a delete marker
                 responseHeaders['x-amz-version-id'] =
                     getVersionIdResHeader(verCfg, objMD);
+                monitoring.promMetrics(
+                    'HEAD', bucketName, 405, 'headObject');
                 return callback(errors.MethodNotAllowed, responseHeaders);
             }
             const headerValResult = validateHeaders(request.headers,
@@ -78,6 +87,7 @@ function objectHead(authInfo, request, log, callback) {
             const responseHeaders =
                 collectResponseHeaders(objMD, corsHeaders, verCfg);
             pushMetric('headObject', log, { authInfo, bucket: bucketName });
+            monitoring.promMetrics('HEAD', bucketName, '200', 'headObject');
             return callback(null, responseHeaders);
         });
 }

lib/api/objectPut.js

@@ -1,6 +1,7 @@
 const async = require('async');
 const { errors, versioning } = require('arsenal');
 
+const constants = require('../../constants');
 const aclUtils = require('../utilities/aclUtils');
 const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
@@ -9,6 +10,7 @@ const { checkQueryVersionId } = require('./apiUtils/object/versioning');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const kms = require('../kms/wrapper');
+const monitoring = require('../utilities/monitoringHandler');
 
 const versionIdUtils = versioning.VersionID;
 
@@ -35,12 +37,21 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
     log.debug('processing request', { method: 'objectPut' });
     if (!aclUtils.checkGrantHeaderValidity(request.headers)) {
         log.trace('invalid acl header');
+        monitoring.promMetrics('PUT', request.bucketName, 400,
+            'putObject');
         return callback(errors.InvalidArgument);
     }
     const queryContainsVersionId = checkQueryVersionId(request.query);
     if (queryContainsVersionId instanceof Error) {
         return callback(queryContainsVersionId);
     }
+    const size = request.parsedContentLength;
+    if (Number.parseInt(size, 10) > constants.maximumAllowedUploadSize) {
+        log.debug('Upload size exceeds maximum allowed for a single PUT',
+            { size });
+        return callback(errors.EntityTooLarge);
+    }
+
     const bucketName = request.bucketName;
     const objectKey = request.objectKey;
     const requestType = 'objectPut';
@@ -57,11 +68,13 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
                 error: err,
                 method: 'metadataValidateBucketAndObj',
             });
+            monitoring.promMetrics('PUT', bucketName, err.code, 'putObject');
             return callback(err, responseHeaders);
         }
         if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) {
             log.trace('deleted flag on bucket and request ' +
                 'from non-owner account');
+            monitoring.promMetrics('PUT', bucketName, 404, 'putObject');
             return callback(errors.NoSuchBucket);
         }
         return async.waterfall([
@@ -86,6 +99,8 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
             },
         ], (err, storingResult) => {
             if (err) {
+                monitoring.promMetrics('PUT', bucketName, err.code,
+                    'putObject');
                 return callback(err, responseHeaders);
             }
             const newByteLength = request.parsedContentLength;
@@ -118,6 +133,8 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
                 newByteLength,
                 oldByteLength: isVersionedObj ? null : oldByteLength,
             });
+            monitoring.promMetrics('PUT', bucketName, '200',
+                'putObject', newByteLength, oldByteLength, isVersionedObj);
             return callback(null, responseHeaders);
         });
     });

lib/api/objectPutACL.js

@@ -10,6 +10,7 @@ const vault = require('../auth/vault');
 const { decodeVersionId, getVersionIdResHeader }
     = require('./apiUtils/object/versioning');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
+const monitoring = require('../utilities/monitoringHandler');
 
 /*
    Format of xml request:
@@ -56,10 +57,12 @@ function objectPutACL(authInfo, request, log, cb) {
     ];
     if (newCannedACL && possibleCannedACL.indexOf(newCannedACL) === -1) {
         log.trace('invalid canned acl argument', { cannedAcl: newCannedACL });
+        monitoring.promMetrics('PUT', bucketName, 400, 'putObjectAcl');
         return cb(errors.InvalidArgument);
     }
     if (!aclUtils.checkGrantHeaderValidity(request.headers)) {
         log.trace('invalid acl header');
+        monitoring.promMetrics('PUT', bucketName, 400, 'putObjectAcl');
         return cb(errors.InvalidArgument);
     }
     const possibleGroups = [
@@ -288,6 +291,8 @@ function objectPutACL(authInfo, request, log, cb) {
                 error: err,
                 method: 'objectPutACL',
             });
+            monitoring.promMetrics(
+                'PUT', bucketName, err.code, 'putObjectAcl');
             return cb(err, resHeaders);
         }
 
@@ -301,6 +306,7 @@ function objectPutACL(authInfo, request, log, cb) {
             bucket: bucketName,
             keys: [objectKey],
         });
+        monitoring.promMetrics('PUT', bucketName, '200', 'putObjectAcl');
         return cb(null, resHeaders);
     });
 }

lib/api/objectPutCopyPart.js

@@ -11,6 +11,7 @@ const logger = require('../utilities/logger');
 const services = require('../services');
 const setUpCopyLocator = require('./apiUtils/object/setUpCopyLocator');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
+const monitoring = require('../utilities/monitoringHandler');
 
 const versionIdUtils = versioning.VersionID;
 
@@ -46,6 +47,8 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
     const partNumber = Number.parseInt(request.query.partNumber, 10);
     // AWS caps partNumbers at 10,000
     if (partNumber > 10000 || !Number.isInteger(partNumber) || partNumber < 1) {
+        monitoring.promMetrics('PUT', destBucketName, 400,
+            'putObjectCopyPart');
         return callback(errors.InvalidArgument);
     }
     // We pad the partNumbers so that the parts will be sorted
@@ -335,6 +338,8 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
         if (err && err !== skipError) {
             log.trace('error from copy part waterfall',
             { error: err });
+            monitoring.promMetrics('PUT', destBucketName, err.code,
+                'putObjectCopyPart');
             return callback(err, null, corsHeaders);
         }
         const xml = [
@@ -363,6 +368,8 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
             newByteLength: copyObjectSize,
             oldByteLength: prevObjectSize,
         });
+        monitoring.promMetrics(
+            'PUT', destBucketName, '200', 'putObjectCopyPart');
         return callback(null, xml, additionalHeaders);
     });
 }

lib/api/objectPutPart.js

@@ -18,6 +18,7 @@ const { config } = require('../Config');
 const multipleBackendGateway = require('../data/multipleBackendGateway');
 const locationConstraintCheck
     = require('./apiUtils/object/locationConstraintCheck');
+const monitoring = require('../utilities/monitoringHandler');
 
 const skipError = new Error('skip');
 
@@ -59,6 +60,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
 
     if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) {
         log.debug('put part size too large', { size });
+        monitoring.promMetrics('PUT', request.bucketName, 400,
+            'putObjectPart');
         return cb(errors.EntityTooLarge);
     }
 
@@ -71,9 +74,13 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
     const partNumber = Number.parseInt(request.query.partNumber, 10);
     // AWS caps partNumbers at 10,000
     if (partNumber > 10000) {
+        monitoring.promMetrics('PUT', request.bucketName, 400,
+            'putObjectPart');
         return cb(errors.TooManyParts);
     }
     if (!Number.isInteger(partNumber) || partNumber < 1) {
+        monitoring.promMetrics('PUT', request.bucketName, 400,
+            'putObjectPart');
         return cb(errors.InvalidArgument);
     }
     const bucketName = request.bucketName;
@@ -218,7 +225,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
                         // if data backend handles MPU, skip to end of waterfall
                         return next(skipError, destinationBucket,
                         partInfo.dataStoreETag);
-                    } else if (partInfo && partInfo.dataStoreType === 'azure') {
+                    } else if (partInfo &&
+                        constants.s3HandledBackends[partInfo.dataStoreType]) {
                         return next(null, destinationBucket,
                             objectLocationConstraint, cipherBundle, splitter,
                             partInfo);
@@ -267,7 +275,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
         (destinationBucket, objectLocationConstraint, cipherBundle,
         partKey, prevObjectSize, oldLocations, partInfo, next) => {
             // NOTE: set oldLocations to null so we do not batchDelete for now
-            if (partInfo && partInfo.dataStoreType === 'azure') {
+            if (partInfo &&
+                constants.skipBatchDeleteBackends[partInfo.dataStoreType]) {
                 // skip to storing metadata
                 return next(null, destinationBucket, partInfo,
                   partInfo.dataStoreETag,
@@ -371,6 +380,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
                 error: err,
                 method: 'objectPutPart',
             });
+            monitoring.promMetrics('PUT', bucketName, err.code,
+                'putObjectPart');
             return cb(err, null, corsHeaders);
         }
         pushMetric('uploadPart', log, {
@@ -380,6 +391,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
             newByteLength: size,
             oldByteLength: prevObjectSize,
         });
+        monitoring.promMetrics('PUT', bucketName,
+            '200', 'putObjectPart', size, prevObjectSize);
         return cb(null, hexDigest, corsHeaders);
     });
 }

lib/api/objectPutTagging.js

@@ -6,6 +6,7 @@ const { decodeVersionId, getVersionIdResHeader } =
 
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const metadata = require('../metadata/wrapper');
@@ -102,12 +103,16 @@ function objectPutTagging(authInfo, request, log, callback) {
         if (err) {
             log.trace('error processing request', { error: err,
                 method: 'objectPutTagging' });
+            monitoring.promMetrics('PUT', bucketName, err.code,
+                'putObjectTagging');
         } else {
             pushMetric('putObjectTagging', log, {
                 authInfo,
                 bucket: bucketName,
                 keys: [objectKey],
             });
+            monitoring.promMetrics(
+                'PUT', bucketName, '200', 'putObjectTagging');
             const verCfg = bucket.getVersioningConfiguration();
             additionalResHeaders['x-amz-version-id'] =
                 getVersionIdResHeader(verCfg, objectMD);

lib/api/serviceGet.js

@@ -2,6 +2,7 @@ const { errors } = require('arsenal');
 
 const constants = require('../../constants');
 const services = require('../services');
+const monitoring = require('../utilities/monitoringHandler');
 
 /*
  *  Format of xml response:
@@ -55,6 +56,8 @@ function serviceGet(authInfo, request, log, callback) {
 
     if (authInfo.isRequesterPublicUser()) {
         log.debug('operation not available for public user');
+        monitoring.promMetrics(
+            'GET', request.bucketName, 403, 'getService');
         return callback(errors.AccessDenied);
     }
     const xml = [];
@@ -73,12 +76,15 @@ function serviceGet(authInfo, request, log, callback) {
     return services.getService(authInfo, request, log, constants.splitter,
         (err, userBuckets, splitter) => {
             if (err) {
+                monitoring.promMetrics(
+                    'GET', userBuckets, err.code, 'getService');
                 return callback(err);
             }
             // TODO push metric for serviceGet
             // pushMetric('getService', log, {
             //      bucket: bucketName,
             // });
+            monitoring.promMetrics('GET', userBuckets, '200', 'getService');
             return callback(null, generateXml(xml, canonicalId, userBuckets,
                 splitter));
         });

lib/api/websiteGet.js

@@ -32,6 +32,8 @@ function _errorActions(err, errorDocument, routingRules,
         objectKey, err.code);
     if (errRoutingRule) {
         // route will redirect
+        monitoring.promMetrics(
+            'GET', bucketName, err.code, 'getObject');
         return callback(err, false, null, corsHeaders, errRoutingRule,
             objectKey);
     }
@@ -42,6 +44,8 @@ function _errorActions(err, errorDocument, routingRules,
                     // error retrieving error document so return original error
                     // and set boolean of error retrieving user's error document
                     // to true
+                    monitoring.promMetrics(
+                        'GET', bucketName, err.code, 'getObject');
                     return callback(err, true, null, corsHeaders);
                 }
                 // return the default error message if the object is private
@@ -49,6 +53,8 @@ function _errorActions(err, errorDocument, routingRules,
                 if (!isObjAuthorized(bucket, errObjMD, 'objectGet',
                     constants.publicId)) {
                     log.trace('errorObj not authorized', { error: err });
+                    monitoring.promMetrics(
+                        'GET', bucketName, err.code, 'getObject');
                     return callback(err, true, null, corsHeaders);
                 }
                 const dataLocator = errObjMD.location;
@@ -67,9 +73,13 @@ function _errorActions(err, errorDocument, routingRules,
                     bucket: bucketName,
                     newByteLength: responseMetaHeaders['Content-Length'],
                 });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'getObject');
                 return callback(err, false, dataLocator, responseMetaHeaders);
             });
     }
+    monitoring.promMetrics(
+        'GET', bucketName, err.code, 'getObject');
     return callback(err, false, null, corsHeaders);
 }
 
@@ -89,16 +99,22 @@ function websiteGet(request, log, callback) {
     return metadata.getBucket(bucketName, log, (err, bucket) => {
         if (err) {
             log.trace('error retrieving bucket metadata', { error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'getObject');
             return callback(err, false);
         }
         if (bucketShield(bucket, 'objectGet')) {
             log.trace('bucket in transient/deleted state so shielding');
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getObject');
             return callback(errors.NoSuchBucket, false);
         }
         const corsHeaders = collectCorsHeaders(request.headers.origin,
             request.method, bucket);
         const websiteConfig = bucket.getWebsiteConfiguration();
         if (!websiteConfig) {
+            monitoring.promMetrics(
+                'GET', bucketName, 404, 'getObject');
             return callback(errors.NoSuchWebsiteConfiguration, false, null,
             corsHeaders);
         }
@@ -142,6 +158,8 @@ function websiteGet(request, log, callback) {
                 if (err) {
                     log.trace('error retrieving object metadata',
                     { error: err });
+                    monitoring.promMetrics(
+                        'GET', bucketName, err.code, 'getObject');
                     let returnErr = err;
                     const bucketAuthorized = isBucketAuthorized(bucket,
                       'bucketGet', constants.publicId);
@@ -205,6 +223,8 @@ function websiteGet(request, log, callback) {
                     bucket: bucketName,
                     newByteLength: responseMetaHeaders['Content-Length'],
                 });
+                monitoring.promMetrics('GET', bucketName, '200',
+                    'getObject', responseMetaHeaders['Content-Length']);
                 return callback(null, false, dataLocator, responseMetaHeaders);
             });
     });

lib/api/websiteHead.js

@@ -12,7 +12,6 @@ const { pushMetric } = require('../utapi/utilities');
 const { isBucketAuthorized, isObjAuthorized } =
     require('./apiUtils/authorization/permissionChecks');
 
-
 /**
  * _errorActions - take a number of actions once have error getting obj
  * @param {object} err - arsenal errors object
@@ -51,10 +50,14 @@ function websiteHead(request, log, callback) {
     return metadata.getBucket(bucketName, log, (err, bucket) => {
         if (err) {
             log.trace('error retrieving bucket metadata', { error: err });
+            monitoring.promMetrics(
+                'HEAD', bucketName, err.code, 'headObject');
             return callback(err);
         }
         if (bucketShield(bucket, 'objectHead')) {
             log.trace('bucket in transient/deleted state so shielding');
+            monitoring.promMetrics(
+                'HEAD', bucketName, 404, 'headObject');
             return callback(errors.NoSuchBucket);
         }
         const corsHeaders = collectCorsHeaders(request.headers.origin,
@@ -63,6 +66,8 @@ function websiteHead(request, log, callback) {
         // head of an object. object ACL's are what matter
         const websiteConfig = bucket.getWebsiteConfiguration();
         if (!websiteConfig) {
+            monitoring.promMetrics(
+                'HEAD', bucketName, 404, 'headObject');
             return callback(errors.NoSuchWebsiteConfiguration);
         }
         // any errors above would be generic header error response
@@ -151,6 +156,8 @@ function websiteHead(request, log, callback) {
                 pushMetric('headObject', log, {
                     bucket: bucketName,
                 });
+                monitoring.promMetrics(
+                    'HEAD', bucketName, '200', 'headObject');
                 return callback(null, responseMetaHeaders);
             });
     });

lib/auth/in_memory/builder.js

@@ -1,17 +1,29 @@
+const serviceAccountPrefix =
+    require('arsenal').constants.zenkoServiceAccount;
+
 /** build simple authdata with only one account
  * @param {string} accessKey - account's accessKey
  * @param {string} secretKey - account's secretKey
+ * @param {string} canonicalId - account's canonical id
+ * @param {string} [serviceName] - service name to use to generate can id
+ * @param {string} userName - account's user name
  * @return {object} authdata - authdata with account's accessKey and secretKey
  */
-function buildAuthDataAccount(accessKey, secretKey) {
+function buildAuthDataAccount(accessKey, secretKey, canonicalId, serviceName,
+userName) {
+    // TODO: remove specific check for clueso and generate unique
+    // canonical id's for accounts
+    const finalCanonicalId = canonicalId ||
+        (serviceName ? `${serviceAccountPrefix}/${serviceName}` :
+            '12349df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47qwer');
+    const shortid = '123456789012';
     return {
         accounts: [{
-            name: 'CustomAccount',
+            name: userName || 'CustomAccount',
             email: 'customaccount1@setbyenv.com',
-            arn: 'arn:aws:iam::123456789012:root',
-            canonicalID: '12349df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d52' +
-            '18e7cd47qwer',
-            shortid: '123456789012',
+            arn: `arn:aws:iam::${shortid}:root`,
+            canonicalID: finalCanonicalId,
+            shortid,
             keys: [{
                 access: accessKey,
                 secret: secretKey,

lib/data/external/AwsClient.js

@@ -1,5 +1,6 @@
-const { errors, s3middleware } = require('arsenal');
 const AWS = require('aws-sdk');
+const { errors, s3middleware } = require('arsenal');
+const werelogs = require('werelogs');
 const MD5Sum = s3middleware.MD5Sum;
 const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
 const createLogger = require('../multipleBackendLogger');
@@ -15,12 +16,48 @@ const missingVerIdInternalError = errors.InternalError.customizeDescription(
 class AwsClient {
     constructor(config) {
         this.clientType = 'aws_s3';
+        this.type = 'AWS';
         this._s3Params = config.s3Params;
         this._awsBucketName = config.bucketName;
         this._bucketMatch = config.bucketMatch;
         this._dataStoreName = config.dataStoreName;
         this._serverSideEncryption = config.serverSideEncryption;
+        this._supportsVersioning = config.supportsVersioning;
         this._client = new AWS.S3(this._s3Params);
+        this._logger = new werelogs.Logger('AwsClient');
+    }
+
+    setup(cb) {
+        // this request implicitly updates the endpoint for the location
+        // the following code explcitly sets it to avoid surprises
+        this._client.getBucketLocation({ Bucket: this._awsBucketName },
+        (err, res) => {
+            if (err && err.code !== 'AuthorizationHeaderMalformed') {
+                this._logger.error('error during setup', {
+                    error: err,
+                    method: 'AwsClient.setup',
+                });
+                return cb(err);
+            }
+            let region;
+            if (err && err.code === 'AuthorizationHeaderMalformed') {
+                // set regional endpoint
+                region = err.region;
+            } else if (res) {
+                region = res.LocationConstraint;
+            }
+            const isAWS = this._s3Params.endpoint.endsWith('amazonaws.com');
+            if (region && isAWS) {
+                const endpoint = `s3.${region}.amazonaws.com`;
+                this._logger.debug('setting regional endpoint', {
+                    method: 'AwsClient.setup',
+                    region,
+                    endpoint,
+                });
+                this._client.endpoint = new AWS.Endpoint(endpoint);
+            }
+            return cb();
+        });
     }
 
     _createAwsKey(requestBucketName, requestObjectKey,
@@ -30,6 +67,14 @@ class AwsClient {
         }
         return `${requestBucketName}/${requestObjectKey}`;
     }
+
+    toObjectGetInfo(objectKey, bucketName) {
+        return {
+            key: this._createAwsKey(bucketName, objectKey, this._bucketMatch),
+            dataStoreName: this._dataStoreName,
+        };
+    }
+
     put(stream, size, keyContext, reqUids, callback) {
         const awsKey = this._createAwsKey(keyContext.bucketName,
            keyContext.objectKey, this._bucketMatch);
@@ -39,16 +84,16 @@ class AwsClient {
         const putCb = (err, data) => {
             if (err) {
                 logHelper(log, 'error', 'err from data backend',
-                  err, this._dataStoreName);
+                  err, this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
-            if (!data.VersionId) {
+            if (!data.VersionId && this._supportsVersioning) {
                 logHelper(log, 'error', 'missing version id for data ' +
                     'backend object', missingVerIdInternalError,
-                    this._dataStoreName);
+                    this._dataStoreName, this.clientType);
                 return callback(missingVerIdInternalError);
             }
             const dataStoreVersionId = data.VersionId;
@@ -106,14 +151,15 @@ class AwsClient {
                 if (err.code === 'NotFound') {
                     const error = errors.ServiceUnavailable
                     .customizeDescription(
-                        'Unexpected error from AWS: "NotFound". Data on AWS ' +
+                        `Unexpected error from ${this.type}: ` +
+                        `"NotFound". Data on ${this.type} ` +
                         'may have been altered outside of CloudServer.'
                     );
                     return callback(error);
                 }
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             return callback();
@@ -128,12 +174,14 @@ class AwsClient {
             VersionId: dataStoreVersionId,
             Range: range ? `bytes=${range[0]}-${range[1]}` : null,
         }).on('success', response => {
-            log.trace('AWS GET request response headers',
-              { responseHeaders: response.httpResponse.headers });
+            log.trace(`${this.type} GET request response headers`,
+              { responseHeaders: response.httpResponse.headers,
+                backendType: this.clientType });
         });
         const stream = request.createReadStream().on('error', err => {
-            logHelper(log, 'error', 'error streaming data from AWS',
-              err, this._dataStoreName);
+            logHelper(log, 'error',
+              `error streaming data from ${this.type}`,
+              err, this._dataStoreName, this.clientType);
             return callback(err);
         });
         return callback(null, stream);
@@ -151,8 +199,8 @@ class AwsClient {
         return this._client.deleteObject(params, err => {
             if (err) {
                 logHelper(log, 'error', 'error deleting object from ' +
-                'datastore', err, this._dataStoreName);
-                if (err.code === 'NoSuchVersion') {
+                'datastore', err, this._dataStoreName, this.clientType);
+                if (err.code === 'NoSuchVersion' || err.code === 'NoSuchKey') {
                     // data may have been deleted directly from the AWS backend
                     // don't want to retry the delete and errors are not
                     // sent back to client anyway, so no need to return err
@@ -160,7 +208,7 @@ class AwsClient {
                 }
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             return callback();
@@ -176,6 +224,12 @@ class AwsClient {
                 awsResp[location] = { error: err, external: true };
                 return callback(null, awsResp);
             }
+            if (!this._supportsVersioning) {
+                awsResp[location] = {
+                    message: 'Congrats! You own the bucket',
+                };
+                return callback(null, awsResp);
+            }
             return this._client.getBucketVersioning({
                 Bucket: this._awsBucketName },
             (err, data) => {
@@ -208,14 +262,13 @@ class AwsClient {
                 metaHeadersTrimmed[headerKey] = metaHeaders[header];
             }
         });
-        Object.assign(metaHeaders, metaHeadersTrimmed);
         const awsBucket = this._awsBucketName;
         const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
         const params = {
             Bucket: awsBucket,
             Key: awsKey,
             WebsiteRedirectLocation: websiteRedirectHeader,
-            Metadata: metaHeaders,
+            Metadata: metaHeadersTrimmed,
             ContentType: contentType,
             CacheControl: cacheControl,
             ContentDisposition: contentDisposition,
@@ -224,10 +277,10 @@ class AwsClient {
         return this._client.createMultipartUpload(params, (err, mpuResObj) => {
             if (err) {
                 logHelper(log, 'error', 'err from data backend',
-                  err, this._dataStoreName);
+                  err, this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             return callback(null, mpuResObj);
@@ -252,10 +305,10 @@ class AwsClient {
         return this._client.uploadPart(params, (err, partResObj) => {
             if (err) {
                 logHelper(log, 'error', 'err from data backend ' +
-                  'on uploadPart', err, this._dataStoreName);
+                  'on uploadPart', err, this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             // Because we manually add quotes to ETag later, remove quotes here
@@ -280,10 +333,10 @@ class AwsClient {
         return this._client.listParts(params, (err, partList) => {
             if (err) {
                 logHelper(log, 'error', 'err from data backend on listPart',
-                  err, this._dataStoreName);
+                  err, this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             // build storedParts object to mimic Scality S3 backend returns
@@ -348,20 +401,20 @@ class AwsClient {
             if (err) {
                 if (mpuError[err.code]) {
                     logHelper(log, 'trace', 'err from data backend on ' +
-                    'completeMPU', err, this._dataStoreName);
+                    'completeMPU', err, this._dataStoreName, this.clientType);
                     return callback(errors[err.code]);
                 }
                 logHelper(log, 'error', 'err from data backend on ' +
-                'completeMPU', err, this._dataStoreName);
+                'completeMPU', err, this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
-            if (!completeMpuRes.VersionId) {
+            if (!completeMpuRes.VersionId && this._supportsVersioning) {
                 logHelper(log, 'error', 'missing version id for data ' +
                 'backend object', missingVerIdInternalError,
-                    this._dataStoreName);
+                    this._dataStoreName, this.clientType);
                 return callback(missingVerIdInternalError);
             }
             // need to get content length of new object to store
@@ -370,17 +423,18 @@ class AwsClient {
             (err, objHeaders) => {
                 if (err) {
                     logHelper(log, 'trace', 'err from data backend on ' +
-                    'headObject', err, this._dataStoreName);
+                    'headObject', err, this._dataStoreName, this.clientType);
                     return callback(errors.ServiceUnavailable
                       .customizeDescription('Error returned from ' +
-                      `AWS: ${err.message}`)
+                      `${this.type}: ${err.message}`)
                     );
                 }
                 // remove quotes from eTag because they're added later
                 completeObjData.eTag = completeMpuRes.ETag
                     .substring(1, completeMpuRes.ETag.length - 1);
                 completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
-                completeObjData.contentLength = objHeaders.ContentLength;
+                completeObjData.contentLength =
+                    Number.parseInt(objHeaders.ContentLength, 10);
                 return callback(null, completeObjData);
             });
         });
@@ -396,10 +450,11 @@ class AwsClient {
             if (err) {
                 logHelper(log, 'error', 'There was an error aborting ' +
                 'the MPU on AWS S3. You should abort directly on AWS S3 ' +
-                'using the same uploadId.', err, this._dataStoreName);
+                'using the same uploadId.', err, this._dataStoreName,
+                this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             return callback();
@@ -424,10 +479,11 @@ class AwsClient {
         return this._client.putObjectTagging(tagParams, err => {
             if (err) {
                 logHelper(log, 'error', 'error from data backend on ' +
-                  'putObjectTagging', err, this._dataStoreName);
+                  'putObjectTagging', err,
+                  this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             return callback();
@@ -446,10 +502,11 @@ class AwsClient {
         return this._client.deleteObjectTagging(tagParams, err => {
             if (err) {
                 logHelper(log, 'error', 'error from data backend on ' +
-                'deleteObjectTagging', err, this._dataStoreName);
+                'deleteObjectTagging', err,
+                this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             return callback();
@@ -482,24 +539,24 @@ class AwsClient {
             if (err) {
                 if (err.code === 'AccessDenied') {
                     logHelper(log, 'error', 'Unable to access ' +
-                    `${sourceAwsBucketName} AWS bucket`, err,
-                    this._dataStoreName);
+                    `${sourceAwsBucketName} ${this.type} bucket`, err,
+                    this._dataStoreName, this.clientType);
                     return callback(errors.AccessDenied
                       .customizeDescription('Error: Unable to access ' +
-                      `${sourceAwsBucketName} AWS bucket`)
+                      `${sourceAwsBucketName} ${this.type} bucket`)
                     );
                 }
                 logHelper(log, 'error', 'error from data backend on ' +
-                'copyObject', err, this._dataStoreName);
+                'copyObject', err, this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
-            if (!copyResult.VersionId) {
+            if (!copyResult.VersionId && this._supportsVersioning) {
                 logHelper(log, 'error', 'missing version id for data ' +
                 'backend object', missingVerIdInternalError,
-                    this._dataStoreName);
+                    this._dataStoreName, this.clientType);
                 return callback(missingVerIdInternalError);
             }
             return callback(null, destAwsKey, copyResult.VersionId);
@@ -532,17 +589,17 @@ class AwsClient {
                 if (err.code === 'AccessDenied') {
                     logHelper(log, 'error', 'Unable to access ' +
                     `${sourceAwsBucketName} AWS bucket`, err,
-                    this._dataStoreName);
+                    this._dataStoreName, this.clientType);
                     return callback(errors.AccessDenied
                       .customizeDescription('Error: Unable to access ' +
                       `${sourceAwsBucketName} AWS bucket`)
                     );
                 }
                 logHelper(log, 'error', 'error from data backend on ' +
-                'uploadPartCopy', err, this._dataStoreName);
+                'uploadPartCopy', err, this._dataStoreName, this.clientType);
                 return callback(errors.ServiceUnavailable
                   .customizeDescription('Error returned from ' +
-                  `AWS: ${err.message}`)
+                  `${this.type}: ${err.message}`)
                 );
             }
             const eTag = removeQuotes(res.CopyPartResult.ETag);

lib/data/external/AzureClient.js

@@ -9,8 +9,12 @@ const { validateAndFilterMpuParts } =
     require('../../api/apiUtils/object/processMpuParts');
 const constants = require('../../../constants');
 const metadata = require('../../metadata/wrapper');
+const packageVersion = require('../../../package.json').version;
 const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
 
+azure.Constants.USER_AGENT_PRODUCT_NAME = constants.productName;
+azure.Constants.USER_AGENT_PRODUCT_VERSION = packageVersion;
+
 class AzureClient {
     constructor(config) {
         this._azureStorageEndpoint = config.azureStorageEndpoint;
@@ -22,12 +26,16 @@ class AzureClient {
             this._azureStorageEndpoint);
         this._dataStoreName = config.dataStoreName;
         this._bucketMatch = config.bucketMatch;
-        if (config.proxy) {
-            const parsedUrl = url.parse(config.proxy);
+        if (config.proxy && config.proxy.url) {
+            const parsedUrl = url.parse(config.proxy.url);
             if (!parsedUrl.port) {
                 parsedUrl.port = 80;
             }
-            this._client.setProxy(parsedUrl);
+            const proxyParams = parsedUrl;
+            if (config.proxy.certs) {
+                Object.assign(proxyParams, config.proxy.certs);
+            }
+            this._client.setProxy(proxyParams);
         }
     }
 
@@ -102,6 +110,13 @@ class AzureClient {
         });
     }
 
+    toObjectGetInfo(objectKey, bucketName) {
+        return {
+            key: this._createAzureKey(bucketName, objectKey, this._bucketMatch),
+            dataStoreName: this._dataStoreName,
+        };
+    }
+
     put(stream, size, keyContext, reqUids, callback) {
         const log = createLogger(reqUids);
         // before blob is put, make sure there is no ongoing MPU with same key

lib/data/external/GCP/GcpApis/abortMPU.js

@@ -0,0 +1,33 @@
+const { errors } = require('arsenal');
+const MpuHelper = require('./mpuHelper');
+const { createMpuKey, logger } = require('../GcpUtils');
+const { logHelper } = require('../../utils');
+
+/**
+ * abortMPU - remove all objects of a GCP Multipart Upload
+ * @param {object} params - abortMPU params
+ * @param {string} params.Bucket - bucket name
+ * @param {string} params.MPU - mpu bucket name
+ * @param {string} params.Key - object key
+ * @param {number} params.UploadId - MPU upload id
+ * @param {function} callback - callback function to call
+ * @return {undefined}
+ */
+function abortMPU(params, callback) {
+    if (!params || !params.Key || !params.UploadId ||
+        !params.Bucket || !params.MPU) {
+        const error = errors.InvalidRequest
+            .customizeDescription('Missing required parameter');
+        logHelper(logger, 'error', 'error in abortMultipartUpload', error);
+        return callback(error);
+    }
+    const mpuHelper = new MpuHelper(this);
+    const delParams = {
+        Bucket: params.Bucket,
+        MPU: params.MPU,
+        Prefix: createMpuKey(params.Key, params.UploadId),
+    };
+    return mpuHelper.removeParts(delParams, callback);
+}
+
+module.exports = abortMPU;

lib/data/external/GCP/GcpApis/completeMPU.js

@@ -0,0 +1,69 @@
+const async = require('async');
+const { errors } = require('arsenal');
+const MpuHelper = require('./mpuHelper');
+const { createMpuKey, logger } = require('../GcpUtils');
+const { logHelper } = require('../../utils');
+
+/**
+ * completeMPU - merges a list of parts into a single object
+ * @param {object} params - completeMPU params
+ * @param {string} params.Bucket - bucket name
+ * @param {string} params.MPU - mpu bucket name
+ * @param {string} params.Key - object key
+ * @param {number} params.UploadId - MPU upload id
+ * @param {Object} params.MultipartUpload - MPU upload object
+ * @param {Object[]} param.MultipartUpload.Parts - a list of parts to merge
+ * @param {function} callback - callback function to call with MPU result
+ * @return {undefined}
+ */
+function completeMPU(params, callback) {
+    if (!params || !params.MultipartUpload ||
+        !params.MultipartUpload.Parts || !params.UploadId ||
+        !params.Bucket || !params.Key) {
+        const error = errors.InvalidRequest
+            .customizeDescription('Missing required parameter');
+        logHelper(logger, 'error', 'error in completeMultipartUpload', error);
+        return callback(error);
+    }
+    const partList = params.MultipartUpload.Parts;
+    // verify that the part list is in order
+    if (params.MultipartUpload.Parts.length === 0) {
+        const error = errors.InvalidRequest
+            .customizeDescription('You must specify at least one part');
+        logHelper(logger, 'error', 'error in completeMultipartUpload', error);
+        return callback(error);
+    }
+    for (let ind = 1; ind < partList.length; ++ind) {
+        if (partList[ind - 1].PartNumber >= partList[ind].PartNumber) {
+            logHelper(logger, 'error', 'error in completeMultipartUpload',
+                errors.InvalidPartOrder);
+            return callback(errors.InvalidPartOrder);
+        }
+    }
+    const mpuHelper = new MpuHelper(this); // this === GcpClient
+    return async.waterfall([
+        next => {
+            // first compose: in mpu bucket
+            // max 10,000 => 313 parts
+            // max component count per object 32
+            logger.trace('completeMultipartUpload: compose',
+                { partCount: partList.length });
+            mpuHelper.splitMerge(params, partList, 'compose', next);
+        },
+        (numParts, next) => mpuHelper.composeFinal(numParts, params, next),
+        (result, next) => mpuHelper.generateMpuResult(result, partList, next),
+        (result, aggregateETag, next) =>
+            mpuHelper.copyToMain(result, aggregateETag, params, next),
+        (mpuResult, next) => {
+            const delParams = {
+                Bucket: params.Bucket,
+                MPU: params.MPU,
+                Prefix: createMpuKey(params.Key, params.UploadId),
+            };
+            return mpuHelper.removeParts(delParams,
+                err => next(err, mpuResult));
+        },
+    ], callback);
+}
+
+module.exports = completeMPU;

lib/data/external/GCP/GcpApis/createMPU.js

@@ -0,0 +1,51 @@
+const uuid = require('uuid/v4');
+const { errors } = require('arsenal');
+const { createMpuKey, logger, getPutTagsMetadata } = require('../GcpUtils');
+const { logHelper } = require('../../utils');
+
+/**
+ * createMPU - creates a MPU upload on GCP (sets a 0-byte object placeholder
+ * with for the final composed object)
+ * @param {object} params - createMPU param
+ * @param {string} params.Bucket - bucket name
+ * @param {string} params.Key - object key
+ * @param {string} params.Metadata - object Metadata
+ * @param {string} params.ContentType - Content-Type header
+ * @param {string} params.CacheControl - Cache-Control header
+ * @param {string} params.ContentDisposition - Content-Disposition header
+ * @param {string} params.ContentEncoding - Content-Encoding header
+ * @param {function} callback - callback function to call with the generated
+ * upload-id for MPU operations
+ * @return {undefined}
+ */
+function createMPU(params, callback) {
+    // As google cloud does not have a create MPU function,
+    // create an empty 'init' object that will temporarily store the
+    // object metadata and return an upload ID to mimic an AWS MPU
+    if (!params || !params.Bucket || !params.Key) {
+        const error = errors.InvalidRequest
+            .customizeDescription('Missing required parameter');
+        logHelper(logger, 'error', 'error in createMultipartUpload', error);
+        return callback(error);
+    }
+    const uploadId = uuid().replace(/-/g, '');
+    const mpuParams = {
+        Bucket: params.Bucket,
+        Key: createMpuKey(params.Key, uploadId, 'init'),
+        Metadata: params.Metadata,
+        ContentType: params.ContentType,
+        CacheControl: params.CacheControl,
+        ContentDisposition: params.ContentDisposition,
+        ContentEncoding: params.ContentEncoding,
+    };
+    mpuParams.Metadata = getPutTagsMetadata(mpuParams.Metadata, params.Tagging);
+    return this.putObject(mpuParams, err => {
+        if (err) {
+            logHelper(logger, 'error', 'error in createMPU - putObject', err);
+            return callback(err);
+        }
+        return callback(null, { UploadId: uploadId });
+    });
+}
+
+module.exports = createMPU;

lib/data/external/GCP/GcpApis/deleteTagging.js

@@ -0,0 +1,24 @@
+const async = require('async');
+
+const { stripTags } = require('../GcpUtils');
+
+function deleteObjectTagging(params, callback) {
+    return async.waterfall([
+        next => this.headObject({
+            Bucket: params.Bucket,
+            Key: params.Key,
+            VersionId: params.VersionId,
+        }, next),
+        (resObj, next) => {
+            const completeMD = stripTags(resObj.Metadata);
+            this.copyObject({
+                Bucket: params.Bucket,
+                Key: params.Key,
+                CopySource: `${params.Bucket}/${params.Key}`,
+                Metadata: completeMD,
+                MetadataDirective: 'REPLACE',
+            }, next);
+        },
+    ], callback);
+}
+module.exports = deleteObjectTagging;

lib/data/external/GCP/GcpApis/getTagging.js

@@ -0,0 +1,19 @@
+const { retrieveTags } = require('../GcpUtils');
+
+function getObjectTagging(params, callback) {
+    const headParams = {
+        Bucket: params.Bucket,
+        Key: params.Key,
+        VersionId: params.VersionId,
+    };
+    this.headObject(headParams, (err, res) => {
+        const TagSet = retrieveTags(res.Metadata);
+        const retObj = {
+            VersionId: res.VersionId,
+            TagSet,
+        };
+        return callback(null, retObj);
+    });
+}
+
+module.exports = getObjectTagging;

lib/data/external/GCP/GcpApis/index.js

@@ -0,0 +1,14 @@
+module.exports = {
+    // mpu functions
+    abortMultipartUpload: require('./abortMPU'),
+    completeMultipartUpload: require('./completeMPU'),
+    createMultipartUpload: require('./createMPU'),
+    listParts: require('./listParts'),
+    uploadPart: require('./uploadPart'),
+    uploadPartCopy: require('./uploadPartCopy'),
+    // object tagging
+    putObject: require('./putObject'),
+    putObjectTagging: require('./putTagging'),
+    getObjectTagging: require('./getTagging'),
+    deleteObjectTagging: require('./deleteTagging'),
+};

lib/data/external/GCP/GcpApis/listParts.js

@@ -0,0 +1,42 @@
+const { errors } = require('arsenal');
+const { createMpuKey, logger } = require('../GcpUtils');
+const { logHelper } = require('../../utils');
+
+/**
+ * listParts - list uploaded MPU parts
+ * @param {object} params - listParts param
+ * @param {string} params.Bucket - bucket name
+ * @param {string} params.Key - object key
+ * @param {string} params.UploadId - MPU upload id
+ * @param {function} callback - callback function to call with the list of parts
+ * @return {undefined}
+ */
+function listParts(params, callback) {
+    if (!params || !params.UploadId || !params.Bucket || !params.Key) {
+        const error = errors.InvalidRequest
+            .customizeDescription('Missing required parameter');
+        logHelper(logger, 'error', 'error in listParts', error);
+        return callback(error);
+    }
+    if (params.PartNumberMarker && params.PartNumberMarker < 0) {
+        return callback(errors.InvalidArgument
+            .customizeDescription('The request specified an invalid marker'));
+    }
+    const mpuParams = {
+        Bucket: params.Bucket,
+        Prefix: createMpuKey(params.Key, params.UploadId, 'parts'),
+        Marker: createMpuKey(params.Key, params.UploadId,
+            params.PartNumberMarker, 'parts'),
+        MaxKeys: params.MaxParts,
+    };
+    return this.listObjects(mpuParams, (err, res) => {
+        if (err) {
+            logHelper(logger, 'error',
+                'error in listParts - listObjects', err);
+            return callback(err);
+        }
+        return callback(null, res);
+    });
+}
+
+module.exports = listParts;

lib/data/external/GCP/GcpApis/mpuHelper.js

@@ -0,0 +1,316 @@
+const async = require('async');
+const Backoff = require('backo');
+const { eachSlice, createMpuKey, createMpuList, logger } =
+    require('../GcpUtils');
+const { logHelper } = require('../../utils');
+const { createAggregateETag } =
+    require('../../../../api/apiUtils/object/processMpuParts');
+
+const BACKOFF_PARAMS = { min: 1000, max: 300000, jitter: 0.1, factor: 1.5 };
+
+class MpuHelper {
+    constructor(service, options = {}) {
+        this.service = service;
+        this.backoffParams = {
+            min: options.min || BACKOFF_PARAMS.min,
+            max: options.max || BACKOFF_PARAMS.max,
+            jitter: options.jitter || BACKOFF_PARAMS.jitter,
+            factor: options.factor || BACKOFF_PARAMS.factor,
+        };
+    }
+
+    _retry(fnName, params, callback) {
+        const backoff = new Backoff(this.backoffParams);
+        const handleFunc = (fnName, params, retry, callback) => {
+            const timeout = backoff.duration();
+            return setTimeout((params, cb) =>
+            this.service[fnName](params, cb), timeout, params,
+            (err, res) => {
+                if (err) {
+                    if (err.statusCode === 429 || err.code === 429) {
+                        if (fnName === 'composeObject') {
+                            logger.trace('composeObject: slow down request',
+                                { retryCount: retry, timeout });
+                        } else if (fnName === 'copyObject') {
+                            logger.trace('copyObject: slow down request',
+                                { retryCount: retry, timeout });
+                        }
+                        return handleFunc(
+                            fnName, params, retry + 1, callback);
+                    }
+                    logHelper(logger, 'error', `${fnName} failed`, err);
+                    return callback(err);
+                }
+                backoff.reset();
+                return callback(null, res);
+            });
+        };
+        handleFunc(fnName, params, 0, callback);
+    }
+
+    /**
+     * retryCompose - exponential backoff retry implementation for the compose
+     * operation
+     * @param {object} params - compose object params
+     * @param {function} callback - callback function to call with the result
+     * of the compose operation
+     * @return {undefined}
+     */
+    retryCompose(params, callback) {
+        this._retry('composeObject', params, callback);
+    }
+
+
+    /**
+     * retryCopy - exponential backoff retry implementation for the copy
+     * operation
+     * @param {object} params - copy object params
+     * @param {function} callback - callback function to call with the result
+     * of the copy operation
+     * @return {undefined}
+     */
+    retryCopy(params, callback) {
+        this._retry('copyObject', params, callback);
+    }
+
+    /**
+     * splitMerge - breaks down the MPU list of parts to be compose on GCP;
+     * splits partList into chunks of 32 objects, the limit of each compose
+     * operation.
+     * @param {object} params - complete MPU params
+     * @param {string} params.Bucket - bucket name
+     * @param {string} params.MPU - mpu bucket name
+     * @param {string} params.Key - object key
+     * @param {string} params.UploadId - MPU upload id
+     * @param {object[]} partList - list of parts for complete multipart upload
+     * @param {string} level - the phase name of the MPU process
+     * @param {function} callback - the callback function to call
+     * @return {undefined}
+     */
+    splitMerge(params, partList, level, callback) {
+        // create composition of slices from the partList array
+        return async.mapLimit(eachSlice.call(partList, 32),
+        this.service._maxConcurrent,
+        (infoParts, cb) => {
+            const mpuPartList = infoParts.Parts.map(item =>
+                ({ PartName: item.PartName }));
+            const partNumber = infoParts.PartNumber;
+            const tmpKey =
+                createMpuKey(params.Key, params.UploadId, partNumber, level);
+            const mergedObject = { PartName: tmpKey };
+            if (mpuPartList.length < 2) {
+                logger.trace(
+                    'splitMerge: parts are fewer than 2, copy instead');
+                // else just perform a copy
+                const copyParams = {
+                    Bucket: params.MPU,
+                    Key: tmpKey,
+                    CopySource: `${params.MPU}/${mpuPartList[0].PartName}`,
+                };
+                return this.service.copyObject(copyParams, (err, res) => {
+                    if (err) {
+                        logHelper(logger, 'error',
+                            'error in splitMerge - copyObject', err);
+                        return cb(err);
+                    }
+                    mergedObject.VersionId = res.VersionId;
+                    mergedObject.ETag = res.ETag;
+                    return cb(null, mergedObject);
+                });
+            }
+            const composeParams = {
+                Bucket: params.MPU,
+                Key: tmpKey,
+                MultipartUpload: { Parts: mpuPartList },
+            };
+            return this.retryCompose(composeParams, (err, res) => {
+                if (err) {
+                    return cb(err);
+                }
+                mergedObject.VersionId = res.VersionId;
+                mergedObject.ETag = res.ETag;
+                return cb(null, mergedObject);
+            });
+        }, (err, res) => {
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, res.length);
+        });
+    }
+
+    /**
+     * removeParts - remove all objects created to perform a multipart upload
+     * @param {object} params - remove parts params
+     * @param {string} params.Bucket - bucket name
+     * @param {string} params.MPU - mpu bucket name
+     * @param {string} params.Key - object key
+     * @param {string} params.UploadId - MPU upload id
+     * @param {function} callback - callback function to call
+     * @return {undefined}
+     */
+    removeParts(params, callback) {
+        const _getObjectVersions = callback => {
+            logger.trace('remove all parts from mpu bucket');
+            let partsList = [];
+            let isTruncated = true;
+            let nextMarker;
+            return async.whilst(() => isTruncated, next => {
+                const listParams = {
+                    Bucket: params.MPU,
+                    Prefix: params.Prefix,
+                    Marker: nextMarker,
+                };
+                return this.service.listVersions(listParams, (err, res) => {
+                    if (err) {
+                        logHelper(logger, 'error', 'error in ' +
+                            'removeParts - listVersions', err);
+                        return next(err);
+                    }
+                    nextMarker = res.NextMarker;
+                    isTruncated = res.IsTruncated;
+                    partsList = partsList.concat(res.Versions);
+                    return next();
+                });
+            }, err => callback(err, partsList));
+        };
+
+        const _deleteObjects = (partsList, callback) => {
+            logger.trace('successfully listed mpu parts', {
+                objectCount: partsList.length,
+            });
+            return async.eachLimit(partsList, 10, (obj, next) => {
+                const delParams = {
+                    Bucket: params.MPU,
+                    Key: obj.Key,
+                    VersionId: obj.VersionId,
+                };
+                this.service.deleteObject(delParams, err => {
+                    if (err) {
+                        logHelper(logger, 'error',
+                            'error deleting object', err);
+                        return next(err);
+                    }
+                    return next();
+                });
+            }, err => callback(err));
+        };
+
+        return async.waterfall([
+            _getObjectVersions,
+            _deleteObjects,
+        ], err => callback(err));
+    }
+
+    composeFinal(numParts, params, callback) {
+        // final compose:
+        // number of parts to compose <= 10
+        // perform final compose in mpu bucket
+        logger.trace('completeMultipartUpload: final compose');
+        const parts = createMpuList(params, 'compose', numParts);
+        const partList = parts.map(item => (
+            { PartName: item.PartName }));
+        if (partList.length < 2) {
+            logger.trace(
+                'fewer than 2 parts, skip to copy phase');
+            return callback(null, partList[0].PartName);
+        }
+        const composeParams = {
+            Bucket: params.MPU,
+            Key: createMpuKey(params.Key, params.UploadId, 'final'),
+            MultipartUpload: { Parts: partList },
+        };
+        return this.retryCompose(composeParams, err => {
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, null);
+        });
+    }
+
+    /*
+     * Create MPU Aggregate ETag
+     */
+    generateMpuResult(res, partList, callback) {
+        const concatETag = partList.reduce((prev, curr) =>
+            prev + curr.ETag.substring(1, curr.ETag.length - 1), '');
+        const aggregateETag = createAggregateETag(concatETag, partList);
+        return callback(null, res, aggregateETag);
+    }
+
+    copyToMain(res, aggregateETag, params, callback) {
+        // move object from mpu bucket into the main bucket
+        // retrieve initial metadata then compose the object
+        const copySource = res ||
+            createMpuKey(params.Key, params.UploadId, 'final');
+        return async.waterfall([
+            next => {
+                // retrieve metadata from init object in mpu bucket
+                const headParams = {
+                    Bucket: params.MPU,
+                    Key: createMpuKey(params.Key, params.UploadId,
+                        'init'),
+                };
+                logger.trace('retrieving object metadata');
+                return this.service.headObject(headParams, (err, res) => {
+                    if (err) {
+                        logHelper(logger, 'error',
+                            'error in createMultipartUpload - headObject',
+                            err);
+                        return next(err);
+                    }
+                    return next(null, res);
+                });
+            },
+            (res, next) => {
+                const metadata = res.Metadata;
+                // copy the final object into the main bucket
+                const copyMetadata = Object.assign({}, metadata);
+                copyMetadata['scal-etag'] = aggregateETag;
+                const copyParams = {
+                    Bucket: params.Bucket,
+                    Key: params.Key,
+                    Metadata: copyMetadata,
+                    MetadataDirective: 'REPLACE',
+                    CopySource: `${params.MPU}/${copySource}`,
+                    ContentType: res.ContentType,
+                    CacheControl: res.CacheControl,
+                    ContentEncoding: res.ContentEncoding,
+                    ContentDisposition: res.ContentDisposition,
+                    ContentLanguage: res.ContentLanguage,
+                };
+                logger.trace('copyParams', { copyParams });
+                this.retryCopy(copyParams, (err, res) => {
+                    if (err) {
+                        logHelper(logger, 'error', 'error in ' +
+                            'createMultipartUpload - final copyObject',
+                            err);
+                        return next(err);
+                    }
+                    const mpuResult = {
+                        Bucket: params.Bucket,
+                        Key: params.Key,
+                        VersionId: res.VersionId,
+                        ETag: `"${aggregateETag}"`,
+                    };
+                    return this.service.headObject({
+                        Bucket: params.Bucket,
+                        Key: params.Key,
+                        VersionId: res.VersionId,
+                    }, (err, res) => {
+                        if (err) {
+                            logHelper(logger, 'error', 'error in ' +
+                                'createMultipartUpload - final head object',
+                                err);
+                            return next(err);
+                        }
+                        mpuResult.ContentLength = res.ContentLength;
+                        return next(null, mpuResult);
+                    });
+                });
+            },
+        ], callback);
+    }
+}
+
+module.exports = MpuHelper;

lib/data/external/GCP/GcpApis/putObject.js

@@ -0,0 +1,11 @@
+const { getPutTagsMetadata } = require('../GcpUtils');
+
+function putObject(params, callback) {
+    const putParams = Object.assign({}, params);
+    putParams.Metadata = getPutTagsMetadata(putParams.Metadata, params.Tagging);
+    delete putParams.Tagging;
+    // error handling will be by the actual putObject request
+    return this.putObjectReq(putParams, callback);
+}
+
+module.exports = putObject;