Use fs.readFileSync to read config file instead of require

FIXME: Ideally, both locations and storage classes should be supported

.dockerignore

@@ -1,3 +1,9 @@
-.git
-.github
 node_modules
+localData/*
+localMetadata/*
+# Keep the .git/HEAD file in order to properly report version
+.git/objects
+.github
+.tox
+coverage
+.DS_Store

.eslintrc

@@ -1,10 +1,54 @@
 {
     "extends": "scality",
+    "plugins": [
+        "mocha"
+    ],
+    "rules": {
+        "import/extensions": "off",
+        "lines-around-directive": "off",
+        "no-underscore-dangle": "off",
+        "indent": "off",
+        "object-curly-newline": "off",
+        "operator-linebreak": "off",
+        "function-paren-newline": "off",
+        "import/newline-after-import": "off",
+        "prefer-destructuring": "off",
+        "implicit-arrow-linebreak": "off",
+        "no-bitwise": "off",
+        "dot-location": "off",
+        "comma-dangle": "off",
+        "no-undef-init": "off",
+        "global-require": "off",
+        "import/no-dynamic-require": "off",
+        "class-methods-use-this": "off",
+        "no-plusplus": "off",
+        "no-else-return": "off",
+        "object-property-newline": "off",
+        "import/order": "off",
+        "no-continue": "off",
+        "no-tabs": "off",
+        "lines-between-class-members": "off",
+        "prefer-spread": "off",
+        "no-lonely-if": "off",
+        "no-useless-escape": "off",
+        "no-restricted-globals": "off",
+        "no-buffer-constructor": "off",
+        "import/no-extraneous-dependencies": "off",
+        "space-unary-ops": "off",
+        "no-useless-return": "off",
+        "no-unexpected-multiline": "off",
+        "no-mixed-operators": "off",
+        "newline-per-chained-call": "off",
+        "operator-assignment": "off",
+        "spaced-comment": "off",
+        "comma-style": "off",
+        "no-restricted-properties": "off",
+        "new-parens": "off",
+        "no-multi-spaces": "off",
+        "quote-props": "off",
+        "mocha/no-exclusive-tests": "error",
+    },
     "parserOptions": {
-        "ecmaVersion": 2018,
-        "sourceType": "module",
-        "ecmaFeatures": {
-            "jsx": true
-        }
+        "ecmaVersion": 2020
     }
 }

.github/ISSUE_TEMPLATE.md

@@ -1,19 +1,32 @@
-# Issue template
+# General support information
 
-If you are reporting a new issue, make sure that we do not have any
-duplicates already open. You can ensure this by searching the issue list for
-this repository. If there is a duplicate, please close your issue and add a
-comment to the existing issue instead.
+GitHub Issues are **reserved** for actionable bug reports (including
+documentation inaccuracies), and feature requests.
+**All questions** (regarding configuration, use cases, performance, community,
+events, setup and usage recommendations, among other things) should be asked on
+the **[Zenko Forum](http://forum.zenko.io/)**.
 
-## General support information
+> Questions opened as GitHub issues will systematically be closed, and moved to
+> the [Zenko Forum](http://forum.zenko.io/).
 
-GitHub Issues are reserved for actionable bug reports and feature requests.
-General questions should be sent to the
-[S3 scality server Forum](http://forum.scality.com/).
+--------------------------------------------------------------------------------
+
+## Avoiding duplicates
+
+When reporting a new issue/requesting a feature, make sure that we do not have
+any duplicates already open:
+
+- search the issue list for this repository (use the search bar, select
+  "Issues" on the left pane after searching);
+- if there is a duplicate, please do not open your issue, and add a comment
+  to the existing issue instead.
+
+--------------------------------------------------------------------------------
 
 ## Bug report information
 
-(delete this section if not applicable)
+(delete this section (everything between the lines) if you're not reporting a bug
+but requesting a feature)
 
 ### Description
 
@@ -29,13 +42,22 @@ Describe the results you received
 
 ### Expected result
 
-Describe the results you expecteds
+Describe the results you expected
 
-### Additional information: (Node.js version, Docker version, etc)
+### Additional information
+
+- Node.js version,
+- Docker version,
+- yarn version,
+- distribution/OS,
+- optional: anything else you deem helpful to us.
+
+--------------------------------------------------------------------------------
 
 ## Feature Request
 
-(delete this section if not applicable)
+(delete this section (everything between the lines) if you're not requesting
+a feature but reporting a bug)
 
 ### Proposal
 
@@ -52,3 +74,14 @@ What you would like to happen
 ### Use case
 
 Please provide use cases for changing the current behavior
+
+### Additional information
+
+- Is this request for your company? Y/N
+  - If Y: Company name:
+  - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N
+- Are you willing to contribute this feature yourself?
+- Position/Title:
+- How did you hear about us?
+
+--------------------------------------------------------------------------------

.github/actions/setup-ci/action.yaml

@@ -15,7 +15,7 @@ runs:
       shell: bash
       run: |-
         set -exu;
-        mkdir -p /tmp/artifacts/${{ github.job }}/;
+        mkdir -p /tmp/artifacts/${JOB_NAME}/;
     - uses: actions/setup-node@v4
       with:
         node-version: '16'
@@ -23,3 +23,21 @@ runs:
     - name: install dependencies
       shell: bash
       run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
+    - uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Setup python2 test environment
+      shell: bash
+      run: |
+        sudo apt-get install -y libdigest-hmac-perl
+        pip install 's3cmd==2.3.0'
+    - name: fix sproxyd.conf permissions
+      shell: bash
+      run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
+    - name: ensure fuse kernel module is loaded (for sproxyd)
+      shell: bash
+      run: sudo modprobe fuse

.github/ceph/Dockerfile

@@ -0,0 +1,25 @@
+FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
+
+ENV CEPH_DAEMON demo
+ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
+
+ENV CEPH_DEMO_UID zenko
+ENV CEPH_DEMO_ACCESS_KEY accessKey1
+ENV CEPH_DEMO_SECRET_KEY verySecretKey1
+ENV CEPH_DEMO_BUCKET zenkobucket
+
+ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
+ENV MON_IP 0.0.0.0
+ENV NETWORK_AUTO_DETECT 4
+ENV RGW_CIVETWEB_PORT 8001
+
+RUN rm /etc/yum.repos.d/tcmu-runner.repo
+
+ADD ./entrypoint-wrapper.sh /
+RUN chmod +x /entrypoint-wrapper.sh && \
+    yum install -y python-pip && \
+    yum clean all && \
+    pip install awscli && \
+    rm -rf /root/.cache/pip
+
+ENTRYPOINT [ "/entrypoint-wrapper.sh" ]

.github/ceph/entrypoint-wrapper.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+touch /artifacts/ceph.log
+mkfifo /tmp/entrypoint_output
+# We run this in the background so that we can tail the RGW log after init,
+# because entrypoint.sh never returns
+
+# The next line will be needed when ceph builds 3.2.2 so I'll leave it here
+# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
+
+bash /entrypoint.sh > /tmp/entrypoint_output &
+entrypoint_pid="$!"
+while read -r line; do
+    echo $line
+    # When we find this line server has started
+    if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
+        break
+    fi
+done < /tmp/entrypoint_output
+
+# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
+s3cmd mb s3://cephbucket s3://cephbucket2
+
+mkdir /root/.aws
+cat > /root/.aws/credentials <<EOF
+[default]
+aws_access_key_id = accessKey1
+aws_secret_access_key = verySecretKey1
+EOF
+
+# Enable versioning on them
+for bucket in cephbucket cephbucket2; do
+    echo "Enabling versiong for $bucket"
+    aws --endpoint http://127.0.0.1:8001 s3api  put-bucket-versioning --bucket $bucket --versioning Status=Enabled
+done
+tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
+wait $entrypoint_pid

.github/ceph/wait_for_ceph.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# This script is needed because RADOS Gateway
+# will open the port before beginning to serve traffic
+# causing wait_for_local_port.bash to exit immediately
+
+echo 'Waiting for ceph'
+while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
+    sleep 1
+    echo -n "."
+done

.github/docker/creds.env

@@ -34,4 +34,3 @@ gcpbackendmismatch_GCP_SERVICE_KEY
 gcpbackend_GCP_SERVICE_KEYFILE
 gcpbackendmismatch_GCP_SERVICE_KEYFILE
 gcpbackendnoproxy_GCP_SERVICE_KEYFILE
-gcpbackendproxy_GCP_SERVICE_KEYFILE

.github/docker/docker-compose.yaml

@@ -20,6 +20,7 @@ services:
       - METADATA_HOST=0.0.0.0
       - S3BACKEND
       - S3DATA
+      - S3METADATA
       - MPU_TESTING
       - S3VAULT
       - S3_LOCATION_FILE
@@ -34,7 +35,16 @@ services:
       - S3KMIP_KEY
       - S3KMIP_CERT
       - S3KMIP_CA
+      - MONGODB_HOSTS=0.0.0.0:27018
+      - MONGODB_RS=rs0
+      - DEFAULT_BUCKET_KEY_FORMAT
+      - METADATA_MAX_CACHED_BUCKETS
       - ENABLE_NULL_VERSION_COMPAT_MODE
+      - SCUBA_HOST
+      - SCUBA_PORT
+      - SCUBA_HEALTHCHECK_FREQUENCY
+      - S3QUOTA
+      - QUOTA_ENABLE_INFLIGHTS
     env_file:
       - creds.env
     depends_on:
@@ -65,3 +75,18 @@ services:
     image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
     volumes:
       - /tmp/artifacts/${JOB_NAME}:/artifacts
+  mongo:
+    network_mode: "host"
+    profiles: ['mongo', 'ceph']
+    image: ${MONGODB_IMAGE}
+  ceph: 
+    network_mode: "host"
+    profiles: ['ceph']
+    image: ghcr.io/scality/cloudserver/ci-ceph 
+  sproxyd:
+    network_mode: "host"
+    profiles: ['sproxyd']
+    image: sproxyd-standalone
+    build: ./sproxyd
+    user: 0:0
+    privileged: yes

.github/docker/mongodb/Dockerfile

@@ -0,0 +1,28 @@
+FROM mongo:5.0.21
+
+ENV USER=scality \
+    HOME_DIR=/home/scality \
+    CONF_DIR=/conf \
+    DATA_DIR=/data
+
+# Set up directories and permissions
+RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
+    mkdir /logs; \
+    adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
+
+# Set up environment variables and directories for scality user
+RUN mkdir ${CONF_DIR} && \
+    chown -R ${USER} ${CONF_DIR} && \
+    chown -R ${USER} ${DATA_DIR}
+
+# copy the mongo config file
+COPY /conf/mongod.conf /conf/mongod.conf
+COPY /conf/mongo-run.sh /conf/mongo-run.sh
+COPY /conf/initReplicaSet /conf/initReplicaSet.js
+
+EXPOSE 27017/tcp
+EXPOSE 27018
+
+# Set up CMD
+ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
+CMD ["bash", "/conf/mongo-run.sh"]

.github/docker/mongodb/conf/initReplicaSet

@@ -0,0 +1,4 @@
+rs.initiate({
+    _id: "rs0",
+    members: [{ _id: 0, host: "127.0.0.1:27018" }]
+});

.github/docker/mongodb/conf/mongo-run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -exo pipefail
+
+init_RS() {
+  sleep 5
+  mongo --port 27018 /conf/initReplicaSet.js
+}
+init_RS &
+
+mongod --bind_ip_all --config=/conf/mongod.conf

.github/docker/mongodb/conf/mongod.conf

@@ -0,0 +1,15 @@
+storage:
+   journal:
+      enabled: true
+   engine: wiredTiger
+   dbPath: "/data/db"
+processManagement:
+   fork: false
+net:
+   port: 27018
+   bindIp: 0.0.0.0
+replication:
+   replSetName: "rs0"
+   enableMajorityReadConcern: true
+security:
+    authorization: disabled

.github/docker/sproxyd/Dockerfile

@@ -0,0 +1,3 @@
+FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
+ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
+RUN chown root:root /conf/sproxyd0.conf

.github/docker/sproxyd/conf/fastcgi_params

@@ -0,0 +1,26 @@
+fastcgi_param  QUERY_STRING       $query_string;
+fastcgi_param  REQUEST_METHOD     $request_method;
+fastcgi_param  CONTENT_TYPE       $content_type;
+fastcgi_param  CONTENT_LENGTH     $content_length;
+
+#fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;
+fastcgi_param  SCRIPT_NAME        /var/www;
+fastcgi_param  PATH_INFO          $document_uri;
+
+fastcgi_param  REQUEST_URI        $request_uri;
+fastcgi_param  DOCUMENT_URI       $document_uri;
+fastcgi_param  DOCUMENT_ROOT      $document_root;
+fastcgi_param  SERVER_PROTOCOL    $server_protocol;
+fastcgi_param  HTTPS              $https if_not_empty;
+
+fastcgi_param  GATEWAY_INTERFACE  CGI/1.1;
+fastcgi_param  SERVER_SOFTWARE    nginx/$nginx_version;
+
+fastcgi_param  REMOTE_ADDR        $remote_addr;
+fastcgi_param  REMOTE_PORT        $remote_port;
+fastcgi_param  SERVER_ADDR        $server_addr;
+fastcgi_param  SERVER_PORT        $server_port;
+fastcgi_param  SERVER_NAME        $server_name;
+
+# PHP only, required if PHP was built with --enable-force-cgi-redirect
+fastcgi_param  REDIRECT_STATUS    200;

.github/docker/sproxyd/conf/nginx.conf

@@ -0,0 +1,88 @@
+worker_processes 1;
+error_log  /logs/error.log;
+user root root;
+events {
+    worker_connections  1000;
+    reuse_port on;
+    multi_accept on;
+}
+worker_rlimit_nofile    20000;
+http {
+    root /var/www/;
+    upstream sproxyds {
+        least_conn;
+        keepalive 40;
+        server 127.0.0.1:20000;
+    }
+    server {
+        client_max_body_size 0;
+        client_body_timeout 150;
+        client_header_timeout 150;
+        postpone_output 0;
+        client_body_postpone_size 0;
+        keepalive_requests 1100;
+        keepalive_timeout 300s;
+        server_tokens off;
+        default_type application/octet-stream;
+        gzip off;
+        tcp_nodelay on;
+        tcp_nopush  on;
+        sendfile on;
+        listen 81;
+        server_name localhost;
+        rewrite ^/arc/(.*)$ /dc1/$1 permanent;
+        location ~* ^/proxy/(.*)$ {
+            rewrite ^/proxy/(.*)$ /$1 last;
+        }
+        allow 127.0.0.1;
+
+        deny all;
+        set $usermd '-';
+        set $sentusermd '-';
+    	set $elapsed_ms '-';
+    	set $now '-';
+    	log_by_lua '
+    		if not(ngx.var.http_x_scal_usermd == nil) and  string.len(ngx.var.http_x_scal_usermd) > 2 then
+    			ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
+    		end
+                    if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
+                            ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
+                    end
+    		local elapsed_ms = tonumber(ngx.var.request_time)
+    		if not (  elapsed_ms == nil) then
+    			elapsed_ms = elapsed_ms * 1000
+    			ngx.var.elapsed_ms = tostring(elapsed_ms)
+    		end
+    		local time = tonumber(ngx.var.msec) * 1000
+    		ngx.var.now = time
+    	';
+        log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
+                       '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
+                       '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
+		       '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
+		       '"contentType":"$content_type","s3Address":"$remote_addr",'
+		       '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
+                       '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
+		       '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
+		       '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
+                       '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
+                       '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
+        access_log /dev/stdout irm;
+    	error_log  /dev/stdout error;
+        location / {
+            proxy_request_buffering off;
+            fastcgi_request_buffering off;
+            fastcgi_no_cache 1;
+            fastcgi_cache_bypass 1;
+            fastcgi_buffering off;
+            fastcgi_ignore_client_abort on;
+            fastcgi_keep_conn on;
+            include fastcgi_params;
+            fastcgi_pass sproxyds;
+            fastcgi_next_upstream error timeout;
+            fastcgi_send_timeout 285s;
+            fastcgi_read_timeout 285s;
+        }
+    }
+}
+

.github/docker/sproxyd/conf/sproxyd0.conf

@@ -0,0 +1,12 @@
+{
+    "general": {
+        "ring": "DATA",
+        "port": 20000,
+        "syslog_facility": "local0"
+    },
+    "ring_driver:0": {
+        "alias": "dc1",
+        "type": "local",
+        "queue_path": "/tmp/ring-objs"
+    },
+}

.github/docker/sproxyd/conf/supervisord.conf

@@ -0,0 +1,43 @@
+[supervisord]
+nodaemon = true
+loglevel = info
+logfile  = %(ENV_LOG_DIR)s/supervisord.log
+pidfile  = %(ENV_SUP_RUN_DIR)s/supervisord.pid
+logfile_maxbytes = 20MB
+logfile_backups = 2
+
+[unix_http_server]
+file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[program:nginx]
+directory=%(ENV_SUP_RUN_DIR)s
+command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart=true
+autostart=true
+user=root
+
+[program:sproxyd]
+directory=%(ENV_SUP_RUN_DIR)s
+process_name=%(program_name)s-%(process_num)s
+numprocs=1
+numprocs_start=0
+command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+redirect_stderr=true
+autorestart=true
+autostart=true
+user=root

.github/workflows/alerts.yaml

@@ -0,0 +1,35 @@
+name: Test alerts
+
+on:
+  push:
+    branches-ignore:
+      - 'development/**'
+      - 'q/*/**'
+
+jobs:
+  run-alert-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tests:
+          - name: 1 minute interval tests
+            file: monitoring/alerts.test.yaml
+          
+          - name: 10 seconds interval tests
+            file: monitoring/alerts.10s.test.yaml
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Render and test ${{ matrix.tests.name }}
+        uses: scality/action-prom-render-test@1.0.3
+        with:
+          alert_file_path: monitoring/alerts.yaml
+          test_file_path: ${{ matrix.tests.file }}
+          alert_inputs: |
+            namespace=zenko
+            service=artesca-data-connector-s3api-metrics
+            reportJob=artesca-data-ops-report-handler
+            replicas=3
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -9,6 +9,9 @@ on:
         description: 'Tag to be released'
         required: true
 
+env:
+  PROJECT_NAME: ${{ github.event.repository.name }}
+
 jobs:
   build-federation-image:
     runs-on: ubuntu-20.04
@@ -34,32 +37,38 @@ jobs:
           cache-from: type=gha,scope=federation
           cache-to: type=gha,mode=max,scope=federation
 
-  build-image:
-    runs-on: ubuntu-20.04
+  release:
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up Docker Buildx
+
+      - name: Set up Docker Buildk
         uses: docker/setup-buildx-action@v3
-      - name: Login to GitHub Registry
+
+      - name: Login to Registry
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ github.token }}
-      - name: Build and push image
+
+      - name: Push dashboards into the production namespace
+        run: |
+          oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
+            dashboard.json:application/grafana-dashboard+json \
+            alerts.yaml:application/prometheus-alerts+yaml
+        working-directory: monitoring
+
+      - name: Build and push
         uses: docker/build-push-action@v5
         with:
-          push: true
           context: .
-          tags: |
-            ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
+          push: true
+          tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-  github-release:
-    runs-on: ubuntu-latest
-    steps:
       - name: Create Release
         uses: softprops/action-gh-release@v2
         env:

.github/workflows/tests.yaml

@@ -2,6 +2,8 @@
 name: tests
 
 on:
+  workflow_dispatch:
+
   push:
     branches-ignore:
       - 'development/**'
@@ -126,6 +128,9 @@ jobs:
 
   build:
     runs-on: ubuntu-20.04
+    permissions:
+      contents: read
+      packages: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -162,39 +167,21 @@ jobs:
             git.commit-sha=${{ github.sha }}
           cache-from: type=gha,scope=pykmip
           cache-to: type=gha,mode=max,scope=pykmip
-
-  build-federation-image:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to GitHub Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ github.token }}
-      - name: Build and push image for federation
+      - name: Build and push MongoDB
         uses: docker/build-push-action@v5
         with:
           push: true
-          context: .
-          file: images/svc-base/Dockerfile
-          tags: |
-            ghcr.io/${{ github.repository }}:${{ github.sha }}-svc-base
-          labels: |
-            git.repository=${{ github.repository }}
-            git.commit-sha=${{ github.sha }}
-          cache-from: type=gha,scope=federation
-          cache-to: type=gha,mode=max,scope=federation
+          context: .github/docker/mongodb
+          tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+          cache-from: type=gha,scope=mongodb
+          cache-to: type=gha,mode=max,scope=mongodb
 
   multiple-backend:
     runs-on: ubuntu-latest
     needs: build
     env:
       CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       S3BACKEND: mem
       S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
       S3DATA: multiple
@@ -202,18 +189,22 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
+    - name: Login to Registry
+      uses: docker/login-action@v3
       with:
-        python-version: 3.9
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Setup CI services
-      run: docker compose up -d
+      run: docker compose --profile sproxyd up -d
       working-directory: .github/docker
     - name: Run multiple backend test
       run: |-
         set -o pipefail;
         bash wait_for_local_port.bash 8000 40
+        bash wait_for_local_port.bash 81 40
         yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
       env:
         S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
@@ -227,14 +218,89 @@ jobs:
         source: /tmp/artifacts
       if: always()
  
+  mongo-v0-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v0
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  mongo-v1-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v1
+      METADATA_MAX_CACHED_BUCKETS: 1
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;        
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+        yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
   file-ft-tests:
     strategy:
       matrix:
         include:
-          - enable-null-compat: ''
-            job-name: file-ft-tests
-          - enable-null-compat: 'true'
-            job-name: file-ft-tests-null-compat
+          - job-name: file-ft-tests
     name: ${{ matrix.job-name }}
     runs-on: ubuntu-latest
     needs: build
@@ -242,15 +308,12 @@ jobs:
       S3BACKEND: file
       S3VAULT: mem
       CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       MPU_TESTING: "yes"
-      ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
       JOB_NAME: ${{ matrix.job-name }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Setup matrix job artifacts directory
@@ -258,10 +321,6 @@ jobs:
       run: |
         set -exu
         mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
-    - name: Setup python test environment
-      run: |
-        sudo apt-get install -y libdigest-hmac-perl
-        pip install 's3cmd==2.3.0'
     - name: Setup CI services
       run: docker compose up -d
       working-directory: .github/docker
@@ -288,13 +347,11 @@ jobs:
       S3BACKEND: mem
       BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
       CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Setup CI services
@@ -315,22 +372,64 @@ jobs:
         source: /tmp/artifacts
       if: always()
 
+  quota-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        inflights:
+          - name: "With Inflights"
+            value: "true"
+          - name: "Without Inflights"
+            value: "false"
+    env:
+      S3METADATA: mongodb
+      S3BACKEND: mem
+      S3QUOTA: scuba
+      QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
+      SCUBA_HOST: localhost
+      SCUBA_PORT: 8100
+      SCUBA_HEALTHCHECK_FREQUENCY: 100
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run quota tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
   kmip-ft-tests:
     runs-on: ubuntu-latest
     needs: build
     env:
       S3BACKEND: file
       S3VAULT: mem
-      MPU_TESTING: true
+      MPU_TESTING: "yes" 
       CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
       PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Copy KMIP certs
@@ -354,3 +453,81 @@ jobs:
         password: ${{ secrets.ARTIFACTS_PASSWORD }}
         source: /tmp/artifacts
       if: always()
+
+  ceph-backend-test:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      S3DATA: multiple
+      S3KMS: file
+      CI_CEPH: 'true'
+      MPU_TESTING: "yes" 
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Login to GitHub Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '2.5.9'
+    - name: Install Ruby dependencies
+      run: |
+        gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
+    - name: Install Java dependencies
+      run: |
+        sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
+    - name: Setup CI services
+      run: docker compose --profile ceph up -d
+      working-directory: .github/docker
+      env: 
+        S3METADATA: mongodb
+    - name: Run Ceph multiple backend tests
+      run: |-
+        set -ex -o pipefail;
+        bash .github/ceph/wait_for_ceph.sh
+        bash wait_for_local_port.bash 27018 40
+        bash wait_for_local_port.bash 8000 40
+        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+        S3METADATA: mem
+    - name: Run Java tests
+      run: |-
+        set -ex -o pipefail;
+        mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
+      working-directory: tests/functional/jaws
+    - name: Run Ruby tests
+      run: |-
+        set -ex -o pipefail;
+        rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
+      working-directory: tests/functional/fog
+    - name: Run Javascript AWS SDK tests
+      run: |-
+        set -ex -o pipefail;
+        yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
+        yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
+        S3BACKEND: file
+        S3VAULT: mem
+        S3METADATA: mongodb
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()

.gitignore

@@ -22,9 +22,14 @@ coverage
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
 
+# Sphinx build dir
+_build
+
 # Dependency directory
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
+yarn.lock
+.tox
 
 # Junit directory
 junit

Dockerfile

@@ -1,22 +1,19 @@
-FROM node:16.13.2-slim
-MAINTAINER Giorgio Regni <gr@scality.com>
+ARG NODE_VERSION=16.20-bullseye-slim
+
+FROM node:${NODE_VERSION} as builder
 
 WORKDIR /usr/src/app
 
-# Keep the .git directory in order to properly report version
-COPY ./package.json yarn.lock ./
-
-ENV PYTHON=python3.9
-ENV PY_VERSION=3.9.7
-
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
-    jq \
-    python \
-    git \
         build-essential \
-    ssh \
         ca-certificates \
+        curl \
+        git \
+        gnupg2 \
+        jq \
+        python3 \
+        ssh \
         wget \
         libffi-dev \
         zlib1g-dev \
@@ -24,28 +21,40 @@ RUN apt-get update \
     && mkdir -p /root/ssh \
     && ssh-keyscan -H github.com > /root/ssh/known_hosts
 
-RUN cd /tmp \
-    && wget https://www.python.org/ftp/python/$PY_VERSION/Python-$PY_VERSION.tgz \
-    && tar -C /usr/local/bin -xzvf Python-$PY_VERSION.tgz \
-    && cd /usr/local/bin/Python-$PY_VERSION \
-    && ./configure --enable-optimizations \
-    && make \
-    && make altinstall \
-    && rm -rf /tmp/Python-$PY_VERSION.tgz
+ENV PYTHON=python3
+COPY package.json yarn.lock /usr/src/app/
+RUN npm install typescript -g
+RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
 
-RUN yarn cache clean \
-    && yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
-    && apt-get autoremove --purge -y python git build-essential \
-    && rm -rf /var/lib/apt/lists/* \
-    && yarn cache clean \
-    && rm -rf ~/.node-gyp \
-    && rm -rf /tmp/yarn-*
+################################################################################
+FROM node:${NODE_VERSION}
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        jq \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV NO_PROXY localhost,127.0.0.1
+ENV no_proxy localhost,127.0.0.1
+
+EXPOSE 8000
+EXPOSE 8002
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        jq \
+        tini \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /usr/src/app
+
+# Keep the .git directory in order to properly report version
+COPY . /usr/src/app
+COPY --from=builder /usr/src/app/node_modules ./node_modules/
 
-COPY ./ ./
 
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
-ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
-CMD [ "yarn", "start" ]
+ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
 
-EXPOSE 8000 8002
+CMD [ "yarn", "start" ]

Healthchecks.md

@@ -1,6 +1,7 @@
 # S3 Healthcheck
 
-Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
+Scality S3 exposes a healthcheck route `/live` on the port used
+for the metrics (defaults to port 8002) which returns a
 response with HTTP code
 
 - 200 OK

README.md

@@ -1,12 +1,7 @@
-# Zenko CloudServer
+# Zenko CloudServer with Vitastor Backend
 
 ![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
 
-[![CircleCI][badgepub]](https://circleci.com/gh/scality/S3)
-[![Scality CI][badgepriv]](http://ci.ironmann.io/gh/scality/S3)
-[![Docker Pulls][badgedocker]](https://hub.docker.com/r/scality/s3server/)
-[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
-
 ## Overview
 
 CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
@@ -16,125 +11,71 @@ Scality’s Open Source Multi-Cloud Data Controller.
 CloudServer provides a single AWS S3 API interface to access multiple
 backend data storage both on-premise or public in the cloud.
 
-CloudServer is useful for Developers, either to run as part of a
-continous integration test environment to emulate the AWS S3 service locally
-or as an abstraction layer to develop object storage enabled
-application on the go.
+This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
+backend support.
 
-## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
+## Quick Start with Vitastor
 
-## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
+Vitastor Backend is in experimental status, however you can already try to
+run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
+it works too 😊.
 
-## Docker
+Installation instructions:
 
-[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/scality/s3server/)
+### Install Vitastor
 
-## Contributing
+Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
 
-In order to contribute, please follow the
-[Contributing Guidelines](
-https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
+### Install Zenko with Vitastor Backend
 
-## Installation
+- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
+- Install dependencies: `npm install --omit dev` or just `npm install`
+- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
+- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
+  You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
+- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
 
-### Dependencies
+### Install and Configure MongoDB
 
-Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
-. Up-to-date versions can be found at
-[Nodesource](https://github.com/nodesource/distributions).
+Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
 
-### Clone source code
+### Setup Zenko
 
-```shell
-git clone https://github.com/scality/S3.git
+- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
+- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
+- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
+- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
+- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
+  and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
+  access keys, but it's not published, so let's use a file for now.
+- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
+  You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
+  in this file.
+
+Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
+instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
+
+### Start Zenko
+
+Start the S3 server with: `node index.js`
+
+If you use default settings, Zenko CloudServer starts on port 8000.
+The default access key is `accessKey1` with a secret key of `verySecretKey1`.
+
+Now you can access your S3 with `s3cmd` or `geesefs`:
+
+```
+s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
 ```
 
-### Install js dependencies
-
-Go to the ./S3 folder,
-
-```shell
-yarn install --frozen-lockfile
+```
+AWS_ACCESS_KEY_ID=accessKey1 \
+    AWS_SECRET_ACCESS_KEY=verySecretKey1 \
+    geesefs --endpoint http://localhost:8000 testbucket mountdir
 ```
 
-If you get an error regarding installation of the diskUsage module,
-please install g++.
+# Author & License
 
-If you get an error regarding level-down bindings, try clearing your yarn cache:
-
-```shell
-yarn cache clean
-```
-
-## Run it with a file backend
-
-```shell
-yarn start
-```
-
-This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
-9991 are also open locally for internal transfer of metadata and data,
-respectively.
-
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-By default the metadata files will be saved in the
-localMetadata directory and the data files will be saved
-in the localData directory within the ./S3 directory on your
-machine.  These directories have been pre-created within the
-repository.  If you would like to save the data or metadata in
-different locations of your choice, you must specify them with absolute paths.
-So, when starting the server:
-
-```shell
-mkdir -m 700 $(pwd)/myFavoriteDataPath
-mkdir -m 700 $(pwd)/myFavoriteMetadataPath
-export S3DATAPATH="$(pwd)/myFavoriteDataPath"
-export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
-yarn start
-```
-
-## Run it with multiple data backends
-
-```shell
-export S3DATA='multiple'
-yarn start
-```
-
-This starts a Zenko CloudServer on port 8000.
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-With multiple backends, you have the ability to
-choose where each object will be saved by setting
-the following header with a locationConstraint on
-a PUT request:
-
-```shell
-'x-amz-meta-scal-location-constraint':'myLocationConstraint'
-```
-
-If no header is sent with a PUT object request, the
-location constraint of the bucket will determine
-where the data is saved. If the bucket has no location
-constraint, the endpoint of the PUT request will be
-used to determine location.
-
-See the Configuration section below to learn how to set
-location constraints.
-
-## Run it with an in-memory backend
-
-```shell
-yarn run mem_backend
-```
-
-This starts a Zenko CloudServer on port 8000.
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
-[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
-[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
-[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
+- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
+- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
+  (a "network copyleft" license based on AGPL/SSPL, but worded in a better way)

_config.yml

@@ -1,2 +1,2 @@
 ---
-theme: jekyll-theme-minimal
+theme: jekyll-theme-modernist

authdata.json.example

@@ -0,0 +1,56 @@
+{
+    "accounts": [{
+        "name": "Bart",
+        "email": "sampleaccount1@sampling.com",
+        "arn": "arn:aws:iam::123456789012:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+        "shortid": "123456789012",
+        "keys": [{
+            "access": "accessKey1",
+            "secret": "verySecretKey1"
+        }]
+    }, {
+        "name": "Lisa",
+        "email": "sampleaccount2@sampling.com",
+        "arn": "arn:aws:iam::123456789013:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
+        "shortid": "123456789013",
+        "keys": [{
+            "access": "accessKey2",
+            "secret": "verySecretKey2"
+        }]
+    },
+    {
+        "name": "Clueso",
+        "email": "inspector@clueso.info",
+        "arn": "arn:aws:iam::123456789014:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/clueso",
+        "shortid": "123456789014",
+        "keys": [{
+            "access": "cluesoKey1",
+            "secret": "cluesoSecretKey1"
+        }]
+    },
+    {
+        "name": "Replication",
+        "email": "inspector@replication.info",
+        "arn": "arn:aws:iam::123456789015:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/replication",
+        "shortid": "123456789015",
+        "keys": [{
+            "access": "replicationKey1",
+            "secret": "replicationSecretKey1"
+        }]
+    },
+    {
+        "name": "Lifecycle",
+        "email": "inspector@lifecycle.info",
+        "arn": "arn:aws:iam::123456789016:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle",
+        "shortid": "123456789016",
+        "keys": [{
+            "access": "lifecycleKey1",
+            "secret": "lifecycleSecretKey1"
+        }]
+    }]
+}

bin/create_bucket_with_NFS_enabled.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict'; // eslint-disable-line strict
+
+require('../lib/nfs/utilities.js').createBucketWithNFSEnabled();

bin/search_bucket.js

@@ -0,0 +1,108 @@
+#!/bin/sh
+// 2>/dev/null ; exec "$(which nodejs 2>/dev/null || which node)" "$0" "$@"
+'use strict'; // eslint-disable-line strict
+
+const { auth } = require('arsenal');
+const commander = require('commander');
+
+const http = require('http');
+const https = require('https');
+const logger = require('../lib/utilities/logger');
+
+function _performSearch(host,
+                        port,
+                        bucketName,
+                        query,
+                        listVersions,
+                        accessKey,
+                        secretKey,
+                        sessionToken,
+                        verbose, ssl) {
+    const escapedSearch = encodeURIComponent(query);
+    const options = {
+        host,
+        port,
+        method: 'GET',
+        path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`,
+        headers: {
+            'Content-Length': 0,
+        },
+        rejectUnauthorized: false,
+        versions: '',
+    };
+    if (sessionToken) {
+        options.headers['x-amz-security-token'] = sessionToken;
+    }
+    const transport = ssl ? https : http;
+    const request = transport.request(options, response => {
+        if (verbose) {
+            logger.info('response status code', {
+                statusCode: response.statusCode,
+            });
+            logger.info('response headers', { headers: response.headers });
+        }
+        const body = [];
+        response.setEncoding('utf8');
+        response.on('data', chunk => body.push(chunk));
+        response.on('end', () => {
+            if (response.statusCode >= 200 && response.statusCode < 300) {
+                logger.info('Success');
+                process.stdout.write(body.join(''));
+                process.exit(0);
+            } else {
+                logger.error('request failed with HTTP Status ', {
+                    statusCode: response.statusCode,
+                    body: body.join(''),
+                });
+                process.exit(1);
+            }
+        });
+    });
+    // generateV4Headers exepects request object with path that does not
+    // include query
+    request.path = `/${bucketName}`;
+    const requestData = listVersions ? { search: query, versions: '' } : { search: query };
+    auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3');
+    request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`;
+    if (verbose) {
+        logger.info('request headers', { headers: request._headers });
+    }
+    request.end();
+}
+
+/**
+ * This function is used as a binary to send a request to S3 to perform a
+ * search on the objects in a bucket
+ *
+ * @return {undefined}
+ */
+function searchBucket() {
+    // TODO: Include other bucket listing possible query params?
+    commander
+        .version('0.0.1')
+        .option('-a, --access-key <accessKey>', 'Access key id')
+        .option('-k, --secret-key <secretKey>', 'Secret access key')
+        .option('-t, --session-token <sessionToken>', 'Session token')
+        .option('-b, --bucket <bucket>', 'Name of the bucket')
+        .option('-q, --query <query>', 'Search query')
+        .option('-h, --host <host>', 'Host of the server')
+        .option('-p, --port <port>', 'Port of the server')
+        .option('-s', '--ssl', 'Enable ssl')
+        .option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
+            'otherwise only list the latest version')
+        .option('-v, --verbose')
+        .parse(process.argv);
+    const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
+        commander;
+
+    if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
+        logger.error('missing parameter');
+        commander.outputHelp();
+        process.exit(1);
+    }
+
+    _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose,
+        ssl);
+}
+
+searchBucket();

circle.yml

@@ -1,45 +0,0 @@
----
-general:
-  branches:
-    ignore:
-      - /^ultron\/.*/   # Ignore ultron/* branches
-  artifacts:
-    - coverage/
-
-machine:
-  node:
-    version: 6.13.1
-  services:
-    - redis
-    - docker
-  ruby:
-    version: "2.4.1"
-  environment:
-    CXX: g++-4.9
-    ENABLE_LOCAL_CACHE: true
-    REPORT_TOKEN: report-token-1
-  hosts:
-    bucketwebsitetester.s3-website-us-east-1.amazonaws.com: 127.0.0.1
-
-dependencies:
-  override:
-    - rm -rf node_modules
-    - yarn install --frozen-lockfile
-  post:
-    - sudo pip install flake8 yamllint
-    - sudo pip install s3cmd==1.6.1
-    # fog and ruby testing dependencies
-    - gem install fog-aws -v 1.3.0
-    - gem install mime-types -v 3.1
-    - gem install rspec -v 3.5
-    - gem install json
-    # java sdk dependencies
-    - sudo apt-get install -y -q default-jdk
-
-
-test:
-  override:
-    - docker run --name squid-proxy -d --net=host
-      --publish 3128:3128 sameersbn/squid:3.3.8-23
-    - bash tests.bash:
-        parallel: true

conf/authdata.json

@@ -1,23 +0,0 @@
-{
-    "accounts": [{
-        "name": "Bart",
-        "email": "sampleaccount1@sampling.com",
-        "arn": "arn:aws:iam::123456789012:root",
-        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
-        "shortid": "123456789012",
-        "keys": [{
-            "access": "accessKey1",
-            "secret": "verySecretKey1"
-        }]
-    }, {
-        "name": "Lisa",
-        "email": "sampleaccount2@sampling.com",
-        "arn": "arn:aws:iam::123456789013:root",
-        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
-        "shortid": "123456789013",
-        "keys": [{
-            "access": "accessKey2",
-            "secret": "verySecretKey2"
-        }]
-    }]
-}

config.json.example

@@ -4,13 +4,16 @@
     "metricsPort": 8002,
     "metricsListenOn": [],
     "replicationGroupId": "RG001",
+    "workers": 4,
     "restEndpoints": {
         "localhost": "us-east-1",
         "127.0.0.1": "us-east-1",
         "cloudserver-front": "us-east-1",
         "s3.docker.test": "us-east-1",
         "127.0.0.2": "us-east-1",
-        "s3.amazonaws.com": "us-east-1"
+        "s3.amazonaws.com": "us-east-1",
+        "zenko-cloudserver-replicator": "us-east-1",
+        "lb": "us-east-1"
     },
     "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
                         "s3-website.us-east-2.amazonaws.com",
@@ -25,7 +28,8 @@
                         "s3-website-eu-west-1.amazonaws.com",
                         "s3-website-sa-east-1.amazonaws.com",
                         "s3-website.localhost",
-                        "s3-website.scality.test"],
+                        "s3-website.scality.test",
+                        "zenkoazuretest.blob.core.windows.net"],
     "replicationEndpoints": [{
         "site": "zenko",
         "servers": ["127.0.0.1:8000"],
@@ -34,6 +38,14 @@
         "site": "us-east-2",
         "type": "aws_s3"
     }],
+    "backbeat": {
+        "host": "localhost",
+        "port": 8900
+    },
+    "workflowEngineOperator": {
+        "host": "localhost",
+        "port": 3001
+    },
     "cdmi": {
         "host": "localhost",
         "port": 81,
@@ -47,7 +59,7 @@
         "host": "localhost",
         "port": 8500
     },
-    "clusters": 10,
+    "clusters": 1,
     "log": {
         "logLevel": "info",
         "dumpLevel": "error"
@@ -63,6 +75,10 @@
         "host": "localhost",
         "port": 9991
     },
+    "pfsClient": {
+        "host": "localhost",
+        "port": 9992
+    },
     "metadataDaemon": {
         "bindAddress": "localhost",
         "port": 9990
@@ -71,10 +87,47 @@
         "bindAddress": "localhost",
         "port": 9991
     },
+    "pfsDaemon": {
+        "bindAddress": "localhost",
+        "port": 9992
+    },
     "recordLog": {
-        "enabled": false,
+        "enabled": true,
         "recordLogName": "s3-recordlog"
     },
+    "mongodb": {
+        "replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
+        "writeConcern": "majority",
+        "replicaSet": "rs0",
+        "readPreference": "primary",
+        "database": "metadata"
+    },
+    "authdata": "authdata.json",
+    "backends": {
+        "auth": "file",
+        "data": "file",
+        "metadata": "mongodb",
+        "kms": "file",
+        "quota": "none"
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
+    },
     "requests": {
         "viaProxy": false,
         "trustedProxyCIDRs": [],

config.json.vitastor

@@ -0,0 +1,71 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "metricsPort": 8002,
+    "metricsListenOn": [],
+    "replicationGroupId": "RG001",
+    "restEndpoints": {
+        "localhost": "STANDARD",
+        "127.0.0.1": "STANDARD",
+        "yourhostname.ru": "STANDARD"
+    },
+    "websiteEndpoints": [
+        "static.yourhostname.ru"
+    ],
+    "replicationEndpoints": [ {
+        "site": "zenko",
+        "servers": ["127.0.0.1:8000"],
+        "default": true
+    } ],
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "backends": {
+        "metadata": "mongodb"
+    },
+    "mongodb": {
+        "replicaSetHosts": "127.0.0.1:27017",
+        "writeConcern": "majority",
+        "replicaSet": "rs0",
+        "readPreference": "primary",
+        "database": "s3",
+        "authCredentials": {
+            "username": "s3",
+            "password": ""
+        }
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
+    },
+    "requests": {
+        "viaProxy": false,
+        "trustedProxyCIDRs": [],
+        "extractClientIPFromHeader": ""
+    },
+    "bucketNotificationDestinations": [
+        {
+            "resource": "target1",
+            "type": "dummy",
+            "host": "localhost:6000"
+        }
+    ]
+}

constants.js

@@ -39,6 +39,8 @@ const constants = {
     // once the multipart upload is complete.
     mpuBucketPrefix: 'mpuShadowBucket',
     blacklistedPrefixes: { bucket: [], object: [] },
+    // GCP Object Tagging Prefix
+    gcpTaggingPrefix: 'aws-tag-',
     // PublicId is used as the canonicalID for a request that contains
     // no authentication information.  Requestor can access
     // only public resources
@@ -64,14 +66,20 @@ const constants = {
     // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
     minimumAllowedPartSize: 5242880,
 
+    // AWS sets a maximum total parts limit
+    // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
+    maximumAllowedPartCount: 10000,
+
+    gcpMaximumAllowedPartCount: 1024,
+
     // Max size on put part or copy part is 5GB. For functional
     // testing use 110 MB as max
     maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :
         5368709120,
 
-    // AWS sets a maximum total parts limit
-    // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
-    maximumAllowedPartCount: 10000,
+    // Max size allowed in a single put object request is 5GB
+    // https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
+    maximumAllowedUploadSize: 5368709120,
 
     // AWS states max size for user-defined metadata (x-amz-meta- headers) is
     // 2 KB: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
@@ -96,7 +104,6 @@ const constants = {
         'policyStatus',
         'publicAccessBlock',
         'requestPayment',
-        'restore',
         'torrent',
     ],
 
@@ -109,11 +116,21 @@ const constants = {
     ],
 
     // user metadata header to set object locationConstraint
-    objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
+    objectLocationConstraintHeader: 'x-amz-storage-class',
     lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
     legacyLocations: ['sproxyd', 'legacy'],
+    // declare here all existing service accounts and their properties
+    // (if any, otherwise an empty object)
+    serviceAccountProperties: {
+        replication: {},
+        lifecycle: {},
+        gc: {},
+        'md-ingestion': {
+            canReplicate: true,
+        },
+    },
     /* eslint-disable camelcase */
-    externalBackends: { aws_s3: true, azure: true, gcp: true },
+    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
     // some of the available data backends  (if called directly rather
     // than through the multiple backend gateway) need a key provided
     // as a string as first parameter of the get/delete methods.
@@ -122,13 +139,19 @@ const constants = {
     // for external backends, don't call unless at least 1 minute
     // (60,000 milliseconds) since last call
     externalBackendHealthCheckInterval: 60000,
-    versioningNotImplBackends: { azure: true },
-    mpuMDStoredExternallyBackend: { aws_s3: true },
+    versioningNotImplBackends: { azure: true, gcp: true },
+    mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
+    skipBatchDeleteBackends: { azure: true, gcp: true },
+    s3HandledBackends: { azure: true, gcp: true },
+    hasCopyPartBackends: { aws_s3: true, gcp: true },
     /* eslint-enable camelcase */
     mpuMDStoredOnS3Backend: { azure: true },
     azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
     base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
         '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
+    productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
+    // location constraint delimiter
+    zenkoSeparator: ':',
     // user metadata applied on zenko objects
     zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
     bucketOwnerActions: [
@@ -181,16 +204,13 @@ const constants = {
         'objectPutRetention',
     ],
     allowedUtapiEventFilterStates: ['allow', 'deny'],
-    // The AWS assumed Role resource type
-    assumedRoleArnResourceType: 'assumed-role',
-    // Session name of the backbeat lifecycle assumed role session.
-    backbeatLifecycleSessionName: 'backbeat-lifecycle',
-    multiObjectDeleteConcurrency: 50,
+    allowedRestoreObjectRequestTierValues: ['Standard'],
     lifecycleListing: {
         CURRENT_TYPE: 'current',
         NON_CURRENT_TYPE: 'noncurrent',
         ORPHAN_DM_TYPE: 'orphan',
     },
+    multiObjectDeleteConcurrency: 50,
     maxScannedLifecycleListingEntries: 10000,
     overheadField: [
         'content-length',
@@ -211,11 +231,18 @@ const constants = {
     ]),
     ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
     ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
+    // The AWS assumed Role resource type
+    assumedRoleArnResourceType: 'assumed-role',
+    // Session name of the backbeat lifecycle assumed role session.
+    backbeatLifecycleSessionName: 'backbeat-lifecycle',
     actionsToConsiderAsObjectPut: [
         'initiateMultipartUpload',
         'objectPutPart',
         'completeMultipartUpload',
     ],
+    // if requester is not bucket owner, bucket policy actions should be denied with
+    // MethodNotAllowed error
+    onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
 };
 
 module.exports = constants;

dataserver.js

@@ -17,13 +17,17 @@ process.on('uncaughtException', err => {
 if (config.backends.data === 'file' ||
     (config.backends.data === 'multiple' &&
      config.backends.metadata !== 'scality')) {
-    const dataServer = new arsenal.network.rest.RESTServer(
-        { bindAddress: config.dataDaemon.bindAddress,
+    const dataServer = new arsenal.network.rest.RESTServer({
+        bindAddress: config.dataDaemon.bindAddress,
         port: config.dataDaemon.port,
-            dataStore: new arsenal.storage.data.file.DataFileStore(
-                { dataPath: config.dataDaemon.dataPath,
-                    log: config.log }),
-            log: config.log });
+        dataStore: new arsenal.storage.data.file.DataFileStore({
+            dataPath: config.dataDaemon.dataPath,
+            log: config.log,
+            noSync: config.dataDaemon.noSync,
+            noCache: config.dataDaemon.noCache,
+        }),
+        log: config.log,
+    });
     dataServer.setup(err => {
         if (err) {
             logger.error('Error initializing REST data server',

docker-entrypoint.sh

@@ -6,14 +6,15 @@ set -e
 # modifying config.json
 JQ_FILTERS_CONFIG="."
 
+# ENDPOINT var can accept comma separated values
+# for multiple endpoint locations
 if [[ "$ENDPOINT" ]]; then
-    HOST_NAME="$ENDPOINT"
-fi
-
-if [[ "$HOST_NAME" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$HOST_NAME\"]=\"us-east-1\""
-    echo "Host name has been modified to $HOST_NAME"
-    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with $HOST_NAME"
+    IFS="," read -ra HOST_NAMES <<< "$ENDPOINT"
+    for host in "${HOST_NAMES[@]}"; do
+        JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$host\"]=\"us-east-1\""
+    done
+    echo "Host name has been modified to ${HOST_NAMES[@]}"
+    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with ${HOST_NAMES[@]}"
 fi
 
 if [[ "$LOG_LEVEL" ]]; then
@@ -25,7 +26,7 @@ if [[ "$LOG_LEVEL" ]]; then
     fi
 fi
 
-if [[ "$SSL" && "$HOST_NAME" ]]; then
+if [[ "$SSL" && "$HOST_NAMES" ]]; then
 # This condition makes sure that the certificates are not generated twice. (for docker restart)
 if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
 # Compute config for utapi tests
@@ -36,15 +37,15 @@ prompt = no
 req_extensions = s3_req
 
 [req_distinguished_name]
-CN = ${HOST_NAME}
+CN = ${HOST_NAMES[0]}
 
 [s3_req]
 subjectAltName = @alt_names
 extendedKeyUsage = serverAuth, clientAuth
 
 [alt_names]
-DNS.1 = *.${HOST_NAME}
-DNS.2 = ${HOST_NAME}
+DNS.1 = *.${HOST_NAMES[0]}
+DNS.2 = ${HOST_NAMES[0]}
 
 EOF
 
@@ -70,9 +71,14 @@ fi
 if [[ "$LISTEN_ADDR" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
 fi
 
+if [[ "$REPLICATION_GROUP_ID" ]] ; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
+fi
+
 if [[ "$DATA_HOST" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
 fi
@@ -81,6 +87,22 @@ if [[ "$METADATA_HOST" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
 fi
 
+if [[ "$PFSD_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
+fi
+
+if [[ "$MONGODB_HOSTS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
+fi
+
+if [[ "$MONGODB_RS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSet=\"$MONGODB_RS\""
+fi
+
+if [[ "$MONGODB_DATABASE" ]]; then
+   JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.database=\"$MONGODB_DATABASE\""
+fi
+
 if [ -z "$REDIS_HA_NAME" ]; then
     REDIS_HA_NAME='mymaster'
 fi
@@ -113,12 +135,68 @@ if [[ "$RECORDLOG_ENABLED" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true"
 fi
 
-if [[ -n "$BUCKET_DENY_FILTER" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
+if [[ "$STORAGE_LIMIT_ENABLED" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.metrics[.utapi.metrics | length]=\"location\""
 fi
 
-if [[ "$BUCKETD_BOOTSTRAP" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .bucketd.bootstrap=[\"$BUCKETD_BOOTSTRAP\""]
+if [[ "$CRR_METRICS_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\""
+fi
+
+if [[ "$CRR_METRICS_PORT" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
+fi
+
+if [[ "$WE_OPERATOR_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
+fi
+
+if [[ "$WE_OPERATOR_PORT" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
+fi
+
+if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
+fi
+
+# external backends http(s) agent config
+
+# AWS
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
+#GCP
+if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
+if [[ -n "$BUCKET_DENY_FILTER" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
 fi
 
 if [[ "$TESTING_MODE" ]]; then
@@ -130,6 +208,10 @@ if [[ $JQ_FILTERS_CONFIG != "." ]]; then
     mv config.json.tmp config.json
 fi
 
+if test -v INITIAL_INSTANCE_ID && test -v S3METADATAPATH && ! test -f ${S3METADATAPATH}/uuid ; then
+    echo -n ${INITIAL_INSTANCE_ID} > ${S3METADATAPATH}/uuid
+fi
+
 # s3 secret credentials for Zenko
 if [ -r /run/secrets/s3-credentials ] ; then
     . /run/secrets/s3-credentials

docs/ARCHITECTURE.rst

@@ -27,7 +27,7 @@ including null versions and delete markers, described in the above
 links.
 
 Implementation of Bucket Versioning in Zenko CloudServer
------------------------------------------
+--------------------------------------------------------
 
 Overview of Metadata and API Component Roles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -66,7 +66,7 @@ The second section, `"Implementation of Bucket Versioning in
 API" <#implementation-of-bucket-versioning-in-api>`__, describes the way
 the metadata options are used in the API within S3 actions to create new
 versions, update their metadata, and delete them. The management of null
-versions and creation of delete markers are also described in this
+versions and creation of delete markers is also described in this
 section.
 
 Implementation of Bucket Versioning in Metadata
@@ -179,12 +179,13 @@ PUT
    the master version with this version.
 -  ``versionId: <versionId>`` create or update a specific version (for updating
    version's ACL or tags, or remote updates in geo-replication)
-   -  if the version identified by ``versionId`` happens to be the latest
+
+   *  if the version identified by ``versionId`` happens to be the latest
       version, the master version will be updated as well
-   -  if the master version is not as recent as the version identified by
+   *  if the master version is not as recent as the version identified by
       ``versionId``, as may happen with cross-region replication, the master
       will be updated as well
-   -  note that with ``versionId`` set to an empty string ``''``, it will
+   *  note that with ``versionId`` set to an empty string ``''``, it will
       overwrite the master version only (same as no options, but the master
       version will have a ``versionId`` property set in its metadata like
       any other version). The ``versionId`` will never be exposed to an
@@ -208,10 +209,13 @@ A deletion targeting the latest version of an object has to:
 -  delete the specified version identified by ``versionId``
 -  replace the master version with a version that is a placeholder for
    deletion
+
     -  this version contains a special keyword, 'isPHD', to indicate the
          master version was deleted and needs to be updated
+
 -  initiate a repair operation to update the value of the master
    version:
+
     -  involves listing the versions of the object and get the latest
        version to replace the placeholder delete version
     -  if no more versions exist, metadata deletes the master version,
@@ -764,7 +768,7 @@ This will open two ports:
 Then, one or more instances of Zenko CloudServer without the dmd can be started
 elsewhere with:
 
-::
+.. code:: sh
 
    yarn run start_s3server
 

docs/CLIENTS.rst

@@ -178,7 +178,7 @@ Ruby
 ~~~~
 
 `AWS SDK for Ruby - Version 2 <http://docs.aws.amazon.com/sdkforruby/api/>`__
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code:: ruby
 
@@ -239,6 +239,7 @@ Python
 Client integration
 
 .. code:: python
+
     import boto3
 
     client = boto3.client(
@@ -253,6 +254,7 @@ Client integration
 Full integration (with object mapping)
 
 .. code:: python
+
     import os
 
     from botocore.utils import fix_s3_host
@@ -293,3 +295,51 @@ Should force path-style requests even though v3 advertises it does by default.
     $client->createBucket(array(
         'Bucket' => 'bucketphp',
     ));
+
+Go
+~~
+
+`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: go
+
+   package main
+
+   import (
+      "context"
+      "fmt"
+      "log"
+      "os"
+      "time"
+
+      "github.com/aws/aws-sdk-go/aws"
+      "github.com/aws/aws-sdk-go/aws/endpoints"
+      "github.com/aws/aws-sdk-go/aws/session"
+      "github.com/aws/aws-sdk-go/service/s3"
+   )
+
+   func main() {
+      os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
+      os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
+      endpoint := "http://localhost:8000"
+      timeout := time.Duration(10) * time.Second
+      sess := session.Must(session.NewSession())
+
+      // Create a context with a timeout that will abort the upload if it takes
+      // more than the passed in timeout.
+      ctx, cancel := context.WithTimeout(context.Background(), timeout)
+      defer cancel()
+
+      svc := s3.New(sess, &aws.Config{
+         Region:   aws.String(endpoints.UsEast1RegionID),
+         Endpoint: &endpoint,
+      })
+
+      out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
+      if err != nil {
+         log.Fatal(err)
+      } else {
+         fmt.Println(out)
+      }
+   }

docs/CONTRIBUTING.rst

@@ -14,7 +14,7 @@ Got an idea? Get started!
 In order to contribute, please follow the `Contributing
 Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__.
 If anything is unclear to you, reach out to us on
-`slack <https://zenko-io.slack.com/>`__ or via a GitHub issue.
+`forum <https://forum.zenko.io/>`__ or via a GitHub issue.
 
 Don't write code? There are other ways to help!
 -----------------------------------------------

docs/DOCKER.rst

@@ -1,11 +1,7 @@
 Docker
 ======
 
--  `Environment Variables <#environment-variables>`__
--  `Tunables and setup tips <#tunables-and-setup-tips>`__
--  `Examples for continuous integration with
-   Docker <#continuous-integration-with-docker-hosted CloudServer>`__
--  `Examples for going in production with Docker <#in-production-with-docker-hosted CloudServer>`__
+.. _environment-variables:
 
 Environment Variables
 ---------------------
@@ -15,25 +11,27 @@ S3DATA
 
 S3DATA=multiple
 ^^^^^^^^^^^^^^^
-Allows you to run Scality Zenko CloudServer with multiple data backends, defined
+
+This variable enables running CloudServer with multiple data backends, defined
 as regions.
-When using multiple data backends, a custom ``locationConfig.json`` file is
-mandatory. It will allow you to set custom regions. You will then need to
-provide associated rest_endpoints for each custom region in your
-``config.json`` file.
-`Learn more about multiple backends configuration <../GETTING_STARTED/#location-configuration>`__
 
-If you are using Scality RING endpoints, please refer to your customer
-documentation.
+For multiple data backends, a custom locationConfig.json file is required.
+This file enables you to set custom regions. You must provide associated 
+rest_endpoints for each custom region in config.json.
 
-Running it with an AWS S3 hosted backend
-""""""""""""""""""""""""""""""""""""""""
-To run CloudServer with an S3 AWS backend, you will have to add a new section
-to your ``locationConfig.json`` file with the ``aws_s3`` location type:
+`Learn more about multiple-backend configurations <GETTING_STARTED.html#location-configuration>`__
+
+If you are using Scality RING endpoints, refer to your customer documentation.
+
+Running CloudServer with an AWS S3-Hosted Backend
+"""""""""""""""""""""""""""""""""""""""""""""""""
+
+To run CloudServer with an S3 AWS backend, add a new section to the 
+``locationConfig.json`` file with the ``aws_s3`` location type:
 
 .. code:: json
 
-(...)
+    (...)
     "awsbackend": {
         "type": "aws_s3",
         "details": {
@@ -43,121 +41,134 @@ to your ``locationConfig.json`` file with the ``aws_s3`` location type:
             "credentialsProfile": "aws_hosted_profile"
         }
     }
-(...)
+    (...)
 
-You will also have to edit your AWS credentials file to be able to use your
-command line tool of choice. This file should mention credentials for all the
-backends you're using. You can use several profiles when using multiple
-profiles.
+Edit your AWS credentials file to enable your preferred command-line tool.
+This file must mention credentials for all backends in use. You can use 
+several profiles if multiple profiles are configured.
 
 .. code:: json
 
-[default]
-aws_access_key_id=accessKey1
-aws_secret_access_key=verySecretKey1
-[aws_hosted_profile]
-aws_access_key_id={{YOUR_ACCESS_KEY}}
-aws_secret_access_key={{YOUR_SECRET_KEY}}
+    [default]
+    aws_access_key_id=accessKey1
+    aws_secret_access_key=verySecretKey1
+    [aws_hosted_profile]
+    aws_access_key_id={{YOUR_ACCESS_KEY}}
+    aws_secret_access_key={{YOUR_SECRET_KEY}}
 
-Just as you need to mount your locationConfig.json, you will need to mount your
-AWS credentials file at run time:
-``-v ~/.aws/credentials:/root/.aws/credentials`` on Linux, OS X, or Unix or
+As with locationConfig.json, the AWS credentials file must be mounted at 
+run time: ``-v ~/.aws/credentials:/root/.aws/credentials`` on Unix-like 
+systems (Linux, OS X, etc.), or 
 ``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows
 
-NOTE: One account can't copy to another account with a source and
-destination on real AWS unless the account associated with the
-access Key/secret Key pairs used for the destination bucket has rights
-to get in the source bucket. ACL's would have to be updated
-on AWS directly to enable this.
+.. note:: One account cannot copy to another account with a source and
+   destination on real AWS unless the account associated with the 
+   accessKey/secretKey pairs used for the destination bucket has source 
+   bucket access privileges. To enable this, update ACLs directly on AWS.
 
 S3BACKEND
-~~~~~~
+~~~~~~~~~
 
 S3BACKEND=file
-^^^^^^^^^^^
-When storing file data, for it to be persistent you must mount docker volumes
-for both data and metadata. See `this section <#using-docker-volumes-in-production>`__
+^^^^^^^^^^^^^^
+
+For stored file data to persist, you must mount Docker volumes
+for both data and metadata. See :ref:`In Production with a Docker-Hosted CloudServer <in-production-w-a-Docker-hosted-cloudserver>`
 
 S3BACKEND=mem
-^^^^^^^^^^
-This is ideal for testing - no data will remain after container is shutdown.
+^^^^^^^^^^^^^
+
+This is ideal for testing: no data remains after the container is shut down.
 
 ENDPOINT
 ~~~~~~~~
 
-This variable specifies your endpoint. If you have a domain such as
-new.host.com, by specifying that here, you and your users can direct s3
-server requests to new.host.com.
+This variable specifies the endpoint. To direct CloudServer requests to 
+new.host.com, for example, specify the endpoint with:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e ENDPOINT=new.host.com scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 -e ENDPOINT=new.host.com zenko/cloudserver
 
-Note: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
-permissions, make sure to associate 127.0.0.1 with ``new.host.com``
+.. note:: On Unix-like systems (Linux, OS X, etc.) edit /etc/hosts
+   to associate 127.0.0.1 with new.host.com.
+
+REMOTE_MANAGEMENT_DISABLE
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CloudServer is a part of `Zenko <https://www.zenko.io/>`__. When you run CloudServer standalone it will still try to connect to Orbit by default (browser-based graphical user interface for Zenko).
+
+Setting this variable to true(1) will default to accessKey1 and verySecretKey1 for credentials and disable the automatic Orbit management:
+
+.. code-block:: shell
+
+    $ docker run -d --name cloudserver -p 8000:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
 
 SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 These variables specify authentication credentials for an account named
-"CustomAccount".
+“CustomAccount”.
 
-You can set credentials for many accounts by editing
-``conf/authdata.json`` (see below for further info), but if you just
-want to specify one set of your own, you can use these environment
-variables.
+Set account credentials for multiple accounts by editing conf/authdata.json
+(see below for further details). To specify one set for personal use, set these 
+environment variables:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey
-    -e SCALITY_SECRET_ACCESS_KEY=newSecretKey scality/s3server
+   $ docker run -d --name cloudserver -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey \
+   -e SCALITY_SECRET_ACCESS_KEY=newSecretKey zenko/cloudserver
 
-Note: Anything in the ``authdata.json`` file will be ignored. Note: The
-old ``ACCESS_KEY`` and ``SECRET_KEY`` environment variables are now
-deprecated
+.. note:: This takes precedence over the contents of the authdata.json 
+	  file. The authdata.json file is ignored. 
+
+.. note:: The ACCESS_KEY and SECRET_KEY environment variables are 
+	  deprecated.
 
 LOG\_LEVEL
 ~~~~~~~~~~
 
-This variable allows you to change the log level: info, debug or trace.
-The default is info. Debug will give you more detailed logs and trace
-will give you the most detailed.
+This variable changes the log level. There are three levels: info, debug, 
+and trace. The default is info. Debug provides more detailed logs, and trace
+provides the most detailed logs.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 -e LOG_LEVEL=trace zenko/cloudserver
 
 SSL
 ~~~
 
-This variable set to true allows you to run S3 with SSL:
+Set true, this variable runs CloudServer with SSL.
 
-**Note1**: You also need to specify the ENDPOINT environment variable.
-**Note2**: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
-permissions, make sure to associate 127.0.0.1 with ``<YOUR_ENDPOINT>``
+If SSL is set true: 
 
-**Warning**: These certs, being self-signed (and the CA being generated
-inside the container) will be untrusted by any clients, and could
-disappear on a container upgrade. That's ok as long as it's for quick
-testing. Also, best security practice for non-testing would be to use an
-extra container to do SSL/TLS termination such as haproxy/nginx/stunnel
-to limit what an exploit on either component could expose, as well as
-certificates in a mounted volume
+* The ENDPOINT environment variable must also be specified.
+
+* On Unix-like systems (Linux, OS X, etc.), 127.0.0.1 must be associated with
+  <YOUR_ENDPOINT> in /etc/hosts.
+
+   .. Warning:: Self-signed certs with a CA generated within the container are 
+      suitable for testing purposes only. Clients cannot trust them, and they may
+      disappear altogether on a container upgrade. The best security practice for 
+      production environments is to use an extra container, such as 
+      haproxy/nginx/stunnel, for SSL/TLS termination and to pull certificates
+      from a mounted volume, limiting what an exploit on either component
+      can expose. 
 
 .. code:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT>
-    scality/s3server
+     $ docker run -d --name cloudserver -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT> \
+     zenko/cloudserver
 
-More information about how to use S3 server with SSL
-`here <https://s3.scality.com/v1.0/page/scality-with-ssl>`__
+  For more information about using ClousdServer with SSL, see `Using SSL <GETTING_STARTED.html#Using SSL>`__
 
 LISTEN\_ADDR
 ~~~~~~~~~~~~
 
-This variable instructs the Zenko CloudServer, and its data and metadata
-components to listen on the specified address. This allows starting the data
-or metadata servers as standalone services, for example.
+This variable causes CloudServer and its data and metadata components to 
+listen on the specified address. This allows starting the data or metadata 
+servers as standalone services, for example.
 
 .. code:: shell
 
@@ -174,8 +185,8 @@ Zenko CloudServer.
 
 .. code:: shell
 
-    docker run -d --name s3server -e DATA_HOST=s3server-data
-    -e METADATA_HOST=s3server-metadata scality/s3server npm run start_s3server
+    $ docker run -d --name cloudserver -e DATA_HOST=cloudserver-data \
+    -e METADATA_HOST=cloudserver-metadata zenko/cloudserver yarn run start_s3server
 
 REDIS\_HOST
 ~~~~~~~~~~~
@@ -185,19 +196,21 @@ localhost.
 
 .. code:: shell
 
-    docker run -d --name s3server -p 8000:8000
-    -e REDIS_HOST=my-redis-server.example.com scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 \
+    -e REDIS_HOST=my-redis-server.example.com zenko/cloudserver
 
 REDIS\_PORT
 ~~~~~~~~~~~
 
-Use this variable to connect to the redis cache server on another port than
-the default 6379.
+Use this variable to connect to the Redis cache server on a port other 
+than the default 6379.
 
 .. code:: shell
 
-    docker run -d --name s3server -p 8000:8000
-    -e REDIS_PORT=6379 scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 \
+    -e REDIS_PORT=6379 zenko/cloudserver
+
+.. _tunables-and-setup-tips:
 
 Tunables and Setup Tips
 -----------------------
@@ -205,60 +218,57 @@ Tunables and Setup Tips
 Using Docker Volumes
 ~~~~~~~~~~~~~~~~~~~~
 
-Zenko CloudServer runs with a file backend by default.
+CloudServer runs with a file backend by default, meaning that data is 
+stored inside the CloudServer’s Docker container.
 
-So, by default, the data is stored inside your Zenko CloudServer Docker
-container.
+For data and metadata to persist, data and metadata must be hosted in Docker 
+volumes outside the CloudServer’s Docker container. Otherwise, the data
+and metadata are destroyed when the container is erased.
 
-However, if you want your data and metadata to persist, you **MUST** use
-Docker volumes to host your data and metadata outside your Zenko CloudServer
-Docker container. Otherwise, the data and metadata will be destroyed
-when you erase the container.
+.. code-block:: shell
 
-.. code:: shell
+    $ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata \
+    -p 8000:8000 ­-d zenko/cloudserver
 
-    docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata
-    -p 8000:8000 ­-d scality/s3server
+This command mounts the ./data host directory to the container
+at /usr/src/app/localData and the ./metadata host directory to
+the container at /usr/src/app/localMetaData. 
 
-This command mounts the host directory, ``./data``, into the container
-at ``/usr/src/app/localData`` and the host directory, ``./metadata``, into
-the container at ``/usr/src/app/localMetaData``. It can also be any host
-mount point, like ``/mnt/data`` and ``/mnt/metadata``.
+.. tip:: These host directories can be mounted to any accessible mount 
+   point, such as /mnt/data and /mnt/metadata, for example.
 
-Adding modifying or deleting accounts or users credentials
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Adding, Modifying, or Deleting Accounts or Credentials
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-1. Create locally a customized ``authdata.json`` based on our ``/conf/authdata.json``.
+1. Create a customized authdata.json file locally based on /conf/authdata.json.
+
+2. Use `Docker volumes <https://docs.docker.com/storage/volumes/>`__
+   to override the default ``authdata.json`` through a Docker file mapping.
 
-2. Use `Docker
-   Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__
-   to override the default ``authdata.json`` through a docker file mapping.
 For example:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
-    scality/s3server
+    $ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d \
+    zenko/cloudserver
 
-Specifying your own host name
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Specifying a Host Name
+~~~~~~~~~~~~~~~~~~~~~~
 
-To specify a host name (e.g. s3.domain.name), you can provide your own
-`config.json <https://github.com/scality/S3/blob/master/config.json>`__
-using `Docker
-Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__.
+To specify a host name (for example, s3.domain.name), provide your own
+`config.json <https://github.com/scality/cloudserver/blob/master/config.json>`__
+file using `Docker volumes <https://docs.docker.com/storage/volumes/>`__.
 
-First add a new key-value pair in the restEndpoints section of your
-config.json. The key in the key-value pair should be the host name you
-would like to add and the value is the default location\_constraint for
-this endpoint.
+First, add a new key-value pair to the restEndpoints section of your
+config.json. Make the key the host name you want, and the value the default 
+location\_constraint for this endpoint.
 
 For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one
 of the ``location_constraints`` listed in your locationConfig.json file
 `here <https://github.com/scality/S3/blob/master/locationConfig.json>`__.
 
-More information about location configuration
-`here <https://github.com/scality/S3/blob/master/README.md#location-configuration>`__
+For more information about location configuration, see:
+`GETTING STARTED <GETTING_STARTED.html#location-configuration>`__
 
 .. code:: json
 
@@ -266,33 +276,33 @@ More information about location configuration
         "localhost": "file",
         "127.0.0.1": "file",
         ...
-        "s3.example.com": "us-east-1"
+        "cloudserver.example.com": "us-east-1"
     },
 
-Then, run your Scality S3 Server using `Docker
-Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
+Next, run CloudServer using a `Docker volume 
+<https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
+    $ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d zenko/cloudserver
 
-Your local ``config.json`` file will override the default one through a
-docker file mapping.
+The local ``config.json`` file overrides the default one through a Docker 
+file mapping.
 
-Running as an unprivileged user
+Running as an Unprivileged User
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Zenko CloudServer runs as root by default.
+CloudServer runs as root by default.
 
-You can change that by modifing the dockerfile and specifying a user
-before the entrypoint.
+To change this, modify the dockerfile and specify a user before the 
+entry point.
 
-The user needs to exist within the container, and own the folder
-**/usr/src/app** for Scality Zenko CloudServer to run properly.
+The user must exist within the container, and must own the 
+/usr/src/app directory for CloudServer to run.
 
-For instance, you can modify these lines in the dockerfile:
+For example, the following dockerfile lines can be modified:
 
-.. code:: shell
+.. code-block:: shell
 
     ...
     && groupadd -r -g 1001 scality \
@@ -304,54 +314,58 @@ For instance, you can modify these lines in the dockerfile:
     USER scality
     ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
 
-Continuous integration with Docker hosted CloudServer
------------------------------------------------------
+.. _continuous-integration-with-docker-hosted-cloudserver:
 
-When you start the Docker Scality Zenko CloudServer image, you can adjust the
-configuration of the Scality Zenko CloudServer instance by passing one or more
-environment variables on the docker run command line.
+Continuous Integration with a Docker-Hosted CloudServer
+-------------------------------------------------------
 
-Sample ways to run it for CI are:
+When you start the Docker CloudServer image, you can adjust the
+configuration of the CloudServer instance by passing one or more
+environment variables on the ``docker run`` command line.
 
-- With custom locations (one in-memory, one hosted on AWS), and custom
-  credentials mounted:
 
-.. code:: shell
+To run CloudServer for CI with custom locations (one in-memory, 
+one hosted on AWS), and custom credentials mounted:
 
-    docker run --name CloudServer -p 8000:8000
-    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-    -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-    -v ~/.aws/credentials:/root/.aws/credentials
-    -e S3DATA=multiple -e S3BACKEND=mem scality/s3server
+.. code-block:: shell
 
-- With custom locations, (one in-memory, one hosted on AWS, one file),
-  and custom credentials set as environment variables
-  (see `this section <#scality-access-key-id-and-scality-secret-access-key>`__):
+   $ docker run --name CloudServer -p 8000:8000 \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
+   -v ~/.aws/credentials:/root/.aws/credentials \
+   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
 
-.. code:: shell
+To run CloudServer for CI with custom locations, (one in-memory, one
+hosted on AWS, and one file), and custom credentials `set as environment 
+variables <GETTING_STARTED.html#scality-access-key-id-and-scality-secret-access-key>`__):
 
-    docker run --name CloudServer -p 8000:8000
-    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-    -v ~/.aws/credentials:/root/.aws/credentials
-    -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-    -e SCALITY_ACCESS_KEY_ID=accessKey1
-    -e SCALITY_SECRET_ACCESS_KEY=verySecretKey1
-    -e S3DATA=multiple -e S3BACKEND=mem scality/s3server
+.. code-block:: shell
 
-In production with Docker hosted CloudServer
---------------------------------------------
+   $ docker run --name CloudServer -p 8000:8000 \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v ~/.aws/credentials:/root/.aws/credentials \
+   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -e SCALITY_ACCESS_KEY_ID=accessKey1 \
+   -e SCALITY_SECRET_ACCESS_KEY=verySecretKey1 \
+   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
 
-In production, we expect that data will be persistent, that you will use the
-multiple backends capabilities of Zenko CloudServer, and that you will have a
-custom endpoint for your local storage, and custom credentials for your local
-storage:
+.. _in-production-w-a-Docker-hosted-cloudserver:
 
-.. code:: shell
+In Production with a Docker-Hosted CloudServer
+----------------------------------------------
 
-    docker run -d --name CloudServer
-    -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-    -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-    -v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple
-    -e ENDPOINT=custom.endpoint.com
-    -p 8000:8000 ­-d scality/s3server
+Because data must persist in production settings, CloudServer offers
+multiple-backend capabilities. This requires a custom endpoint 
+and custom credentials for local storage.
+
+Customize these with:
+
+.. code-block:: shell
+
+   $ docker run -d --name CloudServer \
+   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
+   -v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple \
+   -e ENDPOINT=custom.endpoint.com \
+   -p 8000:8000 ­-d zenko/cloudserver \

docs/GETTING_STARTED.rst

@@ -1,193 +1,201 @@
 Getting Started
-=================
+===============
 
 .. figure:: ../res/scality-cloudserver-logo.png
    :alt: Zenko CloudServer logo
 
-|CircleCI| |Scality CI|
+
+Dependencies
+------------
+
+Building and running the Scality Zenko CloudServer requires node.js 10.x and
+yarn v1.17.x. Up-to-date versions can be found at
+`Nodesource <https://github.com/nodesource/distributions>`__.
 
 Installation
 ------------
 
-Dependencies
-~~~~~~~~~~~~
+1. Clone the source code
 
-Building and running the Scality Zenko CloudServer requires node.js 6.9.5 and
-npm v3 . Up-to-date versions can be found at
-`Nodesource <https://github.com/nodesource/distributions>`__.
+   .. code-block:: shell
 
-Clone source code
-~~~~~~~~~~~~~~~~~
+      $ git clone https://github.com/scality/cloudserver.git
 
-.. code:: shell
+2. Go to the cloudserver directory and use yarn to install the js dependencies.
 
-    git clone https://github.com/scality/S3.git
+   .. code-block:: shell
 
-Install js dependencies
-~~~~~~~~~~~~~~~~~~~~~~~
+      $ cd cloudserver
+      $ yarn install
 
-Go to the ./S3 folder,
+Running CloudServer with a File Backend
+---------------------------------------
 
-.. code:: shell
+.. code-block:: shell
 
-    npm install
+   $ yarn start
 
-Run it with a file backend
---------------------------
+This starts a Zenko CloudServer on port 8000. Two additional ports, 9990 
+and 9991, are also open locally for internal transfer of metadata and 
+data, respectively.
 
-.. code:: shell
+The default access key is accessKey1. The secret key is verySecretKey1.
 
-    npm start
+By default, metadata files are saved in the localMetadata directory and 
+data files are saved in the localData directory in the local ./cloudserver 
+directory. These directories are pre-created within the repository. To 
+save data or metadata in different locations, you must specify them using 
+absolute paths. Thus, when starting the server:
 
-This starts an Zenko CloudServer on port 8000. Two additional ports 9990 and
-9991 are also open locally for internal transfer of metadata and data,
-respectively.
+.. code-block:: shell
 
-The default access key is accessKey1 with a secret key of
-verySecretKey1.
+   $ mkdir -m 700 $(pwd)/myFavoriteDataPath
+   $ mkdir -m 700 $(pwd)/myFavoriteMetadataPath
+   $ export S3DATAPATH="$(pwd)/myFavoriteDataPath"
+   $ export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
+   $ yarn start
 
-By default the metadata files will be saved in the localMetadata
-directory and the data files will be saved in the localData directory
-within the ./S3 directory on your machine. These directories have been
-pre-created within the repository. If you would like to save the data or
-metadata in different locations of your choice, you must specify them
-with absolute paths. So, when starting the server:
+Running CloudServer with Multiple Data Backends
+-----------------------------------------------
 
-.. code:: shell
+.. code-block:: shell
 
-    mkdir -m 700 $(pwd)/myFavoriteDataPath
-    mkdir -m 700 $(pwd)/myFavoriteMetadataPath
-    export S3DATAPATH="$(pwd)/myFavoriteDataPath"
-    export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
-    npm start
+   $ export S3DATA='multiple'
+   $ yarn start
 
-Run it with multiple data backends
-----------------------------------
+This starts a Zenko CloudServer on port 8000. 
 
-.. code:: shell
+The default access key is accessKey1. The secret key is verySecretKey1.
 
-    export S3DATA='multiple'
-    npm start
+With multiple backends, you can choose where each object is saved by setting
+the following header with a location constraint in a PUT request:
 
-This starts an Zenko CloudServer on port 8000. The default access key is
-accessKey1 with a secret key of verySecretKey1.
-
-With multiple backends, you have the ability to choose where each object
-will be saved by setting the following header with a locationConstraint
-on a PUT request:
-
-.. code:: shell
+.. code-block:: shell
 
     'x-amz-meta-scal-location-constraint':'myLocationConstraint'
 
-If no header is sent with a PUT object request, the location constraint
-of the bucket will determine where the data is saved. If the bucket has
-no location constraint, the endpoint of the PUT request will be used to
-determine location.
+If no header is sent with a PUT object request, the bucket’s location
+constraint determines where the data is saved. If the bucket has no
+location constraint, the endpoint of the PUT request determines location.
 
-See the Configuration section below to learn how to set location
-constraints.
+See the Configuration_ section to set location constraints.
 
-Run it with an in-memory backend
---------------------------------
+Run CloudServer with an In-Memory Backend
+-----------------------------------------
+
+.. code-block:: shell
+
+   $ yarn run mem_backend
+
+This starts a Zenko CloudServer on port 8000. 
+
+The default access key is accessKey1. The secret key is verySecretKey1.
+
+Run CloudServer with Vault User Management
+------------------------------------------
 
 .. code:: shell
 
-    npm run mem_backend
+   export S3VAULT=vault
+   yarn start
 
-This starts an Zenko CloudServer on port 8000. The default access key is
-accessKey1 with a secret key of verySecretKey1.
+Note: Vault is proprietary and must be accessed separately.
+This starts a Zenko CloudServer using Vault for user management.
 
-Run it for continuous integration testing or in production with Docker
-----------------------------------------------------------------------
+Run CloudServer for Continuous Integration Testing or in Production with Docker
+-------------------------------------------------------------------------------
 
-`DOCKER <../DOCKER/>`__
+Run Cloudserver with `DOCKER <DOCKER.html>`__
 
 Testing
--------
+~~~~~~~
 
-You can run the unit tests with the following command:
+Run unit tests with the command:
 
-.. code:: shell
+.. code-block:: shell
 
-    npm test
+   $ yarn test
 
-You can run the multiple backend unit tests with:
+Run multiple-backend unit tests with:
 
-.. code:: shell
-    CI=true S3DATA=multiple npm start
-    npm run multiple_backend_test
+.. code-block:: shell
 
-You can run the linter with:
+   $ CI=true S3DATA=multiple yarn start
+   $ yarn run multiple_backend_test
 
-.. code:: shell
+Run the linter with:
 
-    npm run lint
+.. code-block:: shell
 
-Running functional tests locally:
+   $ yarn run lint
 
-For the AWS backend and Azure backend tests to pass locally,
-you must modify tests/locationConfigTests.json so that awsbackend
-specifies a bucketname of a bucket you have access to based on
-your credentials profile and modify "azurebackend" with details
-for your Azure account.
+Running Functional Tests Locally
+--------------------------------
+
+To pass AWS and Azure backend tests locally, modify 
+tests/locationConfig/locationConfigTests.json so that ``awsbackend`` 
+specifies the bucketname of a bucket you have access to based on your
+credentials, and modify ``azurebackend`` with details for your Azure account.
 
 The test suite requires additional tools, **s3cmd** and **Redis**
 installed in the environment the tests are running in.
 
--  Install `s3cmd <http://s3tools.org/download>`__
--  Install `redis <https://redis.io/download>`__ and start Redis.
--  Add localCache section to your ``config.json``:
+1. Install `s3cmd <http://s3tools.org/download>`__
 
-::
+2. Install `redis <https://redis.io/download>`__ and start Redis.
+
+3. Add localCache section to ``config.json``:
+
+   .. code:: json
 
       "localCache": {
         "host": REDIS_HOST,
         "port": REDIS_PORT
 	}
 
-where ``REDIS_HOST`` is your Redis instance IP address (``"127.0.0.1"``
-if your Redis is running locally) and ``REDIS_PORT`` is your Redis
-instance port (``6379`` by default)
+   where ``REDIS_HOST`` is the Redis instance IP address (``"127.0.0.1"``
+   if Redis is running locally) and ``REDIS_PORT`` is the Redis instance
+   port (``6379`` by default)
 
--  Add the following to the etc/hosts file on your machine:
+4. Add the following to the local etc/hosts file:
 
-.. code:: shell
+   .. code-block:: shell
 
       127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
 
--  Start the Zenko CloudServer in memory and run the functional tests:
+5. Start Zenko CloudServer in memory and run the functional tests:
 
-.. code:: shell
+   .. code-block:: shell
 
-    CI=true npm run mem_backend
-    CI=true npm run ft_test
+      $ CI=true yarn run mem_backend
+      $ CI=true yarn run ft_test
+
+.. _Configuration:
 
 Configuration
 -------------
 
-There are three configuration files for your Scality Zenko CloudServer:
+There are three configuration files for Zenko CloudServer:
 
-1. ``conf/authdata.json``, described above for authentication
+* ``conf/authdata.json``, for authentication.
 
-2. ``locationConfig.json``, to set up configuration options for
+* ``locationConfig.json``, to configure where data is saved.
 
-   where data will be saved
+* ``config.json``, for general configuration options.
 
-3. ``config.json``, for general configuration options
+.. _location-configuration:
 
 Location Configuration
 ~~~~~~~~~~~~~~~~~~~~~~
 
-You must specify at least one locationConstraint in your
-locationConfig.json (or leave as pre-configured).
+You must specify at least one locationConstraint in locationConfig.json
+(or leave it as pre-configured).
 
-You must also specify 'us-east-1' as a locationConstraint so if you only
-define one locationConstraint, that would be it. If you put a bucket to
-an unknown endpoint and do not specify a locationConstraint in the put
-bucket call, us-east-1 will be used.
+You must also specify 'us-east-1' as a locationConstraint. If you put a 
+bucket to an unknown endpoint and do not specify a locationConstraint in
+the PUT bucket call, us-east-1 is used.
 
-For instance, the following locationConstraint will save data sent to
+For instance, the following locationConstraint saves data sent to
 ``myLocationConstraint`` to the file backend:
 
 .. code:: json
@@ -198,17 +206,16 @@ For instance, the following locationConstraint will save data sent to
        "details": {}
    },
 
-Each locationConstraint must include the ``type``,
-``legacyAwsBehavior``, and ``details`` keys. ``type`` indicates which
-backend will be used for that region. Currently, mem, file, and scality
-are the supported backends. ``legacyAwsBehavior`` indicates whether the
-region will have the same behavior as the AWS S3 'us-east-1' region. If
-the locationConstraint type is scality, ``details`` should contain
-connector information for sproxyd. If the locationConstraint type is mem
-or file, ``details`` should be empty.
+Each locationConstraint must include the ``type``, ``legacyAwsBehavior``,
+and ``details`` keys. ``type`` indicates which backend is used for that
+region. Supported backends are mem, file, and scality.``legacyAwsBehavior``
+indicates whether the region behaves the same as the AWS S3 'us-east-1' 
+region. If the locationConstraint type is ``scality``, ``details`` must 
+contain connector information for sproxyd. If the locationConstraint type
+is ``mem`` or ``file``, ``details`` must be empty.
 
-Once you have your locationConstraints in your locationConfig.json, you
-can specify a default locationConstraint for each of your endpoints.
+Once locationConstraints is set in locationConfig.json, specify a default
+locationConstraint for each endpoint.
 
 For instance, the following sets the ``localhost`` endpoint to the
 ``myLocationConstraint`` data backend defined above:
@@ -219,26 +226,24 @@ For instance, the following sets the ``localhost`` endpoint to the
          "localhost": "myLocationConstraint"
     },
 
-If you would like to use an endpoint other than localhost for your
-Scality Zenko CloudServer, that endpoint MUST be listed in your
-``restEndpoints``. Otherwise if your server is running with a:
+To use an endpoint other than localhost for Zenko CloudServer, the endpoint
+must be listed in ``restEndpoints``. Otherwise, if the server is running
+with a:
 
--  **file backend**: your default location constraint will be ``file``
-
--  **memory backend**: your default location constraint will be ``mem``
+*  **file backend**: The default location constraint is ``file``
+*  **memory backend**: The default location constraint is ``mem``
 
 Endpoints
 ~~~~~~~~~
 
-Note that our Zenko CloudServer supports both:
+The Zenko CloudServer supports endpoints that are rendered in either:
 
--  path-style: http://myhostname.com/mybucket
--  hosted-style: http://mybucket.myhostname.com
+* path style: http://myhostname.com/mybucket or
+* hosted style: http://mybucket.myhostname.com
 
-However, hosted-style requests will not hit the server if you are using
-an ip address for your host. So, make sure you are using path-style
-requests in that case. For instance, if you are using the AWS SDK for
-JavaScript, you would instantiate your client like this:
+However, if an IP address is specified for the host, hosted-style requests
+cannot reach the server. Use path-style requests in that case. For example,
+if you are using the AWS SDK for JavaScript, instantiate your client like this:
 
 .. code:: js
 
@@ -247,87 +252,99 @@ JavaScript, you would instantiate your client like this:
        s3ForcePathStyle: true,
     });
 
-Setting your own access key and secret key pairs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Setting Your Own Access and Secret Key Pairs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can set credentials for many accounts by editing
-``conf/authdata.json`` but if you want to specify one set of your own
-credentials, you can use ``SCALITY_ACCESS_KEY_ID`` and
-``SCALITY_SECRET_ACCESS_KEY`` environment variables.
+Credentials can be set for many accounts by editing ``conf/authdata.json``, 
+but use the ``SCALITY_ACCESS_KEY_ID`` and ``SCALITY_SECRET_ACCESS_KEY`` 
+environment variables to specify your own credentials.
+
+_`scality-access-key-id-and-scality-secret-access-key`
 
 SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 These variables specify authentication credentials for an account named
-"CustomAccount".
+“CustomAccount”.
 
-Note: Anything in the ``authdata.json`` file will be ignored.
+.. note:: Anything in the ``authdata.json`` file is ignored.
 
-.. code:: shell
+.. code-block:: shell
 
-    SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey npm start
+   $ SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey yarn start
 
+.. _Using_SSL:
 
-Scality with SSL
-~~~~~~~~~~~~~~~~~~~~~~
+Using SSL
+~~~~~~~~~
 
-If you wish to use https with your local Zenko CloudServer, you need to set up
-SSL certificates. Here is a simple guide of how to do it.
+To use https with your local CloudServer, you must set up
+SSL certificates. 
 
-Deploying Zenko CloudServer
-^^^^^^^^^^^^^^^^^^^
+1. Deploy CloudServer using `our DockerHub page
+   <https://hub.docker.com/r/zenko/cloudserver/>`__ (run it with a file
+   backend).
 
-First, you need to deploy **Zenko CloudServer**. This can be done very easily
-via `our **DockerHub**
-page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
-with a file backend).
+   .. Note:: If Docker is not installed locally, follow the
+      `instructions to install it for your distribution 
+      <https://docs.docker.com/engine/installation/>`__
 
-    *Note:* *- If you don't have docker installed on your machine, here
-    are the `instructions to install it for your
-    distribution <https://docs.docker.com/engine/installation/>`__*
+2. Update the CloudServer container’s config 
 
-Updating your Zenko CloudServer container's config
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   Add your certificates to your container. To do this, 
+   #. exec inside the CloudServer container. 
 
-You're going to add your certificates to your container. In order to do
-so, you need to exec inside your Zenko CloudServer container. Run a
-``$> docker ps`` and find your container's id (the corresponding image
-name should be ``scality/s3server``. Copy the corresponding container id
-(here we'll use ``894aee038c5e``, and run:
+   #. Run ``$> docker ps`` to find the container’s ID (the corresponding 
+      image name is ``scality/cloudserver``. 
       
-.. code:: sh
+   #. Copy the corresponding container ID (``894aee038c5e`` in the present
+      example), and run:
+
+      .. code-block:: shell
 
          $> docker exec -it 894aee038c5e bash
 
-You're now inside your container, using an interactive terminal :)
+      This puts you inside your container, using an interactive terminal.
 
-Generate SSL key and certificates
-**********************************
+3. Generate the SSL key and certificates. The paths where the different
+   files are stored are defined after the ``-out`` option in each of the 
+   following commands.
 
-There are 5 steps to this generation. The paths where the different
-files are stored are defined after the ``-out`` option in each command
+    #. Generate a private key for your certificate signing request (CSR):
 
-.. code:: sh
+       .. code-block:: shell
 
-    # Generate a private key for your CSR
 	  $> openssl genrsa -out ca.key 2048
-    # Generate a self signed certificate for your local Certificate Authority
+
+    #. Generate a self-signed certificate for your local certificate 
+       authority (CA):
+
+       .. code:: shell
+
 	  $> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
 
-    # Generate a key for Zenko CloudServer
+    #. Generate a key for the CloudServer:
+
+       .. code:: shell
+
           $> openssl genrsa -out test.key 2048
-    # Generate a Certificate Signing Request for S3 Server
+
+    #. Generate a CSR for CloudServer:
+
+       .. code:: shell
+
           $> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
-    # Generate a local-CA-signed certificate for S3 Server
+
+    #. Generate a certificate for CloudServer signed by the local CA:
+
+       .. code:: shell
+
           $> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
 
-Update Zenko CloudServer ``config.json``
-**********************************
+4. Update Zenko CloudServer ``config.json``. Add a ``certFilePaths`` 
+   section to ``./config.json`` with appropriate paths:
 
-Add a ``certFilePaths`` section to ``./config.json`` with the
-appropriate paths:
-
-.. code:: json
+   .. code:: json
 
         "certFilePaths": {
             "key": "./test.key",
@@ -335,42 +352,36 @@ appropriate paths:
             "ca": "./ca.crt"
         }
 
-Run your container with the new config
-****************************************
+5. Run your container with the new config. 
 
-First, you need to exit your container. Simply run ``$> exit``. Then,
-you need to restart your container. Normally, a simple
-``$> docker restart s3server`` should do the trick.
+   #. Exit the container by running ``$> exit``. 
 
-Update your host config
-^^^^^^^^^^^^^^^^^^^^^^^
+   #. Restart the container with ``$> docker restart cloudserver``.
 
-Associates local IP addresses with hostname
-*******************************************
+6. Update the host configuration by adding s3.scality.test 
+   to /etc/hosts:
 
-In your ``/etc/hosts`` file on Linux, OS X, or Unix (with root
-permissions), edit the line of localhost so it looks like this:
-
-::
+   .. code:: bash
 
       127.0.0.1      localhost s3.scality.test
 
-Copy the local certificate authority from your container
-*********************************************************
+7. Copy the local certificate authority (ca.crt in step 4) from your 
+   container. Choose the path to save this file to (in the present 
+   example, ``/root/ca.crt``), and run:
 
-In the above commands, it's the file named ``ca.crt``. Choose the path
-you want to save this file at (here we chose ``/root/ca.crt``), and run
-something like:
-
-.. code:: sh
+   .. code:: shell
 
       $> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
 
-Test your config
-^^^^^^^^^^^^^^^^^
+   .. note:: Your container ID will be different, and your path to 
+	     ca.crt may be different.
 
-If you do not have aws-sdk installed, run ``$> npm install aws-sdk``. In
-a ``test.js`` file, paste the following script:
+Test the Config
+^^^^^^^^^^^^^^^
+
+If aws-sdk is not installed, run ``$> yarn install aws-sdk``. 
+
+Paste the following script into a file named "test.js":
 
 .. code:: js
 
@@ -410,8 +421,13 @@ a ``test.js`` file, paste the following script:
         });
     });
 
-Now run that script with ``$> nodejs test.js``. If all goes well, it
-should output ``SSL is cool!``. Enjoy that added security!
+Now run this script with:
+
+.. code::
+
+   $> nodejs test.js
+ 
+On success, the script outputs ``SSL is cool!``.
 
 
 .. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg

docs/INTEGRATIONS.rst

@@ -4,40 +4,34 @@ Integrations
 High Availability
 =================
 
-`Docker swarm <https://docs.docker.com/engine/swarm/>`__ is a
-clustering tool developped by Docker and ready to use with its
-containers. It allows to start a service, which we define and use as a
-means to ensure Zenko CloudServer's continuous availability to the end user.
-Indeed, a swarm defines a manager and n workers among n+1 servers. We
-will do a basic setup in this tutorial, with just 3 servers, which
-already provides a strong service resiliency, whilst remaining easy to
-do as an individual. We will use NFS through docker to share data and
+`Docker Swarm <https://docs.docker.com/engine/swarm/>`__ is a clustering tool
+developed by Docker for use with its containers. It can be used to start
+services, which we define to ensure CloudServer's continuous availability to
+end users. A swarm defines a manager and *n* workers among *n* + 1 servers.
+
+This tutorial shows how to perform a basic setup with three servers, which
+provides strong service resiliency, while remaining easy to use and
+maintain. We will use NFS through Docker to share data and
 metadata between the different servers.
 
-You will see that the steps of this tutorial are defined as **On
-Server**, **On Clients**, **On All Machines**. This refers respectively
-to NFS Server, NFS Clients, or NFS Server and Clients. In our example,
-the IP of the Server will be **10.200.15.113**, while the IPs of the
-Clients will be **10.200.15.96 and 10.200.15.97**
+Sections are labeled **On Server**, **On Clients**, or
+**On All Machines**, referring respectively to NFS server, NFS clients, or
+NFS server and clients. In the present example, the server’s IP address is
+**10.200.15.113** and the client IP addresses are **10.200.15.96** and
+**10.200.15.97**
 
-Installing docker
------------------
+1. Install Docker (on All Machines)
 
-Any version from docker 1.12.6 onwards should work; we used Docker
-17.03.0-ce for this tutorial.
+   Docker 17.03.0-ce is used for this tutorial. Docker 1.12.6 and later will
+   likely work, but is not tested.
 
-On All Machines
-~~~~~~~~~~~~~~~
+   * On Ubuntu 14.04
+     Install Docker CE for Ubuntu as `documented at Docker
+     <https://docs.docker.com/install/linux/docker-ce/ubuntu/>`__.
+     Install the aufs dependency as recommended by Docker. The required
+     commands are:
 
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
-
-The docker website has `solid
-documentation <https://docs.docker.com/engine/installation/linux/ubuntu/>`__.
-We have chosen to install the aufs dependency, as recommended by Docker.
-Here are the required commands:
-
-.. code:: sh
+     .. code:: sh
 
       $> sudo apt-get update
       $> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
@@ -47,14 +41,12 @@ Here are the required commands:
       $> sudo apt-get update
       $> sudo apt-get install docker-ce
 
-On CentOS 7
-^^^^^^^^^^^
+    * On CentOS 7
+      Install Docker CE as `documented at Docker
+      <https://docs.docker.com/install/linux/docker-ce/centos/>`__.
+      The required commands are:
 
-The docker website has `solid
-documentation <https://docs.docker.com/engine/installation/linux/centos/>`__.
-Here are the required commands:
-
-.. code:: sh
+      .. code:: sh
 
         $> sudo yum install -y yum-utils
         $> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
@@ -62,31 +54,24 @@ Here are the required commands:
         $> sudo yum install docker-ce
         $> sudo systemctl start docker
 
-Configure NFS
--------------
+2. Install NFS on Client(s)
 
-On Clients
-~~~~~~~~~~
+   NFS clients mount Docker volumes over the NFS server’s shared folders.
+   If the NFS commons are installed, manual mounts are no longer needed.
 
-Your NFS Clients will mount Docker volumes over your NFS Server's shared
-folders. Hence, you don't have to mount anything manually, you just have
-to install the NFS commons:
+   * On Ubuntu 14.04
 
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
+     Install the NFS commons with apt-get:
 
-Simply install the NFS commons:
-
-.. code:: sh
+     .. code:: sh
 
        $> sudo apt-get install nfs-common
 
-On CentOS 7
-^^^^^^^^^^^
+   * On CentOS 7
 
-Install the NFS utils, and then start the required services:
+     Install the NFS utils; then start required services:
 
-.. code:: sh
+     .. code:: sh
 
       $> yum install nfs-utils
       $> sudo systemctl enable rpcbind
@@ -98,29 +83,24 @@ Install the NFS utils, and then start the required services:
       $> sudo systemctl start nfs-lock
       $> sudo systemctl start nfs-idmap
 
-On Server
-~~~~~~~~~
+3. Install NFS (on Server)
 
-Your NFS Server will be the machine to physically host the data and
-metadata. The package(s) we will install on it is slightly different
-from the one we installed on the clients.
+   The NFS server hosts the data and metadata. The package(s) to install on it
+   differs from the package installed on the clients.
 
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
+   * On Ubuntu 14.04
 
-Install the NFS server specific package and the NFS commons:
+     Install the NFS server-specific package and the NFS commons:
 
-.. code:: sh
+     .. code:: sh
 
       $> sudo apt-get install nfs-kernel-server nfs-common
 
-On CentOS 7
-^^^^^^^^^^^
+   * On CentOS 7
 
-Same steps as with the client: install the NFS utils and start the
-required services:
+     Install the NFS utils and start the required services:
 
-.. code:: sh
+     .. code:: sh
 
        $> yum install nfs-utils
        $> sudo systemctl enable rpcbind
@@ -132,111 +112,88 @@ required services:
        $> sudo systemctl start nfs-lock
        $> sudo systemctl start nfs-idmap
 
-On Ubuntu 14.04 and CentOS 7
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   For both distributions:
 
-Choose where your shared data and metadata from your local `Zenko CloudServer
-<http://www.zenko.io/cloudserver/>`__ will be stored.
-We chose to go with /var/nfs/data and /var/nfs/metadata. You also need
-to set proper sharing permissions for these folders as they'll be shared
-over NFS:
+   #. Choose where shared data and metadata from the local
+      `CloudServer <http://www.zenko.io/cloudserver/>`__ shall be stored (The
+      present example uses /var/nfs/data and /var/nfs/metadata). Set permissions
+      for these folders for
+      sharing over NFS:
 
-.. code:: sh
+      .. code:: sh
 
         $> mkdir -p /var/nfs/data /var/nfs/metadata
         $> chmod -R 777 /var/nfs/
 
-Now you need to update your **/etc/exports** file. This is the file that
-configures network permissions and rwx permissions for NFS access. By
-default, Ubuntu applies the no\_subtree\_check option, so we declared
-both folders with the same permissions, even though they're in the same
-tree:
+   #. The /etc/exports file configures network permissions and r-w-x permissions
+      for NFS access. Edit /etc/exports, adding the following lines:
 
-.. code:: sh
-
-    $> sudo vim /etc/exports
-
-In this file, add the following lines:
-
-.. code:: sh
+      .. code:: sh
 
         /var/nfs/data        10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
         /var/nfs/metadata    10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
 
-Export this new NFS table:
+      Ubuntu applies the no\_subtree\_check option by default, so both
+      folders are declared with the same permissions, even though they’re in
+      the same tree.
 
-.. code:: sh
+   #. Export this new NFS table:
+
+      .. code:: sh
 
         $> sudo exportfs -a
 
-Eventually, you need to allow for NFS mount from Docker volumes on other
-machines. You need to change the Docker config in
-**/lib/systemd/system/docker.service**:
+   #. Edit the ``MountFlags`` option in the Docker config in
+      /lib/systemd/system/docker.service to enable NFS mount from Docker volumes
+      on other machines:
 
-.. code:: sh
-
-    $> sudo vim /lib/systemd/system/docker.service
-
-In this file, change the **MountFlags** option:
-
-.. code:: sh
+      .. code:: sh
 
         MountFlags=shared
 
-Now you just need to restart the NFS server and docker daemons so your
-changes apply.
+   #. Restart the NFS server and Docker daemons to apply these changes.
 
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
+      * On Ubuntu 14.04
 
-Restart your NFS Server and docker services:
-
-.. code:: sh
+        .. code:: sh
 
           $> sudo service nfs-kernel-server restart
           $> sudo service docker restart
 
-On CentOS 7
-^^^^^^^^^^^
+      * On CentOS 7
 
-Restart your NFS Server and docker daemons:
-
-.. code:: sh
+        .. code:: sh
 
          $> sudo systemctl restart nfs-server
          $> sudo systemctl daemon-reload
          $> sudo systemctl restart docker
 
-Set up your Docker Swarm service
---------------------------------
 
-On All Machines
-~~~~~~~~~~~~~~~
+4. Set Up a Docker Swarm
 
-On Ubuntu 14.04 and CentOS 7
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  * On all machines and distributions:
 
-We will now set up the Docker volumes that will be mounted to the NFS
-Server and serve as data and metadata storage for Zenko CloudServer. These two
-commands have to be replicated on all machines:
+    Set up the Docker volumes to be mounted to the NFS server for CloudServer’s
+    data and metadata storage. The following commands must be replicated on all
+    machines:
 
-.. code:: sh
+    .. code:: sh
 
      $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
      $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
 
-There is no need to ""docker exec" these volumes to mount them: the
-Docker Swarm manager will do it when the Docker service will be started.
+    There is no need to ``docker exec`` these volumes to mount them: the
+    Docker Swarm manager does this when the Docker service is started.
 
-On Server
-^^^^^^^^^
+  * On a server:
 
-To start a Docker service on a Docker Swarm cluster, you first have to
-initialize that cluster (i.e.: define a manager), then have the
-workers/nodes join in, and then start the service. Initialize the swarm
-cluster, and look at the response:
+    To start a Docker service on a Docker Swarm cluster, initialize the cluster
+    (that is, define a manager), prompt workers/nodes to join in, and then start
+    the service.
 
-.. code:: sh
+    Initialize the swarm cluster, and review its response:
+
+    .. code:: sh
 
       $> docker swarm init --advertise-addr 10.200.15.113
 
@@ -250,204 +207,180 @@ cluster, and look at the response:
 
       To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
 
-On Clients
-^^^^^^^^^^
+  * On clients:
 
-Simply copy/paste the command provided by your docker swarm init. When
-all goes well, you'll get something like this:
+    Copy and paste the command provided by your Docker Swarm init. A successful
+    request/response will resemble:
 
-.. code:: sh
+    .. code:: sh
 
       $> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
 
       This node joined a swarm as a worker.
 
-On Server
-^^^^^^^^^
+Set Up Docker Swarm on Clients on a Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Start the service on your swarm cluster!
+Start the service on the Swarm cluster.
 
 .. code:: sh
 
-    $> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/s3server
+  $> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/cloudserver
 
-If you run a docker service ls, you should have the following output:
+On a successful installation, ``docker service ls`` returns the following
+output:
 
 .. code:: sh
 
     $> docker service ls
     ID            NAME  MODE        REPLICAS  IMAGE
-    ocmggza412ft  s3    replicated  1/1       scality/s3server:latest
+    ocmggza412ft  s3    replicated  1/1       scality/cloudserver:latest
 
-If your service won't start, consider disabling apparmor/SELinux.
+If the service does not start, consider disabling apparmor/SELinux.
 
-Testing your High Availability S3Server
----------------------------------------
+Testing the High-Availability CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-On All Machines
-~~~~~~~~~~~~~~~
-
-On Ubuntu 14.04 and CentOS 7
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Try to find out where your Scality Zenko CloudServer is actually running using
-the **docker ps** command. It can be on any node of the swarm cluster,
-manager or worker. When you find it, you can kill it, with **docker stop
-<container id>** and you'll see it respawn on a different node of the
-swarm cluster. Now you see, if one of your servers falls, or if docker
-stops unexpectedly, your end user will still be able to access your
-local Zenko CloudServer.
+On all machines (client/server) and distributions (Ubuntu and CentOS),
+determine where CloudServer is running using ``docker ps``. CloudServer can
+operate on any node of the Swarm cluster, manager or worker. When you find
+it, you can kill it with ``docker stop <container id>``. It will respawn
+on a different node. Now, if one server falls, or if Docker stops
+unexpectedly, the end user will still be able to access your the local CloudServer.
 
 Troubleshooting
----------------
+~~~~~~~~~~~~~~~
 
-To troubleshoot the service you can run:
+To troubleshoot the service, run:
 
 .. code:: sh
 
     $> docker service ps s3docker service ps s3
     ID                         NAME      IMAGE             NODE                               DESIRED STATE  CURRENT STATE       ERROR
-    0ar81cw4lvv8chafm8pw48wbc  s3.1      scality/s3server  localhost.localdomain.localdomain  Running        Running 7 days ago
-    cvmf3j3bz8w6r4h0lf3pxo6eu   \_ s3.1  scality/s3server  localhost.localdomain.localdomain  Shutdown       Failed 7 days ago   "task: non-zero exit (137)"
+    0ar81cw4lvv8chafm8pw48wbc  s3.1      scality/cloudserver  localhost.localdomain.localdomain  Running        Running 7 days ago
+    cvmf3j3bz8w6r4h0lf3pxo6eu   \_ s3.1  scality/cloudserver  localhost.localdomain.localdomain  Shutdown       Failed 7 days ago   "task: non-zero exit (137)"
 
-If the error is truncated it is possible to have a more detailed view of
-the error by inspecting the docker task ID:
+If the error is truncated, view the error in detail by inspecting the
+Docker task ID:
 
 .. code:: sh
 
     $> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu
 
 Off you go!
------------
-
-Let us know what you use this functionality for, and if you'd like any
-specific developments around it. Or, even better: come and contribute to
-our `Github repository <https://github.com/scality/s3/>`__! We look
-forward to meeting you!
+~~~~~~~~~~~
 
+Let us know how you use this and if you'd like any specific developments 
+around it. Even better: come and contribute to our `Github repository 
+<https://github.com/scality/s3/>`__! We look forward to meeting you!
 
 S3FS
 ====
-Export your buckets as a filesystem with s3fs on top of Zenko CloudServer
+
+You can export buckets as a filesystem with s3fs on CloudServer.
 
 `s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source
-tool that allows you to mount an S3 bucket on a filesystem-like backend.
-It is available both on Debian and RedHat distributions. For this
-tutorial, we used an Ubuntu 14.04 host to deploy and use s3fs over
-Scality's Zenko CloudServer.
+tool, available both on Debian and RedHat distributions, that enables
+you to mount an S3 bucket on a filesystem-like backend. This tutorial uses
+an Ubuntu 14.04 host to deploy and use s3fs over CloudServer.
 
 Deploying Zenko CloudServer with SSL
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-First, you need to deploy **Zenko CloudServer**. This can be done very easily
-via `our DockerHub
-page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
-with a file backend).
+First, deploy CloudServer with a file backend using `our DockerHub page
+<https://hub.docker.com/r/zenko/cloudserver>`__.
 
-    *Note:* *- If you don't have docker installed on your machine, here
-    are the `instructions to install it for your
-    distribution <https://docs.docker.com/engine/installation/>`__*
+.. note::
 
-You also necessarily have to set up SSL with Zenko CloudServer to use s3fs. We
-have a nice
-`tutorial <https://s3.scality.com/v1.0/page/scality-with-ssl>`__ to help
-you do it.
+  If Docker is not installed on your machine, follow
+  `these instructions <https://docs.docker.com/engine/installation/>`__
+  to install it for your distribution.
 
-s3fs setup
-----------
+You must also set up SSL with CloudServer to use s3fs. See `Using SSL
+<./GETTING_STARTED#Using_SSL>`__ for instructions.
+
+s3fs Setup
+~~~~~~~~~~
 
 Installing s3fs
-~~~~~~~~~~~~~~~
+---------------
 
-s3fs has quite a few dependencies. As explained in their
-`README <https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation>`__,
-the following commands should install everything for Ubuntu 14.04:
+Follow the instructions in the s3fs `README
+<https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation-from-pre-built-packages>`__,
 
-.. code:: sh
-
-    $> sudo apt-get install automake autotools-dev g++ git libcurl4-gnutls-dev
-    $> sudo apt-get install libfuse-dev libssl-dev libxml2-dev make pkg-config
-
-Now you want to install s3fs per se:
-
-.. code:: sh
-
-    $> git clone https://github.com/s3fs-fuse/s3fs-fuse.git
-    $> cd s3fs-fuse
-    $> ./autogen.sh
-    $> ./configure
-    $> make
-    $> sudo make install
-
-Check that s3fs is properly installed by checking its version. it should
-answer as below:
+Check that s3fs is properly installed. A version check should return
+a response resembling:
 
 .. code:: sh
 
     $> s3fs --version
 
     Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL
+    Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
+    License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>
+    This is free software: you are free to change and redistribute it.
+    There is NO WARRANTY, to the extent permitted by law.
 
 Configuring s3fs
-~~~~~~~~~~~~~~~~
+----------------
 
 s3fs expects you to provide it with a password file. Our file is
 ``/etc/passwd-s3fs``. The structure for this file is
-``ACCESSKEYID:SECRETKEYID``, so, for S3Server, you can run:
+``ACCESSKEYID:SECRETKEYID``, so, for CloudServer, you can run:
 
 .. code:: sh
 
     $> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs
     $> chmod 600 /etc/passwd-s3fs
 
-Using Zenko CloudServer with s3fs
-------------------------
+Using CloudServer with s3fs
+---------------------------
 
-First, you're going to need a mountpoint; we chose ``/mnt/tests3fs``:
+1. Use /mnt/tests3fs as a mount point.
 
-.. code:: sh
+   .. code:: sh
 
     $> mkdir /mnt/tests3fs
 
-Then, you want to create a bucket on your local Zenko CloudServer; we named it
-``tests3fs``:
+2. Create a bucket on your local CloudServer. In the present example it is
+   named “tests3fs”.
 
-.. code:: sh
+   .. code:: sh
 
     $> s3cmd mb s3://tests3fs
 
-    *Note:* *- If you've never used s3cmd with our Zenko CloudServer, our README
-    provides you with a `recommended
-    config <https://github.com/scality/S3/blob/master/README.md#s3cmd>`__*
+3. Mount the bucket to your mount point with s3fs:
 
-Now you can mount your bucket to your mountpoint with s3fs:
-
-.. code:: sh
+   .. code:: sh
 
     $> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style
 
-    *If you're curious, the structure of this command is*
-    ``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``\ *, and the
-    options are mandatory and serve the following purposes:
-    * ``passwd_file``\ *: specifiy path to password file;
-    * ``url``\ *: specify the hostname used by your SSL provider;
-    * ``use_path_request_style``\ *: force path style (by default, s3fs
-    uses subdomains (DNS style)).*
+   The structure of this command is:
+   ``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``. Of these mandatory
+   options:
 
-| From now on, you can either add files to your mountpoint, or add
-  objects to your bucket, and they'll show in the other.
-| For example, let's' create two files, and then a directory with a file
-  in our mountpoint:
+   * ``passwd_file`` specifies the path to the password file.
+   * ``url`` specifies the host name used by your SSL provider.
+   * ``use_path_request_style`` forces the path style (by default,
+       s3fs uses DNS-style subdomains).
 
-.. code:: sh
+Once the bucket is mounted, files added to the mount point or
+objects added to the bucket will appear in both locations.
+
+Example
+-------
+
+   Create two files, and then a directory with a file in our mount point:
+
+   .. code:: sh
 
       $> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
       $> mkdir /mnt/tests3fs/dir1
       $> touch /mnt/tests3fs/dir1/file3
 
-Now, I can use s3cmd to show me what is actually in S3Server:
+   Now, use s3cmd to show what is in CloudServer:
 
-.. code:: sh
+   .. code:: sh
 
       $> s3cmd ls -r s3://tests3fs
 
@@ -456,35 +389,21 @@ Now, I can use s3cmd to show me what is actually in S3Server:
       2017-02-28 17:28         0   s3://tests3fs/file1
       2017-02-28 17:28         0   s3://tests3fs/file2
 
-Now you can enjoy a filesystem view on your local Zenko CloudServer!
+   Now you can enjoy a filesystem view on your local CloudServer.
 
 
 Duplicity
 =========
 
-How to backup your files with Zenko CloudServer.
+How to back up your files with CloudServer.
 
-Installing
------------
-
-Installing Duplicity and its dependencies
+Installing Duplicity and its Dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Second, you want to install
-`Duplicity <http://duplicity.nongnu.org/index.html>`__. You have to
-download `this
-tarball <https://code.launchpad.net/duplicity/0.7-series/0.7.11/+download/duplicity-0.7.11.tar.gz>`__,
-decompress it, and then checkout the README inside, which will give you
-a list of dependencies to install. If you're using Ubuntu 14.04, this is
-your lucky day: here is a lazy step by step install.
-
-.. code:: sh
-
-    $> apt-get install librsync-dev gnupg
-    $> apt-get install python-dev python-pip python-lockfile
-    $> pip install -U boto
-
-Then you want to actually install Duplicity:
+To install `Duplicity <http://duplicity.nongnu.org/>`__,
+go to `this site <https://code.launchpad.net/duplicity/0.7-series>`__.
+Download the latest tarball. Decompress it and follow the instructions
+in the README.
 
 .. code:: sh
 
@@ -492,22 +411,29 @@ Then you want to actually install Duplicity:
    $> cd duplicity-0.7.11
    $> python setup.py install
 
-Using
-------
+You may receive error messages indicating the need to install some or all
+of the following dependencies:
 
-Testing your installation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. code:: sh
 
-First, we're just going to quickly check that Zenko CloudServer is actually
-running. To do so, simply run ``$> docker ps`` . You should see one
-container named ``scality/s3server``. If that is not the case, try
-``$> docker start s3server``, and check again.
+    $> apt-get install librsync-dev gnupg
+    $> apt-get install python-dev python-pip python-lockfile
+    $> pip install -U boto
 
-Secondly, as you probably know, Duplicity uses a module called **Boto**
-to send requests to S3. Boto requires a configuration file located in
-**``/etc/boto.cfg``** to have your credentials and preferences. Here is
-a minimalistic config `that you can finetune following these
-instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
+Testing the Installation
+------------------------
+
+1. Check that CloudServer is running. Run ``$> docker ps``. You should
+   see one container named ``scality/cloudserver``. If you do not, run
+   ``$> docker start cloudserver`` and check again.
+
+
+2. Duplicity uses a module called “Boto” to send requests to S3. Boto
+   requires a configuration file located in ``/etc/boto.cfg`` to store
+   your credentials and preferences. A minimal configuration
+   you can fine tune `following these instructions
+   <http://boto.cloudhackers.com/en/latest/getting_started.html>`__ is
+   shown here:
 
 ::
 
@@ -521,26 +447,25 @@ instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
     # If using SSL, unmute and provide absolute path to local CA certificate
     # ca_certificates_file = /absolute/path/to/ca.crt
 
-    *Note:* *If you want to set up SSL with Zenko CloudServer, check out our
-    `tutorial <http://link/to/SSL/tutorial>`__*
+ .. note:: To set up SSL with CloudServer, check out our `Using SSL
+	   <./GETTING_STARTED#Using_SSL>`__ in GETTING STARTED.
 
-At this point, we've met all the requirements to start running Zenko CloudServer
-as a backend to Duplicity. So we should be able to back up a local
-folder/file to local S3. Let's try with the duplicity decompressed
-folder:
+3. At this point all requirements to run CloudServer as a backend to Duplicity
+   have been met. A local folder/file should back up to the local S3.
+   Try it with the decompressed Duplicity folder:
 
 .. code:: sh
 
     $> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/"
 
-    *Note:* *Duplicity will prompt you for a symmetric encryption
-    passphrase. Save it somewhere as you will need it to recover your
-    data. Alternatively, you can also add the ``--no-encryption`` flag
-    and the data will be stored plain.*
+.. note:: Duplicity will prompt for a symmetric encryption passphrase.
+	  Save it carefully, as you will need it to recover your data.
+	  Alternatively, you can add the ``--no-encryption`` flag
+	  and the data will be stored plain.
 
-If this command is succesful, you will get an output looking like this:
+   If this command is successful, you will receive an output resembling:
 
-::
+   .. code:: sh
 
       --------------[ Backup Statistics ]--------------
       StartTime 1486486547.13 (Tue Feb  7 16:55:47 2017)
@@ -560,15 +485,13 @@ If this command is succesful, you will get an output looking like this:
       Errors 0
       -------------------------------------------------
 
-Congratulations! You can now backup to your local S3 through duplicity
-:)
+Congratulations! You can now back up to your local S3 through Duplicity.
 
-Automating backups
-~~~~~~~~~~~~~~~~~~~
+Automating Backups
+------------------
 
-Now you probably want to back up your files periodically. The easiest
-way to do this is to write a bash script and add it to your crontab.
-Here is my suggestion for such a file:
+The easiest way to back up files periodically is to write a bash script
+and add it to your crontab. A suggested script follows.
 
 .. code:: sh
 
@@ -577,33 +500,33 @@ Here is my suggestion for such a file:
     # Export your passphrase so you don't have to type anything
     export PASSPHRASE="mypassphrase"
 
-    # If you want to use a GPG Key, put it here and unmute the line below
+    # To use a GPG key, put it here and uncomment the line below
     #GPG_KEY=
 
     # Define your backup bucket, with localhost specified
-    DEST="s3://127.0.0.1:8000/testbuckets3server/"
+    DEST="s3://127.0.0.1:8000/testbucketcloudserver/"
 
-    # Define the absolute path to the folder you want to backup
+    # Define the absolute path to the folder to back up
     SOURCE=/root/testfolder
 
     # Set to "full" for full backups, and "incremental" for incremental backups
-    # Warning: you have to perform one full backup befor you can perform
+    # Warning: you must perform one full backup befor you can perform
     # incremental ones on top of it
     FULL=incremental
 
-    # How long to keep backups for; if you don't want to delete old
-    # backups, keep empty; otherwise, syntax is "1Y" for one year, "1M"
-    # for one month, "1D" for one day
+    # How long to keep backups. If you don't want to delete old backups, keep
+    # this value empty; otherwise, the syntax is "1Y" for one year, "1M" for
+    # one month, "1D" for one day.
     OLDER_THAN="1Y"
 
-    # is_running checks whether duplicity is currently completing a task
+    # is_running checks whether Duplicity is currently completing a task
     is_running=$(ps -ef | grep duplicity  | grep python | wc -l)
 
-    # If duplicity is already completing a task, this will simply not run
+    # If Duplicity is already completing a task, this will not run
     if [ $is_running -eq 0 ]; then
         echo "Backup for ${SOURCE} started"
 
-        # If you want to delete backups older than a certain time, we do it here
+        # To delete backups older than a certain time, do it here
         if [ "$OLDER_THAN" != "" ]; then
             echo "Removing backups older than ${OLDER_THAN}"
             duplicity remove-older-than ${OLDER_THAN} ${DEST}
@@ -626,17 +549,17 @@ Here is my suggestion for such a file:
     # Forget the passphrase...
     unset PASSPHRASE
 
-So let's say you put this file in ``/usr/local/sbin/backup.sh.`` Next
-you want to run ``crontab -e`` and paste your configuration in the file
-that opens. If you're unfamiliar with Cron, here is a good `How
-To <https://help.ubuntu.com/community/CronHowto>`__. The folder I'm
-backing up is a folder I modify permanently during my workday, so I want
-incremental backups every 5mn from 8AM to 9PM monday to friday. Here is
-the line I will paste in my crontab:
+Put this file in ``/usr/local/sbin/backup.sh``. Run ``crontab -e`` and
+paste your configuration into the file that opens. If you're unfamiliar
+with Cron, here is a good `HowTo
+<https://help.ubuntu.com/community/CronHowto>`__. If the folder being
+backed up is a folder to be modified permanently during the work day,
+we can set incremental backups every 5 minutes from 8 AM to 9 PM Monday
+through Friday by pasting the following line into crontab:
 
-.. code:: cron
+.. code:: sh
 
     */5 8-20 * * 1-5 /usr/local/sbin/backup.sh
 
-Now I can try and add / remove files from the folder I'm backing up, and
-I will see incremental backups in my bucket.
+Adding or removing files from the folder being backed up will result in
+incremental backups in the bucket.

docs/MD_SEARCH.rst

@@ -0,0 +1,263 @@
+Metadata Search Documentation
+=============================
+
+Description
+-----------
+
+This feature enables metadata search to be performed on the metadata of objects
+stored in Zenko.
+
+Requirements
+------------
+
+* MongoDB
+
+Design
+------
+
+The Metadata Search feature expands on the existing :code:`GET Bucket` S3 API by
+enabling users to conduct metadata searches by adding the custom Zenko query
+string parameter, :code:`search`. The :code:`search` parameter is structured as a pseudo
+SQL WHERE clause, and supports basic SQL operators. For example:
+:code:`"A=1 AND B=2 OR C=3"` (complex queries can be built using nesting
+operators, :code:`(` and :code:`)`).
+
+The search process is as follows:
+
+* Zenko receives a :code:`GET` request.
+
+  .. code::
+
+    # regular getBucket request
+    GET /bucketname HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # getBucket versions request
+    GET /bucketname?versions HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # search getBucket request
+    GET /bucketname?search=key%3Dsearch-item HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+* If the request does *not* contain the :code:`search` query parameter, Zenko performs
+  a normal bucket listing and returns an XML result containing the list of
+  objects.
+* If the request *does* contain the :code:`search` query parameter, Zenko parses and
+  validates the search string.
+
+  - If the search string is invalid, Zenko returns an :code:`InvalidArgument` error.
+
+    .. code::
+
+      <?xml version=\"1.0\" encoding=\"UTF-8\"?>
+      <Error>
+        <Code>InvalidArgument</Code>
+        <Message>Invalid sql where clause sent as search query</Message>
+        <Resource></Resource>
+        <RequestId>d1d6afc64345a8e1198e</RequestId>
+      </Error>
+
+  - If the search string is valid, Zenko parses it and generates an abstract
+    syntax tree (AST). The AST is then passed to the MongoDB backend to be
+    used as the query filter for retrieving objects from a bucket that
+    satisfies the requested search conditions. Zenko parses the filtered
+    results and returns them as the response.
+
+Metadata search results have the same structure as a :code:`GET Bucket` response:
+
+.. code:: xml
+
+  <?xml version="1.0" encoding="UTF-8"?>
+  <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+      <Name>bucketname</Name>
+      <Prefix/>
+      <Marker/>
+      <MaxKeys>1000</MaxKeys>
+      <IsTruncated>false</IsTruncated>
+      <Contents>
+          <Key>objectKey</Key>
+          <LastModified>2018-04-19T18:31:49.426Z</LastModified>
+          <ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
+          <Size>0</Size>
+          <Owner>
+              <ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
+              <DisplayName>Bart</DisplayName>
+          </Owner>
+          <StorageClass>STANDARD</StorageClass>
+      </Contents>
+      <Contents>
+          ...
+      </Contents>
+  </ListBucketResult>
+
+Performing Metadata Searches with Zenko
+---------------------------------------
+
+You can perform metadata searches by:
+
++ Using the :code:`search_bucket` tool in the
+  `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository.
++ Creating a signed HTTP request to Zenko in your preferred programming
+  language.
+
+Using the S3 Tool
++++++++++++++++++
+
+After cloning the `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository
+and installing the necessary dependencies, run the following command in the S3
+project’s root directory to access the search tool:
+
+.. code::
+
+  node bin/search_bucket
+
+This generates the following output:
+
+.. code::
+
+    Usage: search_bucket [options]
+
+    Options:
+
+      -V, --version                 output the version number
+      -a, --access-key <accessKey>  Access key id
+      -k, --secret-key <secretKey>  Secret access key
+      -b, --bucket <bucket>         Name of the bucket
+      -q, --query <query>           Search query
+      -h, --host <host>             Host of the server
+      -p, --port <port>             Port of the server
+      -s                            --ssl
+      -v, --verbose
+      -h, --help                    output usage information
+
+In the following examples, Zenko Server is accessible on endpoint
+:code:`http://127.0.0.1:8000` and contains the bucket :code:`zenkobucket`.
+
+.. code::
+
+    # search for objects with metadata "blue"
+    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+        -q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
+
+    # search for objects tagged with "type=color"
+    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+        -q "tags.type=color" -h 127.0.0.1 -p 8000
+
+Coding Examples
++++++++++++++++
+
+Search requests can be also performed by making HTTP requests authenticated
+with one of the AWS Signature schemes: version 2 or version 4. \
+For more about authentication scheme, see:
+
+* https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
+* http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
+* http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+
+You can also view examples for making requests with Auth V4 in various
+languages `here <../../../examples>`__.
+
+Specifying Metadata Fields
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To search system metadata headers:
+
+.. code::
+
+    {system-metadata-key}{supported SQL op}{search value}
+    # example
+    key = blueObject
+    size > 0
+    key LIKE "blue.*"
+
+To search custom user metadata:
+
+.. code::
+
+    # metadata must be prefixed with "x-amz-meta-"
+    x-amz-meta-{user-metadata-key}{supported SQL op}{search value}
+
+    # example
+    x-amz-meta-color = blue
+    x-amz-meta-color != red
+    x-amz-meta-color LIKE "b.*"
+
+To search tags:
+
+.. code::
+
+    # tag searches must be prefixed with "tags."
+    tags.{tag-key}{supported SQL op}{search value}
+    # example
+    tags.type = color
+
+Examples queries:
+
+.. code::
+
+    # searching for objects with custom metadata "color"=blue" and are tagged
+    # "type"="color"
+
+    tags.type="color" AND  x-amz-meta-color="blue"
+
+    # searching for objects with the object key containing the substring "blue"
+    # or (custom metadata "color"=blue" and are tagged "type"="color")
+
+    key LIKE '.*blue.*' OR (x-amz-meta-color="blue" AND tags.type="color")
+
+Differences from SQL
+++++++++++++++++++++
+
+Zenko metadata search queries are similar to SQL-query :code:`WHERE` clauses, but
+differ in that:
+
+* They follow the :code:`PCRE` format
+* They do not require values with hyphens to be enclosed in
+  backticks, :code:``(`)``
+
+  .. code::
+
+        # SQL query
+        `x-amz-meta-search-item` = `ice-cream-cone`
+
+        # MD Search query
+        x-amz-meta-search-item = ice-cream-cone
+
+* Search queries do not support all SQL operators.
+
+  .. code::
+
+    # Supported SQL operators:
+    =, <, >, <=, >=, !=, AND, OR, LIKE, <>
+
+    # Unsupported SQL operators:
+    NOT, BETWEEN, IN, IS, +, -, %, ^, /, *, !
+
+Using Regular Expressions in Metadata Search
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Regular expressions in Zenko metadata search differ from SQL in the following
+ways:
+
++ Wildcards are represented with :code:`.*` instead of :code:`%`.
++ Regex patterns must be wrapped in quotes. Failure to do this can lead to
+  misinterpretation of patterns.
++ As with :code:`PCRE`, regular expressions can be entered in either the
+  :code:`/pattern/` syntax or as the pattern itself if regex options are
+  not required.
+
+Example regular expressions:
+
+.. code::
+
+    # search for strings containing word substring "helloworld"
+        ".*helloworld.*"
+        "/.*helloworld.*/"
+        "/.*helloworld.*/i"

docs/Makefile

@@ -0,0 +1,21 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Zenko
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+

docs/RELEASE.md

@@ -0,0 +1,73 @@
+# Cloudserver Release Plan
+
+## Docker Image Generation
+
+Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
+CloudServer has a few images there:
+
+* Cloudserver container image: ghcr.io/scality/cloudserver
+* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
+* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
+
+With every CI build, the CI will push images, tagging the
+content with the developer branch's short SHA-1 commit hash.
+This allows those images to be used by developers, CI builds,
+build chain and so on.
+
+Tagged versions of cloudserver will be stored in the production namespace.
+
+## How to Pull Docker Images
+
+```sh
+docker pull ghcr.io/scality/cloudserver:<commit hash>
+docker pull ghcr.io/scality/cloudserver:<tag>
+```
+
+## Release Process
+
+To release a production image:
+
+* Create a PR to bump the package version
+  Update Cloudserver's `package.json` by bumping it to the relevant next
+  version in a new PR. Per example if the last released version was
+  `8.4.7`, the next version would be `8.4.8`.
+
+```js
+{
+  "name": "cloudserver",
+  "version": "8.4.8", <--- Here
+  [...]
+}
+```
+
+* Review & merge the PR
+
+* Create the release on GitHub
+  
+  * Go the Release tab (https://github.com/scality/cloudserver/releases);
+  * Click on the `Draft new release button`;
+  * In the `tag` field, type the name of the release (`8.4.8`), and confirm
+    to create the tag on publish;
+  * Click on `Generate release notes` button to fill the fields;
+  * Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
+  * Click to `Publish the release` to create the GitHub release and git tag
+
+  Notes:
+  * the Git tag will be created automatically.
+  * this should be done as soon as the PR is merged, so that the tag
+    is put on the "version bump" commit.
+
+* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
+
+  * Branch Name: The one used for the tag earlier. In this example `development/8.4`
+  * Override Stage: 'release'
+  * Extra properties:
+    * name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
+
+* Release the release version on Jira
+
+  * Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
+  * Create a next version
+    * Name: `[next version]`, in this example `8.4.9`
+  * Click `...` and select `Release` on the recently released version (`8.4.8`)
+  * Fill in the field to move incomplete version to the next one

docs/USING_PUBLIC_CLOUDS.rst

@@ -1,10 +1,12 @@
+.. _use-public-cloud:
+
 Using Public Clouds as data backends
 ====================================
 
 Introduction
 ------------
 
-As stated in our `GETTING STARTED guide <../GETTING_STARTED/#location-configuration>`__,
+As stated in our `GETTING STARTED guide <GETTING_STARTED.html#location-configuration>`__,
 new data backends can be added by creating a region (also called location
 constraint) with the right endpoint and credentials.
 This section of the documentation shows you how to set up our currently
@@ -160,8 +162,8 @@ CloudServer.
    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
    -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
    -v ~/.aws/credentials:/root/.aws/credentials \
-   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-   -d scality/s3server
+   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 \
+   -d scality/cloudserver
 
 Testing: put an object to AWS S3 using CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -243,7 +245,7 @@ There are a few configurable options here:
   this region should behave like any other AWS S3 region (in the case of MS Azure
   hosted data, this is mostly relevant for the format of errors);
 - :code:`azureStorageEndpoint` : set to your storage account's endpoint, usually
-  :code:`https://{{storageAccountName}}.blob.core.windows.name`;
+  :code:`https://{{storageAccountName}}.blob.core.windows.net`;
 - :code:`azureContainerName` : set to an *existing container* in your MS Azure
   storage account; this is the container in which your data will be stored for
   this location constraint;
@@ -322,7 +324,7 @@ CloudServer.
    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
    -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
    -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-   -d scality/s3server
+   -d scality/cloudserver
 
 Testing: put an object to MS Azure using CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/developers/getting-started.rst

@@ -0,0 +1,79 @@
+============================================
+Add New Backend Storage To Zenko CloudServer
+============================================
+
+This set of documents aims at bootstrapping developers with Zenko's CloudServer
+module, so they can then go on and contribute features.
+
+.. toctree::
+    :maxdepth: 2
+
+    non-s3-compatible-backend
+    s3-compatible-backend
+
+We always encourage our community to offer new extensions to Zenko,
+and new backend support is paramount to meeting more community needs.
+If that is something you want to contribute (or just do on your own
+version of the cloudserver image), this is the guid to read. Please
+make sure you follow our `Contributing Guidelines`_/.
+
+If you need help with anything, please search our `forum`_ for more
+information.
+
+Add support for a new backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently the main public cloud protocols are supported and more can
+be added. There are two main types of backend: those compatible with
+Amazon's S3 protocol and those not compatible.
+
+================= ========== ============ ===========
+Backend type      Supported  Active WIP   Not started
+================= ========== ============ ===========
+Private disk/fs       x
+AWS S3                x
+Microsoft Azure       x
+Backblaze B2                    x
+Google Cloud          x
+Openstack Swift                               x
+================= ========== ============ ===========
+
+.. important:: Should you want to request for a new backend to be
+               supported, please do so by opening a `Github issue`_,
+               and filling out the "Feature Request" section of our
+               template.
+
+To add support for a new backend support to CloudServer official
+repository, please follow these steps:
+
+- familiarize yourself with our `Contributing Guidelines`_
+- open a `Github issue`_ and fill out Feature Request form, and
+  specify you would like to contribute it yourself;
+- wait for our core team to get back to you with an answer on whether
+  we are interested in taking that contribution in (and hence
+  committing to maintaining it over time);
+- once approved, fork the repository and start your development;
+- use the `forum`_ with any question you may have during the
+  development process;
+- when you think it's ready, let us know so that we create a feature
+  branch against which we'll compare and review your code;
+- open a pull request with your changes against that dedicated feature
+  branch;
+- once that pull request gets merged, you're done.
+
+.. tip::
+
+    While we do take care of the final rebase (when we merge your feature
+    branch on the latest default branch), we do ask that you keep up to date with our latest default branch
+    until then.
+
+.. important::
+
+    If we do not approve your feature request, you may of course still
+    work on supporting a new backend: all our "no" means is that we do not
+    have the resources, as part of our core development team, to maintain
+    this feature for the moment.
+
+.. _GitHub issue: https://github.com/scality/S3/issues
+.. _Contributing Guidelines: https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md
+.. _forum: https://forum.zenko.io

docs/developers/new-backend.rst

@@ -0,0 +1,53 @@
+=================
+Add A New Backend
+=================
+
+Supporting all possible public cloud storage APIs is CloudServer's
+ultimate goal. As an open source project, contributions are welcome.
+
+The first step is to get familiar with building a custom Docker image
+for CloudServer.
+
+Build a Custom Docker Image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clone Zenko's CloudServer, install all dependencies and start the
+service:
+
+.. code-block:: shell
+
+  $ git clone https://github.com/scality/cloudserver
+  $ cd cloudserver
+  $ yarn install
+  $ yarn start
+
+.. tip::
+
+    Some optional dependencies may fail, resulting in you seeing `yarn
+    WARN` messages; these can safely be ignored.  Refer to the User
+    documentation for all available options.
+
+Build the Docker image:
+
+.. code-block:: shell
+
+     # docker build . -t
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+Push the newly created Docker image to your own hub:
+
+.. code-block:: shell
+
+     # docker push
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+.. note::
+
+    To perform this last operation, you need to be authenticated with DockerHub
+
+There are two main types of backend you could want Zenko to support:
+
+== link:S3_COMPATIBLE_BACKENDS.adoc[S3 compatible data backends]
+
+== link:NON_S3_COMPATIBLE_BACKENDS.adoc[Data backends using another protocol than the S3 protocol]
+

docs/developers/non-s3-compatible-backend.rst

@@ -0,0 +1,530 @@
+==========================================================
+Adding support for data backends not supporting the S3 API
+==========================================================
+
+These backends abstract the complexity of multiple APIs to let users
+work on a single common namespace across multiple clouds.
+
+This documents aims at introducing you to the right files in
+CloudServer (the Zenko stack's subcomponent in charge of API
+translation, among other things) to add support to your own backend of
+choice.
+
+General configuration
+~~~~~~~~~~~~~~~~~~~~~
+
+There are a number of constants and environment variables to define to support a
+new data backend; here is a list and where to find them:
+
+:file:`/constants.js`
+---------------------
+
+ * give your backend type a name, as part of the `externalBackends` object;
+ * specify whether versioning is implemented, as part of the
+   `versioningNotImplemented` object;
+
+:file:`/lib/Config.js`
+----------------------
+
+ * this is where you should put common utility functions, like the ones to parse
+   the location object from `locationConfig.json`;
+ * make sure you define environment variables (like `GCP_SERVICE_EMAIL` as we'll
+   use those internally for the CI to test against the real remote backend;
+
+:file:`/lib/data/external/{{backendName}}Client.js`
+---------------------------------------------------
+
+ * this file is where you'll instantiate your backend client; this should be a
+   class with a constructor taking the config object built in `/lib/Config.js` as
+   parameter;
+ * over time, you may need some utility functions which we've defined in the
+   folder `/api/apiUtils`, and in the file `/lib/data/external/utils`;
+
+:file:`/lib/data/external/utils.js`
+-----------------------------------
+
+ * make sure to add options for `sourceLocationConstraintType` to be equal to
+   the name you gave your backend in :file:`/constants.js`;
+
+:file:`/lib/data/external/{{BackendName}}_lib/`
+-----------------------------------------------
+
+ * this folder is where you'll put the functions needed for supporting your
+   backend; keep your files as atomic as possible;
+
+:file:`/tests/locationConfig/locationConfigTests.json`
+------------------------------------------------------
+
+ * this file is where you'll create location profiles to be used by your
+   functional tests;
+
+:file:`/lib/data/locationConstraintParser.js`
+---------------------------------------------
+
+ * this is where you'll instantiate your client if the operation the end user
+   sent effectively writes to your backend; everything happens inside the
+   function `parseLC()`; you should add a condition that executes if
+   `locationObj.type` is the name of your backend (that you defined in
+   `constants.js`), and instantiates a client of yours. See pseudocode below,
+   assuming location type name is `ztore`:
+
+
+.. code-block:: js
+   :linenos:
+   :emphasize-lines: 12
+
+    (...) //<1>
+    const ZtoreClient = require('./external/ZtoreClient');
+    const { config } = require('../Config'); //<1>
+
+    function parseLC(){ //<1>
+    (...) //<1>
+        Object.keys(config.locationConstraints).forEach(location => { //<1>
+            const locationObj = config.locationConstraints[location]; //<1>
+            (...) //<1>
+            if (locationObj.type === 'ztore' {
+                const ztoreEndpoint = config.getZtoreEndpoint(location);
+                const ztoreCredentials = config.getZtoreCredentials(location); //<2>
+                clients[location] = new ZtoreClient({
+                    ztoreEndpoint,
+                    ztoreCredentials,
+                    ztoreBucketname: locationObj.details.ztoreBucketName,
+                    bucketMatch:  locationObj.details.BucketMatch,
+                    dataStoreName: location,
+                }); //<3>
+                clients[location].clientType = 'ztore';
+            });
+            (...) //<1>
+        });
+    }
+
+
+1. Code that is already there
+2. You may need more utility functions depending on your backend specs
+3. You may have more fields required in your constructor object depending on
+   your backend specs
+
+Operation of type PUT
+~~~~~~~~~~~~~~~~~~~~~
+
+PUT routes are usually where people get started, as it's the easiest to check!
+Simply go on your remote backend console and you'll be able to see whether your
+object actually went up in the cloud...
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `put()` function is also called
+  `put()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `put(stream, size, keyContext, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+  //TODO: generate this from jsdoc
+
+ - `stream`: the stream of data you want to put in the cloud; if you're
+   unfamiliar with node.js streams, we suggest you start training, as we use
+   them a lot !
+
+ - `size`: the size of the object you're trying to put;
+
+ - `keyContext`: an object with metadata about the operation; common entries are
+   `namespace`, `buckerName`, `owner`, `cipherBundle`, and `tagging`; if these
+   are not sufficient for your integration, contact us to get architecture
+   validation before adding new entries;
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your PUT operation, and
+  then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/put/put{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+  function returning ajusted `keys[]` to your tests.
+
+Operation of type GET
+~~~~~~~~~~~~~~~~~~~~~
+
+GET routes are easy to test after PUT routes are implemented, hence why we're
+covering them second.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `get()` function is also called
+  `get()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `get(objectGetInfo, range, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+
+ - `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+
+ - `range`: the range of bytes you will get, for "get-by-range" operations (we
+   recommend you do simple GETs first, and then look at this);
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your GET operation, and
+  then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type DELETE
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+DELETE routes are easy to test after PUT routes are implemented, and they are
+similar to GET routes in our implementation, hence why we're covering them
+third.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `delete()` function is also called
+  `delete()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `delete(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your DELETE operation,
+  and then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/delete/delete{{BackendName}}js`
+-----------------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the
+          tutorial in order (that is, if you have covered the PUT operation
+          already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type HEAD
+~~~~~~~~~~~~~~~~~~~~~~
+
+HEAD routes are very similar to DELETE routes in our implementation, hence why
+we're covering them fourth.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `head()` function is also called
+  `head()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `head(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+// TODO:: generate this from jsdoc
+
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your HEAD operation,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Healthcheck
+~~~~~~~~~~~
+
+Healtchecks are used to make sure failure to write to a remote cloud is due to
+a problem on that remote cloud, an not on Zenko's side.
+This is usually done by trying to create a bucket that already exists, and
+making sure you get the expected answer.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `healthcheck()` function is called
+  `checkExternalBackend()` and it's defined in
+  :file:`/lib/data/multipleBackendGateway.js`; you will need to add your own;
+- your healtcheck function should get `location` as a parameter, which is an
+  object comprising:`
+
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/{{backendName}}_create_bucket.js`
+-------------------------------------------------------------------------------
+
+- this is where you should write the function performing the actual bucket
+  creation;
+
+:file:`/lib/data/external/{{backendName}}_lib/utils.js`
+-------------------------------------------------------
+
+- add an object named per your backend's name to the `backendHealth` dictionary,
+  with proper `response` and `time` entries;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `healthcheck` function to add your location's array, and call your
+  healthcheck; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+
+.. code-block:: js
+   :linenos:
+
+    (...) //<1>
+
+        healthcheck: (flightCheckOnStartUp, log, callback) => { //<1>
+            (...) //<1>
+            const ztoreArray = []; //<2>
+            async.each(Object.keys(clients), (location, cb) => { //<1>
+                (...) //<1>
+                } else if (client.clientType === 'ztore' {
+                    ztoreArray.push(location); //<3>
+                    return cb();
+                }
+            (...) //<1>
+            multBackendResp[location] = { code: 200, message: 'OK' }; //<1>
+            return cb();
+        }, () => { //<1>
+            async.parallel([
+                (...) //<1>
+                next => checkExternalBackend( //<4>
+                    clients, ztoreArray, 'ztore', flightCheckOnStartUp,
+                    externalBackendHealthCheckInterval, next),
+            ] (...) //<1>
+            });
+            (...) //<1>
+        });
+    }
+
+1. Code that is already there
+2. The array that will store all locations of type 'ztore'
+3. Where you add locations of type 'ztore' to the array
+4. Where you actually call the healthcheck function on all 'ztore' locations
+
+Multipart upload (MPU)
+~~~~~~~~~~~~~~~~~~~~~~
+
+This is the final part to supporting a new backend! MPU is far from
+the easiest subject, but you've come so far it shouldn't be a problem.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+You'll be creating four functions with template signatures:
+
+- `createMPU(Key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
+  cacheControl, contentDisposition, contentEncoding, log, callback)` will
+  initiate the multi part upload process; now, here, all parameters are
+  metadata headers except for:
+
+ * `Key`, the key id for the final object (collection of all parts);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `uploadPart(request, streamingV4Params, stream, size, key, uploadId, partNumber, bucketName, log, callback)`
+   will be called for each part; the parameters can be explicited as follow:
+
+ * `request`, the request object for putting the part;
+ * `streamingV4Params`, parameters for auth V4 parameters against S3;
+ * `stream`, the node.js readable stream used to put the part;
+ * `size`, the size of the part;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `partNumber`, the number of the part in this MPU (ordered);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback)` will
+   end the MPU process once all parts are uploaded; parameters can be explicited
+   as follows:
+
+ * `jsonList`, user-sent list of parts to include in final mpu object;
+ * `mdInfo`, object containing 3 keys: storedParts, mpuOverviewKey, and
+   splitter;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance:
+
+- `abortMPU(key, uploadId, bucketName, log, callback)` will handle errors, and
+  make sure that all parts that may have been uploaded will be deleted if the
+  MPU ultimately fails; the parameters are:
+
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance.
+
+:file:`/lib/api/objectPutPart.js`
+---------------------------------
+
+- you'll need to add your backend type in appropriate sections (simply look for
+  other backends already implemented).
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your MPU operations,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `createMPU` function to add your location type, and call your
+  `createMPU()`; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+.. code-block:: javascript
+   :linenos:
+
+    (...) //<1>
+        createMPU:(key, metaHeaders, bucketName, websiteRedirectHeader, //<1>
+         location, contentType, cacheControl, contentDisposition,
+         contentEncoding, log, cb) => {
+            const client = clients[location]; //<1>
+            if (client.clientType === 'aws_s3') { //<1>
+                return client.createMPU(key, metaHeaders, bucketName,
+                websiteRedirectHeader, contentType, cacheControl,
+                contentDisposition, contentEncoding, log, cb);
+            } else if (client.clientType === 'ztore') { //<2>
+                return client.createMPU(key, metaHeaders, bucketName,
+                  websiteRedirectHeader, contentType, cacheControl,
+                  contentDisposition, contentEncoding, log, cb);
+            }
+            return cb();
+        };
+    (...) //<1>
+
+1. Code that is already there
+2. Where the `createMPU()` of your client is actually called
+
+Add functional tests
+~~~~~~~~~~~~~~~~~~~~
+
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/initMPU/{{BackendName}}InitMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/listParts/{{BackendName}}ListPart.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/{{BackendName}}AbortMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/{{BackendName}}CompleteMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/{{BackendName}}UploadPart.js`
+
+Adding support in Orbit, Zenko's UI for simplified Multi Cloud Management
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can only be done by our core developers' team. Once your backend
+integration is merged, you may open a feature request on the
+`Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/developers/s3-compatible-backend.rst

@@ -0,0 +1,43 @@
+======================
+S3-Compatible Backends
+======================
+
+
+Adding Support in CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is the easiest case for backend support integration: there is nothing to do
+but configuration!  Follow the steps described in our
+:ref:`use-public-cloud` and make sure you:
+
+- set ``details.awsEndpoint`` to your storage provider endpoint;
+
+- use ``details.credentials`` and *not* ``details.credentialsProfile`` to set your
+  credentials for that S3-compatible backend.
+
+For example, if you’re using a Wasabi bucket as a backend, then your region
+definition for that backend will look something like:
+::
+
+    "wasabi-bucket-zenkobucket": {
+    "type": "aws_s3",
+    "legacyAwsBehavior": true,
+    "details": {
+    "awsEndpoint": "s3.wasabisys.com",
+    "bucketName": "zenkobucket",
+    "bucketMatch": true,
+    "credentials": {
+    "accessKey": "\\{YOUR_WASABI_ACCESS_KEY}",
+    "secretKey": "\\{YOUR_WASABI_SECRET_KEY}"
+    }
+    }
+    },
+
+Adding Support in Zenko Orbit
+#############################
+
+This can only be done by our core developpers' team. If that’s what you’re
+after, open a feature request on the `Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/index.rst

@@ -1,11 +1,12 @@
 Scality Zenko CloudServer
-==================
+=========================
 
 .. _user-docs:
 
 .. toctree::
    :maxdepth: 2
    :caption: Documentation
+   :glob:
 
    CONTRIBUTING
    GETTING_STARTED
@@ -14,3 +15,4 @@ Scality Zenko CloudServer
    DOCKER
    INTEGRATIONS
    ARCHITECTURE
+   developers/*

docs/requirements.in

@@ -0,0 +1,2 @@
+Sphinx >= 1.7.5
+recommonmark >= 0.4.0

docs/requirements.txt

@@ -0,0 +1,119 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    tox -e pip-compile
+#
+alabaster==0.7.12 \
+    --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
+    --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 \
+    # via sphinx
+babel==2.6.0 \
+    --hash=sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669 \
+    --hash=sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23 \
+    # via sphinx
+certifi==2018.10.15 \
+    --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
+    --hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+commonmark==0.5.4 \
+    --hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5 \
+    # via recommonmark
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
+    # via recommonmark, sphinx
+idna==2.7 \
+    --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
+    --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
+    # via requests
+imagesize==1.1.0 \
+    --hash=sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8 \
+    --hash=sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5 \
+    # via sphinx
+jinja2==2.10 \
+    --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
+    --hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
+    # via sphinx
+markupsafe==1.1.0 \
+    --hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \
+    --hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \
+    --hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \
+    --hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \
+    --hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \
+    --hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \
+    --hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \
+    --hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \
+    --hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \
+    --hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \
+    --hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \
+    --hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \
+    --hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \
+    --hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \
+    --hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \
+    --hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \
+    --hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \
+    --hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \
+    --hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \
+    --hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \
+    --hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \
+    --hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \
+    --hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \
+    --hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \
+    --hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \
+    --hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \
+    --hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \
+    --hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 \
+    # via jinja2
+packaging==18.0 \
+    --hash=sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807 \
+    --hash=sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9 \
+    # via sphinx
+pygments==2.2.0 \
+    --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
+    --hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc \
+    # via sphinx
+pyparsing==2.3.0 \
+    --hash=sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b \
+    --hash=sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592 \
+    # via packaging
+pytz==2018.7 \
+    --hash=sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca \
+    --hash=sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6 \
+    # via babel
+recommonmark==0.4.0 \
+    --hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d \
+    --hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7
+requests==2.20.1 \
+    --hash=sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54 \
+    --hash=sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263 \
+    # via sphinx
+six==1.11.0 \
+    --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
+    --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \
+    # via packaging, sphinx
+snowballstemmer==1.2.1 \
+    --hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 \
+    --hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \
+    # via sphinx
+sphinx==1.8.2 \
+    --hash=sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea \
+    --hash=sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64
+sphinxcontrib-websupport==1.1.0 \
+    --hash=sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd \
+    --hash=sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9 \
+    # via sphinx
+typing==3.6.6 \
+    --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \
+    --hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
+    --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
+    # via sphinx
+urllib3==1.24.1 \
+    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
+    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via requests

examples/go-md-search.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+    "fmt"
+    "time"
+    "bytes"
+    "net/http"
+    "net/url"
+    "io/ioutil"
+    "github.com/aws/aws-sdk-go/aws/credentials"
+    "github.com/aws/aws-sdk-go/aws/signer/v4"
+)
+
+func main() {
+    // Input AWS access key, secret key
+    aws_access_key_id := "accessKey1"
+    aws_secret_access_key := "verySecretKey1"
+    endpoint := "http://localhost:8000"
+    bucket_name := "bucketname"
+    searchQuery := url.QueryEscape("x-amz-meta-color=blue")
+    buf := bytes.NewBuffer([]byte{})
+
+    requestUrl := fmt.Sprintf("%s/%s?search=%s",
+        endpoint, bucket_name, searchQuery)
+
+    request, err := http.NewRequest("GET", requestUrl, buf)
+    if err != nil {
+        panic(err)
+    }
+    reader := bytes.NewReader(buf.Bytes())
+    credentials := credentials.NewStaticCredentials(aws_access_key_id,
+        aws_secret_access_key, "")
+    signer := v4.NewSigner(credentials)
+    signer.Sign(request, reader, "s3", "us-east-1", time.Now())
+    client := &http.Client{}
+    resp, err := client.Do(request)
+    if err != nil {
+        panic(err)
+    }
+    defer resp.Body.Close()
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println(string(body))
+}

examples/node-md-search.js

@@ -0,0 +1,28 @@
+const { S3 } = require('aws-sdk');
+const config = {
+    sslEnabled: false,
+    endpoint: 'http://127.0.0.1:8000',
+    signatureCache: false,
+    signatureVersion: 'v4',
+    region: 'us-east-1',
+    s3ForcePathStyle: true,
+    accessKeyId: 'accessKey1',
+    secretAccessKey: 'verySecretKey1',
+};
+const s3Client = new S3(config);
+
+const encodedSearch =
+    encodeURIComponent('x-amz-meta-color="blue"');
+const req = s3Client.listObjects({ Bucket: 'bucketname' });
+
+// the build event
+req.on('build', () => {
+    req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
+});
+req.on('success', res => {
+    process.stdout.write(`Result ${res.data}`);
+});
+req.on('error', err => {
+    process.stdout.write(`Error ${err}`);
+});
+req.send();

examples/python-md-search.py

@@ -0,0 +1,79 @@
+import datetime
+import hashlib
+import hmac
+import urllib
+# pip install requests
+import requests
+
+access_key = 'accessKey1'
+secret_key = 'verySecretKey1'
+
+method = 'GET'
+service = 's3'
+host = 'localhost:8000'
+region = 'us-east-1'
+canonical_uri = '/bucketname'
+query = 'x-amz-meta-color=blue'
+canonical_querystring = 'search=%s' % (urllib.quote(query))
+algorithm = 'AWS4-HMAC-SHA256'
+
+t = datetime.datetime.utcnow()
+amz_date = t.strftime('%Y%m%dT%H%M%SZ')
+date_stamp = t.strftime('%Y%m%d')
+
+# Key derivation functions. See:
+# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
+
+
+def sign(key, msg):
+    return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
+
+
+def getSignatureKey(key, date_stamp, regionName, serviceName):
+    kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
+    kRegion = sign(kDate, regionName)
+    kService = sign(kRegion, serviceName)
+    kSigning = sign(kService, 'aws4_request')
+    return kSigning
+
+
+payload_hash = hashlib.sha256('').hexdigest()
+
+canonical_headers = \
+    'host:{0}\nx-amz-content-sha256:{1}\nx-amz-date:{2}\n' \
+    .format(host, payload_hash, amz_date)
+
+signed_headers = 'host;x-amz-content-sha256;x-amz-date'
+
+canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
+    .format(method, canonical_uri, canonical_querystring, canonical_headers,
+            signed_headers, payload_hash)
+print(canonical_request)
+
+credential_scope = '{0}/{1}/{2}/aws4_request' \
+    .format(date_stamp, region, service)
+
+string_to_sign = '{0}\n{1}\n{2}\n{3}' \
+    .format(algorithm, amz_date, credential_scope,
+            hashlib.sha256(canonical_request).hexdigest())
+
+signing_key = getSignatureKey(secret_key, date_stamp, region, service)
+
+signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
+                     hashlib.sha256).hexdigest()
+
+authorization_header = \
+    '{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}' \
+    .format(algorithm, access_key, credential_scope, signed_headers, signature)
+
+# The 'host' header is added automatically by the Python 'requests' library.
+headers = {
+    'X-Amz-Content-Sha256': payload_hash,
+    'X-Amz-Date': amz_date,
+    'Authorization': authorization_header
+}
+
+endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
+
+r = requests.get(endpoint, headers=headers)
+print(r.text)

images/svc-base/Dockerfile

@@ -14,8 +14,10 @@ RUN rm -f ~/.gitconfig && \
     git config --global --add safe.directory . && \
     git lfs install && \
     GIT_LFS_SKIP_SMUDGE=1 && \
+    yarn global add typescript && \
     yarn install --frozen-lockfile --production --network-concurrency 1 && \
-    yarn cache clean --all
+    yarn cache clean --all && \
+    yarn global remove typescript
 
 # run symlinking separately to avoid yarn installation errors
 # we might have to check if the symlinking is really needed!

lib/api/api.js

@@ -7,7 +7,8 @@ const bucketDeleteEncryption = require('./bucketDeleteEncryption');
 const bucketDeleteWebsite = require('./bucketDeleteWebsite');
 const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
 const bucketDeletePolicy = require('./bucketDeletePolicy');
-const bucketGet = require('./bucketGet');
+const bucketDeleteQuota = require('./bucketDeleteQuota');
+const { bucketGet } = require('./bucketGet');
 const bucketGetACL = require('./bucketGetACL');
 const bucketGetCors = require('./bucketGetCors');
 const bucketGetVersioning = require('./bucketGetVersioning');
@@ -17,6 +18,7 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
 const bucketGetNotification = require('./bucketGetNotification');
 const bucketGetObjectLock = require('./bucketGetObjectLock');
 const bucketGetPolicy = require('./bucketGetPolicy');
+const bucketGetQuota = require('./bucketGetQuota');
 const bucketGetEncryption = require('./bucketGetEncryption');
 const bucketHead = require('./bucketHead');
 const { bucketPut } = require('./bucketPut');
@@ -33,6 +35,7 @@ const bucketPutNotification = require('./bucketPutNotification');
 const bucketPutEncryption = require('./bucketPutEncryption');
 const bucketPutPolicy = require('./bucketPutPolicy');
 const bucketPutObjectLock = require('./bucketPutObjectLock');
+const bucketUpdateQuota = require('./bucketUpdateQuota');
 const bucketGetReplication = require('./bucketGetReplication');
 const bucketDeleteReplication = require('./bucketDeleteReplication');
 const corsPreflight = require('./corsPreflight');
@@ -40,10 +43,11 @@ const completeMultipartUpload = require('./completeMultipartUpload');
 const initiateMultipartUpload = require('./initiateMultipartUpload');
 const listMultipartUploads = require('./listMultipartUploads');
 const listParts = require('./listParts');
+const metadataSearch = require('./metadataSearch');
 const { multiObjectDelete } = require('./multiObjectDelete');
 const multipartDelete = require('./multipartDelete');
 const objectCopy = require('./objectCopy');
-const objectDelete = require('./objectDelete');
+const { objectDelete } = require('./objectDelete');
 const objectDeleteTagging = require('./objectDeleteTagging');
 const objectGet = require('./objectGet');
 const objectGetACL = require('./objectGetACL');
@@ -58,6 +62,7 @@ const objectPutTagging = require('./objectPutTagging');
 const objectPutPart = require('./objectPutPart');
 const objectPutCopyPart = require('./objectPutCopyPart');
 const objectPutRetention = require('./objectPutRetention');
+const objectRestore = require('./objectRestore');
 const prepareRequestContexts
     = require('./apiUtils/authorization/prepareRequestContexts');
 const serviceGet = require('./serviceGet');
@@ -67,6 +72,7 @@ const writeContinue = require('../utilities/writeContinue');
 const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
 const parseCopySource = require('./apiUtils/object/parseCopySource');
 const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
+const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
 const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
 
 const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
@@ -79,6 +85,10 @@ const api = {
         // Attach the apiMethod method to the request, so it can used by monitoring in the server
         // eslint-disable-next-line no-param-reassign
         request.apiMethod = apiMethod;
+        // Array of end of API callbacks, used to perform some logic
+        // at the end of an API.
+        // eslint-disable-next-line no-param-reassign
+        request.finalizerHooks = [];
 
         const actionLog = monitoringMap[apiMethod];
         if (!actionLog &&
@@ -187,21 +197,27 @@ const api = {
 
         return async.waterfall([
             next => auth.server.doAuth(
-                request, log, (err, userInfo, authorizationResults, streamingV4Params) => {
+                request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
                     if (err) {
+                        // VaultClient returns standard errors, but the route requires
+                        // Arsenal errors
+                        const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
                         log.trace('authentication error', { error: err });
-                        return next(err);
+                        return next(arsenalError);
                     }
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                 }, 's3', requestContexts),
-            (userInfo, authorizationResults, streamingV4Params, next) => {
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => {
                 const authNames = { accountName: userInfo.getAccountDisplayName() };
                 if (userInfo.isRequesterAnIAMUser()) {
                     authNames.userName = userInfo.getIAMdisplayName();
                 }
+                if (isRequesterASessionUser(userInfo)) {
+                    authNames.sessionName = userInfo.getShortid().split(':')[1];
+                }
                 log.addDefaultFields(authNames);
                 if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                 }
                 // issue 100 Continue to the client
                 writeContinue(request, response);
@@ -232,12 +248,12 @@ const api = {
                     }
                     // Convert array of post buffers into one string
                     request.post = Buffer.concat(post, postLength).toString();
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                 });
                 return undefined;
             },
             // Tag condition keys require information from CloudServer for evaluation
-            (userInfo, authorizationResults, streamingV4Params, next) => tagConditionKeyAuth(
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
                 authorizationResults,
                 request,
                 requestContexts,
@@ -248,13 +264,14 @@ const api = {
                         log.trace('tag authentication error', { error: err });
                         return next(err);
                     }
-                    return next(null, userInfo, authResultsWithTags, streamingV4Params);
+                    return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
                 },
             ),
-        ], (err, userInfo, authorizationResults, streamingV4Params) => {
+        ], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
             if (err) {
                 return callback(err);
             }
+            request.accountQuotas = infos?.accountQuota;
             if (authorizationResults) {
                 const checkedResults = checkAuthResults(authorizationResults);
                 if (checkedResults instanceof Error) {
@@ -271,19 +288,23 @@ const api = {
                     return acc;
                 }, {});
             }
+            const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
+                    (hook, done) => hook(err, done),
+                    () => callback(err, ...results));
+
             if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
                 request._response = response;
                 return this[apiMethod](userInfo, request, streamingV4Params,
-                    log, callback, authorizationResults);
+                    log, methodCallback, authorizationResults);
             }
             if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
                 return this[apiMethod](userInfo, request, sourceBucket,
-                    sourceObject, sourceVersionId, log, callback);
+                    sourceObject, sourceVersionId, log, methodCallback);
             }
             if (apiMethod === 'objectGet') {
                 return this[apiMethod](userInfo, request, returnTagCount, log, callback);
             }
-            return this[apiMethod](userInfo, request, log, callback);
+            return this[apiMethod](userInfo, request, log, methodCallback);
         });
     },
     bucketDelete,
@@ -304,18 +325,21 @@ const api = {
     bucketPutCors,
     bucketPutVersioning,
     bucketPutTagging,
+    bucketDeleteTagging,
     bucketGetTagging,
     bucketPutWebsite,
     bucketPutReplication,
     bucketGetReplication,
     bucketDeleteReplication,
+    bucketDeleteQuota,
     bucketPutLifecycle,
+    bucketUpdateQuota,
     bucketGetLifecycle,
     bucketDeleteLifecycle,
     bucketPutPolicy,
     bucketGetPolicy,
+    bucketGetQuota,
     bucketDeletePolicy,
-    bucketDeleteTagging,
     bucketPutObjectLock,
     bucketPutNotification,
     bucketGetNotification,
@@ -325,6 +349,7 @@ const api = {
     initiateMultipartUpload,
     listMultipartUploads,
     listParts,
+    metadataSearch,
     multiObjectDelete,
     multipartDelete,
     objectDelete,
@@ -343,6 +368,7 @@ const api = {
     objectPutPart,
     objectPutCopyPart,
     objectPutRetention,
+    objectRestore,
     serviceGet,
     websiteGet: website,
     websiteHead: website,

lib/api/apiUtils/authorization/permissionChecks.js

@@ -5,8 +5,13 @@ const constants = require('../../../../constants');
 const { config } = require('../../../Config');
 
 const {
-    allAuthedUsersId, bucketOwnerActions, logId, publicId,
-    assumedRoleArnResourceType, backbeatLifecycleSessionName, arrayOfAllowed,
+    allAuthedUsersId,
+    bucketOwnerActions,
+    logId,
+    publicId,
+    arrayOfAllowed,
+    assumedRoleArnResourceType,
+    backbeatLifecycleSessionName,
     actionsToConsiderAsObjectPut,
 } = constants;
 
@@ -14,6 +19,25 @@ const {
 const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
     ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
 
+function getServiceAccountProperties(canonicalID) {
+    const canonicalIDArray = canonicalID.split('/');
+    const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
+    return constants.serviceAccountProperties[serviceName];
+}
+
+function isServiceAccount(canonicalID) {
+    return getServiceAccountProperties(canonicalID) !== undefined;
+}
+
+function isRequesterASessionUser(authInfo) {
+    const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
+    return regexpAssumedRoleArn.test(authInfo.getArn());
+}
+
+function isRequesterNonAccountUser(authInfo) {
+    return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
+}
+
 /**
  * Checks the access control for a given bucket based on the request type and user's canonical ID.
  *
@@ -364,11 +388,11 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo
         let requesterIsNotUser = true;
         let arn = null;
         if (authInfo) {
-            requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
+            requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
             arn = authInfo.getArn();
         }
         // if the bucket owner is an account, users should not have default access
-        if ((bucket.getOwner() === canonicalID) && requesterIsNotUser) {
+        if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
             results[_requestType] = actionImplicitDenies[_requestType] === false;
             return results[_requestType];
         }
@@ -439,11 +463,11 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI
         let arn = null;
         let isUserUnauthenticated = false;
         if (authInfo) {
-            requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
+            requesterIsNotUser =  !isRequesterNonAccountUser(authInfo);
             arn = authInfo.getArn();
             isUserUnauthenticated = arn === undefined;
         }
-        if (objectMD['owner-id'] === canonicalID && requesterIsNotUser) {
+        if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
             results[_requestType] = actionImplicitDenies[_requestType] === false;
             return results[_requestType];
         }
@@ -514,7 +538,7 @@ function checkIp(value) {
         }
 
         // Apply the existing IP validation logic to each element
-        const validateIpRegex = (ip) => {
+        const validateIpRegex = ip => {
             if (constants.ipv4Regex.test(ip)) {
                 return ip.split('.').every(part => parseInt(part, 10) <= 255);
             }
@@ -553,6 +577,7 @@ function validatePolicyConditions(policy) {
         if (s.Condition) {
             const conditionOperators = Object.keys(s.Condition);
             // there can be multiple condition operations in the Condition enclosure
+            // eslint-disable-next-line no-restricted-syntax
             for (const conditionOperator of conditionOperators) {
                 const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
                 const conditionValue = s.Condition[conditionOperator][conditionKey];
@@ -603,6 +628,10 @@ function isLifecycleSession(arn) {
 module.exports = {
     isBucketAuthorized,
     isObjAuthorized,
+    getServiceAccountProperties,
+    isServiceAccount,
+    isRequesterASessionUser,
+    isRequesterNonAccountUser,
     checkBucketAcls,
     checkObjectAcls,
     validatePolicyResource,

lib/api/apiUtils/authorization/prepareRequestContexts.js

@@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
             apiMethod, 's3');
     }
 
-    if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
+    if (apiMethod === 'bucketPut') {
         return null;
     }
 
@@ -65,7 +65,17 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
 
     const requestContexts = [];
 
-    if (apiMethodAfterVersionCheck === 'objectCopy'
+    if (apiMethod === 'multiObjectDelete') {
+        // MultiObjectDelete does not require any authorization when evaluating
+        // the API. Instead, we authorize each object passed.
+        // But in order to get any relevant information from the authorization service
+        // for example, the account quota, we must send a request context object
+        // with no `specificResource`. We expect the result to be an implicit deny.
+        // In the API, we then ignore these authorization results, and we can use
+        // any information returned, e.g., the quota.
+        const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
+          requestContexts.push(requestContextMultiObjectDelete);
+    } else if (apiMethodAfterVersionCheck === 'objectCopy'
         || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
         const objectGetAction = sourceVersionId ? 'objectGetVersion' :
           'objectGet';
@@ -147,6 +157,13 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
             generateRequestContext('objectGet');
         requestContexts.push(getObjectRequestContext);
     } else if (apiMethodAfterVersionCheck === 'objectPut') {
+        // if put object with version
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
             const putRequestContext =
               generateRequestContext(apiMethodAfterVersionCheck);
             requestContexts.push(putRequestContext);
@@ -161,6 +178,38 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
                     generateRequestContext('objectPutLegalHold');
                 requestContexts.push(putLegalHoldStatusAction);
             }
+            // if put object (versioning) with ACL
+            if (isHeaderAcl(request.headers)) {
+                const putAclRequestContext =
+                  generateRequestContext('objectPutACL');
+                requestContexts.push(putAclRequestContext);
+            }
+            if (request.headers['x-amz-object-lock-mode']) {
+                const putObjectLockRequestContext =
+                  generateRequestContext('objectPutRetention');
+                requestContexts.push(putObjectLockRequestContext);
+            }
+            if (request.headers['x-amz-version-id']) {
+                const putObjectVersionRequestContext =
+                    generateRequestContext('objectPutTaggingVersion');
+                requestContexts.push(putObjectVersionRequestContext);
+            }
+        }
+    } else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
+      apiMethodAfterVersionCheck === 'objectPutPart' ||
+      apiMethodAfterVersionCheck === 'completeMultipartUpload'
+      ) {
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
+            const putRequestContext =
+              generateRequestContext(apiMethodAfterVersionCheck);
+            requestContexts.push(putRequestContext);
+        }
+
         // if put object (versioning) with ACL
         if (isHeaderAcl(request.headers)) {
             const putAclRequestContext =

lib/api/apiUtils/authorization/tagConditionKeys.js

@@ -60,6 +60,8 @@ function updateRequestContextsWithTags(request, requestContexts, apiMethod, log,
             log.trace('error processing tag condition key evaluation');
             return cb(err);
         }
+        // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
+        // eslint-disable-next-line no-restricted-syntax
         for (const rc of requestContexts) {
             rc.setNeedTagEval(true);
             if (requestTagsQuery) {

lib/api/apiUtils/bucket/bucketCreation.js

@@ -10,13 +10,15 @@ const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
 const metadata = require('../../../metadata/wrapper');
 const kms = require('../../../kms/wrapper');
 const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
+const { isServiceAccount } = require('../authorization/permissionChecks');
 
 const usersBucket = constants.usersBucket;
 const oldUsersBucket = constants.oldUsersBucket;
+const zenkoSeparator = constants.zenkoSeparator;
 const userBucketOwner = 'admin';
 
 
-function addToUsersBucket(canonicalID, bucketName, log, cb) {
+function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
     // BACKWARD: Simplify once do not have to deal with old
     // usersbucket name and old splitter
 
@@ -28,7 +30,10 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
         const splitter = usersBucketAttrs ?
             constants.splitter : constants.oldSplitter;
         let key = createKeyForUserBucket(canonicalID, splitter, bucketName);
-        const omVal = { creationDate: new Date().toJSON() };
+        const omVal = {
+            creationDate: new Date().toJSON(),
+            ingestion: bucketMD.getIngestion(),
+        };
         // If the new format usersbucket does not exist, try to put the
         // key in the old usersBucket using the old splitter.
         // Otherwise put the key in the new format usersBucket
@@ -36,7 +41,7 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
             usersBucket : oldUsersBucket;
         return metadata.putObjectMD(usersBucketBeingCalled, key,
             omVal, {}, log, err => {
-                if (err && err.is.NoSuchBucket) {
+                if (err?.is?.NoSuchBucket) {
                     // There must be no usersBucket so createBucket
                     // one using the new format
                     log.trace('users bucket does not exist, ' +
@@ -91,7 +96,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
             return callback(err);
         }
         log.trace('created bucket in metadata');
-        return addToUsersBucket(canonicalID, bucketName, log, err => {
+        return addToUsersBucket(canonicalID, bucketName, bucket, log, err => {
             if (err) {
                 return callback(err);
             }
@@ -114,7 +119,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
  */
 function cleanUpBucket(bucketMD, canonicalID, log, callback) {
     const bucketName = bucketMD.getName();
-    return addToUsersBucket(canonicalID, bucketName, log, err => {
+    return addToUsersBucket(canonicalID, bucketName, bucketMD, log, err => {
         if (err) {
             return callback(err);
         }
@@ -165,15 +170,28 @@ function createBucket(authInfo, bucketName, headers,
     const ownerDisplayName =
         authInfo.getAccountDisplayName();
     const creationDate = new Date().toJSON();
+    const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
     const headerObjectLock = headers['x-amz-bucket-object-lock-enabled'];
     const objectLockEnabled
         = headerObjectLock && headerObjectLock.toLowerCase() === 'true';
     const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName,
         creationDate, BucketInfo.currentModelVersion(), null, null, null, null,
-        null, null, null, null, null, null, null, null, objectLockEnabled);
+        null, null, null, null, null, null, null, null, null, isNFSEnabled,
+        null, null, objectLockEnabled);
+    let locationConstraintVal = null;
 
-    if (locationConstraint !== undefined) {
-        bucket.setLocationConstraint(locationConstraint);
+    if (locationConstraint) {
+        const [locationConstraintStr, ingestion] =
+            locationConstraint.split(zenkoSeparator);
+        if (locationConstraintStr) {
+            locationConstraintVal = locationConstraintStr;
+            bucket.setLocationConstraint(locationConstraintStr);
+        }
+        if (ingestion === 'ingest') {
+            bucket.enableIngestion();
+            //automatically enable versioning for ingestion buckets
+            bucket.setVersioningConfiguration({ Status: 'Enabled' });
+        }
     }
     if (objectLockEnabled) {
         // default versioning configuration AWS sets
@@ -223,7 +241,8 @@ function createBucket(authInfo, bucketName, headers,
         }
         const existingBucketMD = results.getAnyExistingBucketInfo;
         if (existingBucketMD instanceof BucketInfo &&
-            existingBucketMD.getOwner() !== canonicalID) {
+            existingBucketMD.getOwner() !== canonicalID &&
+            !isServiceAccount(canonicalID)) {
             // return existingBucketMD to collect cors headers
             return cb(errors.BucketAlreadyExists, existingBucketMD);
         }
@@ -254,7 +273,7 @@ function createBucket(authInfo, bucketName, headers,
         // error unless old AWS behavior (us-east-1)
         // Existing locationConstraint must have legacyAwsBehavior === true
         // New locationConstraint should have legacyAwsBehavior === true
-        if (isLegacyAWSBehavior(locationConstraint) &&
+        if (isLegacyAWSBehavior(locationConstraintVal) &&
             isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) {
             log.trace('returning 200 instead of 409 to mirror us-east-1');
             return cb(null, existingBucketMD);

lib/api/apiUtils/bucket/bucketDeletion.js

@@ -93,7 +93,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
                 log, (err, objectsListRes) => {
                     // If no shadow bucket ever created, no ongoing MPU's, so
                     // continue with deletion
-                    if (err && err.is.NoSuchBucket) {
+                    if (err?.is.NoSuchBucket) {
                         return next();
                     }
                     if (err) {

lib/api/apiUtils/bucket/checkPreferredLocations.js

@@ -0,0 +1,26 @@
+const { errors } = require('arsenal');
+
+function checkPreferredLocations(location, locationConstraints, log) {
+    const retError = loc => {
+        const errMsg = 'value of the location you are attempting to set - ' +
+        `${loc} - is not listed in the locationConstraint config`;
+        log.trace(`locationConstraint is invalid - ${errMsg}`,
+          { locationConstraint: loc });
+        return errors.InvalidLocationConstraint.customizeDescription(errMsg);
+    };
+    if (typeof location === 'string' && !locationConstraints[location]) {
+        return retError(location);
+    }
+    if (typeof location === 'object') {
+        const { read, write } = location;
+        if (!locationConstraints[read]) {
+            return retError(read);
+        }
+        if (!locationConstraints[write]) {
+            return retError(write);
+        }
+    }
+    return null;
+}
+
+module.exports = checkPreferredLocations;

lib/api/apiUtils/bucket/deleteUserBucketEntry.js

@@ -11,11 +11,11 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
     metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
         // If the object representing the bucket is not in the
         // users bucket just continue
-        if (error && error.is.NoSuchKey) {
+        if (error?.is.NoSuchKey) {
             return cb(null);
         // BACKWARDS COMPATIBILITY: Remove this once no longer
         // have old user bucket format
-        } else if (error && error.NoSuchBucket) {
+        } else if (error?.is.NoSuchBucket) {
             const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
                 oldSplitter, bucketName);
             return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,

lib/api/apiUtils/bucket/getNotificationConfiguration.js

@@ -15,11 +15,21 @@ function getNotificationConfiguration(parsedXml) {
     }
     const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource));
     const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]);
-    if (!notifConfigTargets.every(t => targets.has(t))) {
-        // TODO: match the error message to AWS's response along with
-        // the request destination name in the response
-        const errDesc = 'Unable to validate the destination configuration';
-        return { error: errors.InvalidArgument.customizeDescription(errDesc) };
+    // getting invalid targets
+    const invalidTargets = [];
+    notifConfigTargets.forEach((t, i) => {
+        if (!targets.has(t)) {
+            invalidTargets.push({
+                ArgumentName: notifConfig.queueConfig[i].queueArn,
+                ArgumentValue: 'The destination queue does not exist',
+            });
+        }
+    });
+    if (invalidTargets.length > 0) {
+        const errDesc = 'Unable to validate the following destination configurations';
+        let error = errors.InvalidArgument.customizeDescription(errDesc);
+        error = error.addMetadataEntry('invalidArguments', invalidTargets);
+        return { error };
     }
     return notifConfig;
 }

lib/api/apiUtils/bucket/parseLikeExpression.js

@@ -0,0 +1,19 @@
+/**
+ * parse LIKE expressions
+ * @param {string} regex - regex pattern
+ * @return {object} MongoDB search object
+ */
+function parseLikeExpression(regex) {
+    if (typeof regex !== 'string') {
+        return null;
+    }
+    const split = regex.split('/');
+    if (split.length < 3 || split[0] !== '') {
+        return { $regex: regex };
+    }
+    const pattern = split.slice(1, split.length - 1).join('/');
+    const regexOpt = split[split.length - 1];
+    return { $regex: new RegExp(pattern), $options: regexOpt };
+}
+
+module.exports = parseLikeExpression;

lib/api/apiUtils/bucket/parseWhere.js

@@ -0,0 +1,85 @@
+const parseLikeExpression = require('./parseLikeExpression');
+
+/*
+  This code is based on code from https://github.com/olehch/sqltomongo
+  with the following license:
+
+  The MIT License (MIT)
+
+  Copyright (c) 2016 Oleh
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+/**
+ * A helper object to map SQL-like naming to MongoDB query syntax
+ */
+const exprMapper = {
+    '=': '$eq',
+    '!=': '$ne',
+    '<>': '$ne',
+    '>': '$gt',
+    '<': '$lt',
+    '>=': '$gte',
+    '<=': '$lte',
+    'LIKE': '$regex',
+};
+
+/*
+ * Parses object with WHERE clause recursively
+ * and generates MongoDB `find` query object
+ */
+function parseWhere(root) {
+    const operator = Object.keys(root)[0];
+
+    // extract leaf binary expressions
+    if (operator === 'AND') {
+        const e1 = parseWhere(root[operator][0]);
+        const e2 = parseWhere(root[operator][1]);
+
+        // eslint-disable-next-line
+        return { '$and' : [
+            e1,
+            e2,
+        ] };
+    } else if (operator === 'OR') {
+        const e1 = parseWhere(root[operator][0]);
+        const e2 = parseWhere(root[operator][1]);
+
+        // eslint-disable-next-line
+        return  { '$or' : [
+            e1,
+            e2,
+        ] };
+    }
+    const field = root[operator][0];
+    const value = root[operator][1];
+    const expr = exprMapper[operator];
+    const obj = {};
+
+    if (operator === 'LIKE') {
+        obj[`value.${field}`] = parseLikeExpression(value);
+    } else {
+        obj[`value.${field}`] = { [expr]: value };
+    }
+
+    return obj;
+}
+
+module.exports = parseWhere;

lib/api/apiUtils/bucket/validateReplicationConfig.js

@@ -0,0 +1,32 @@
+const config = require('../../../Config').config;
+
+/**
+ * Validates that the replication configuration contains a preferred
+ * read location if the bucket location is a transient source
+ *
+ * @param {object} repConfig - replication configuration
+ * @param {object} bucket - bucket metadata
+ *
+ * @return {boolean} validity of replication configuration with
+ * transient source
+ */
+function validateReplicationConfig(repConfig, bucket) {
+    const bucketLocationName = bucket.getLocationConstraint();
+    if (!repConfig || !repConfig.rules) {
+        return false;
+    }
+    const bucketLocation = config.locationConstraints[bucketLocationName];
+    if (!bucketLocation.isTransient) {
+        return true;
+    }
+    return repConfig.rules.every(rule => {
+        if (!rule.storageClass) {
+            return true;
+        }
+        const storageClasses = rule.storageClass.split(',');
+        return storageClasses.some(
+            site => site.endsWith(':preferred_read'));
+    });
+}
+
+module.exports = validateReplicationConfig;

lib/api/apiUtils/bucket/validateSearch.js

@@ -0,0 +1,96 @@
+const Parser = require('sql-where-parser');
+const { errors } = require('arsenal');
+const objModel = require('arsenal').models.ObjectMD;
+
+const BINARY_OP = 2;
+const sqlConfig = {
+    operators: [
+        {
+            '=': BINARY_OP,
+            '<': BINARY_OP,
+            '>': BINARY_OP,
+            '<>': BINARY_OP,
+            '<=': BINARY_OP,
+            '>=': BINARY_OP,
+            '!=': BINARY_OP,
+        },
+        { LIKE: BINARY_OP },
+        { AND: BINARY_OP },
+        { OR: BINARY_OP },
+    ],
+    tokenizer: {
+        shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'],
+        shouldMatch: ['"', '\'', '`'],
+        shouldDelimitBy: [' ', '\n', '\r', '\t'],
+    },
+};
+const parser = new Parser(sqlConfig);
+
+function _validateTree(whereClause, possibleAttributes) {
+    let invalidAttribute;
+
+    function _searchTree(node) {
+        if (typeof node !== 'object') {
+            invalidAttribute = node;
+        } else {
+            const operator = Object.keys(node)[0];
+            if (operator === 'AND' || operator === 'OR') {
+                _searchTree(node[operator][0]);
+                _searchTree(node[operator][1]);
+            } else {
+                const field = node[operator][0];
+                if (!field.startsWith('tags.') &&
+                    !possibleAttributes[field] &&
+                    !field.startsWith('replicationInfo.') &&
+                    !field.startsWith('x-amz-meta-')) {
+                    invalidAttribute = field;
+                }
+            }
+        }
+    }
+    _searchTree(whereClause);
+    return invalidAttribute;
+}
+
+/**
+ * validateSearchParams - validate value of ?search= in request
+ * @param  {string} searchParams - value of search params in request
+ *  which should be jsu sql where clause
+ * For metadata: x-amz-meta-color=\"blue\"
+ * For tags: tags.x-amz-meta-color=\"blue\"
+ * For replication status : replication-status=\"PENDING\"
+ * For any other attribute: `content-length`=5
+ * @return {undefined | error} undefined if validates or arsenal error if not
+ */
+function validateSearchParams(searchParams) {
+    let ast;
+    try {
+        // allow using 'replicationStatus' as search param to increase
+        // ease of use, pending metadata search rework
+        // eslint-disable-next-line no-param-reassign
+        searchParams = searchParams.replace(
+            'replication-status', 'replicationInfo.status');
+        ast = parser.parse(searchParams);
+    } catch (e) {
+        if (e) {
+            return {
+                error: errors.InvalidArgument
+                .customizeDescription('Invalid sql where clause ' +
+                'sent as search query'),
+            };
+        }
+    }
+    const possibleAttributes = objModel.getAttributes();
+    const invalidAttribute = _validateTree(ast, possibleAttributes);
+    if (invalidAttribute) {
+        return {
+            error: errors.InvalidArgument
+                .customizeDescription('Search param ' +
+                `contains unknown attribute: ${invalidAttribute}`) };
+    }
+    return {
+        ast,
+    };
+}
+
+module.exports = validateSearchParams;

lib/api/apiUtils/object/BackendInfo.js

@@ -1,233 +0,0 @@
-const { config } = require('../../../Config');
-const { legacyLocations } = require('../../../../constants.js');
-const { locationConstraints } = config;
-const escapeForXml = require('arsenal').s3middleware.escapeForXml;
-
-class BackendInfo {
-    /**
-    * Represents the info necessary to evaluate which data backend to use
-    * on a data put call.
-    * @constructor
-    * @param {string | undefined} objectLocationConstraint - location constraint
-    * for object based on user meta header
-    * @param {string | undefined } bucketLocationConstraint - location
-    * constraint for bucket based on bucket metadata
-    * @param {string} requestEndpoint - endpoint to which request was made
-    * @param {string | undefined } legacyLocationConstraint - legacy location
-    * constraint
-    */
-    constructor(objectLocationConstraint, bucketLocationConstraint,
-        requestEndpoint, legacyLocationConstraint) {
-        this._objectLocationConstraint = objectLocationConstraint;
-        this._bucketLocationConstraint = bucketLocationConstraint;
-        this._requestEndpoint = requestEndpoint;
-        this._legacyLocationConstraint = legacyLocationConstraint;
-        return this;
-    }
-
-    /**
-     * validate proposed location constraint against config
-     * @param {string | undefined} locationConstraint - value of user
-     * metadata location constraint header or bucket location constraint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if valid, false if not
-     */
-    static isValidLocationConstraint(locationConstraint, log) {
-        if (Object.keys(config.locationConstraints).
-            indexOf(locationConstraint) < 0) {
-            log.trace('proposed locationConstraint is invalid',
-                { locationConstraint });
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * validate that request endpoint is listed in the restEndpoint config
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if present, false if not
-     */
-    static isRequestEndpointPresent(requestEndpoint, log) {
-        if (Object.keys(config.restEndpoints).indexOf(requestEndpoint) < 0) {
-            log.trace('requestEndpoint does not match config restEndpoints',
-              { requestEndpoint });
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * validate that locationConstraint for request Endpoint matches
-     * one config locationConstraint
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if matches, false if not
-     */
-    static isRequestEndpointValueValid(requestEndpoint, log) {
-        if (Object.keys(config.locationConstraints).indexOf(config
-            .restEndpoints[requestEndpoint]) < 0) {
-            log.trace('the default locationConstraint for request' +
-                'Endpoint does not match any config locationConstraint',
-                { requestEndpoint });
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * validate that s3 server is running with a file or memory backend
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if running with file/mem backend, false if not
-     */
-    static isMemOrFileBackend(requestEndpoint, log) {
-        if (config.backends.data === 'mem' ||
-        config.backends.data === 'file') {
-            log.trace('use data backend for the location', {
-                dataBackend: config.backends.data,
-                method: 'isMemOrFileBackend',
-            });
-            return true;
-        }
-        return false;
-    }
-
-    /**
-     * validate requestEndpoint against config or mem/file data backend
-     * - if there is no match for the request endpoint in the config
-     * restEndpoints and data backend is set to mem or file we will use this
-     * data backend for the location.
-     * - if locationConstraint for request Endpoint does not match
-     * any config locationConstraint, we will return an error
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if valid, false if not
-     */
-    static isValidRequestEndpointOrBackend(requestEndpoint, log) {
-        if (!BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
-            return BackendInfo.isMemOrFileBackend(requestEndpoint, log);
-        }
-        return BackendInfo.isRequestEndpointValueValid(requestEndpoint, log);
-    }
-
-    /**
-     * validate controlling BackendInfo Parameter
-     * @param {string | undefined} objectLocationConstraint - value of user
-     * metadata location constraint header
-     * @param {string | null} bucketLocationConstraint - location
-     * constraint from bucket metadata
-     * @param {string} requestEndpoint - endpoint of request
-     * @param {object} log - werelogs logger
-     * @return {object} - location constraint validity
-     */
-    static controllingBackendParam(objectLocationConstraint,
-        bucketLocationConstraint, requestEndpoint, log) {
-        if (objectLocationConstraint) {
-            if (BackendInfo.isValidLocationConstraint(objectLocationConstraint,
-                log)) {
-                log.trace('objectLocationConstraint is valid');
-                return { isValid: true };
-            }
-            log.trace('objectLocationConstraint is invalid');
-            return { isValid: false, description: 'Object Location Error - ' +
-            `Your object location "${escapeForXml(objectLocationConstraint)}"` +
-            'is not  in your location config - Please update.' };
-        }
-        if (bucketLocationConstraint) {
-            if (BackendInfo.isValidLocationConstraint(bucketLocationConstraint,
-                log)) {
-                log.trace('bucketLocationConstraint is valid');
-                return { isValid: true };
-            }
-            log.trace('bucketLocationConstraint is invalid');
-            return { isValid: false, description: 'Bucket Location Error - ' +
-            `Your bucket location "${escapeForXml(bucketLocationConstraint)}"` +
-            ' is not in your location config - Please update.' };
-        }
-        const legacyLocationConstraint =
-        BackendInfo.getLegacyLocationConstraint();
-        if (legacyLocationConstraint) {
-            log.trace('legacy location is valid');
-            return { isValid: true, legacyLocationConstraint };
-        }
-        if (!BackendInfo.isValidRequestEndpointOrBackend(requestEndpoint,
-          log)) {
-            return { isValid: false, description: 'Endpoint Location Error - ' +
-            `Your endpoint "${requestEndpoint}" is not in restEndpoints ` +
-            'in your config OR the default location constraint for request ' +
-            `endpoint "${escapeForXml(requestEndpoint)}" does not ` +
-            'match any config locationConstraint - Please update.' };
-        }
-        if (BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
-            return { isValid: true };
-        }
-        return { isValid: true, defaultedToDataBackend: true };
-    }
-
-    /**
-    * Return legacyLocationConstraint
-    * @return {string | undefined} legacyLocationConstraint;
-    */
-    static getLegacyLocationConstraint() {
-        return legacyLocations.find(ll => locationConstraints[ll]);
-    }
-
-    /**
-    * Return objectLocationConstraint
-    * @return {string | undefined} objectLocationConstraint;
-    */
-    getObjectLocationConstraint() {
-        return this._objectLocationConstraint;
-    }
-
-    /**
-    * Return bucketLocationConstraint
-    * @return {string | undefined} bucketLocationConstraint;
-    */
-    getBucketLocationConstraint() {
-        return this._bucketLocationConstraint;
-    }
-
-    /**
-    * Return requestEndpoint
-    * @return {string} requestEndpoint;
-    */
-    getRequestEndpoint() {
-        return this._requestEndpoint;
-    }
-
-    /**
-    * Return locationConstraint that should be used with put request
-    * Order of priority is:
-    * (1) objectLocationConstraint,
-    * (2) bucketLocationConstraint,
-    * (3) legacyLocationConstraint,
-    * (4) default locationConstraint for requestEndpoint  if requestEndpoint
-    *     is listed in restEndpoints in config.json
-    * (5) default data backend
-    * @return {string} locationConstraint;
-    */
-    getControllingLocationConstraint() {
-        const objectLC = this.getObjectLocationConstraint();
-        const bucketLC = this.getBucketLocationConstraint();
-        const reqEndpoint = this.getRequestEndpoint();
-        if (objectLC) {
-            return objectLC;
-        }
-        if (bucketLC) {
-            return bucketLC;
-        }
-        if (this._legacyLocationConstraint) {
-            return this._legacyLocationConstraint;
-        }
-        if (config.restEndpoints[reqEndpoint]) {
-            return config.restEndpoints[reqEndpoint];
-        }
-        return config.backends.data;
-    }
-}
-
-module.exports = {
-    BackendInfo,
-};

lib/api/apiUtils/object/abortMultipartUpload.js

@@ -73,15 +73,6 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
                 return next(null, mpuBucket, destBucket, skipDataDelete);
             });
         },
-        function sendAbortPut(mpuBucket, destBucket, skipDataDelete, next) {
-            services.sendAbortMPUPut(bucketName, objectKey, uploadId, log,
-                err => {
-                    if (err) {
-                        return next(err, destBucket);
-                    }
-                    return next(null, mpuBucket, destBucket, skipDataDelete);
-                });
-        },
         function getPartLocations(mpuBucket, destBucket, skipDataDelete,
         next) {
             services.getMPUparts(mpuBucket.getName(), uploadId, log,
@@ -96,7 +87,6 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
         },
         function deleteData(mpuBucket, storedParts, destBucket,
         skipDataDelete, next) {
-            // for Azure we do not need to delete data
             if (skipDataDelete) {
                 return next(null, mpuBucket, storedParts, destBucket);
             }

lib/api/apiUtils/object/applyZenkoUserMD.js

@@ -0,0 +1,19 @@
+const { zenkoIDHeader } = require('arsenal').constants;
+
+const _config = require('../../../Config').config;
+
+/**
+ * applyZenkoUserMD - if request is within a Zenko deployment, apply user
+ * metadata called "zenko-source" to the object
+ * @param {Object} metaHeaders - user metadata object
+ * @return {undefined}
+ */
+function applyZenkoUserMD(metaHeaders) {
+    if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' &&
+        !metaHeaders[zenkoIDHeader]) {
+        // eslint-disable-next-line no-param-reassign
+        metaHeaders[zenkoIDHeader] = _config.getPublicInstanceId();
+    }
+}
+
+module.exports = applyZenkoUserMD;

lib/api/apiUtils/object/checkReadLocation.js

@@ -0,0 +1,27 @@
+/**
+ * checkReadLocation - verify that a bucket's default read location exists
+ * for a specified read data locator
+ * @param {Config} config - Config object
+ * @param {string} locationName - location constraint
+ * @param {string} objectKey - object key
+ * @param {string} bucketName - bucket name
+ * @return {Object | null} return object containing location information
+ * if location exists; otherwise, null
+ */
+function checkReadLocation(config, locationName, objectKey, bucketName) {
+    const readLocation = config.getLocationConstraint(locationName);
+    if (readLocation) {
+        const bucketMatch = readLocation.details &&
+            readLocation.details.bucketMatch;
+        const backendKey = bucketMatch ? objectKey :
+            `${bucketName}/${objectKey}`;
+        return {
+            location: locationName,
+            key: backendKey,
+            locationType: readLocation.type,
+        };
+    }
+    return null;
+}
+
+module.exports = checkReadLocation;

lib/api/apiUtils/object/coldStorage.js

@@ -0,0 +1,247 @@
+/*
+ * Code based on Yutaka Oishi (Fujifilm) contributions
+ * Date: 11 Sep 2020
+ */
+const { ObjectMDArchive } = require('arsenal').models;
+const errors = require('arsenal').errors;
+const { config } = require('../../../Config');
+const { locationConstraints } = config;
+
+const { scaledMsPerDay } = config.getTimeOptions();
+
+/**
+ * Get response header "x-amz-restore"
+ * Be called by objectHead.js
+ * @param {object} objMD - object's metadata
+ * @returns {string|undefined} x-amz-restore
+ */
+function getAmzRestoreResHeader(objMD) {
+    if (objMD.archive &&
+        objMD.archive.restoreRequestedAt &&
+        !objMD.archive.restoreCompletedAt) {
+        // Avoid race condition by relying on the `archive` MD of the object
+        // and return the right header after a RESTORE request.
+        // eslint-disable-next-line
+        return `ongoing-request="true"`;
+    }
+    if (objMD['x-amz-restore']) {
+        if (objMD['x-amz-restore']['expiry-date']) {
+            const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString();
+            // eslint-disable-next-line
+            return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
+        }
+    }
+    return undefined;
+}
+
+/**
+ * Check if restore can be done.
+ *
+ * @param {ObjectMD} objectMD - object metadata
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
+ */
+function _validateStartRestore(objectMD, log) {
+    if (objectMD.archive?.restoreCompletedAt) {
+        if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
+            // return InvalidObjectState error if the restored object is expired
+            // but restore info md of this object has not yet been cleared
+            log.debug('The restored object already expired.',
+                {
+                    archive: objectMD.archive,
+                    method: '_validateStartRestore',
+                });
+            return errors.InvalidObjectState;
+        }
+
+        // If object is already restored, no further check is needed
+        // Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have
+        // been reset.
+        return undefined;
+    }
+    const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
+    if (!isLocationCold) {
+        // return InvalidObjectState error if the object is not in cold storage,
+        // not in cold storage means either location cold flag not exists or cold flag is explicit false
+        log.debug('The bucket of the object is not in a cold storage location.',
+            {
+                isLocationCold,
+                method: '_validateStartRestore',
+            });
+        return errors.InvalidObjectState;
+    }
+    if (objectMD.archive?.restoreRequestedAt) {
+        // return RestoreAlreadyInProgress error if the object is currently being restored
+        // check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
+        log.debug('The object is currently being restored.',
+            {
+                archive: objectMD.archive,
+                method: '_validateStartRestore',
+            });
+        return errors.RestoreAlreadyInProgress;
+    }
+    return undefined;
+}
+
+/**
+ * Check if "put version id" is allowed
+ *
+ * @param {ObjectMD} objMD - object metadata
+ * @param {string} versionId - object's version id
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} - undefined if "put version id" is allowed
+ */
+function validatePutVersionId(objMD, versionId, log) {
+    if (!objMD) {
+        const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
+        log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
+        return err;
+    }
+
+    if (objMD.isDeleteMarker) {
+        log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
+        return errors.MethodNotAllowed;
+    }
+
+    const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
+    if (!isLocationCold) {
+        log.error('The object data is not stored in a cold storage location.',
+            {
+                isLocationCold,
+                dataStoreName: objMD.dataStoreName,
+                method: 'validatePutVersionId',
+            });
+        return errors.InvalidObjectState;
+    }
+
+    // make sure object archive restoration is in progress
+    // NOTE: we do not use putObjectVersion to update the restoration period.
+    if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
+        || objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
+        log.error('object archive restoration is not in progress',
+            { method: 'validatePutVersionId', versionId });
+        return errors.InvalidObjectState;
+    }
+
+    return undefined;
+}
+
+/**
+ * Check if the object is already restored, and update the expiration date accordingly:
+ * > After restoring an archived object, you can update the restoration period by reissuing the
+ * > request with a new period. Amazon S3 updates the restoration period relative to the current
+ * > time.
+ *
+ * @param {ObjectMD} objectMD - object metadata
+ * @param {object} log - werelogs logger
+ * @return {boolean} - true if the object is already restored
+ */
+function _updateObjectExpirationDate(objectMD, log) {
+    // Check if restoreCompletedAt field exists
+    // Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
+    // checked earlier in the process, so checking again here would create weird states
+    const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
+    log.debug('The restore status of the object.', {
+        isObjectAlreadyRestored,
+        method: 'isObjectAlreadyRestored'
+    });
+    if (isObjectAlreadyRestored) {
+        const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
+        expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
+
+        /* eslint-disable no-param-reassign */
+        objectMD.archive.restoreWillExpireAt = expiryDate;
+        objectMD['x-amz-restore'] = {
+            'ongoing-request': false,
+            'expiry-date': expiryDate,
+        };
+        /* eslint-enable no-param-reassign */
+    }
+    return isObjectAlreadyRestored;
+}
+
+/**
+ * update restore expiration date.
+ *
+ * @param {ObjectMD} objectMD - objectMD instance
+ * @param {object} restoreParam - restore param
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} internal error if object MD is not valid
+ *
+ */
+function _updateRestoreInfo(objectMD, restoreParam, log) {
+    if (!objectMD.archive) {
+        log.debug('objectMD.archive doesn\'t exits', {
+            objectMD,
+            method: '_updateRestoreInfo'
+        });
+        return errors.InternalError.customizeDescription('Archive metadata is missing.');
+    }
+    /* eslint-disable no-param-reassign */
+    objectMD.archive.restoreRequestedAt = new Date();
+    objectMD.archive.restoreRequestedDays = restoreParam.days;
+    objectMD.originOp = 's3:ObjectRestore:Post';
+    /* eslint-enable no-param-reassign */
+    if (!ObjectMDArchive.isValid(objectMD.archive)) {
+        log.debug('archive is not valid', {
+            archive: objectMD.archive,
+            method: '_updateRestoreInfo'
+        });
+        return errors.InternalError.customizeDescription('Invalid archive metadata.');
+    }
+    return undefined;
+}
+
+/**
+ * start to restore object.
+ * If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
+ * calculate restore expiry-date and add it to objectMD.
+ * Be called by objectRestore.js
+ *
+ * @param {ObjectMD} objectMD - objectMd instance
+ * @param {object} restoreParam - bucket name
+ * @param {object} log - werelogs logger
+ * @param {function} cb - bucket name
+ * @return {undefined}
+ *
+ */
+function startRestore(objectMD, restoreParam, log, cb) {
+    log.info('Validating if restore can be done or not.');
+    const checkResultError = _validateStartRestore(objectMD, log);
+    if (checkResultError) {
+        return cb(checkResultError);
+    }
+    log.info('Updating restore information.');
+    const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
+    if (updateResultError) {
+        return cb(updateResultError);
+    }
+    const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
+    return cb(null, isObjectAlreadyRestored);
+}
+
+/**
+ * checks if object data is available or if it's in cold storage
+ * @param {ObjectMD} objMD Object metadata
+ * @returns {ArsenalError|null} error if object data is not available
+ */
+function verifyColdObjectAvailable(objMD) {
+    // return error when object is cold
+    if (objMD.archive &&
+        // Object is in cold backend
+        (!objMD.archive.restoreRequestedAt ||
+            // Object is being restored
+            (objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
+        const err = errors.InvalidObjectState
+            .customizeDescription('The operation is not valid for the object\'s storage class');
+        return err;
+    }
+    return null;
+}
+
+module.exports = {
+    startRestore,
+    getAmzRestoreResHeader,
+    validatePutVersionId,
+    verifyColdObjectAvailable,
+};

lib/api/apiUtils/object/createAndStoreObject.js

@@ -7,43 +7,17 @@ const { data } = require('../../../data/wrapper');
 const services = require('../../../services');
 const { dataStore } = require('./storeObject');
 const locationConstraintCheck = require('./locationConstraintCheck');
-const { versioningPreprocessing } = require('./versioning');
+const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
 const removeAWSChunked = require('./removeAWSChunked');
 const getReplicationInfo = require('./getReplicationInfo');
 const { config } = require('../../../Config');
 const validateWebsiteHeader = require('./websiteServing')
     .validateWebsiteHeader;
-const {
-    externalBackends, versioningNotImplBackends, zenkoIDHeader,
-} = constants;
+const applyZenkoUserMD = require('./applyZenkoUserMD');
+const { externalBackends, versioningNotImplBackends } = constants;
 
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
-'a versioned object to a location-constraint of type Azure.';
-
-/**
- * Retro-propagation is where S3C ingestion will re-ingest an object whose
- * request originated from Zenko.
- * To avoid this, Zenko requests which create objects/versions will be tagged
- * with a user-metadata header defined in constants.zenkoIDHeader. When
- * ingesting objects into Zenko, we can determine if this object has already
- * been created in Zenko.
- * Delete marker requests cannot specify user-metadata fields, so we instead
- * rely on checking the "user-agent" to see the origin of a request.
- * If delete marker, and user-agent came from a Zenko client, we add the
- * user-metadata field to the object metadata.
- * @param {Object} metaHeaders - user metadata object
- * @param {http.ClientRequest} request - client request with user-agent header
- * @param {Boolean} isDeleteMarker - delete marker indicator
- * @return {undefined}
- */
-function _checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker) {
-    const userAgent = request.headers['user-agent'];
-
-    if (isDeleteMarker && userAgent && userAgent.includes('Zenko')) {
-        // eslint-disable-next-line no-param-reassign
-        metaHeaders[zenkoIDHeader] = 'zenko';
-    }
-}
+'a versioned object to a location-constraint of type Azure or GCP.';
 
 function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
     metadataStoreParams, dataToDelete, log, requestMethod, callback) {
@@ -78,6 +52,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
  * credentialScope (to be used for streaming v4 auth if applicable)
  * @param {(object|null)} overheadField - fields to be included in metadata overhead
  * @param {RequestLogger} log - logger instance
+ * @param {string} originOp - Origin operation
  * @param {function} callback - callback function
  * @return {undefined} and call callback with (err, result) -
  * result.contentMD5 - content md5 of new object or version
@@ -85,7 +60,10 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
  */
 function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
-        overheadField, log, callback) {
+        overheadField, log, originOp, callback) {
+    const putVersionId = request.headers['x-scal-s3-version-id'];
+    const isPutVersion = putVersionId || putVersionId === '';
+
     const size = isDeleteMarker ? 0 : request.parsedContentLength;
     // although the request method may actually be 'DELETE' if creating a
     // delete marker, for our purposes we consider this to be a 'PUT'
@@ -108,9 +86,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         });
         return process.nextTick(() => callback(metaHeaders));
     }
-    // if receiving a request from Zenko for a delete marker, we place a
-    // user-metadata field on the object
-    _checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker);
+    // if the request occurs within a Zenko deployment, we place a user-metadata
+    // field on the object
+    applyZenkoUserMD(metaHeaders);
 
     log.trace('meta headers', { metaHeaders, method: 'objectPut' });
     const objectKeyContext = {
@@ -136,11 +114,26 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         size,
         headers,
         isDeleteMarker,
-        replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
+        replicationInfo: getReplicationInfo(
+            objectKey, bucketMD, false, size, null, null, authInfo),
         overheadField,
         log,
     };
 
+    // For Azure BlobStorage API compatability
+    // If an object already exists copy/repair creation-time
+    // creation-time must remain static after an object is created
+    //  --> EVEN FOR VERSIONS <--
+    if (objMD) {
+        if (objMD['creation-time']) {
+            metadataStoreParams.creationTime = objMD['creation-time'];
+        } else {
+            // If creation-time is not set (for old objects)
+            // fall back to the last modified and store it back to the db
+            metadataStoreParams.creationTime = objMD['last-modified'];
+        }
+    }
+
     if (!isDeleteMarker) {
         metadataStoreParams.contentType = request.headers['content-type'];
         metadataStoreParams.cacheControl = request.headers['cache-control'];
@@ -150,7 +143,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             removeAWSChunked(request.headers['content-encoding']);
         metadataStoreParams.expires = request.headers.expires;
         metadataStoreParams.tagging = request.headers['x-amz-tagging'];
-        metadataStoreParams.originOp = 's3:ObjectCreated:Put';
+        metadataStoreParams.originOp = originOp;
         const defaultObjectLockConfiguration
             = bucketMD.getObjectLockConfiguration();
         if (defaultObjectLockConfiguration) {
@@ -165,7 +158,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         // eslint-disable-next-line no-param-reassign
         request.headers[constants.objectLocationConstraintHeader] =
             objMD[constants.objectLocationConstraintHeader];
-        metadataStoreParams.originOp = 's3:ObjectRemoved:DeleteMarkerCreated';
+        metadataStoreParams.originOp = originOp;
     }
 
     const backendInfoObj =
@@ -204,12 +197,42 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
     const dontSkipBackend = externalBackends;
     /* eslint-enable camelcase */
 
+    const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
+    const mdOnlySize = request.headers['x-amz-meta-size'];
+
     return async.waterfall([
         function storeData(next) {
-            if (size === 0 && !dontSkipBackend[locationType]) {
+            if (size === 0) {
+                if (!dontSkipBackend[locationType]) {
                     metadataStoreParams.contentMD5 = constants.emptyFileMd5;
                     return next(null, null, null);
                 }
+                // Handle mdOnlyHeader as a metadata only operation. If
+                // the object in question is actually 0 byte or has a body size
+                // then handle normally.
+                if (mdOnlyHeader === 'true' && mdOnlySize > 0) {
+                    log.debug('metadata only operation x-amz-meta-mdonly');
+                    const md5 = request.headers['x-amz-meta-md5chksum']
+                        ? new Buffer(request.headers['x-amz-meta-md5chksum'],
+                        'base64').toString('hex') : null;
+                    const numParts = request.headers['x-amz-meta-md5numparts'];
+                    let _md5;
+                    if (numParts === undefined) {
+                        _md5 = md5;
+                    } else {
+                        _md5 = `${md5}-${numParts}`;
+                    }
+                    const versionId = request.headers['x-amz-meta-version-id'];
+                    const dataGetInfo = {
+                        key: objectKey,
+                        dataStoreName: location,
+                        dataStoreType: locationType,
+                        dataStoreVersionId: versionId,
+                        dataStoreMD5: _md5,
+                    };
+                    return next(null, dataGetInfo, _md5);
+                }
+            }
             return dataStore(objectKeyContext, cipherBundle, request, size,
                     streamingV4Params, backendInfo, log, next);
         },
@@ -233,10 +256,19 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
                 dataGetInfoArr[0].cipheredDataKey =
                     cipherBundle.cipheredDataKey;
             }
+            if (mdOnlyHeader === 'true') {
+                metadataStoreParams.size = mdOnlySize;
+                dataGetInfoArr[0].size = mdOnlySize;
+            }
             metadataStoreParams.contentMD5 = calculatedHash;
             return next(null, dataGetInfoArr);
         },
         function getVersioningInfo(infoArr, next) {
+            // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
+            if (isPutVersion) {
+                const options = overwritingVersioning(objMD, metadataStoreParams);
+                return process.nextTick(() => next(null, options, infoArr));
+            }
             return versioningPreprocessing(bucketName, bucketMD,
                 metadataStoreParams.objectKey, objMD, log, (err, options) => {
                     if (err) {

lib/api/apiUtils/object/expirationHeaders.js

@@ -4,23 +4,25 @@ const {
     LifecycleDateTime,
     LifecycleUtils,
 } = require('arsenal').s3middleware.lifecycleHelpers;
+const { config } = require('../../../Config');
 
-// moves lifecycle transition deadlines 1 day earlier, mostly for testing
-const transitionOneDayEarlier = process.env.TRANSITION_ONE_DAY_EARLIER === 'true';
-// moves lifecycle expiration deadlines 1 day earlier, mostly for testing
-const expireOneDayEarlier = process.env.EXPIRE_ONE_DAY_EARLIER === 'true';
+const {
+    expireOneDayEarlier,
+    transitionOneDayEarlier,
+    timeProgressionFactor,
+    scaledMsPerDay,
+} = config.getTimeOptions();
 
 const lifecycleDateTime = new LifecycleDateTime({
     transitionOneDayEarlier,
     expireOneDayEarlier,
+    timeProgressionFactor,
 });
 
-const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime);
-
-const oneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
+const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor);
 
 function calculateDate(objDate, expDays, datetime) {
-    return new Date(datetime.getTimestamp(objDate) + expDays * oneDay);
+    return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay));
 }
 
 function formatExpirationHeader(date, id) {

lib/api/apiUtils/object/getReplicationBackendDataLocator.js

@@ -0,0 +1,51 @@
+const { errors } = require('arsenal');
+
+/**
+ * getReplicationBackendDataLocator - compares given location constraint to
+ * replication backends
+ * @param {object} locationObj - object containing location information
+ * @param {string} locationObj.location - name of location constraint
+ * @param {string} locationObj.key - keyname of object in location constraint
+ * @param {string} locationObj.locationType - type of location constraint
+ * @param {object} replicationInfo - information about object replication
+ * @param {array} replicationInfo.backends - array containing information about
+ * each replication location
+ * @param {string} replicationInfo.backends[].site - name of replication
+ * location
+ * @param {string} replicationInfo.backends[].status - status of replication
+ * @param {string} replicationInfo.backends[].dataStoreVersionId - version id
+ * of object at replication location
+ * @return {object} res - response object
+ *     {array} [res.dataLocator] - if COMPLETED status: array
+ *                                 containing the cloud location,
+ *                                 undefined otherwise
+ *     {string} [res.status] - replication status if no error
+ *     {string} [res.reason] - reason message if PENDING/FAILED
+ *     {Error} [res.error] - defined if object is not replicated to
+ *                           location passed in locationObj
+ */
+function getReplicationBackendDataLocator(locationObj, replicationInfo) {
+    const repBackendResult = {};
+    const locMatch = replicationInfo.backends.find(
+        backend => backend.site === locationObj.location);
+    if (!locMatch) {
+        repBackendResult.error = errors.InvalidLocationConstraint.
+            customizeDescription('Object is not replicated to location ' +
+            'passed in location header');
+        return repBackendResult;
+    }
+    repBackendResult.status = locMatch.status;
+    if (['PENDING', 'FAILED'].includes(locMatch.status)) {
+        repBackendResult.reason =
+            `Object replication to specified backend is ${locMatch.status}`;
+        return repBackendResult;
+    }
+    repBackendResult.dataLocator = [{
+        key: locationObj.key,
+        dataStoreName: locationObj.location,
+        dataStoreType: locationObj.locationType,
+        dataStoreVersionId: locMatch.dataStoreVersionId }];
+    return repBackendResult;
+}
+
+module.exports = getReplicationBackendDataLocator;

lib/api/apiUtils/object/getReplicationInfo.js

@@ -1,5 +1,7 @@
 const s3config = require('../../../Config').config;
-const { isLifecycleSession } = require('../authorization/permissionChecks.js');
+const { isServiceAccount, getServiceAccountProperties } =
+      require('../authorization/permissionChecks');
+const { replicationBackends } = require('arsenal').constants;
 
 function _getBackend(objectMD, site) {
     const backends = objectMD ? objectMD.replicationInfo.backends : [];
@@ -21,25 +23,31 @@ function _getStorageClasses(rule) {
     }
     const { replicationEndpoints } = s3config;
     // If no storage class, use the given default endpoint or the sole endpoint
-    if (replicationEndpoints.length > 1) {
+    if (replicationEndpoints.length > 0) {
         const endPoint =
-            replicationEndpoints.find(endpoint => endpoint.default);
+            replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
         return [endPoint.site];
     }
-    return [replicationEndpoints[0].site];
+    return undefined;
 }
 
 function _getReplicationInfo(rule, replicationConfig, content, operationType,
-    objectMD) {
+    objectMD, bucketMD) {
     const storageTypes = [];
     const backends = [];
     const storageClasses = _getStorageClasses(rule);
+    if (!storageClasses) {
+        return undefined;
+    }
     storageClasses.forEach(storageClass => {
-        const location = s3config.locationConstraints[storageClass];
-        if (location && ['aws_s3', 'azure'].includes(location.type)) {
+        const storageClassName =
+              storageClass.endsWith(':preferred_read') ?
+              storageClass.split(':')[0] : storageClass;
+        const location = s3config.locationConstraints[storageClassName];
+        if (location && replicationBackends[location.type]) {
             storageTypes.push(location.type);
         }
-        backends.push(_getBackend(objectMD, storageClass));
+        backends.push(_getBackend(objectMD, storageClassName));
     });
     if (storageTypes.length > 0 && operationType) {
         content.push(operationType);
@@ -52,6 +60,7 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
         storageClass: storageClasses.join(','),
         role: replicationConfig.role,
         storageType: storageTypes.join(','),
+        isNFS: bucketMD.isNFS(),
     };
 }
 
@@ -65,26 +74,45 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
  * @param {string} operationType - The type of operation to replicate
  * @param {object} objectMD - The object metadata
  * @param {AuthInfo} [authInfo] - authentication info of object owner
- * @param {boolean} [isDeleteMarker] - whether creating a delete marker
  * @return {undefined}
  */
 function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
-    objectMD, authInfo, isDeleteMarker) {
+    objectMD, authInfo) {
     const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
     const config = bucketMD.getReplicationConfiguration();
-    // If bucket does not have a replication configuration, do not replicate.
-    if (config) {
-        // If delete an object due to a lifecycle action,
-        // the delete marker is not replicated to the destination buckets.
-        if (isDeleteMarker && authInfo && isLifecycleSession(authInfo.getArn())) {
-            return undefined;
-        }
 
-        const rule = config.rules.find(rule =>
-            (objKey.startsWith(rule.prefix) && rule.enabled));
+    // Do not replicate object in the following cases:
+    //
+    // - bucket does not have a replication configuration
+    //
+    // - replication configuration does not apply to the object
+    //   (i.e. no rule matches object prefix)
+    //
+    // - replication configuration applies to the object (i.e. a rule matches
+    //   object prefix) but the status is disabled
+    //
+    // - object owner is an internal service account like Lifecycle,
+    //   unless the account properties explicitly allow it to
+    //   replicate like MD ingestion (because we do not want to
+    //   replicate objects created from actions triggered by internal
+    //   services, by design)
+
+    if (config) {
+        let doReplicate = false;
+        if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) {
+            doReplicate = true;
+        } else {
+            const serviceAccountProps = getServiceAccountProperties(
+                authInfo.getCanonicalID());
+            doReplicate = serviceAccountProps.canReplicate;
+        }
+        if (doReplicate) {
+            const rule = config.rules.find(
+                rule => (objKey.startsWith(rule.prefix) && rule.enabled));
             if (rule) {
-            return _getReplicationInfo(rule, config, content, operationType,
-                objectMD);
+                return _getReplicationInfo(
+                    rule, config, content, operationType, objectMD, bucketMD);
+            }
         }
     }
     return undefined;

lib/api/apiUtils/object/locationConstraintCheck.js

@@ -1,6 +1,7 @@
-const { errors } = require('arsenal');
+const { errors, models } = require('arsenal');
 
-const { BackendInfo } = require('./BackendInfo');
+const { BackendInfo } = models;
+const { config } = require('../../../Config');
 const constants = require('../../../../constants');
 
 /**
@@ -29,7 +30,7 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) {
     const bucketLocationConstraint = bucket.getLocationConstraint();
     const requestEndpoint = request.parsedHost;
 
-    const controllingBackend = BackendInfo.controllingBackendParam(
+    const controllingBackend = BackendInfo.controllingBackendParam(config,
         objectLocationConstraint, bucketLocationConstraint,
         requestEndpoint, log);
     if (!controllingBackend.isValid) {
@@ -39,7 +40,7 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) {
         };
         return backendInfoObj;
     }
-    const backendInfo = new BackendInfo(objectLocationConstraint,
+    const backendInfo = new BackendInfo(config, objectLocationConstraint,
         bucketLocationConstraint, requestEndpoint,
         controllingBackend.legacyLocationConstraint);
     backendInfoObj = {

lib/api/apiUtils/object/locationHeaderCheck.js

@@ -0,0 +1,37 @@
+const { errors } = require('arsenal');
+
+const { config } = require('../../../Config');
+
+/**
+ * locationHeaderCheck - compares 'x-amz-location-constraint' header
+ * to location constraints in config
+ * @param {object} headers - request headers
+ * @param {string} objectKey - key name of object
+ * @param {string} bucketName - name of bucket
+ * @return {undefined|Object} returns error, object, or undefined
+ * @return {string} return.location - name of location constraint
+ * @return {string} return.key - name of object at location constraint
+ * @return {string} - return.locationType - type of location constraint
+ */
+function locationHeaderCheck(headers, objectKey, bucketName) {
+    const location = headers['x-amz-location-constraint'];
+    if (location) {
+        const validLocation = config.locationConstraints[location];
+        if (!validLocation) {
+            return errors.InvalidLocationConstraint.customizeDescription(
+                'Invalid location constraint specified in header');
+        }
+        const bucketMatch = validLocation.details.bucketMatch;
+        const backendKey = bucketMatch ? objectKey :
+            `${bucketName}/${objectKey}`;
+        return {
+            location,
+            key: backendKey,
+            locationType: validLocation.type,
+        };
+    }
+    // no location header was passed
+    return undefined;
+}
+
+module.exports = locationHeaderCheck;

lib/api/apiUtils/object/locationKeysHaveChanged.js

@@ -22,8 +22,13 @@ function locationKeysHaveChanged(prev, curr) {
         return curr.every(v => v.key !== prev);
     }
     const keysMap = {};
-    prev.forEach(v => { keysMap[v.key] = true; });
-    return curr.every(v => !keysMap[v.key]);
+    prev.forEach(v => {
+        if (!keysMap[v.dataStoreType]) {
+            keysMap[v.dataStoreType] = {};
+        }
+        keysMap[v.dataStoreType][v.key] = true;
+    });
+    return curr.every(v => !(keysMap[v.dataStoreType] && keysMap[v.dataStoreType][v.key]));
 }
 
 module.exports = locationKeysHaveChanged;

lib/api/apiUtils/object/objectLockHelpers.js

@@ -5,6 +5,7 @@ const { config } = require('../../../Config');
 const vault = require('../../../auth/vault');
 const { evaluateBucketPolicyWithIAM } = require('../authorization/permissionChecks');
 
+const { scaledMsPerDay } = config.getTimeOptions();
 /**
  * Calculates retain until date for the locked object version
  * @param {object} retention - includes days or years retention period
@@ -20,8 +21,9 @@ function calculateRetainUntilDate(retention) {
     const date = moment();
     // Calculate the number of days to retain the lock on the object
     const retainUntilDays = days || years * 365;
+    const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay;
     const retainUntilDate
-        = date.add(retainUntilDays, 'days');
+        = date.add(retainUntilDaysInMs, 'ms');
     return retainUntilDate.toISOString();
 }
 /**

lib/api/apiUtils/object/objectRestore.js

@@ -0,0 +1,172 @@
+const async = require('async');
+const { errors, s3middleware } = require('arsenal');
+
+const { allowedRestoreObjectRequestTierValues } = require('../../../../constants');
+const coldStorage = require('./coldStorage');
+const monitoring = require('../../../utilities/monitoringHandler');
+const { pushMetric } = require('../../../utapi/utilities');
+const { decodeVersionId } = require('./versioning');
+const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
+const { parseRestoreRequestXml } = s3middleware.objectRestore;
+const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
+
+/**
+ * Check if tier is supported
+ * @param {object} restoreInfo - restore information
+ * @returns {ArsenalError|undefined} return NotImplemented error if tier not support
+ */
+function checkTierSupported(restoreInfo) {
+    if (!allowedRestoreObjectRequestTierValues.includes(restoreInfo.tier)) {
+        return errors.NotImplemented;
+    }
+    return undefined;
+}
+
+/**
+ * POST Object restore process
+ *
+ * @param {MetadataWrapper} metadata - metadata wrapper
+ * @param {object} mdUtils - utility object to treat metadata
+ * @param {AuthInfo} userInfo - Instance of AuthInfo class with requester's info
+ * @param {IncomingMessage} request - request info
+ * @param {object} log - Werelogs logger
+ * @param {function} callback callback function
+ * @return {undefined}
+ */
+function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
+    const METHOD = 'objectRestore';
+
+    const { bucketName, objectKey } = request;
+
+    log.debug('processing request', { method: METHOD });
+
+    const decodedVidResult = decodeVersionId(request.query);
+    if (decodedVidResult instanceof Error) {
+        log.trace('invalid versionId query',
+            {
+                method: METHOD,
+                versionId: request.query.versionId,
+                error: decodedVidResult,
+            });
+        return process.nextTick(() => callback(decodedVidResult));
+    }
+
+    let isObjectRestored = false;
+
+    const mdValueParams = {
+        authInfo: userInfo,
+        bucketName,
+        objectKey,
+        versionId: decodedVidResult,
+        requestType: request.apiMethods || 'restoreObject',
+        /**
+         * Restoring an object might not cause any impact on
+         * the storage, if the object is already restored: in
+         * this case, the duration is extended. We disable the
+         * quota evaluation and trigger it manually.
+         */
+        checkQuota: false,
+        request,
+    };
+
+    return async.waterfall([
+            // get metadata of bucket and object
+            function validateBucketAndObject(next) {
+                return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies,
+                log, (err, bucketMD, objectMD) => {
+                    if (err) {
+                        log.trace('request authorization failed', { method: METHOD, error: err });
+                        return next(err);
+                    }
+                    // Call back error if object metadata could not be obtained
+                    if (!objectMD) {
+                        const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey;
+                        log.trace('error no object metadata found', { method: METHOD, error: err });
+                        return next(err, bucketMD);
+                    }
+                    // If object metadata is delete marker,
+                    // call back NoSuchKey or MethodNotAllowed depending on specifying versionId
+                    if (objectMD.isDeleteMarker) {
+                        let err = errors.NoSuchKey;
+                        if (decodedVidResult) {
+                            err = errors.MethodNotAllowed;
+                        }
+                        log.trace('version is a delete marker', { method: METHOD, error: err });
+                        return next(err, bucketMD, objectMD);
+                    }
+                    log.info('it acquired the object metadata.', {
+                        'method': METHOD,
+                    });
+                    return next(null, bucketMD, objectMD);
+                });
+            },
+
+            // generate restore param obj from xml of request body and check tier validity
+            function parseRequestXmlAndCheckTier(bucketMD, objectMD, next) {
+                log.trace('parsing object restore information');
+                return parseRestoreRequestXml(request.post, log, (err, restoreInfo) => {
+                    if (err) {
+                        return next(err, bucketMD, objectMD, restoreInfo);
+                    }
+                    log.info('it parsed xml of the request body.', { method: METHOD, value: restoreInfo });
+                    const checkTierResult = checkTierSupported(restoreInfo);
+                    if (checkTierResult instanceof Error) {
+                        return next(checkTierResult);
+                    }
+                    return next(null, bucketMD, objectMD, restoreInfo);
+                });
+            },
+            // start restore process
+            function startRestore(bucketMD, objectMD, restoreInfo, next) {
+                return coldStorage.startRestore(objectMD, restoreInfo, log,
+                    (err, _isObjectRestored) => {
+                        isObjectRestored = _isObjectRestored;
+                        return next(err, bucketMD, objectMD);
+                    });
+            },
+            function evaluateQuotas(bucketMD, objectMD, next) {
+                if (isObjectRestored) {
+                    return next(null, bucketMD, objectMD);
+                }
+                const actions = Array.isArray(mdValueParams.requestType) ?
+                    mdValueParams.requestType : [mdValueParams.requestType];
+                const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
+                return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
+                    false, log, err => next(err, bucketMD, objectMD));
+            },
+            function updateObjectMD(bucketMD, objectMD, next) {
+                const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
+                metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
+                    log, err => next(err, bucketMD, objectMD));
+            },
+        ],
+        (err, bucketMD) => {
+            // generate CORS response header
+            const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD);
+            if (err) {
+                log.trace('error processing request',
+                    {
+                        method: METHOD,
+                        error: err,
+                    });
+                monitoring.promMetrics(
+                    'POST', bucketName, err.code, 'restoreObject');
+                return callback(err, err.code, responseHeaders);
+            }
+            pushMetric('restoreObject', log, {
+                userInfo,
+                bucket: bucketName,
+            });
+            if (isObjectRestored) {
+                monitoring.promMetrics(
+                    'POST', bucketName, '200', 'restoreObject');
+                return callback(null, 200, responseHeaders);
+            }
+            monitoring.promMetrics(
+                'POST', bucketName, '202', 'restoreObject');
+            return callback(null, 202, responseHeaders);
+        });
+}
+
+
+module.exports = objectRestore;

lib/api/apiUtils/object/processMpuParts.js

@@ -1,241 +0,0 @@
-const { errors } = require('arsenal');
-const crypto = require('crypto');
-const constants = require('../../../../constants');
-
-/**
- * createAggregateETag - creates ETag from concatenated MPU part ETags to
- * mimic AWS
- * @param {string} concatETags - string of concatenated MPU part ETags
- * @param {array} partList - list of parts to complete MPU with
- * @return {string} aggregateETag - final complete MPU obj ETag
- */
-function createAggregateETag(concatETags, partList) {
-    // AWS documentation is unclear on what the MD5 is that it returns
-    // in the response for a complete multipart upload request.
-    // The docs state that they might or might not
-    // return the MD5 of the complete object. It appears
-    // they are returning the MD5 of the parts' MD5s so that is
-    // what we have done here. We:
-    // 1) concatenate the hex version of the
-    // individual ETags
-    // 2) convert the concatenated hex to binary
-    // 3) take the md5 of the binary
-    // 4) create the hex digest of the md5
-    // 5) add '-' plus the number of parts at the end
-
-    // Convert the concatenated hex ETags to binary
-    const bufferedHex = Buffer.from(concatETags, 'hex');
-    // Convert the buffer to a binary string
-    const binaryString = bufferedHex.toString('binary');
-    // Get the md5 of the binary string
-    const md5Hash = crypto.createHash('md5');
-    md5Hash.update(binaryString, 'binary');
-    // Get the hex digest of the md5
-    let aggregateETag = md5Hash.digest('hex');
-    // Add the number of parts at the end
-    aggregateETag = `${aggregateETag}-${partList.length}`;
-
-    return aggregateETag;
-}
-
-/**
- * generateMpuPartStorageInfo - generates info needed for storage of
- * completed MPU object
- * @param {array} filteredPartList - list of parts filtered from metadata
- * @return {object} partsInfo - contains three keys: aggregateETag,
- * dataLocations, and calculatedSize
- */
-function generateMpuPartStorageInfo(filteredPartList) {
-    // Assemble array of part locations, aggregate size
-    // and build string to create aggregate ETag
-    let calculatedSize = 0;
-    const dataLocations = [];
-    let concatETags = '';
-    const partsInfo = {};
-
-    filteredPartList.forEach((storedPart, index) => {
-        const partETagWithoutQuotes =
-            storedPart.ETag.slice(1, -1);
-        const dataStoreETag = `${index + 1}:${partETagWithoutQuotes}`;
-        concatETags += partETagWithoutQuotes;
-
-        // If part was put by a regular put part rather than a
-        // copy it is always one location.  With a put part
-        // copy, could be multiple locations so loop over array
-        // of locations.
-        for (let j = 0; j < storedPart.locations.length; j++) {
-            // If the piece has parts (was a put part object
-            // copy) each piece will have a size attribute.
-            // Otherwise, the piece was put by a regular put
-            // part and the size the of the piece is the full
-            // part size.
-            const location = storedPart.locations[j];
-            // If there is no location, move on
-            if (!location || typeof location !== 'object') {
-                continue;
-            }
-            let pieceSize = Number.parseInt(storedPart.size, 10);
-            if (location.size) {
-                pieceSize = Number.parseInt(location.size, 10);
-            }
-            const pieceRetrievalInfo = {
-                key: location.key,
-                size: pieceSize,
-                start: calculatedSize,
-                dataStoreName: location.dataStoreName,
-                dataStoreETag,
-                cryptoScheme: location.sseCryptoScheme,
-                cipheredDataKey: location.sseCipheredDataKey,
-            };
-            dataLocations.push(pieceRetrievalInfo);
-            // eslint-disable-next-line no-param-reassign
-            calculatedSize += pieceSize;
-        }
-    });
-
-    partsInfo.aggregateETag =
-        createAggregateETag(concatETags, filteredPartList);
-    partsInfo.dataLocations = dataLocations;
-    partsInfo.calculatedSize = calculatedSize;
-    return partsInfo;
-}
-
-/**
- * validateAndFilterMpuParts - validates part list sent by user and filters
- * parts stored in metadata against user part list
- * @param {array} storedParts - array of parts stored in metadata
- * @param {array} jsonList - array of parts sent by user for completion
- * @param {string} mpuOverviewKey - metadata mpu key
- * @param {string} splitter - mpu key divider
- * @param {object} log - Werelogs instance
- * @return {object} filtersPartsObj - contains 3 keys: partList, keysToDelete,
- * and extraPartLocations
- */
-function validateAndFilterMpuParts(storedParts, jsonList, mpuOverviewKey,
-splitter, log) {
-    let storedPartsCopy = [];
-    const filteredPartsObj = {};
-    filteredPartsObj.partList = [];
-
-    const keysToDelete = [];
-    storedParts.forEach(item => {
-        keysToDelete.push(item.key);
-        storedPartsCopy.push({
-            // In order to delete the part listing in the shadow
-            // bucket, need the full key
-            key: item.key,
-            ETag: `"${item.value.ETag}"`,
-            size: item.value.Size,
-            locations: Array.isArray(item.value.partLocations) ?
-                item.value.partLocations : [item.value.partLocations],
-        });
-    });
-    keysToDelete.push(mpuOverviewKey);
-
-    // Check list sent to make sure valid
-    const partLength = jsonList.Part.length;
-    // A user can put more parts than they end up including
-    // in the completed MPU but there cannot be more
-    // parts in the complete message than were already put
-    if (partLength > storedPartsCopy.length) {
-        filteredPartsObj.error = errors.InvalidPart;
-        return filteredPartsObj;
-    }
-
-    let extraParts = [];
-    const extraPartLocations = [];
-
-    for (let i = 0; i < partLength; i++) {
-        const part = jsonList.Part[i];
-        const partNumber = Number.parseInt(part.PartNumber[0], 10);
-        // If the complete list of parts sent with
-        // the complete multipart upload request is not
-        // in ascending order return an error
-        if (i > 0) {
-            const previousPartNumber =
-                Number.parseInt(jsonList.Part[i - 1].PartNumber[0], 10);
-            if (partNumber <= previousPartNumber) {
-                filteredPartsObj.error = errors.InvalidPartOrder;
-                return filteredPartsObj;
-            }
-        }
-
-        let isPartUploaded = false;
-        while (storedPartsCopy.length > 0 && !isPartUploaded) {
-            const storedPart = storedPartsCopy[0];
-            const storedPartNumber =
-                Number.parseInt(storedPart.key.split(splitter)[1], 10);
-
-            if (storedPartNumber === partNumber) {
-                isPartUploaded = true;
-                filteredPartsObj.partList.push(storedPart);
-
-                let partETag = part.ETag[0].replace(/['"]/g, '');
-                // some clients send base64, convert to hex
-                // 32 chars = 16 bytes(2 chars-per-byte) = 128 bits of
-                // MD5 hex
-                if (partETag.length !== 32) {
-                    const buffered = Buffer.from(part.ETag[0], 'base64')
-                        .toString('hex');
-                    partETag = `${buffered}`;
-                }
-                partETag = `"${partETag}"`;
-                // If list of parts sent with complete mpu request contains
-                // a part ETag that does not match the ETag for the part
-                // stored in metadata, return an error
-                if (partETag !== storedPart.ETag) {
-                    filteredPartsObj.error = errors.InvalidPart;
-                    return filteredPartsObj;
-                }
-
-                // If any part other than the last part is less than
-                // 5MB, return an error
-                const storedPartSize =
-                    Number.parseInt(storedPart.size, 10);
-                // allow smaller parts for testing
-                if (process.env.MPU_TESTING) {
-                    log.info('MPU_TESTING env variable setting',
-                        { setting: process.env.MPU_TESTING });
-                }
-                if (process.env.MPU_TESTING !== 'yes' &&
-                i < jsonList.Part.length - 1 &&
-                storedPartSize < constants.minimumAllowedPartSize) {
-                    log.debug('part too small on complete mpu');
-                    filteredPartsObj.error = errors.EntityTooSmall;
-                    return filteredPartsObj;
-                }
-
-                storedPartsCopy = storedPartsCopy.splice(1);
-            } else {
-                extraParts.push(storedPart);
-                storedPartsCopy = storedPartsCopy.splice(1);
-            }
-        }
-        if (!isPartUploaded) {
-            filteredPartsObj.error = errors.InvalidPart;
-            return filteredPartsObj;
-        }
-    }
-    extraParts = extraParts.concat(storedPartsCopy);
-    // if extra parts, need to delete the data when done with completing
-    // mpu so extract the info to delete here
-    if (extraParts.length > 0) {
-        extraParts.forEach(part => {
-            const locations = part.locations;
-            locations.forEach(location => {
-                if (!location || typeof location !== 'object') {
-                    return;
-                }
-                extraPartLocations.push(location);
-            });
-        });
-    }
-    filteredPartsObj.keysToDelete = keysToDelete;
-    filteredPartsObj.extraPartLocations = extraPartLocations;
-    return filteredPartsObj;
-}
-
-module.exports = {
-    generateMpuPartStorageInfo,
-    validateAndFilterMpuParts,
-};

lib/api/apiUtils/object/versioning.js

@@ -4,13 +4,40 @@ const async = require('async');
 const metadata = require('../../../metadata/wrapper');
 const { config } = require('../../../Config');
 
+const { scaledMsPerDay } = config.getTimeOptions();
+
 const versionIdUtils = versioning.VersionID;
 // Use Arsenal function to generate a version ID used internally by metadata
 // for null versions that are created before bucket versioning is configured
 const nonVersionedObjId =
     versionIdUtils.getInfVid(config.replicationGroupId);
 
-/** decodedVidResult - decode the version id from a query object
+/** decodeVID - decode the version id
+ * @param {string} versionId - version ID
+ * @return {(Error|string|undefined)} - return Invalid Argument if decryption
+ * fails due to improper format, otherwise undefined or the decoded version id
+ */
+function decodeVID(versionId) {
+    if (versionId === 'null') {
+        return versionId;
+    }
+
+    let decoded;
+    const invalidErr = errors.InvalidArgument.customizeDescription('Invalid version id specified');
+    try {
+        decoded = versionIdUtils.decode(versionId);
+    } catch (err) {
+        return invalidErr;
+    }
+
+    if (decoded instanceof Error) {
+        return invalidErr;
+    }
+
+    return decoded;
+}
+
+/** decodeVersionId - decode the version id from a query object
  * @param {object} [reqQuery] - request query object
  * @param {string} [reqQuery.versionId] - version ID sent in request query
  * @return {(Error|string|undefined)} - return Invalid Argument if decryption
@@ -20,16 +47,7 @@ function decodeVersionId(reqQuery) {
     if (!reqQuery || !reqQuery.versionId) {
         return undefined;
     }
-    let versionId = reqQuery.versionId;
-    if (versionId === 'null') {
-        return versionId;
-    }
-    versionId = versionIdUtils.decode(versionId);
-    if (versionId instanceof Error) {
-        return errors.InvalidArgument
-            .customizeDescription('Invalid version id specified');
-    }
-    return versionId;
+    return decodeVID(reqQuery.versionId);
 }
 
 /** getVersionIdResHeader - return encrypted version ID if appropriate
@@ -43,8 +61,7 @@ function getVersionIdResHeader(verCfg, objectMD) {
         if (objectMD.isNull || !objectMD.versionId) {
             return 'null';
         }
-        return versionIdUtils.encode(objectMD.versionId,
-                                     config.versionIdEncodingType);
+        return versionIdUtils.encode(objectMD.versionId);
     }
     return undefined;
 }
@@ -443,6 +460,93 @@ function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersio
     return options;
 }
 
+/**
+ * Keep metadatas when the object is restored from cold storage
+ * but remove the specific ones we don't want to keep
+ * @param {object} objMD - obj metadata
+ * @param {object} metadataStoreParams - custom built object containing resource details.
+ * @return {undefined}
+ */
+function restoreMetadata(objMD, metadataStoreParams) {
+    /* eslint-disable no-param-reassign */
+    const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt'];
+    // We need to keep user metadata and tags
+    Object.keys(objMD).forEach(key => {
+        if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) {
+            metadataStoreParams.metaHeaders[key] = objMD[key];
+        }
+    });
+
+    if (objMD['x-amz-website-redirect-location']) {
+        if (!metadataStoreParams.headers) {
+            metadataStoreParams.headers = {};
+        }
+        metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location'];
+    }
+
+    if (objMD.replicationInfo) {
+        metadataStoreParams.replicationInfo = objMD.replicationInfo;
+    }
+
+    if (objMD.legalHold) {
+        metadataStoreParams.legalHold = objMD.legalHold;
+    }
+
+    if (objMD.acl) {
+        metadataStoreParams.acl = objMD.acl;
+    }
+
+    metadataStoreParams.creationTime = objMD['creation-time'];
+    metadataStoreParams.lastModifiedDate = objMD['last-modified'];
+    metadataStoreParams.taggingCopy = objMD.tags;
+}
+
+/** overwritingVersioning - return versioning information for S3 to handle
+ * storing version metadata with a specific version id.
+ * @param {object} objMD - obj metadata
+ * @param {object} metadataStoreParams - custom built object containing resource details.
+ * @return {object} options
+ * options.versionId - specific versionId to overwrite in metadata
+ * options.isNull - (true/undefined) whether new version is null or not
+ * options.nullVersionId - if storing a null version in version history, the
+ *  version id of the null version
+ */
+function overwritingVersioning(objMD, metadataStoreParams) {
+    metadataStoreParams.updateMicroVersionId = true;
+    metadataStoreParams.amzStorageClass = objMD['x-amz-storage-class'];
+
+    // set correct originOp
+    metadataStoreParams.originOp = 's3:ObjectRestore:Completed';
+
+    // update restore
+    const days = objMD.archive?.restoreRequestedDays;
+    const now = Date.now();
+    metadataStoreParams.archive = {
+        archiveInfo: objMD.archive?.archiveInfo,
+        restoreRequestedAt: objMD.archive?.restoreRequestedAt,
+        restoreRequestedDays: objMD.archive?.restoreRequestedDays,
+        restoreCompletedAt: new Date(now),
+        restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
+    };
+
+    /* eslint-enable no-param-reassign */
+
+    const versionId = objMD.versionId || undefined;
+    const options = {
+        versionId,
+        isNull: objMD.isNull,
+    };
+    if (objMD.nullVersionId) {
+        options.extraMD = {
+            nullVersionId: objMD.nullVersionId,
+        };
+    }
+
+    restoreMetadata(objMD, metadataStoreParams);
+
+    return options;
+}
+
 module.exports = {
     decodeVersionId,
     getVersionIdResHeader,
@@ -452,4 +556,6 @@ module.exports = {
     versioningPreprocessing,
     getVersionSpecificMetadataOptions,
     preprocessingVersioningDelete,
+    overwritingVersioning,
+    decodeVID,
 };

lib/api/apiUtils/quotas/quotaUtils.js

@@ -0,0 +1,314 @@
+const async = require('async');
+const { errors } = require('arsenal');
+const monitoring = require('../../../utilities/monitoringHandler');
+const {
+    actionNeedQuotaCheckCopy,
+    actionNeedQuotaCheck,
+    actionWithDataDeletion,
+} = require('arsenal').policies;
+const { config } = require('../../../Config');
+const QuotaService = require('../../../quotas/quotas');
+
+/**
+ * Process the bytes to write based on the request and object metadata
+ * @param {string} apiMethod - api method
+ * @param {BucketInfo} bucket - bucket info
+ * @param {string} versionId - version id of the object
+ * @param {number} contentLength - content length of the object
+ * @param {object} objMD - object metadata
+ * @param {object} destObjMD - destination object metadata
+ * @return {number} processed content length
+ */
+function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
+    let bytes = contentLength;
+    if (apiMethod === 'objectRestore') {
+        // object is being restored
+        bytes = Number.parseInt(objMD['content-length'], 10);
+    } else if (!bytes && objMD?.['content-length']) {
+        if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
+            if (!destObjMD || bucket.isVersioningEnabled()) {
+                // object is being copied
+                bytes = Number.parseInt(objMD['content-length'], 10);
+            } else if (!bucket.isVersioningEnabled()) {
+                // object is being copied and replaces the target
+                bytes = Number.parseInt(objMD['content-length'], 10) -
+                    Number.parseInt(destObjMD['content-length'], 10);
+            }
+        } else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
+            // object is being deleted
+            bytes = -Number.parseInt(objMD['content-length'], 10);
+        }
+    } else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
+        // object is being replaced: store the diff, if the bucket is not versioned
+        bytes = bytes - Number.parseInt(objMD['content-length'], 10);
+    }
+    return bytes || 0;
+}
+
+/**
+ * Checks if a metric is stale based on the provided parameters.
+ *
+ * @param {Object} metric - The metric object to check.
+ * @param {string} resourceType - The type of the resource.
+ * @param {string} resourceName - The name of the resource.
+ * @param {string} action - The action being performed.
+ * @param {number} inflight - The number of inflight requests.
+ * @param {Object} log - The logger object.
+ * @returns {boolean} Returns true if the metric is stale, false otherwise.
+ */
+function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
+    if (metric.date && Date.now() - new Date(metric.date).getTime() >
+        QuotaService.maxStaleness) {
+        log.warn('Stale metrics from the quota service, allowing the request', {
+            resourceType,
+            resourceName,
+            action,
+            inflight,
+        });
+        monitoring.requestWithQuotaMetricsUnavailable.inc();
+        return true;
+    }
+    return false;
+}
+
+/**
+ * Evaluates quotas for a bucket and an account and update inflight count.
+ *
+ * @param {number} bucketQuota - The quota limit for the bucket.
+ * @param {number} accountQuota - The quota limit for the account.
+ * @param {object} bucket - The bucket object.
+ * @param {object} account - The account object.
+ * @param {number} inflight - The number of inflight requests.
+ * @param {number} inflightForCheck - The number of inflight requests for checking quotas.
+ * @param {string} action - The action being performed.
+ * @param {object} log - The logger object.
+ * @param {function} callback - The callback function to be called when evaluation is complete.
+ * @returns {object} - The result of the evaluation.
+ */
+function _evaluateQuotas(
+    bucketQuota,
+    accountQuota,
+    bucket,
+    account,
+    inflight,
+    inflightForCheck,
+    action,
+    log,
+    callback,
+) {
+    let bucketQuotaExceeded = false;
+    let accountQuotaExceeded = false;
+    const creationDate = new Date(bucket.getCreationDate()).getTime();
+    return async.parallel({
+        bucketQuota: parallelDone => {
+            if (bucketQuota > 0) {
+                return QuotaService.getUtilizationMetrics('bucket',
+                    `${bucket.getName()}_${creationDate}`, null, {
+                    action,
+                    inflight,
+                }, (err, bucketMetrics) => {
+                    if (err || inflight < 0) {
+                        return parallelDone(err);
+                    }
+                    if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
+                        bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
+                        log.debug('Bucket quota exceeded', {
+                            bucket: bucket.getName(),
+                            action,
+                            inflight,
+                            quota: bucketQuota,
+                            bytesTotal: bucketMetrics.bytesTotal,
+                        });
+                        bucketQuotaExceeded = true;
+                    }
+                    return parallelDone();
+                });
+            }
+            return parallelDone();
+        },
+        accountQuota: parallelDone => {
+            if (accountQuota > 0 && account?.account) {
+                return QuotaService.getUtilizationMetrics('account',
+                    account.account, null, {
+                    action,
+                    inflight,
+                }, (err, accountMetrics) => {
+                    if (err || inflight < 0) {
+                        return parallelDone(err);
+                    }
+                    if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
+                        accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
+                        log.debug('Account quota exceeded', {
+                            accountId: account.account,
+                            action,
+                            inflight,
+                            quota: accountQuota,
+                            bytesTotal: accountMetrics.bytesTotal,
+                        });
+                        accountQuotaExceeded = true;
+                    }
+                    return parallelDone();
+                });
+            }
+            return parallelDone();
+        },
+    }, err => {
+        if (err) {
+            log.warn('Error evaluating quotas', {
+                error: err.name,
+                description: err.message,
+                isInflightDeletion: inflight < 0,
+            });
+        }
+        return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
+    });
+}
+
+/**
+ * Monitors the duration of quota evaluation for a specific API method.
+ *
+ * @param {string} apiMethod - The name of the API method being monitored.
+ * @param {string} type - The type of quota being evaluated.
+ * @param {string} code - The code associated with the quota being evaluated.
+ * @param {number} duration - The duration of the quota evaluation in nanoseconds.
+ * @returns {undefined} - Returns nothing.
+ */
+function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
+    monitoring.quotaEvaluationDuration.labels({
+        action: apiMethod,
+        type,
+        code,
+    }).observe(duration / 1e9);
+}
+
+/**
+ *
+ * @param {Request} request - request object
+ * @param {BucketInfo} bucket - bucket object
+ * @param {Account} account - account object
+ * @param {array} apiNames - action names: operations to authorize
+ * @param {string} apiMethod - the main API call
+ * @param {number} inflight - inflight bytes
+ * @param {boolean} isStorageReserved - Flag to check if the current quota, minus
+ * the incoming bytes, are under the limit.
+ * @param {Logger} log - logger
+ * @param {function} callback - callback function
+ * @returns {boolean} - true if the quota is valid, false otherwise
+ */
+function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
+    if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
+        return callback(null);
+    }
+    let type;
+    let bucketQuotaExceeded = false;
+    let accountQuotaExceeded = false;
+    let quotaEvaluationDuration;
+    const requestStartTime = process.hrtime.bigint();
+    const bucketQuota = bucket.getQuota();
+    const accountQuota = account?.quota || 0;
+    const shouldSendInflights = config.isQuotaInflightEnabled();
+
+    if (bucketQuota && accountQuota) {
+        type = 'bucket+account';
+    } else if (bucketQuota) {
+        type = 'bucket';
+    } else {
+        type = 'account';
+    }
+
+    if (actionWithDataDeletion[apiMethod]) {
+        type = 'delete';
+    }
+
+    if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
+        if (bucketQuota > 0 || accountQuota > 0) {
+            log.warn('quota is set for a bucket, but the quota service is disabled', {
+                bucketName: bucket.getName(),
+            });
+            monitoring.requestWithQuotaMetricsUnavailable.inc();
+        }
+        return callback(null);
+    }
+
+    if (isStorageReserved) {
+        // eslint-disable-next-line no-param-reassign
+        inflight = 0;
+    }
+
+    return async.forEach(apiNames, (apiName, done) => {
+        // Object copy operations first check the target object,
+        // meaning the source object, containing the current bytes,
+        // is checked second. This logic handles these APIs calls by
+        // ensuring the bytes are positives (i.e., not an object
+        // replacement).
+        if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
+            // eslint-disable-next-line no-param-reassign
+            inflight = Math.abs(inflight);
+        } else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
+            return done();
+        }
+        // When inflights are disabled, the sum of the current utilization metrics
+        // and the current bytes are compared with the quota. The current bytes
+        // are not sent to the utilization service. When inflights are enabled,
+        // the sum of the current utilization metrics only are compared with the
+        // quota. They include the current inflight bytes sent in the request.
+        let _inflights = shouldSendInflights ? inflight : undefined;
+        const inflightForCheck = shouldSendInflights ? 0 : inflight;
+        return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
+            inflightForCheck, apiName, log,
+            (err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
+                if (err) {
+                    return done(err);
+                }
+
+                bucketQuotaExceeded = _bucketQuotaExceeded;
+                accountQuotaExceeded = _accountQuotaExceeded;
+
+                // Inflights are inverted: in case of cleanup, we just re-issue
+                // the same API call.
+                if (_inflights) {
+                    _inflights = -_inflights;
+                }
+
+                request.finalizerHooks.push((errorFromAPI, _done) => {
+                    const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
+                    const quotaCleanUpStartTime = process.hrtime.bigint();
+                    // Quotas are cleaned only in case of error in the API
+                    async.waterfall([
+                        cb => {
+                            if (errorFromAPI) {
+                                return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
+                                    null, apiName, log, cb);
+                            }
+                            return cb();
+                        },
+                    ], () => {
+                        monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
+                            Number(process.hrtime.bigint() - quotaCleanUpStartTime));
+                        return _done();
+                    });
+                });
+
+                return done();
+            });
+    }, err => {
+        quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
+        if (err) {
+            log.warn('Error getting metrics from the quota service, allowing the request', {
+                error: err.name,
+                description: err.message,
+            });
+        }
+        if (!actionWithDataDeletion[apiMethod] &&
+            (bucketQuotaExceeded || accountQuotaExceeded)) {
+            return callback(errors.QuotaExceeded);
+        }
+        return callback();
+    });
+}
+
+module.exports = {
+    processBytesToWrite,
+    isMetricStale,
+    validateQuotas,
+};

lib/api/backbeat/listLifecycleCurrents.js

@@ -3,7 +3,7 @@ const constants = require('../../../constants');
 const services = require('../../services');
 const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
 const { pushMetric } = require('../../utapi/utilities');
-const monitoring = require('../../utilities/metrics');
+const monitoring = require('../../utilities/monitoringHandler');
 const { getLocationConstraintErrorMessage, processCurrents,
     validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
 const { config } = require('../../Config');

lib/api/backbeat/listLifecycleNonCurrents.js

@@ -4,7 +4,7 @@ const services = require('../../services');
 const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
 const { pushMetric } = require('../../utapi/utilities');
 const versionIdUtils = versioning.VersionID;
-const monitoring = require('../../utilities/metrics');
+const monitoring = require('../../utilities/monitoringHandler');
 const { getLocationConstraintErrorMessage, processNonCurrents,
     validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
 const { config } = require('../../Config');

lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js

@@ -3,7 +3,7 @@ const constants = require('../../../constants');
 const services = require('../../services');
 const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
 const { pushMetric } = require('../../utapi/utilities');
-const monitoring = require('../../utilities/metrics');
+const monitoring = require('../../utilities/monitoringHandler');
 const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
 const { config } = require('../../Config');
 

lib/api/bucketDelete.js

@@ -4,7 +4,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const deleteBucket = require('./apiUtils/bucket/bucketDeletion');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
-const monitoring = require('../utilities/metrics');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketDelete - DELETE bucket (currently supports only non-versioned buckets)

lib/api/bucketDeleteCors.js

@@ -6,7 +6,7 @@ const { isBucketAuthorized } =
     require('./apiUtils/authorization/permissionChecks');
 const metadata = require('../metadata/wrapper');
 const { pushMetric } = require('../utapi/utilities');
-const monitoring = require('../utilities/metrics');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketDeleteCors';
 

lib/api/bucketDeleteLifecycle.js

@@ -2,7 +2,7 @@ const metadata = require('../metadata/wrapper');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
-const monitoring = require('../utilities/metrics');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketDeleteLifecycle - Delete the bucket Lifecycle configuration

lib/api/bucketDeleteQuota.js

@@ -0,0 +1,58 @@
+const { waterfall } = require('async');
+const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
+const metadata = require('../metadata/wrapper');
+const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
+
+const requestType = 'bucketDeleteQuota';
+
+/**
+ * Bucket Update Quota - Update bucket quota
+ * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
+ * @param {object} request - http request object
+ * @param {object} log - Werelogs logger
+ * @param {function} callback - callback to server
+ * @return {undefined}
+ */
+function bucketDeleteQuota(authInfo, request, log, callback) {
+    log.debug('processing request', { method: 'bucketDeleteQuota' });
+
+    const { bucketName } = request;
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: request.apiMethods || requestType,
+        request,
+    };
+    return waterfall([
+        next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
+            (err, bucket) => next(err, bucket)),
+        (bucket, next) => {
+            bucket.setQuota(0);
+            metadata.updateBucket(bucket.getName(), bucket, log, err =>
+                next(err, bucket));
+        },
+    ], (err, bucket) => {
+        const corsHeaders = collectCorsHeaders(request.headers.origin,
+            request.method, bucket);
+        if (err) {
+            log.debug('error processing request', {
+                error: err,
+                method: 'bucketDeleteQuota'
+            });
+            monitoring.promMetrics('DELETE', bucketName, err.code,
+                'bucketDeleteQuota');
+            return callback(err, err.code, corsHeaders);
+        }
+        monitoring.promMetrics(
+            'DELETE', bucketName, '204', 'bucketDeleteQuota');
+        pushMetric('bucketDeleteQuota', log, {
+            authInfo,
+            bucket: bucketName,
+        });
+        return callback(null, 204, corsHeaders);
+    });
+}
+
+module.exports = bucketDeleteQuota;

lib/api/bucketDeleteReplication.js

@@ -2,7 +2,7 @@ const metadata = require('../metadata/wrapper');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
-const monitoring = require('../utilities/metrics');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketDeleteReplication - Delete the bucket replication configuration

lib/api/bucketDeleteTagging.js

@@ -1,9 +1,9 @@
+const { waterfall } = require('async');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 const metadata = require('../metadata/wrapper');
-const util = require('node:util');
-const monitoring = require('../utilities/metrics');
 
 /**
  * Bucket Delete Tagging - Delete a bucket's Tagging
@@ -13,16 +13,10 @@ const monitoring = require('../utilities/metrics');
  * @param {function} callback - callback to server
  * @return {undefined}
  */
-async function bucketDeleteTagging(authInfo, request, log, callback) {
+function bucketDeleteTagging(authInfo, request, log, callback) {
     const bucketName = request.bucketName;
-    let error = null;
     log.debug('processing request', { method: 'bucketDeleteTagging', bucketName });
 
-    let bucket;
-    const metadataValidateBucketPromise = util.promisify(standardMetadataValidateBucket);
-    let updateBucketPromise = util.promisify(metadata.updateBucket);
-    // necessary to bind metadata as updateBucket calls 'this', causing undefined otherwise
-    updateBucketPromise = updateBucketPromise.bind(metadata);
     const metadataValParams = {
         authInfo,
         bucketName,
@@ -30,27 +24,39 @@ async function bucketDeleteTagging(authInfo, request, log, callback) {
         request,
     };
 
-    try {
-        bucket = await metadataValidateBucketPromise(metadataValParams, request.actionImplicitDenies, log);
+    let bucket = null;
+    return waterfall([
+        next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
+            (err, b) => {
+                if (err) {
+                    return next(err);
+                }
+                bucket = b;
                 bucket.setTags([]);
-        // eslint-disable-next-line no-unused-expressions
-        await updateBucketPromise(bucket.getName(), bucket, log);
+                return next();
+            }),
+        next => metadata.updateBucket(bucket.getName(), bucket, log, next),
+    ], err => {
+        const corsHeaders = collectCorsHeaders(request.headers.origin,
+            request.method, bucket);
+        if (err) {
+            log.error('error processing request', {
+                error: err,
+                method: 'deleteBucketTagging',
+                bucketName
+            });
+            monitoring.promMetrics('DELETE', bucketName, err.code,
+                'deleteBucketTagging');
+            return callback(err, corsHeaders);
+        }
         pushMetric('deleteBucketTagging', log, {
             authInfo,
             bucket: bucketName,
         });
         monitoring.promMetrics(
             'DELETE', bucketName, '200', 'deleteBucketTagging');
-    } catch (err) {
-        error = err;
-        log.error('error processing request', { error: err,
-            method: 'deleteBucketTagging', bucketName });
-        monitoring.promMetrics('DELETE', bucketName, err.code,
-            'deleteBucketTagging');
-    }
-    const corsHeaders = collectCorsHeaders(request.headers.origin,
-        request.method, bucket);
-    return callback(error, corsHeaders);
+        return callback(err, corsHeaders);
+    });
 }
 
 module.exports = bucketDeleteTagging;

lib/api/bucketDeleteWebsite.js

@@ -6,7 +6,7 @@ const { isBucketAuthorized } =
     require('./apiUtils/authorization/permissionChecks');
 const metadata = require('../metadata/wrapper');
 const { pushMetric } = require('../utapi/utilities');
-const monitoring = require('../utilities/metrics');
+const monitoring = require('../utilities/monitoringHandler');
 
 const requestType = 'bucketDeleteWebsite';
 

lib/api/bucketGet.js

@@ -1,18 +1,15 @@
 const querystring = require('querystring');
 const { errors, versioning, s3middleware } = require('arsenal');
-
 const constants = require('../../constants');
 const services = require('../services');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const escapeForXml = s3middleware.escapeForXml;
 const { pushMetric } = require('../utapi/utilities');
-const monitoring = require('../utilities/metrics');
-
 const versionIdUtils = versioning.VersionID;
+const monitoring = require('../utilities/monitoringHandler');
 const { generateToken, decryptToken }
     = require('../api/apiUtils/object/continueToken');
-const { config } = require('../Config');
 
 // do not url encode the continuation tokens
 const skipUrlEncoding = new Set([
@@ -37,8 +34,7 @@ const skipUrlEncoding = new Set([
 </ListBucketResult>
 */
 
-//  Sample XML response for GET bucket objects:
-/*
+/*	Sample XML response for GET bucket objects:
 <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
   <Name>example-bucket</Name>
   <Prefix></Prefix>
@@ -68,7 +64,6 @@ const skipUrlEncoding = new Set([
 // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETVersion.html#RESTBucketGET_Examples
 /*
 <?xml version="1.0" encoding="UTF-8"?>
-
 <ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
     <Name>bucket</Name>
     <Prefix>my</Prefix>
@@ -109,7 +104,7 @@ const skipUrlEncoding = new Set([
 */
 /* eslint-enable max-len */
 
-function processVersions(bucketName, listParams, list, encType) {
+function processVersions(bucketName, listParams, list) {
     const xml = [];
     xml.push(
         '<?xml version="1.0" encoding="UTF-8"?>',
@@ -134,7 +129,7 @@ function processVersions(bucketName, listParams, list, encType) {
     xmlParams.forEach(p => {
         if (p.value) {
             const val = p.tag !== 'NextVersionIdMarker' || p.value === 'null' ?
-                p.value : versionIdUtils.encode(p.value, encType);
+                p.value : versionIdUtils.encode(p.value);
             xml.push(`<${p.tag}>${escapeXmlFn(val)}</${p.tag}>`);
         }
     });
@@ -150,7 +145,7 @@ function processVersions(bucketName, listParams, list, encType) {
             `<Key>${objectKey}</Key>`,
             '<VersionId>',
             (v.IsNull || v.VersionId === undefined) ?
-                'null' : versionIdUtils.encode(v.VersionId, encType),
+                'null' : versionIdUtils.encode(v.VersionId),
             '</VersionId>',
             `<IsLatest>${isLatest}</IsLatest>`,
             `<LastModified>${v.LastModified}</LastModified>`,
@@ -268,8 +263,7 @@ function handleResult(listParams, requestMaxKeys, encoding, authInfo,
     listParams.encoding = encoding;
     let res;
     if (listParams.listingType === 'DelimiterVersions') {
-        res = processVersions(bucketName, listParams, list,
-                              config.versionIdEncodingType);
+        res = processVersions(bucketName, listParams, list);
     } else {
         res = processMasterVersions(bucketName, listParams, list);
     }
@@ -392,4 +386,8 @@ function bucketGet(authInfo, request, log, callback) {
     return undefined;
 }
 
-module.exports = bucketGet;
+module.exports = {
+    processVersions,
+    processMasterVersions,
+    bucketGet,
+};

lib/api/bucketGetACL.js

@@ -3,7 +3,7 @@ const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const vault = require('../auth/vault');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const { pushMetric } = require('../utapi/utilities');
-const monitoring = require('../utilities/metrics');
+const monitoring = require('../utilities/monitoringHandler');
 
 //  Sample XML response:
 /*