Use fs.readFileSync to read config file instead of require

FIXME: Ideally, both locations and storage classes should be supported

.dockerignore

@@ -1,3 +1,9 @@
-.git
+node_modules
+localData/*
+localMetadata/*
+# Keep the .git/HEAD file in order to properly report version
+.git/objects
 .github
-node_modules
+.tox
+coverage
+.DS_Store

.eslintrc

@@ -1 +1,54 @@
-{ "extends": "scality" }
+{
+    "extends": "scality",
+    "plugins": [
+        "mocha"
+    ],
+    "rules": {
+        "import/extensions": "off",
+        "lines-around-directive": "off",
+        "no-underscore-dangle": "off",
+        "indent": "off",
+        "object-curly-newline": "off",
+        "operator-linebreak": "off",
+        "function-paren-newline": "off",
+        "import/newline-after-import": "off",
+        "prefer-destructuring": "off",
+        "implicit-arrow-linebreak": "off",
+        "no-bitwise": "off",
+        "dot-location": "off",
+        "comma-dangle": "off",
+        "no-undef-init": "off",
+        "global-require": "off",
+        "import/no-dynamic-require": "off",
+        "class-methods-use-this": "off",
+        "no-plusplus": "off",
+        "no-else-return": "off",
+        "object-property-newline": "off",
+        "import/order": "off",
+        "no-continue": "off",
+        "no-tabs": "off",
+        "lines-between-class-members": "off",
+        "prefer-spread": "off",
+        "no-lonely-if": "off",
+        "no-useless-escape": "off",
+        "no-restricted-globals": "off",
+        "no-buffer-constructor": "off",
+        "import/no-extraneous-dependencies": "off",
+        "space-unary-ops": "off",
+        "no-useless-return": "off",
+        "no-unexpected-multiline": "off",
+        "no-mixed-operators": "off",
+        "newline-per-chained-call": "off",
+        "operator-assignment": "off",
+        "spaced-comment": "off",
+        "comma-style": "off",
+        "no-restricted-properties": "off",
+        "new-parens": "off",
+        "no-multi-spaces": "off",
+        "quote-props": "off",
+        "mocha/no-exclusive-tests": "error",
+    },
+    "parserOptions": {
+        "ecmaVersion": 2020
+    }
+}

.github/ISSUE_TEMPLATE.md

@@ -1,19 +1,32 @@
-# Issue template
+# General support information
 
-If you are reporting a new issue, make sure that we do not have any
-duplicates already open. You can ensure this by searching the issue list for
-this repository. If there is a duplicate, please close your issue and add a
-comment to the existing issue instead.
+GitHub Issues are **reserved** for actionable bug reports (including
+documentation inaccuracies), and feature requests.
+**All questions** (regarding configuration, use cases, performance, community,
+events, setup and usage recommendations, among other things) should be asked on
+the **[Zenko Forum](http://forum.zenko.io/)**.
 
-## General support information
+> Questions opened as GitHub issues will systematically be closed, and moved to
+> the [Zenko Forum](http://forum.zenko.io/).
 
-GitHub Issues are reserved for actionable bug reports and feature requests.
-General questions should be sent to the
-[S3 scality server Forum](http://forum.scality.com/).
+--------------------------------------------------------------------------------
+
+## Avoiding duplicates
+
+When reporting a new issue/requesting a feature, make sure that we do not have
+any duplicates already open:
+
+- search the issue list for this repository (use the search bar, select
+  "Issues" on the left pane after searching);
+- if there is a duplicate, please do not open your issue, and add a comment
+  to the existing issue instead.
+
+--------------------------------------------------------------------------------
 
 ## Bug report information
 
-(delete this section if not applicable)
+(delete this section (everything between the lines) if you're not reporting a bug
+but requesting a feature)
 
 ### Description
 
@@ -29,13 +42,22 @@ Describe the results you received
 
 ### Expected result
 
-Describe the results you expecteds
+Describe the results you expected
 
-### Additional information: (Node.js version, Docker version, etc)
+### Additional information
+
+- Node.js version,
+- Docker version,
+- yarn version,
+- distribution/OS,
+- optional: anything else you deem helpful to us.
+
+--------------------------------------------------------------------------------
 
 ## Feature Request
 
-(delete this section if not applicable)
+(delete this section (everything between the lines) if you're not requesting
+a feature but reporting a bug)
 
 ### Proposal
 
@@ -52,3 +74,14 @@ What you would like to happen
 ### Use case
 
 Please provide use cases for changing the current behavior
+
+### Additional information
+
+- Is this request for your company? Y/N
+  - If Y: Company name:
+  - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N
+- Are you willing to contribute this feature yourself?
+- Position/Title:
+- How did you hear about us?
+
+--------------------------------------------------------------------------------

.github/actions/setup-ci/action.yaml

@@ -15,18 +15,29 @@ runs:
       shell: bash
       run: |-
         set -exu;
-        mkdir -p /tmp/artifacts/${{ github.job }}/;
-    - uses: actions/setup-node@v2
+        mkdir -p /tmp/artifacts/${JOB_NAME}/;
+    - uses: actions/setup-node@v4
       with:
         node-version: '16'
         cache: 'yarn'
     - name: install dependencies
       shell: bash
       run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip
-    - name: Install python deps
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Setup python2 test environment
       shell: bash
-      run: pip install docker-compose
+      run: |
+        sudo apt-get install -y libdigest-hmac-perl
+        pip install 's3cmd==2.3.0'
+    - name: fix sproxyd.conf permissions
+      shell: bash
+      run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
+    - name: ensure fuse kernel module is loaded (for sproxyd)
+      shell: bash
+      run: sudo modprobe fuse

.github/ceph/Dockerfile

@@ -0,0 +1,25 @@
+FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
+
+ENV CEPH_DAEMON demo
+ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
+
+ENV CEPH_DEMO_UID zenko
+ENV CEPH_DEMO_ACCESS_KEY accessKey1
+ENV CEPH_DEMO_SECRET_KEY verySecretKey1
+ENV CEPH_DEMO_BUCKET zenkobucket
+
+ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
+ENV MON_IP 0.0.0.0
+ENV NETWORK_AUTO_DETECT 4
+ENV RGW_CIVETWEB_PORT 8001
+
+RUN rm /etc/yum.repos.d/tcmu-runner.repo
+
+ADD ./entrypoint-wrapper.sh /
+RUN chmod +x /entrypoint-wrapper.sh && \
+    yum install -y python-pip && \
+    yum clean all && \
+    pip install awscli && \
+    rm -rf /root/.cache/pip
+
+ENTRYPOINT [ "/entrypoint-wrapper.sh" ]

.github/ceph/entrypoint-wrapper.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+touch /artifacts/ceph.log
+mkfifo /tmp/entrypoint_output
+# We run this in the background so that we can tail the RGW log after init,
+# because entrypoint.sh never returns
+
+# The next line will be needed when ceph builds 3.2.2 so I'll leave it here
+# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
+
+bash /entrypoint.sh > /tmp/entrypoint_output &
+entrypoint_pid="$!"
+while read -r line; do
+    echo $line
+    # When we find this line server has started
+    if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
+        break
+    fi
+done < /tmp/entrypoint_output
+
+# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
+s3cmd mb s3://cephbucket s3://cephbucket2
+
+mkdir /root/.aws
+cat > /root/.aws/credentials <<EOF
+[default]
+aws_access_key_id = accessKey1
+aws_secret_access_key = verySecretKey1
+EOF
+
+# Enable versioning on them
+for bucket in cephbucket cephbucket2; do
+    echo "Enabling versiong for $bucket"
+    aws --endpoint http://127.0.0.1:8001 s3api  put-bucket-versioning --bucket $bucket --versioning Status=Enabled
+done
+tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
+wait $entrypoint_pid

.github/ceph/wait_for_ceph.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# This script is needed because RADOS Gateway
+# will open the port before beginning to serve traffic
+# causing wait_for_local_port.bash to exit immediately
+
+echo 'Waiting for ceph'
+while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
+    sleep 1
+    echo -n "."
+done

.github/docker/creds.env

@@ -34,4 +34,3 @@ gcpbackendmismatch_GCP_SERVICE_KEY
 gcpbackend_GCP_SERVICE_KEYFILE
 gcpbackendmismatch_GCP_SERVICE_KEYFILE
 gcpbackendnoproxy_GCP_SERVICE_KEYFILE
-gcpbackendproxy_GCP_SERVICE_KEYFILE

.github/docker/docker-compose.yaml

@@ -20,6 +20,7 @@ services:
       - METADATA_HOST=0.0.0.0
       - S3BACKEND
       - S3DATA
+      - S3METADATA
       - MPU_TESTING
       - S3VAULT
       - S3_LOCATION_FILE
@@ -34,6 +35,16 @@ services:
       - S3KMIP_KEY
       - S3KMIP_CERT
       - S3KMIP_CA
+      - MONGODB_HOSTS=0.0.0.0:27018
+      - MONGODB_RS=rs0
+      - DEFAULT_BUCKET_KEY_FORMAT
+      - METADATA_MAX_CACHED_BUCKETS
+      - ENABLE_NULL_VERSION_COMPAT_MODE
+      - SCUBA_HOST
+      - SCUBA_PORT
+      - SCUBA_HEALTHCHECK_FREQUENCY
+      - S3QUOTA
+      - QUOTA_ENABLE_INFLIGHTS
     env_file:
       - creds.env
     depends_on:
@@ -61,6 +72,21 @@ services:
   pykmip:
     network_mode: "host"
     profiles: ['pykmip']
-    image: registry.scality.com/cloudserver-dev/pykmip
+    image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
     volumes:
       - /tmp/artifacts/${JOB_NAME}:/artifacts
+  mongo:
+    network_mode: "host"
+    profiles: ['mongo', 'ceph']
+    image: ${MONGODB_IMAGE}
+  ceph: 
+    network_mode: "host"
+    profiles: ['ceph']
+    image: ghcr.io/scality/cloudserver/ci-ceph 
+  sproxyd:
+    network_mode: "host"
+    profiles: ['sproxyd']
+    image: sproxyd-standalone
+    build: ./sproxyd
+    user: 0:0
+    privileged: yes

.github/docker/mongodb/Dockerfile

@@ -0,0 +1,28 @@
+FROM mongo:5.0.21
+
+ENV USER=scality \
+    HOME_DIR=/home/scality \
+    CONF_DIR=/conf \
+    DATA_DIR=/data
+
+# Set up directories and permissions
+RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
+    mkdir /logs; \
+    adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
+
+# Set up environment variables and directories for scality user
+RUN mkdir ${CONF_DIR} && \
+    chown -R ${USER} ${CONF_DIR} && \
+    chown -R ${USER} ${DATA_DIR}
+
+# copy the mongo config file
+COPY /conf/mongod.conf /conf/mongod.conf
+COPY /conf/mongo-run.sh /conf/mongo-run.sh
+COPY /conf/initReplicaSet /conf/initReplicaSet.js
+
+EXPOSE 27017/tcp
+EXPOSE 27018
+
+# Set up CMD
+ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
+CMD ["bash", "/conf/mongo-run.sh"]

.github/docker/mongodb/conf/initReplicaSet

@@ -0,0 +1,4 @@
+rs.initiate({
+    _id: "rs0",
+    members: [{ _id: 0, host: "127.0.0.1:27018" }]
+});

.github/docker/mongodb/conf/mongo-run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -exo pipefail
+
+init_RS() {
+  sleep 5
+  mongo --port 27018 /conf/initReplicaSet.js
+}
+init_RS &
+
+mongod --bind_ip_all --config=/conf/mongod.conf

.github/docker/mongodb/conf/mongod.conf

@@ -0,0 +1,15 @@
+storage:
+   journal:
+      enabled: true
+   engine: wiredTiger
+   dbPath: "/data/db"
+processManagement:
+   fork: false
+net:
+   port: 27018
+   bindIp: 0.0.0.0
+replication:
+   replSetName: "rs0"
+   enableMajorityReadConcern: true
+security:
+    authorization: disabled

.github/docker/sproxyd/Dockerfile

@@ -0,0 +1,3 @@
+FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
+ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
+RUN chown root:root /conf/sproxyd0.conf

.github/docker/sproxyd/conf/fastcgi_params

@@ -0,0 +1,26 @@
+fastcgi_param  QUERY_STRING       $query_string;
+fastcgi_param  REQUEST_METHOD     $request_method;
+fastcgi_param  CONTENT_TYPE       $content_type;
+fastcgi_param  CONTENT_LENGTH     $content_length;
+
+#fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;
+fastcgi_param  SCRIPT_NAME        /var/www;
+fastcgi_param  PATH_INFO          $document_uri;
+
+fastcgi_param  REQUEST_URI        $request_uri;
+fastcgi_param  DOCUMENT_URI       $document_uri;
+fastcgi_param  DOCUMENT_ROOT      $document_root;
+fastcgi_param  SERVER_PROTOCOL    $server_protocol;
+fastcgi_param  HTTPS              $https if_not_empty;
+
+fastcgi_param  GATEWAY_INTERFACE  CGI/1.1;
+fastcgi_param  SERVER_SOFTWARE    nginx/$nginx_version;
+
+fastcgi_param  REMOTE_ADDR        $remote_addr;
+fastcgi_param  REMOTE_PORT        $remote_port;
+fastcgi_param  SERVER_ADDR        $server_addr;
+fastcgi_param  SERVER_PORT        $server_port;
+fastcgi_param  SERVER_NAME        $server_name;
+
+# PHP only, required if PHP was built with --enable-force-cgi-redirect
+fastcgi_param  REDIRECT_STATUS    200;

.github/docker/sproxyd/conf/nginx.conf

@@ -0,0 +1,88 @@
+worker_processes 1;
+error_log  /logs/error.log;
+user root root;
+events {
+    worker_connections  1000;
+    reuse_port on;
+    multi_accept on;
+}
+worker_rlimit_nofile    20000;
+http {
+    root /var/www/;
+    upstream sproxyds {
+        least_conn;
+        keepalive 40;
+        server 127.0.0.1:20000;
+    }
+    server {
+        client_max_body_size 0;
+        client_body_timeout 150;
+        client_header_timeout 150;
+        postpone_output 0;
+        client_body_postpone_size 0;
+        keepalive_requests 1100;
+        keepalive_timeout 300s;
+        server_tokens off;
+        default_type application/octet-stream;
+        gzip off;
+        tcp_nodelay on;
+        tcp_nopush  on;
+        sendfile on;
+        listen 81;
+        server_name localhost;
+        rewrite ^/arc/(.*)$ /dc1/$1 permanent;
+        location ~* ^/proxy/(.*)$ {
+            rewrite ^/proxy/(.*)$ /$1 last;
+        }
+        allow 127.0.0.1;
+
+        deny all;
+        set $usermd '-';
+        set $sentusermd '-';
+    	set $elapsed_ms '-';
+    	set $now '-';
+    	log_by_lua '
+    		if not(ngx.var.http_x_scal_usermd == nil) and  string.len(ngx.var.http_x_scal_usermd) > 2 then
+    			ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
+    		end
+                    if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
+                            ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
+                    end
+    		local elapsed_ms = tonumber(ngx.var.request_time)
+    		if not (  elapsed_ms == nil) then
+    			elapsed_ms = elapsed_ms * 1000
+    			ngx.var.elapsed_ms = tostring(elapsed_ms)
+    		end
+    		local time = tonumber(ngx.var.msec) * 1000
+    		ngx.var.now = time
+    	';
+        log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
+                       '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
+                       '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
+		       '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
+		       '"contentType":"$content_type","s3Address":"$remote_addr",'
+		       '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
+                       '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
+		       '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
+		       '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
+                       '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
+                       '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
+        access_log /dev/stdout irm;
+    	error_log  /dev/stdout error;
+        location / {
+            proxy_request_buffering off;
+            fastcgi_request_buffering off;
+            fastcgi_no_cache 1;
+            fastcgi_cache_bypass 1;
+            fastcgi_buffering off;
+            fastcgi_ignore_client_abort on;
+            fastcgi_keep_conn on;
+            include fastcgi_params;
+            fastcgi_pass sproxyds;
+            fastcgi_next_upstream error timeout;
+            fastcgi_send_timeout 285s;
+            fastcgi_read_timeout 285s;
+        }
+    }
+}
+

.github/docker/sproxyd/conf/sproxyd0.conf

@@ -0,0 +1,12 @@
+{
+    "general": {
+        "ring": "DATA",
+        "port": 20000,
+        "syslog_facility": "local0"
+    },
+    "ring_driver:0": {
+        "alias": "dc1",
+        "type": "local",
+        "queue_path": "/tmp/ring-objs"
+    },
+}

.github/docker/sproxyd/conf/supervisord.conf

@@ -0,0 +1,43 @@
+[supervisord]
+nodaemon = true
+loglevel = info
+logfile  = %(ENV_LOG_DIR)s/supervisord.log
+pidfile  = %(ENV_SUP_RUN_DIR)s/supervisord.pid
+logfile_maxbytes = 20MB
+logfile_backups = 2
+
+[unix_http_server]
+file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[program:nginx]
+directory=%(ENV_SUP_RUN_DIR)s
+command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart=true
+autostart=true
+user=root
+
+[program:sproxyd]
+directory=%(ENV_SUP_RUN_DIR)s
+process_name=%(program_name)s-%(process_num)s
+numprocs=1
+numprocs_start=0
+command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+redirect_stderr=true
+autorestart=true
+autostart=true
+user=root

.github/workflows/alerts.yaml

@@ -0,0 +1,35 @@
+name: Test alerts
+
+on:
+  push:
+    branches-ignore:
+      - 'development/**'
+      - 'q/*/**'
+
+jobs:
+  run-alert-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tests:
+          - name: 1 minute interval tests
+            file: monitoring/alerts.test.yaml
+          
+          - name: 10 seconds interval tests
+            file: monitoring/alerts.10s.test.yaml
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Render and test ${{ matrix.tests.name }}
+        uses: scality/action-prom-render-test@1.0.3
+        with:
+          alert_file_path: monitoring/alerts.yaml
+          test_file_path: ${{ matrix.tests.file }}
+          alert_inputs: |
+            namespace=zenko
+            service=artesca-data-connector-s3api-metrics
+            reportJob=artesca-data-ops-report-handler
+            replicas=3
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql.yaml

@@ -3,7 +3,7 @@ name: codeQL
 
 on:
   push:
-    branches: [development/*, stabilization/*, hotfix/*]
+    branches: [w/**, q/*]
   pull_request:
     branches: [development/*, stabilization/*, hotfix/*]
   workflow_dispatch:
@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: javascript, python, ruby
 
       - name: Build and analyze
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/dependency-review.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v3
+        uses: actions/dependency-review-action@v4

.github/workflows/release.yaml

@@ -1,5 +1,6 @@
 ---
 name: release
+run-name: release ${{ inputs.tag }}
 
 on:
   workflow_dispatch:
@@ -8,38 +9,70 @@ on:
         description: 'Tag to be released'
         required: true
 
+env:
+  PROJECT_NAME: ${{ github.event.repository.name }}
+
 jobs:
   build-federation-image:
-    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
-    secrets: inherit
-    with:
-      push: true
-      registry: registry.scality.com
-      namespace: ${{ github.event.repository.name }}
-      name: ${{ github.event.repository.name }}
-      context: .
-      file: images/svc-base/Dockerfile
-      tag: ${{ github.event.inputs.tag }}-svc-base
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push image for federation
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          file: images/svc-base/Dockerfile
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
+          cache-from: type=gha,scope=federation
+          cache-to: type=gha,mode=max,scope=federation
 
-  build-image:
-    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
-    secrets: inherit
-    with:
-      push: true
-      registry: registry.scality.com
-      namespace: ${{ github.event.repository.name }}
-      name: ${{ github.event.repository.name }}
-      context: .
-      file: Dockerfile
-      tag: ${{ github.event.inputs.tag }}
-
-  github-release:
+  release:
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildk
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Push dashboards into the production namespace
+        run: |
+          oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
+            dashboard.json:application/grafana-dashboard+json \
+            alerts.yaml:application/prometheus-alerts+yaml
+        working-directory: monitoring
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Create Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ github.token }}
         with:
           name: Release ${{ github.event.inputs.tag }}
           tag_name: ${{ github.event.inputs.tag }}

.github/workflows/tests.yaml

@@ -2,6 +2,8 @@
 name: tests
 
 on:
+  workflow_dispatch:
+
   push:
     branches-ignore:
       - 'development/**'
@@ -65,23 +67,24 @@ env:
   ENABLE_LOCAL_CACHE: "true"
   REPORT_TOKEN: "report-token-1"
   REMOTE_MANAGEMENT_DISABLE: "1"
-
+  # https://github.com/git-lfs/git-lfs/issues/5749
+  GIT_CLONE_PROTECTION_ACTIVE: 'false'
 jobs:
   linting-coverage:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
-    - uses: actions/setup-node@v2
+      uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
       with:
         node-version: '16'
         cache: yarn
     - name: install dependencies
       run: yarn install --frozen-lockfile --network-concurrency 1
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.9'
-    - uses: actions/cache@v2
+    - uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip
@@ -114,7 +117,7 @@ jobs:
         find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
       if: always()
     - name: Upload files to artifacts
-      uses: scality/action-artifacts@v2
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -125,76 +128,166 @@ jobs:
 
   build:
     runs-on: ubuntu-20.04
+    permissions:
+      contents: read
+      packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1.6.0
+        uses: docker/setup-buildx-action@v3
       - name: Login to GitHub Registry
-        uses: docker/login-action@v1.10.0
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Login to Registry
-        uses: docker/login-action@v1
-        with:
-          registry: registry.scality.com
-          username: ${{ secrets.REGISTRY_LOGIN }}
-          password: ${{ secrets.REGISTRY_PASSWORD }}
+          password: ${{ github.token }}
       - name: Build and push cloudserver image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           push: true
           context: .
           provenance: false
           tags: |
-            ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
-            registry.scality.com/cloudserver-dev/cloudserver:${{ github.sha }}
+            ghcr.io/${{ github.repository }}:${{ github.sha }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
           cache-from: type=gha,scope=cloudserver
           cache-to: type=gha,mode=max,scope=cloudserver
-
-  build-federation-image:
-    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
-    secrets: inherit
-    with:
-      push: true
-      registry: registry.scality.com
-      namespace: cloudserver-dev
-      name: cloudserver
-      context: .
-      file: images/svc-base/Dockerfile
-      tag: ${{ github.sha }}-svc-base
+      - name: Build and push pykmip image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .github/pykmip
+          tags: |
+            ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=pykmip
+          cache-to: type=gha,mode=max,scope=pykmip
+      - name: Build and push MongoDB
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .github/docker/mongodb
+          tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+          cache-from: type=gha,scope=mongodb
+          cache-to: type=gha,mode=max,scope=mongodb
 
   multiple-backend:
     runs-on: ubuntu-latest
     needs: build
     env:
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       S3BACKEND: mem
       S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
       S3DATA: multiple
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+      uses: actions/checkout@v4
+    - name: Login to Registry
+      uses: docker/login-action@v3
       with:
-        python-version: 3.9
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Setup CI services
-      run: docker-compose up -d
+      run: docker compose --profile sproxyd up -d
       working-directory: .github/docker
     - name: Run multiple backend test
       run: |-
         set -o pipefail;
         bash wait_for_local_port.bash 8000 40
+        bash wait_for_local_port.bash 81 40
         yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
       env:
         S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+ 
+  mongo-v0-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v0
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  mongo-v1-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v1
+      METADATA_MAX_CACHED_BUCKETS: 1
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;        
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+        yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -204,36 +297,40 @@ jobs:
       if: always()
 
   file-ft-tests:
+    strategy:
+      matrix:
+        include:
+          - job-name: file-ft-tests
+    name: ${{ matrix.job-name }}
     runs-on: ubuntu-latest
     needs: build
     env:
       S3BACKEND: file
       S3VAULT: mem
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       MPU_TESTING: "yes"
-      JOB_NAME: ${{ github.job }}
+      JOB_NAME: ${{ matrix.job-name }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
+      uses: actions/checkout@v4
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
-    - name: Setup python test environment
+    - name: Setup matrix job artifacts directory
+      shell: bash
       run: |
-        sudo apt-get install -y libdigest-hmac-perl
-        pip install 's3cmd==2.3.0'
+        set -exu
+        mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
     - name: Setup CI services
-      run: docker-compose up -d
+      run: docker compose up -d
       working-directory: .github/docker
     - name: Run file ft tests
       run: |-
         set -o pipefail;
         bash wait_for_local_port.bash 8000 40
-        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+        yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -247,20 +344,18 @@ jobs:
     needs: build
     env:
       ENABLE_UTAPI_V2: t
-      S3BACKEND: mem 
+      S3BACKEND: mem
       BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
+      uses: actions/checkout@v4
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Setup CI services
-      run: docker-compose up -d
+      run: docker compose up -d
       working-directory: .github/docker
     - name: Run file utapi v2 tests
       run: |-
@@ -268,7 +363,51 @@ jobs:
         bash wait_for_local_port.bash 8000 40
         yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  quota-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        inflights:
+          - name: "With Inflights"
+            value: "true"
+          - name: "Without Inflights"
+            value: "false"
+    env:
+      S3METADATA: mongodb
+      S3BACKEND: mem
+      S3QUOTA: scuba
+      QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
+      SCUBA_HOST: localhost
+      SCUBA_PORT: 8100
+      SCUBA_HEALTHCHECK_FREQUENCY: 100
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run quota tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net
@@ -283,22 +422,21 @@ jobs:
     env:
       S3BACKEND: file
       S3VAULT: mem
-      MPU_TESTING: true
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
+      MPU_TESTING: "yes" 
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
       JOB_NAME: ${{ github.job }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
+      uses: actions/checkout@v4
     - name: Setup CI environment
       uses: ./.github/actions/setup-ci
     - name: Copy KMIP certs
       run: cp -r ./certs /tmp/ssl-kmip
       working-directory: .github/pykmip
     - name: Setup CI services
-      run: docker-compose --profile pykmip up -d
+      run: docker compose --profile pykmip up -d
       working-directory: .github/docker
     - name: Run file KMIP tests
       run: |-
@@ -307,7 +445,85 @@ jobs:
         bash wait_for_local_port.bash 5696 40
         yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
     - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v3
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  ceph-backend-test:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      S3DATA: multiple
+      S3KMS: file
+      CI_CEPH: 'true'
+      MPU_TESTING: "yes" 
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Login to GitHub Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '2.5.9'
+    - name: Install Ruby dependencies
+      run: |
+        gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
+    - name: Install Java dependencies
+      run: |
+        sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
+    - name: Setup CI services
+      run: docker compose --profile ceph up -d
+      working-directory: .github/docker
+      env: 
+        S3METADATA: mongodb
+    - name: Run Ceph multiple backend tests
+      run: |-
+        set -ex -o pipefail;
+        bash .github/ceph/wait_for_ceph.sh
+        bash wait_for_local_port.bash 27018 40
+        bash wait_for_local_port.bash 8000 40
+        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+        S3METADATA: mem
+    - name: Run Java tests
+      run: |-
+        set -ex -o pipefail;
+        mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
+      working-directory: tests/functional/jaws
+    - name: Run Ruby tests
+      run: |-
+        set -ex -o pipefail;
+        rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
+      working-directory: tests/functional/fog
+    - name: Run Javascript AWS SDK tests
+      run: |-
+        set -ex -o pipefail;
+        yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
+        yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
+        S3BACKEND: file
+        S3VAULT: mem
+        S3METADATA: mongodb
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
       with:
         method: upload
         url: https://artifacts.scality.net

.gitignore

@@ -22,9 +22,14 @@ coverage
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
 
+# Sphinx build dir
+_build
+
 # Dependency directory
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
+yarn.lock
+.tox
 
 # Junit directory
 junit

Dockerfile

@@ -1,51 +1,60 @@
-FROM node:16.13.2-slim
-MAINTAINER Giorgio Regni <gr@scality.com>
+ARG NODE_VERSION=16.20-bullseye-slim
+
+FROM node:${NODE_VERSION} as builder
 
 WORKDIR /usr/src/app
 
-# Keep the .git directory in order to properly report version
-COPY ./package.json yarn.lock ./
-
-ENV PYTHON=python3.9
-ENV PY_VERSION=3.9.7
-
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
-    jq \
-    python \
-    git \
-    build-essential \
-    ssh \
-    ca-certificates \
-    wget \
-    libffi-dev \
-    zlib1g-dev \
+        build-essential \
+        ca-certificates \
+        curl \
+        git \
+        gnupg2 \
+        jq \
+        python3 \
+        ssh \
+        wget \
+        libffi-dev \
+        zlib1g-dev \
     && apt-get clean \
     && mkdir -p /root/ssh \
     && ssh-keyscan -H github.com > /root/ssh/known_hosts
 
-RUN cd /tmp \
-    && wget https://www.python.org/ftp/python/$PY_VERSION/Python-$PY_VERSION.tgz \
-    && tar -C /usr/local/bin -xzvf Python-$PY_VERSION.tgz \
-    && cd /usr/local/bin/Python-$PY_VERSION \
-    && ./configure --enable-optimizations \
-    && make \
-    && make altinstall \
-    && rm -rf /tmp/Python-$PY_VERSION.tgz
+ENV PYTHON=python3
+COPY package.json yarn.lock /usr/src/app/
+RUN npm install typescript -g
+RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
 
-RUN yarn cache clean \
-    && yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
-    && apt-get autoremove --purge -y python git build-essential \
-    && rm -rf /var/lib/apt/lists/* \
-    && yarn cache clean \
-    && rm -rf ~/.node-gyp \
-    && rm -rf /tmp/yarn-*
+################################################################################
+FROM node:${NODE_VERSION}
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        jq \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV NO_PROXY localhost,127.0.0.1
+ENV no_proxy localhost,127.0.0.1
+
+EXPOSE 8000
+EXPOSE 8002
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        jq \
+        tini \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /usr/src/app
+
+# Keep the .git directory in order to properly report version
+COPY . /usr/src/app
+COPY --from=builder /usr/src/app/node_modules ./node_modules/
 
-COPY ./ ./
 
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
-ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
-CMD [ "yarn", "start" ]
+ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
 
-EXPOSE 8000
+CMD [ "yarn", "start" ]

Healthchecks.md

@@ -1,6 +1,7 @@
 # S3 Healthcheck
 
-Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
+Scality S3 exposes a healthcheck route `/live` on the port used
+for the metrics (defaults to port 8002) which returns a
 response with HTTP code
 
 - 200 OK

README.md

@@ -1,12 +1,7 @@
-# Zenko CloudServer
+# Zenko CloudServer with Vitastor Backend
 
 ![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
 
-[![CircleCI][badgepub]](https://circleci.com/gh/scality/S3)
-[![Scality CI][badgepriv]](http://ci.ironmann.io/gh/scality/S3)
-[![Docker Pulls][badgedocker]](https://hub.docker.com/r/scality/s3server/)
-[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
-
 ## Overview
 
 CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
@@ -16,125 +11,71 @@ Scality’s Open Source Multi-Cloud Data Controller.
 CloudServer provides a single AWS S3 API interface to access multiple
 backend data storage both on-premise or public in the cloud.
 
-CloudServer is useful for Developers, either to run as part of a
-continous integration test environment to emulate the AWS S3 service locally
-or as an abstraction layer to develop object storage enabled
-application on the go.
+This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
+backend support.
 
-## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
+## Quick Start with Vitastor
 
-## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
+Vitastor Backend is in experimental status, however you can already try to
+run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
+it works too 😊.
 
-## Docker
+Installation instructions:
 
-[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/scality/s3server/)
+### Install Vitastor
 
-## Contributing
+Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
 
-In order to contribute, please follow the
-[Contributing Guidelines](
-https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
+### Install Zenko with Vitastor Backend
 
-## Installation
+- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
+- Install dependencies: `npm install --omit dev` or just `npm install`
+- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
+- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
+  You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
+- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
 
-### Dependencies
+### Install and Configure MongoDB
 
-Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
-. Up-to-date versions can be found at
-[Nodesource](https://github.com/nodesource/distributions).
+Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
 
-### Clone source code
+### Setup Zenko
 
-```shell
-git clone https://github.com/scality/S3.git
+- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
+- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
+- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
+- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
+- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
+  and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
+  access keys, but it's not published, so let's use a file for now.
+- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
+  You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
+  in this file.
+
+Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
+instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
+
+### Start Zenko
+
+Start the S3 server with: `node index.js`
+
+If you use default settings, Zenko CloudServer starts on port 8000.
+The default access key is `accessKey1` with a secret key of `verySecretKey1`.
+
+Now you can access your S3 with `s3cmd` or `geesefs`:
+
+```
+s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
 ```
 
-### Install js dependencies
-
-Go to the ./S3 folder,
-
-```shell
-yarn install --frozen-lockfile
+```
+AWS_ACCESS_KEY_ID=accessKey1 \
+    AWS_SECRET_ACCESS_KEY=verySecretKey1 \
+    geesefs --endpoint http://localhost:8000 testbucket mountdir
 ```
 
-If you get an error regarding installation of the diskUsage module,
-please install g++.
+# Author & License
 
-If you get an error regarding level-down bindings, try clearing your yarn cache:
-
-```shell
-yarn cache clean
-```
-
-## Run it with a file backend
-
-```shell
-yarn start
-```
-
-This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
-9991 are also open locally for internal transfer of metadata and data,
-respectively.
-
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-By default the metadata files will be saved in the
-localMetadata directory and the data files will be saved
-in the localData directory within the ./S3 directory on your
-machine.  These directories have been pre-created within the
-repository.  If you would like to save the data or metadata in
-different locations of your choice, you must specify them with absolute paths.
-So, when starting the server:
-
-```shell
-mkdir -m 700 $(pwd)/myFavoriteDataPath
-mkdir -m 700 $(pwd)/myFavoriteMetadataPath
-export S3DATAPATH="$(pwd)/myFavoriteDataPath"
-export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
-yarn start
-```
-
-## Run it with multiple data backends
-
-```shell
-export S3DATA='multiple'
-yarn start
-```
-
-This starts a Zenko CloudServer on port 8000.
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-With multiple backends, you have the ability to
-choose where each object will be saved by setting
-the following header with a locationConstraint on
-a PUT request:
-
-```shell
-'x-amz-meta-scal-location-constraint':'myLocationConstraint'
-```
-
-If no header is sent with a PUT object request, the
-location constraint of the bucket will determine
-where the data is saved. If the bucket has no location
-constraint, the endpoint of the PUT request will be
-used to determine location.
-
-See the Configuration section below to learn how to set
-location constraints.
-
-## Run it with an in-memory backend
-
-```shell
-yarn run mem_backend
-```
-
-This starts a Zenko CloudServer on port 8000.
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
-[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
-[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
-[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
+- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
+- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
+  (a "network copyleft" license based on AGPL/SSPL, but worded in a better way)

_config.yml

@@ -1,2 +1,2 @@
 ---
-theme: jekyll-theme-minimal
+theme: jekyll-theme-modernist

authdata.json.example

@@ -0,0 +1,56 @@
+{
+    "accounts": [{
+        "name": "Bart",
+        "email": "sampleaccount1@sampling.com",
+        "arn": "arn:aws:iam::123456789012:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+        "shortid": "123456789012",
+        "keys": [{
+            "access": "accessKey1",
+            "secret": "verySecretKey1"
+        }]
+    }, {
+        "name": "Lisa",
+        "email": "sampleaccount2@sampling.com",
+        "arn": "arn:aws:iam::123456789013:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
+        "shortid": "123456789013",
+        "keys": [{
+            "access": "accessKey2",
+            "secret": "verySecretKey2"
+        }]
+    },
+    {
+        "name": "Clueso",
+        "email": "inspector@clueso.info",
+        "arn": "arn:aws:iam::123456789014:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/clueso",
+        "shortid": "123456789014",
+        "keys": [{
+            "access": "cluesoKey1",
+            "secret": "cluesoSecretKey1"
+        }]
+    },
+    {
+        "name": "Replication",
+        "email": "inspector@replication.info",
+        "arn": "arn:aws:iam::123456789015:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/replication",
+        "shortid": "123456789015",
+        "keys": [{
+            "access": "replicationKey1",
+            "secret": "replicationSecretKey1"
+        }]
+    },
+    {
+        "name": "Lifecycle",
+        "email": "inspector@lifecycle.info",
+        "arn": "arn:aws:iam::123456789016:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle",
+        "shortid": "123456789016",
+        "keys": [{
+            "access": "lifecycleKey1",
+            "secret": "lifecycleSecretKey1"
+        }]
+    }]
+}

bin/create_bucket_with_NFS_enabled.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict'; // eslint-disable-line strict
+
+require('../lib/nfs/utilities.js').createBucketWithNFSEnabled();

bin/search_bucket.js

@@ -0,0 +1,108 @@
+#!/bin/sh
+// 2>/dev/null ; exec "$(which nodejs 2>/dev/null || which node)" "$0" "$@"
+'use strict'; // eslint-disable-line strict
+
+const { auth } = require('arsenal');
+const commander = require('commander');
+
+const http = require('http');
+const https = require('https');
+const logger = require('../lib/utilities/logger');
+
+function _performSearch(host,
+                        port,
+                        bucketName,
+                        query,
+                        listVersions,
+                        accessKey,
+                        secretKey,
+                        sessionToken,
+                        verbose, ssl) {
+    const escapedSearch = encodeURIComponent(query);
+    const options = {
+        host,
+        port,
+        method: 'GET',
+        path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`,
+        headers: {
+            'Content-Length': 0,
+        },
+        rejectUnauthorized: false,
+        versions: '',
+    };
+    if (sessionToken) {
+        options.headers['x-amz-security-token'] = sessionToken;
+    }
+    const transport = ssl ? https : http;
+    const request = transport.request(options, response => {
+        if (verbose) {
+            logger.info('response status code', {
+                statusCode: response.statusCode,
+            });
+            logger.info('response headers', { headers: response.headers });
+        }
+        const body = [];
+        response.setEncoding('utf8');
+        response.on('data', chunk => body.push(chunk));
+        response.on('end', () => {
+            if (response.statusCode >= 200 && response.statusCode < 300) {
+                logger.info('Success');
+                process.stdout.write(body.join(''));
+                process.exit(0);
+            } else {
+                logger.error('request failed with HTTP Status ', {
+                    statusCode: response.statusCode,
+                    body: body.join(''),
+                });
+                process.exit(1);
+            }
+        });
+    });
+    // generateV4Headers exepects request object with path that does not
+    // include query
+    request.path = `/${bucketName}`;
+    const requestData = listVersions ? { search: query, versions: '' } : { search: query };
+    auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3');
+    request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`;
+    if (verbose) {
+        logger.info('request headers', { headers: request._headers });
+    }
+    request.end();
+}
+
+/**
+ * This function is used as a binary to send a request to S3 to perform a
+ * search on the objects in a bucket
+ *
+ * @return {undefined}
+ */
+function searchBucket() {
+    // TODO: Include other bucket listing possible query params?
+    commander
+        .version('0.0.1')
+        .option('-a, --access-key <accessKey>', 'Access key id')
+        .option('-k, --secret-key <secretKey>', 'Secret access key')
+        .option('-t, --session-token <sessionToken>', 'Session token')
+        .option('-b, --bucket <bucket>', 'Name of the bucket')
+        .option('-q, --query <query>', 'Search query')
+        .option('-h, --host <host>', 'Host of the server')
+        .option('-p, --port <port>', 'Port of the server')
+        .option('-s', '--ssl', 'Enable ssl')
+        .option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
+            'otherwise only list the latest version')
+        .option('-v, --verbose')
+        .parse(process.argv);
+    const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
+        commander;
+
+    if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
+        logger.error('missing parameter');
+        commander.outputHelp();
+        process.exit(1);
+    }
+
+    _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose,
+        ssl);
+}
+
+searchBucket();

circle.yml

@@ -1,45 +0,0 @@
----
-general:
-  branches:
-    ignore:
-      - /^ultron\/.*/   # Ignore ultron/* branches
-  artifacts:
-    - coverage/
-
-machine:
-  node:
-    version: 6.13.1
-  services:
-    - redis
-    - docker
-  ruby:
-    version: "2.4.1"
-  environment:
-    CXX: g++-4.9
-    ENABLE_LOCAL_CACHE: true
-    REPORT_TOKEN: report-token-1
-  hosts:
-    bucketwebsitetester.s3-website-us-east-1.amazonaws.com: 127.0.0.1
-
-dependencies:
-  override:
-    - rm -rf node_modules
-    - yarn install --frozen-lockfile
-  post:
-    - sudo pip install flake8 yamllint
-    - sudo pip install s3cmd==1.6.1
-    # fog and ruby testing dependencies
-    - gem install fog-aws -v 1.3.0
-    - gem install mime-types -v 3.1
-    - gem install rspec -v 3.5
-    - gem install json
-    # java sdk dependencies
-    - sudo apt-get install -y -q default-jdk
-
-
-test:
-  override:
-    - docker run --name squid-proxy -d --net=host
-      --publish 3128:3128 sameersbn/squid:3.3.8-23
-    - bash tests.bash:
-        parallel: true

conf/authdata.json

@@ -1,23 +0,0 @@
-{
-    "accounts": [{
-        "name": "Bart",
-        "email": "sampleaccount1@sampling.com",
-        "arn": "arn:aws:iam::123456789012:root",
-        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
-        "shortid": "123456789012",
-        "keys": [{
-            "access": "accessKey1",
-            "secret": "verySecretKey1"
-        }]
-    }, {
-        "name": "Lisa",
-        "email": "sampleaccount2@sampling.com",
-        "arn": "arn:aws:iam::123456789013:root",
-        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
-        "shortid": "123456789013",
-        "keys": [{
-            "access": "accessKey2",
-            "secret": "verySecretKey2"
-        }]
-    }]
-}

config.json.example

@@ -1,14 +1,19 @@
 {
     "port": 8000,
     "listenOn": [],
+    "metricsPort": 8002,
+    "metricsListenOn": [],
     "replicationGroupId": "RG001",
+    "workers": 4,
     "restEndpoints": {
         "localhost": "us-east-1",
         "127.0.0.1": "us-east-1",
         "cloudserver-front": "us-east-1",
         "s3.docker.test": "us-east-1",
         "127.0.0.2": "us-east-1",
-        "s3.amazonaws.com": "us-east-1"
+        "s3.amazonaws.com": "us-east-1",
+        "zenko-cloudserver-replicator": "us-east-1",
+        "lb": "us-east-1"
     },
     "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
                         "s3-website.us-east-2.amazonaws.com",
@@ -23,7 +28,8 @@
                         "s3-website-eu-west-1.amazonaws.com",
                         "s3-website-sa-east-1.amazonaws.com",
                         "s3-website.localhost",
-                        "s3-website.scality.test"],
+                        "s3-website.scality.test",
+                        "zenkoazuretest.blob.core.windows.net"],
     "replicationEndpoints": [{
         "site": "zenko",
         "servers": ["127.0.0.1:8000"],
@@ -32,6 +38,14 @@
         "site": "us-east-2",
         "type": "aws_s3"
     }],
+    "backbeat": {
+        "host": "localhost",
+        "port": 8900
+    },
+    "workflowEngineOperator": {
+        "host": "localhost",
+        "port": 3001
+    },
     "cdmi": {
         "host": "localhost",
         "port": 81,
@@ -45,7 +59,7 @@
         "host": "localhost",
         "port": 8500
     },
-    "clusters": 10,
+    "clusters": 1,
     "log": {
         "logLevel": "info",
         "dumpLevel": "error"
@@ -61,6 +75,10 @@
         "host": "localhost",
         "port": 9991
     },
+    "pfsClient": {
+        "host": "localhost",
+        "port": 9992
+    },
     "metadataDaemon": {
         "bindAddress": "localhost",
         "port": 9990
@@ -69,10 +87,47 @@
         "bindAddress": "localhost",
         "port": 9991
     },
+    "pfsDaemon": {
+        "bindAddress": "localhost",
+        "port": 9992
+    },
     "recordLog": {
-        "enabled": false,
+        "enabled": true,
         "recordLogName": "s3-recordlog"
     },
+    "mongodb": {
+        "replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
+        "writeConcern": "majority",
+        "replicaSet": "rs0",
+        "readPreference": "primary",
+        "database": "metadata"
+    },
+    "authdata": "authdata.json",
+    "backends": {
+        "auth": "file",
+        "data": "file",
+        "metadata": "mongodb",
+        "kms": "file",
+        "quota": "none"
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
+    },
     "requests": {
         "viaProxy": false,
         "trustedProxyCIDRs": [],

config.json.vitastor

@@ -0,0 +1,71 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "metricsPort": 8002,
+    "metricsListenOn": [],
+    "replicationGroupId": "RG001",
+    "restEndpoints": {
+        "localhost": "STANDARD",
+        "127.0.0.1": "STANDARD",
+        "yourhostname.ru": "STANDARD"
+    },
+    "websiteEndpoints": [
+        "static.yourhostname.ru"
+    ],
+    "replicationEndpoints": [ {
+        "site": "zenko",
+        "servers": ["127.0.0.1:8000"],
+        "default": true
+    } ],
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "backends": {
+        "metadata": "mongodb"
+    },
+    "mongodb": {
+        "replicaSetHosts": "127.0.0.1:27017",
+        "writeConcern": "majority",
+        "replicaSet": "rs0",
+        "readPreference": "primary",
+        "database": "s3",
+        "authCredentials": {
+            "username": "s3",
+            "password": ""
+        }
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
+    },
+    "requests": {
+        "viaProxy": false,
+        "trustedProxyCIDRs": [],
+        "extractClientIPFromHeader": ""
+    },
+    "bucketNotificationDestinations": [
+        {
+            "resource": "target1",
+            "type": "dummy",
+            "host": "localhost:6000"
+        }
+    ]
+}

constants.js

@@ -39,6 +39,8 @@ const constants = {
     // once the multipart upload is complete.
     mpuBucketPrefix: 'mpuShadowBucket',
     blacklistedPrefixes: { bucket: [], object: [] },
+    // GCP Object Tagging Prefix
+    gcpTaggingPrefix: 'aws-tag-',
     // PublicId is used as the canonicalID for a request that contains
     // no authentication information.  Requestor can access
     // only public resources
@@ -64,14 +66,20 @@ const constants = {
     // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
     minimumAllowedPartSize: 5242880,
 
+    // AWS sets a maximum total parts limit
+    // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
+    maximumAllowedPartCount: 10000,
+
+    gcpMaximumAllowedPartCount: 1024,
+
     // Max size on put part or copy part is 5GB. For functional
     // testing use 110 MB as max
     maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :
         5368709120,
 
-    // AWS sets a maximum total parts limit
-    // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
-    maximumAllowedPartCount: 10000,
+    // Max size allowed in a single put object request is 5GB
+    // https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
+    maximumAllowedUploadSize: 5368709120,
 
     // AWS states max size for user-defined metadata (x-amz-meta- headers) is
     // 2 KB: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
@@ -96,7 +104,6 @@ const constants = {
         'policyStatus',
         'publicAccessBlock',
         'requestPayment',
-        'restore',
         'torrent',
     ],
 
@@ -109,11 +116,21 @@ const constants = {
     ],
 
     // user metadata header to set object locationConstraint
-    objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
+    objectLocationConstraintHeader: 'x-amz-storage-class',
     lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
     legacyLocations: ['sproxyd', 'legacy'],
+    // declare here all existing service accounts and their properties
+    // (if any, otherwise an empty object)
+    serviceAccountProperties: {
+        replication: {},
+        lifecycle: {},
+        gc: {},
+        'md-ingestion': {
+            canReplicate: true,
+        },
+    },
     /* eslint-disable camelcase */
-    externalBackends: { aws_s3: true, azure: true, gcp: true },
+    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
     // some of the available data backends  (if called directly rather
     // than through the multiple backend gateway) need a key provided
     // as a string as first parameter of the get/delete methods.
@@ -122,13 +139,19 @@ const constants = {
     // for external backends, don't call unless at least 1 minute
     // (60,000 milliseconds) since last call
     externalBackendHealthCheckInterval: 60000,
-    versioningNotImplBackends: { azure: true },
-    mpuMDStoredExternallyBackend: { aws_s3: true },
+    versioningNotImplBackends: { azure: true, gcp: true },
+    mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
+    skipBatchDeleteBackends: { azure: true, gcp: true },
+    s3HandledBackends: { azure: true, gcp: true },
+    hasCopyPartBackends: { aws_s3: true, gcp: true },
     /* eslint-enable camelcase */
     mpuMDStoredOnS3Backend: { azure: true },
     azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
     base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
         '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
+    productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
+    // location constraint delimiter
+    zenkoSeparator: ':',
     // user metadata applied on zenko objects
     zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
     bucketOwnerActions: [
@@ -162,7 +185,6 @@ const constants = {
     // Bucket specific queries supported by AWS that we do not currently support
     // these queries may or may not be supported at object level
     unsupportedBucketQueries: [
-        'tagging',
     ],
     suppressedUtapiEventFields: [
         'object',
@@ -182,10 +204,45 @@ const constants = {
         'objectPutRetention',
     ],
     allowedUtapiEventFilterStates: ['allow', 'deny'],
+    allowedRestoreObjectRequestTierValues: ['Standard'],
+    lifecycleListing: {
+        CURRENT_TYPE: 'current',
+        NON_CURRENT_TYPE: 'noncurrent',
+        ORPHAN_DM_TYPE: 'orphan',
+    },
+    multiObjectDeleteConcurrency: 50,
+    maxScannedLifecycleListingEntries: 10000,
+    overheadField: [
+        'content-length',
+        'owner-id',
+        'versionId',
+        'isNull',
+        'isDeleteMarker',
+    ],
+    unsupportedSignatureChecksums: new Set([
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
+        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
+        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
+        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
+    ]),
+    supportedSignatureChecksums: new Set([
+        'UNSIGNED-PAYLOAD',
+        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
+    ]),
+    ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
+    ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
     // The AWS assumed Role resource type
     assumedRoleArnResourceType: 'assumed-role',
     // Session name of the backbeat lifecycle assumed role session.
     backbeatLifecycleSessionName: 'backbeat-lifecycle',
+    actionsToConsiderAsObjectPut: [
+        'initiateMultipartUpload',
+        'objectPutPart',
+        'completeMultipartUpload',
+    ],
+    // if requester is not bucket owner, bucket policy actions should be denied with
+    // MethodNotAllowed error
+    onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
 };
 
 module.exports = constants;

dataserver.js

@@ -17,13 +17,17 @@ process.on('uncaughtException', err => {
 if (config.backends.data === 'file' ||
     (config.backends.data === 'multiple' &&
      config.backends.metadata !== 'scality')) {
-    const dataServer = new arsenal.network.rest.RESTServer(
-        { bindAddress: config.dataDaemon.bindAddress,
-            port: config.dataDaemon.port,
-            dataStore: new arsenal.storage.data.file.DataFileStore(
-                { dataPath: config.dataDaemon.dataPath,
-                    log: config.log }),
-            log: config.log });
+    const dataServer = new arsenal.network.rest.RESTServer({
+        bindAddress: config.dataDaemon.bindAddress,
+        port: config.dataDaemon.port,
+        dataStore: new arsenal.storage.data.file.DataFileStore({
+            dataPath: config.dataDaemon.dataPath,
+            log: config.log,
+            noSync: config.dataDaemon.noSync,
+            noCache: config.dataDaemon.noCache,
+        }),
+        log: config.log,
+    });
     dataServer.setup(err => {
         if (err) {
             logger.error('Error initializing REST data server',

docker-entrypoint.sh

@@ -6,14 +6,15 @@ set -e
 # modifying config.json
 JQ_FILTERS_CONFIG="."
 
+# ENDPOINT var can accept comma separated values
+# for multiple endpoint locations
 if [[ "$ENDPOINT" ]]; then
-    HOST_NAME="$ENDPOINT"
-fi
-
-if [[ "$HOST_NAME" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$HOST_NAME\"]=\"us-east-1\""
-    echo "Host name has been modified to $HOST_NAME"
-    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with $HOST_NAME"
+    IFS="," read -ra HOST_NAMES <<< "$ENDPOINT"
+    for host in "${HOST_NAMES[@]}"; do
+        JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$host\"]=\"us-east-1\""
+    done
+    echo "Host name has been modified to ${HOST_NAMES[@]}"
+    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with ${HOST_NAMES[@]}"
 fi
 
 if [[ "$LOG_LEVEL" ]]; then
@@ -25,7 +26,7 @@ if [[ "$LOG_LEVEL" ]]; then
     fi
 fi
 
-if [[ "$SSL" && "$HOST_NAME" ]]; then
+if [[ "$SSL" && "$HOST_NAMES" ]]; then
 # This condition makes sure that the certificates are not generated twice. (for docker restart)
 if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
 # Compute config for utapi tests
@@ -36,15 +37,15 @@ prompt = no
 req_extensions = s3_req
 
 [req_distinguished_name]
-CN = ${HOST_NAME}
+CN = ${HOST_NAMES[0]}
 
 [s3_req]
 subjectAltName = @alt_names
 extendedKeyUsage = serverAuth, clientAuth
 
 [alt_names]
-DNS.1 = *.${HOST_NAME}
-DNS.2 = ${HOST_NAME}
+DNS.1 = *.${HOST_NAMES[0]}
+DNS.2 = ${HOST_NAMES[0]}
 
 EOF
 
@@ -70,9 +71,14 @@ fi
 if [[ "$LISTEN_ADDR" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
 fi
 
+if [[ "$REPLICATION_GROUP_ID" ]] ; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
+fi
+
 if [[ "$DATA_HOST" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
 fi
@@ -81,6 +87,22 @@ if [[ "$METADATA_HOST" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
 fi
 
+if [[ "$PFSD_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
+fi
+
+if [[ "$MONGODB_HOSTS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
+fi
+
+if [[ "$MONGODB_RS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSet=\"$MONGODB_RS\""
+fi
+
+if [[ "$MONGODB_DATABASE" ]]; then
+   JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.database=\"$MONGODB_DATABASE\""
+fi
+
 if [ -z "$REDIS_HA_NAME" ]; then
     REDIS_HA_NAME='mymaster'
 fi
@@ -113,12 +135,68 @@ if [[ "$RECORDLOG_ENABLED" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true"
 fi
 
-if [[ -n "$BUCKET_DENY_FILTER" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
+if [[ "$STORAGE_LIMIT_ENABLED" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.metrics[.utapi.metrics | length]=\"location\""
 fi
 
-if [[ "$BUCKETD_BOOTSTRAP" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .bucketd.bootstrap=[\"$BUCKETD_BOOTSTRAP\""]
+if [[ "$CRR_METRICS_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\""
+fi
+
+if [[ "$CRR_METRICS_PORT" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
+fi
+
+if [[ "$WE_OPERATOR_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
+fi
+
+if [[ "$WE_OPERATOR_PORT" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
+fi
+
+if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
+fi
+
+# external backends http(s) agent config
+
+# AWS
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
+#GCP
+if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
+if [[ -n "$BUCKET_DENY_FILTER" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
 fi
 
 if [[ "$TESTING_MODE" ]]; then
@@ -130,6 +208,10 @@ if [[ $JQ_FILTERS_CONFIG != "." ]]; then
     mv config.json.tmp config.json
 fi
 
+if test -v INITIAL_INSTANCE_ID && test -v S3METADATAPATH && ! test -f ${S3METADATAPATH}/uuid ; then
+    echo -n ${INITIAL_INSTANCE_ID} > ${S3METADATAPATH}/uuid
+fi
+
 # s3 secret credentials for Zenko
 if [ -r /run/secrets/s3-credentials ] ; then
     . /run/secrets/s3-credentials

docs/ARCHITECTURE.rst

@@ -27,7 +27,7 @@ including null versions and delete markers, described in the above
 links.
 
 Implementation of Bucket Versioning in Zenko CloudServer
------------------------------------------
+--------------------------------------------------------
 
 Overview of Metadata and API Component Roles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -66,7 +66,7 @@ The second section, `"Implementation of Bucket Versioning in
 API" <#implementation-of-bucket-versioning-in-api>`__, describes the way
 the metadata options are used in the API within S3 actions to create new
 versions, update their metadata, and delete them. The management of null
-versions and creation of delete markers are also described in this
+versions and creation of delete markers is also described in this
 section.
 
 Implementation of Bucket Versioning in Metadata
@@ -179,12 +179,13 @@ PUT
    the master version with this version.
 -  ``versionId: <versionId>`` create or update a specific version (for updating
    version's ACL or tags, or remote updates in geo-replication)
-   -  if the version identified by ``versionId`` happens to be the latest
+
+   *  if the version identified by ``versionId`` happens to be the latest
       version, the master version will be updated as well
-   -  if the master version is not as recent as the version identified by
+   *  if the master version is not as recent as the version identified by
       ``versionId``, as may happen with cross-region replication, the master
       will be updated as well
-   -  note that with ``versionId`` set to an empty string ``''``, it will
+   *  note that with ``versionId`` set to an empty string ``''``, it will
       overwrite the master version only (same as no options, but the master
       version will have a ``versionId`` property set in its metadata like
       any other version). The ``versionId`` will never be exposed to an
@@ -208,10 +209,13 @@ A deletion targeting the latest version of an object has to:
 -  delete the specified version identified by ``versionId``
 -  replace the master version with a version that is a placeholder for
    deletion
+
     -  this version contains a special keyword, 'isPHD', to indicate the
          master version was deleted and needs to be updated
+
 -  initiate a repair operation to update the value of the master
    version:
+
     -  involves listing the versions of the object and get the latest
        version to replace the placeholder delete version
     -  if no more versions exist, metadata deletes the master version,
@@ -755,16 +759,16 @@ command in the Zenko CloudServer directory:
 
 This will open two ports:
 
--  one is based on socket.io and is used for metadata transfers (9990 by
+- one is based on socket.io and is used for metadata transfers (9990 by
   default)
 
--  the other is a REST interface used for data transfers (9991 by
+- the other is a REST interface used for data transfers (9991 by
   default)
 
 Then, one or more instances of Zenko CloudServer without the dmd can be started
 elsewhere with:
 
-::
+.. code:: sh
 
    yarn run start_s3server
 
@@ -792,10 +796,10 @@ access:
 
 To run a remote dmd, you have to do the following:
 
--  change both ``"host"`` attributes to the IP or host name where the
+- change both ``"host"`` attributes to the IP or host name where the
   dmd is run.
 
--  Modify the ``"bindAddress"`` attributes in ``"metadataDaemon"`` and
+- Modify the ``"bindAddress"`` attributes in ``"metadataDaemon"`` and
   ``"dataDaemon"`` sections where the dmd is run to accept remote
   connections (e.g. ``"::"``)
 
@@ -831,13 +835,13 @@ and ``createReadStream``. They more or less map the parameters accepted
 by the corresponding calls in the LevelUp implementation of LevelDB.
 They differ in the following:
 
--  The ``sync`` option is ignored (under the hood, puts are gathered
+- The ``sync`` option is ignored (under the hood, puts are gathered
   into batches which have their ``sync`` property enforced when they
   are committed to the storage)
 
--  Some additional versioning-specific options are supported
+- Some additional versioning-specific options are supported
 
--  ``createReadStream`` becomes asynchronous, takes an additional
+- ``createReadStream`` becomes asynchronous, takes an additional
   callback argument and returns the stream in the second callback
   parameter
 
@@ -847,10 +851,10 @@ with ``DEBUG='socket.io*'`` environment variable set.
 One parameter controls the timeout value after which RPC commands sent
 end with a timeout error, it can be changed either:
 
--  via the ``DEFAULT_CALL_TIMEOUT_MS`` option in
+- via the ``DEFAULT_CALL_TIMEOUT_MS`` option in
   ``lib/network/rpc/rpc.js``
 
--  or in the constructor call of the ``MetadataFileClient`` object (in
+- or in the constructor call of the ``MetadataFileClient`` object (in
   ``lib/metadata/bucketfile/backend.js`` as ``callTimeoutMs``.
 
 Default value is 30000.
@@ -864,10 +868,10 @@ can tune the behavior (for better throughput or getting it more robust
 on weak networks), they have to be set in ``mdserver.js`` file directly,
 as there is no support in ``config.json`` for now for those options:
 
--  ``streamMaxPendingAck``: max number of pending ack events not yet
+- ``streamMaxPendingAck``: max number of pending ack events not yet
   received (default is 5)
 
--  ``streamAckTimeoutMs``: timeout for receiving an ack after an output
+- ``streamAckTimeoutMs``: timeout for receiving an ack after an output
   stream packet is sent to the client (default is 5000)
 
 Data exchange through the REST data port
@@ -918,17 +922,17 @@ Listing Types
 We use three different types of metadata listing for various operations.
 Here are the scenarios we use each for:
 
--  'Delimiter' - when no versions are possible in the bucket since it is
+- 'Delimiter' - when no versions are possible in the bucket since it is
   an internally-used only bucket which is not exposed to a user.
   Namely,
 
 1. to list objects in the "user's bucket" to respond to a GET SERVICE
-  request and
+   request and
 2. to do internal listings on an MPU shadow bucket to complete multipart
-  upload operations.
+   upload operations.
 
--  'DelimiterVersion' - to list all versions in a bucket
--  'DelimiterMaster' - to list just the master versions of objects in a
+- 'DelimiterVersion' - to list all versions in a bucket
+- 'DelimiterMaster' - to list just the master versions of objects in a
   bucket
 
 Algorithms

docs/CLIENTS.rst

@@ -178,7 +178,7 @@ Ruby
 ~~~~
 
 `AWS SDK for Ruby - Version 2 <http://docs.aws.amazon.com/sdkforruby/api/>`__
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code:: ruby
 
@@ -239,6 +239,7 @@ Python
 Client integration
 
 .. code:: python
+
     import boto3
 
     client = boto3.client(
@@ -253,6 +254,7 @@ Client integration
 Full integration (with object mapping)
 
 .. code:: python
+
     import os
 
     from botocore.utils import fix_s3_host
@@ -293,3 +295,51 @@ Should force path-style requests even though v3 advertises it does by default.
     $client->createBucket(array(
         'Bucket' => 'bucketphp',
     ));
+
+Go
+~~
+
+`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: go
+
+   package main
+
+   import (
+      "context"
+      "fmt"
+      "log"
+      "os"
+      "time"
+
+      "github.com/aws/aws-sdk-go/aws"
+      "github.com/aws/aws-sdk-go/aws/endpoints"
+      "github.com/aws/aws-sdk-go/aws/session"
+      "github.com/aws/aws-sdk-go/service/s3"
+   )
+
+   func main() {
+      os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
+      os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
+      endpoint := "http://localhost:8000"
+      timeout := time.Duration(10) * time.Second
+      sess := session.Must(session.NewSession())
+
+      // Create a context with a timeout that will abort the upload if it takes
+      // more than the passed in timeout.
+      ctx, cancel := context.WithTimeout(context.Background(), timeout)
+      defer cancel()
+
+      svc := s3.New(sess, &aws.Config{
+         Region:   aws.String(endpoints.UsEast1RegionID),
+         Endpoint: &endpoint,
+      })
+
+      out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
+      if err != nil {
+         log.Fatal(err)
+      } else {
+         fmt.Println(out)
+      }
+   }

docs/CONTRIBUTING.rst

@@ -14,7 +14,7 @@ Got an idea? Get started!
 In order to contribute, please follow the `Contributing
 Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__.
 If anything is unclear to you, reach out to us on
-`slack <https://zenko-io.slack.com/>`__ or via a GitHub issue.
+`forum <https://forum.zenko.io/>`__ or via a GitHub issue.
 
 Don't write code? There are other ways to help!
 -----------------------------------------------

docs/DOCKER.rst

@@ -1,11 +1,7 @@
 Docker
 ======
 
--  `Environment Variables <#environment-variables>`__
--  `Tunables and setup tips <#tunables-and-setup-tips>`__
--  `Examples for continuous integration with
-   Docker <#continuous-integration-with-docker-hosted CloudServer>`__
--  `Examples for going in production with Docker <#in-production-with-docker-hosted CloudServer>`__
+.. _environment-variables:
 
 Environment Variables
 ---------------------
@@ -15,25 +11,27 @@ S3DATA
 
 S3DATA=multiple
 ^^^^^^^^^^^^^^^
-Allows you to run Scality Zenko CloudServer with multiple data backends, defined
+
+This variable enables running CloudServer with multiple data backends, defined
 as regions.
-When using multiple data backends, a custom ``locationConfig.json`` file is
-mandatory. It will allow you to set custom regions. You will then need to
-provide associated rest_endpoints for each custom region in your
-``config.json`` file.
-`Learn more about multiple backends configuration <../GETTING_STARTED/#location-configuration>`__
 
-If you are using Scality RING endpoints, please refer to your customer
-documentation.
+For multiple data backends, a custom locationConfig.json file is required.
+This file enables you to set custom regions. You must provide associated 
+rest_endpoints for each custom region in config.json.
 
-Running it with an AWS S3 hosted backend
-""""""""""""""""""""""""""""""""""""""""
-To run CloudServer with an S3 AWS backend, you will have to add a new section
-to your ``locationConfig.json`` file with the ``aws_s3`` location type:
+`Learn more about multiple-backend configurations <GETTING_STARTED.html#location-configuration>`__
+
+If you are using Scality RING endpoints, refer to your customer documentation.
+
+Running CloudServer with an AWS S3-Hosted Backend
+"""""""""""""""""""""""""""""""""""""""""""""""""
+
+To run CloudServer with an S3 AWS backend, add a new section to the 
+``locationConfig.json`` file with the ``aws_s3`` location type:
 
 .. code:: json
 
-(...)
+    (...)
     "awsbackend": {
         "type": "aws_s3",
         "details": {
@@ -43,121 +41,134 @@ to your ``locationConfig.json`` file with the ``aws_s3`` location type:
             "credentialsProfile": "aws_hosted_profile"
         }
     }
-(...)
+    (...)
 
-You will also have to edit your AWS credentials file to be able to use your
-command line tool of choice. This file should mention credentials for all the
-backends you're using. You can use several profiles when using multiple
-profiles.
+Edit your AWS credentials file to enable your preferred command-line tool.
+This file must mention credentials for all backends in use. You can use 
+several profiles if multiple profiles are configured.
 
 .. code:: json
 
-[default]
-aws_access_key_id=accessKey1
-aws_secret_access_key=verySecretKey1
-[aws_hosted_profile]
-aws_access_key_id={{YOUR_ACCESS_KEY}}
-aws_secret_access_key={{YOUR_SECRET_KEY}}
+    [default]
+    aws_access_key_id=accessKey1
+    aws_secret_access_key=verySecretKey1
+    [aws_hosted_profile]
+    aws_access_key_id={{YOUR_ACCESS_KEY}}
+    aws_secret_access_key={{YOUR_SECRET_KEY}}
 
-Just as you need to mount your locationConfig.json, you will need to mount your
-AWS credentials file at run time:
-``-v ~/.aws/credentials:/root/.aws/credentials`` on Linux, OS X, or Unix or
+As with locationConfig.json, the AWS credentials file must be mounted at 
+run time: ``-v ~/.aws/credentials:/root/.aws/credentials`` on Unix-like 
+systems (Linux, OS X, etc.), or 
 ``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows
 
-NOTE: One account can't copy to another account with a source and
-destination on real AWS unless the account associated with the
-access Key/secret Key pairs used for the destination bucket has rights
-to get in the source bucket. ACL's would have to be updated
-on AWS directly to enable this.
+.. note:: One account cannot copy to another account with a source and
+   destination on real AWS unless the account associated with the 
+   accessKey/secretKey pairs used for the destination bucket has source 
+   bucket access privileges. To enable this, update ACLs directly on AWS.
 
 S3BACKEND
-~~~~~~
+~~~~~~~~~
 
 S3BACKEND=file
-^^^^^^^^^^^
-When storing file data, for it to be persistent you must mount docker volumes
-for both data and metadata. See `this section <#using-docker-volumes-in-production>`__
+^^^^^^^^^^^^^^
+
+For stored file data to persist, you must mount Docker volumes
+for both data and metadata. See :ref:`In Production with a Docker-Hosted CloudServer <in-production-w-a-Docker-hosted-cloudserver>`
 
 S3BACKEND=mem
-^^^^^^^^^^
-This is ideal for testing - no data will remain after container is shutdown.
+^^^^^^^^^^^^^
+
+This is ideal for testing: no data remains after the container is shut down.
 
 ENDPOINT
 ~~~~~~~~
 
-This variable specifies your endpoint. If you have a domain such as
-new.host.com, by specifying that here, you and your users can direct s3
-server requests to new.host.com.
+This variable specifies the endpoint. To direct CloudServer requests to 
+new.host.com, for example, specify the endpoint with:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e ENDPOINT=new.host.com scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 -e ENDPOINT=new.host.com zenko/cloudserver
 
-Note: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
-permissions, make sure to associate 127.0.0.1 with ``new.host.com``
+.. note:: On Unix-like systems (Linux, OS X, etc.) edit /etc/hosts
+   to associate 127.0.0.1 with new.host.com.
+
+REMOTE_MANAGEMENT_DISABLE
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CloudServer is a part of `Zenko <https://www.zenko.io/>`__. When you run CloudServer standalone it will still try to connect to Orbit by default (browser-based graphical user interface for Zenko).
+
+Setting this variable to true(1) will default to accessKey1 and verySecretKey1 for credentials and disable the automatic Orbit management:
+
+.. code-block:: shell
+
+    $ docker run -d --name cloudserver -p 8000:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
 
 SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 These variables specify authentication credentials for an account named
-"CustomAccount".
+“CustomAccount”.
 
-You can set credentials for many accounts by editing
-``conf/authdata.json`` (see below for further info), but if you just
-want to specify one set of your own, you can use these environment
-variables.
+Set account credentials for multiple accounts by editing conf/authdata.json
+(see below for further details). To specify one set for personal use, set these 
+environment variables:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey
-    -e SCALITY_SECRET_ACCESS_KEY=newSecretKey scality/s3server
+   $ docker run -d --name cloudserver -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey \
+   -e SCALITY_SECRET_ACCESS_KEY=newSecretKey zenko/cloudserver
 
-Note: Anything in the ``authdata.json`` file will be ignored. Note: The
-old ``ACCESS_KEY`` and ``SECRET_KEY`` environment variables are now
-deprecated
+.. note:: This takes precedence over the contents of the authdata.json 
+	  file. The authdata.json file is ignored. 
+
+.. note:: The ACCESS_KEY and SECRET_KEY environment variables are 
+	  deprecated.
 
 LOG\_LEVEL
 ~~~~~~~~~~
 
-This variable allows you to change the log level: info, debug or trace.
-The default is info. Debug will give you more detailed logs and trace
-will give you the most detailed.
+This variable changes the log level. There are three levels: info, debug, 
+and trace. The default is info. Debug provides more detailed logs, and trace
+provides the most detailed logs.
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 -e LOG_LEVEL=trace zenko/cloudserver
 
 SSL
 ~~~
 
-This variable set to true allows you to run S3 with SSL:
+Set true, this variable runs CloudServer with SSL.
 
-**Note1**: You also need to specify the ENDPOINT environment variable.
-**Note2**: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
-permissions, make sure to associate 127.0.0.1 with ``<YOUR_ENDPOINT>``
+If SSL is set true: 
 
-**Warning**: These certs, being self-signed (and the CA being generated
-inside the container) will be untrusted by any clients, and could
-disappear on a container upgrade. That's ok as long as it's for quick
-testing. Also, best security practice for non-testing would be to use an
-extra container to do SSL/TLS termination such as haproxy/nginx/stunnel
-to limit what an exploit on either component could expose, as well as
-certificates in a mounted volume
+* The ENDPOINT environment variable must also be specified.
+
+* On Unix-like systems (Linux, OS X, etc.), 127.0.0.1 must be associated with
+  <YOUR_ENDPOINT> in /etc/hosts.
+
+   .. Warning:: Self-signed certs with a CA generated within the container are 
+      suitable for testing purposes only. Clients cannot trust them, and they may
+      disappear altogether on a container upgrade. The best security practice for 
+      production environments is to use an extra container, such as 
+      haproxy/nginx/stunnel, for SSL/TLS termination and to pull certificates
+      from a mounted volume, limiting what an exploit on either component
+      can expose. 
 
 .. code:: shell
 
-    docker run -d --name s3server -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT>
-    scality/s3server
+     $ docker run -d --name cloudserver -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT> \
+     zenko/cloudserver
 
-More information about how to use S3 server with SSL
-`here <https://s3.scality.com/v1.0/page/scality-with-ssl>`__
+  For more information about using ClousdServer with SSL, see `Using SSL <GETTING_STARTED.html#Using SSL>`__
 
 LISTEN\_ADDR
 ~~~~~~~~~~~~
 
-This variable instructs the Zenko CloudServer, and its data and metadata
-components to listen on the specified address. This allows starting the data
-or metadata servers as standalone services, for example.
+This variable causes CloudServer and its data and metadata components to 
+listen on the specified address. This allows starting the data or metadata 
+servers as standalone services, for example.
 
 .. code:: shell
 
@@ -174,8 +185,8 @@ Zenko CloudServer.
 
 .. code:: shell
 
-    docker run -d --name s3server -e DATA_HOST=s3server-data
-    -e METADATA_HOST=s3server-metadata scality/s3server npm run start_s3server
+    $ docker run -d --name cloudserver -e DATA_HOST=cloudserver-data \
+    -e METADATA_HOST=cloudserver-metadata zenko/cloudserver yarn run start_s3server
 
 REDIS\_HOST
 ~~~~~~~~~~~
@@ -185,19 +196,21 @@ localhost.
 
 .. code:: shell
 
-    docker run -d --name s3server -p 8000:8000
-    -e REDIS_HOST=my-redis-server.example.com scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 \
+    -e REDIS_HOST=my-redis-server.example.com zenko/cloudserver
 
 REDIS\_PORT
 ~~~~~~~~~~~
 
-Use this variable to connect to the redis cache server on another port than
-the default 6379.
+Use this variable to connect to the Redis cache server on a port other 
+than the default 6379.
 
 .. code:: shell
 
-    docker run -d --name s3server -p 8000:8000
-    -e REDIS_PORT=6379 scality/s3server
+    $ docker run -d --name cloudserver -p 8000:8000 \
+    -e REDIS_PORT=6379 zenko/cloudserver
+
+.. _tunables-and-setup-tips:
 
 Tunables and Setup Tips
 -----------------------
@@ -205,60 +218,57 @@ Tunables and Setup Tips
 Using Docker Volumes
 ~~~~~~~~~~~~~~~~~~~~
 
-Zenko CloudServer runs with a file backend by default.
+CloudServer runs with a file backend by default, meaning that data is 
+stored inside the CloudServer’s Docker container.
 
-So, by default, the data is stored inside your Zenko CloudServer Docker
-container.
+For data and metadata to persist, data and metadata must be hosted in Docker 
+volumes outside the CloudServer’s Docker container. Otherwise, the data
+and metadata are destroyed when the container is erased.
 
-However, if you want your data and metadata to persist, you **MUST** use
-Docker volumes to host your data and metadata outside your Zenko CloudServer
-Docker container. Otherwise, the data and metadata will be destroyed
-when you erase the container.
+.. code-block:: shell
 
-.. code:: shell
+    $ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata \
+    -p 8000:8000 ­-d zenko/cloudserver
 
-    docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata
-    -p 8000:8000 ­-d scality/s3server
+This command mounts the ./data host directory to the container
+at /usr/src/app/localData and the ./metadata host directory to
+the container at /usr/src/app/localMetaData. 
 
-This command mounts the host directory, ``./data``, into the container
-at ``/usr/src/app/localData`` and the host directory, ``./metadata``, into
-the container at ``/usr/src/app/localMetaData``. It can also be any host
-mount point, like ``/mnt/data`` and ``/mnt/metadata``.
+.. tip:: These host directories can be mounted to any accessible mount 
+   point, such as /mnt/data and /mnt/metadata, for example.
 
-Adding modifying or deleting accounts or users credentials
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Adding, Modifying, or Deleting Accounts or Credentials
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-1. Create locally a customized ``authdata.json`` based on our ``/conf/authdata.json``.
+1. Create a customized authdata.json file locally based on /conf/authdata.json.
+
+2. Use `Docker volumes <https://docs.docker.com/storage/volumes/>`__
+   to override the default ``authdata.json`` through a Docker file mapping.
 
-2. Use `Docker
-   Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__
-   to override the default ``authdata.json`` through a docker file mapping.
 For example:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
-    scality/s3server
+    $ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d \
+    zenko/cloudserver
 
-Specifying your own host name
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Specifying a Host Name
+~~~~~~~~~~~~~~~~~~~~~~
 
-To specify a host name (e.g. s3.domain.name), you can provide your own
-`config.json <https://github.com/scality/S3/blob/master/config.json>`__
-using `Docker
-Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__.
+To specify a host name (for example, s3.domain.name), provide your own
+`config.json <https://github.com/scality/cloudserver/blob/master/config.json>`__
+file using `Docker volumes <https://docs.docker.com/storage/volumes/>`__.
 
-First add a new key-value pair in the restEndpoints section of your
-config.json. The key in the key-value pair should be the host name you
-would like to add and the value is the default location\_constraint for
-this endpoint.
+First, add a new key-value pair to the restEndpoints section of your
+config.json. Make the key the host name you want, and the value the default 
+location\_constraint for this endpoint.
 
 For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one
 of the ``location_constraints`` listed in your locationConfig.json file
 `here <https://github.com/scality/S3/blob/master/locationConfig.json>`__.
 
-More information about location configuration
-`here <https://github.com/scality/S3/blob/master/README.md#location-configuration>`__
+For more information about location configuration, see:
+`GETTING STARTED <GETTING_STARTED.html#location-configuration>`__
 
 .. code:: json
 
@@ -266,33 +276,33 @@ More information about location configuration
         "localhost": "file",
         "127.0.0.1": "file",
         ...
-        "s3.example.com": "us-east-1"
+        "cloudserver.example.com": "us-east-1"
     },
 
-Then, run your Scality S3 Server using `Docker
-Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
+Next, run CloudServer using a `Docker volume 
+<https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
 
-.. code:: shell
+.. code-block:: shell
 
-    docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
+    $ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d zenko/cloudserver
 
-Your local ``config.json`` file will override the default one through a
-docker file mapping.
+The local ``config.json`` file overrides the default one through a Docker 
+file mapping.
 
-Running as an unprivileged user
+Running as an Unprivileged User
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Zenko CloudServer runs as root by default.
+CloudServer runs as root by default.
 
-You can change that by modifing the dockerfile and specifying a user
-before the entrypoint.
+To change this, modify the dockerfile and specify a user before the 
+entry point.
 
-The user needs to exist within the container, and own the folder
-**/usr/src/app** for Scality Zenko CloudServer to run properly.
+The user must exist within the container, and must own the 
+/usr/src/app directory for CloudServer to run.
 
-For instance, you can modify these lines in the dockerfile:
+For example, the following dockerfile lines can be modified:
 
-.. code:: shell
+.. code-block:: shell
 
     ...
     && groupadd -r -g 1001 scality \
@@ -304,54 +314,58 @@ For instance, you can modify these lines in the dockerfile:
     USER scality
     ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
 
-Continuous integration with Docker hosted CloudServer
------------------------------------------------------
+.. _continuous-integration-with-docker-hosted-cloudserver:
 
-When you start the Docker Scality Zenko CloudServer image, you can adjust the
-configuration of the Scality Zenko CloudServer instance by passing one or more
-environment variables on the docker run command line.
+Continuous Integration with a Docker-Hosted CloudServer
+-------------------------------------------------------
 
-Sample ways to run it for CI are:
+When you start the Docker CloudServer image, you can adjust the
+configuration of the CloudServer instance by passing one or more
+environment variables on the ``docker run`` command line.
 
-- With custom locations (one in-memory, one hosted on AWS), and custom
-  credentials mounted:
 
-.. code:: shell
+To run CloudServer for CI with custom locations (one in-memory, 
+one hosted on AWS), and custom credentials mounted:
 
-    docker run --name CloudServer -p 8000:8000
-    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-    -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-    -v ~/.aws/credentials:/root/.aws/credentials
-    -e S3DATA=multiple -e S3BACKEND=mem scality/s3server
+.. code-block:: shell
 
-- With custom locations, (one in-memory, one hosted on AWS, one file),
-  and custom credentials set as environment variables
-  (see `this section <#scality-access-key-id-and-scality-secret-access-key>`__):
+   $ docker run --name CloudServer -p 8000:8000 \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
+   -v ~/.aws/credentials:/root/.aws/credentials \
+   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
 
-.. code:: shell
+To run CloudServer for CI with custom locations, (one in-memory, one
+hosted on AWS, and one file), and custom credentials `set as environment 
+variables <GETTING_STARTED.html#scality-access-key-id-and-scality-secret-access-key>`__):
 
-    docker run --name CloudServer -p 8000:8000
-    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-    -v ~/.aws/credentials:/root/.aws/credentials
-    -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-    -e SCALITY_ACCESS_KEY_ID=accessKey1
-    -e SCALITY_SECRET_ACCESS_KEY=verySecretKey1
-    -e S3DATA=multiple -e S3BACKEND=mem scality/s3server
+.. code-block:: shell
 
-In production with Docker hosted CloudServer
---------------------------------------------
+   $ docker run --name CloudServer -p 8000:8000 \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v ~/.aws/credentials:/root/.aws/credentials \
+   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -e SCALITY_ACCESS_KEY_ID=accessKey1 \
+   -e SCALITY_SECRET_ACCESS_KEY=verySecretKey1 \
+   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
 
-In production, we expect that data will be persistent, that you will use the
-multiple backends capabilities of Zenko CloudServer, and that you will have a
-custom endpoint for your local storage, and custom credentials for your local
-storage:
+.. _in-production-w-a-Docker-hosted-cloudserver:
 
-.. code:: shell
+In Production with a Docker-Hosted CloudServer
+----------------------------------------------
 
-    docker run -d --name CloudServer
-    -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-    -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-    -v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple
-    -e ENDPOINT=custom.endpoint.com
-    -p 8000:8000 ­-d scality/s3server
+Because data must persist in production settings, CloudServer offers
+multiple-backend capabilities. This requires a custom endpoint 
+and custom credentials for local storage.
+
+Customize these with:
+
+.. code-block:: shell
+
+   $ docker run -d --name CloudServer \
+   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
+   -v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple \
+   -e ENDPOINT=custom.endpoint.com \
+   -p 8000:8000 ­-d zenko/cloudserver \

docs/GETTING_STARTED.rst

@@ -1,214 +1,221 @@
 Getting Started
-=================
+===============
 
 .. figure:: ../res/scality-cloudserver-logo.png
    :alt: Zenko CloudServer logo
 
-|CircleCI| |Scality CI|
+
+Dependencies
+------------
+
+Building and running the Scality Zenko CloudServer requires node.js 10.x and
+yarn v1.17.x. Up-to-date versions can be found at
+`Nodesource <https://github.com/nodesource/distributions>`__.
 
 Installation
 ------------
 
-Dependencies
-~~~~~~~~~~~~
+1. Clone the source code
 
-Building and running the Scality Zenko CloudServer requires node.js 6.9.5 and
-npm v3 . Up-to-date versions can be found at
-`Nodesource <https://github.com/nodesource/distributions>`__.
+   .. code-block:: shell
 
-Clone source code
-~~~~~~~~~~~~~~~~~
+      $ git clone https://github.com/scality/cloudserver.git
 
-.. code:: shell
+2. Go to the cloudserver directory and use yarn to install the js dependencies.
 
-    git clone https://github.com/scality/S3.git
+   .. code-block:: shell
 
-Install js dependencies
-~~~~~~~~~~~~~~~~~~~~~~~
+      $ cd cloudserver
+      $ yarn install
 
-Go to the ./S3 folder,
+Running CloudServer with a File Backend
+---------------------------------------
 
-.. code:: shell
+.. code-block:: shell
 
-    npm install
+   $ yarn start
 
-Run it with a file backend
---------------------------
+This starts a Zenko CloudServer on port 8000. Two additional ports, 9990 
+and 9991, are also open locally for internal transfer of metadata and 
+data, respectively.
 
-.. code:: shell
+The default access key is accessKey1. The secret key is verySecretKey1.
 
-    npm start
+By default, metadata files are saved in the localMetadata directory and 
+data files are saved in the localData directory in the local ./cloudserver 
+directory. These directories are pre-created within the repository. To 
+save data or metadata in different locations, you must specify them using 
+absolute paths. Thus, when starting the server:
 
-This starts an Zenko CloudServer on port 8000. Two additional ports 9990 and
-9991 are also open locally for internal transfer of metadata and data,
-respectively.
+.. code-block:: shell
 
-The default access key is accessKey1 with a secret key of
-verySecretKey1.
+   $ mkdir -m 700 $(pwd)/myFavoriteDataPath
+   $ mkdir -m 700 $(pwd)/myFavoriteMetadataPath
+   $ export S3DATAPATH="$(pwd)/myFavoriteDataPath"
+   $ export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
+   $ yarn start
 
-By default the metadata files will be saved in the localMetadata
-directory and the data files will be saved in the localData directory
-within the ./S3 directory on your machine. These directories have been
-pre-created within the repository. If you would like to save the data or
-metadata in different locations of your choice, you must specify them
-with absolute paths. So, when starting the server:
+Running CloudServer with Multiple Data Backends
+-----------------------------------------------
 
-.. code:: shell
+.. code-block:: shell
 
-    mkdir -m 700 $(pwd)/myFavoriteDataPath
-    mkdir -m 700 $(pwd)/myFavoriteMetadataPath
-    export S3DATAPATH="$(pwd)/myFavoriteDataPath"
-    export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
-    npm start
+   $ export S3DATA='multiple'
+   $ yarn start
 
-Run it with multiple data backends
-----------------------------------
+This starts a Zenko CloudServer on port 8000. 
 
-.. code:: shell
+The default access key is accessKey1. The secret key is verySecretKey1.
 
-    export S3DATA='multiple'
-    npm start
+With multiple backends, you can choose where each object is saved by setting
+the following header with a location constraint in a PUT request:
 
-This starts an Zenko CloudServer on port 8000. The default access key is
-accessKey1 with a secret key of verySecretKey1.
-
-With multiple backends, you have the ability to choose where each object
-will be saved by setting the following header with a locationConstraint
-on a PUT request:
-
-.. code:: shell
+.. code-block:: shell
 
     'x-amz-meta-scal-location-constraint':'myLocationConstraint'
 
-If no header is sent with a PUT object request, the location constraint
-of the bucket will determine where the data is saved. If the bucket has
-no location constraint, the endpoint of the PUT request will be used to
-determine location.
+If no header is sent with a PUT object request, the bucket’s location
+constraint determines where the data is saved. If the bucket has no
+location constraint, the endpoint of the PUT request determines location.
 
-See the Configuration section below to learn how to set location
-constraints.
+See the Configuration_ section to set location constraints.
 
-Run it with an in-memory backend
---------------------------------
+Run CloudServer with an In-Memory Backend
+-----------------------------------------
+
+.. code-block:: shell
+
+   $ yarn run mem_backend
+
+This starts a Zenko CloudServer on port 8000. 
+
+The default access key is accessKey1. The secret key is verySecretKey1.
+
+Run CloudServer with Vault User Management
+------------------------------------------
 
 .. code:: shell
 
-    npm run mem_backend
+   export S3VAULT=vault
+   yarn start
 
-This starts an Zenko CloudServer on port 8000. The default access key is
-accessKey1 with a secret key of verySecretKey1.
+Note: Vault is proprietary and must be accessed separately.
+This starts a Zenko CloudServer using Vault for user management.
 
-Run it for continuous integration testing or in production with Docker
-----------------------------------------------------------------------
+Run CloudServer for Continuous Integration Testing or in Production with Docker
+-------------------------------------------------------------------------------
 
-`DOCKER <../DOCKER/>`__
+Run Cloudserver with `DOCKER <DOCKER.html>`__
 
 Testing
--------
+~~~~~~~
 
-You can run the unit tests with the following command:
+Run unit tests with the command:
 
-.. code:: shell
+.. code-block:: shell
 
-    npm test
+   $ yarn test
 
-You can run the multiple backend unit tests with:
+Run multiple-backend unit tests with:
 
-.. code:: shell
-    CI=true S3DATA=multiple npm start
-    npm run multiple_backend_test
+.. code-block:: shell
 
-You can run the linter with:
+   $ CI=true S3DATA=multiple yarn start
+   $ yarn run multiple_backend_test
 
-.. code:: shell
+Run the linter with:
 
-    npm run lint
+.. code-block:: shell
 
-Running functional tests locally:
+   $ yarn run lint
 
-For the AWS backend and Azure backend tests to pass locally,
-you must modify tests/locationConfigTests.json so that awsbackend
-specifies a bucketname of a bucket you have access to based on
-your credentials profile and modify "azurebackend" with details
-for your Azure account.
+Running Functional Tests Locally
+--------------------------------
+
+To pass AWS and Azure backend tests locally, modify 
+tests/locationConfig/locationConfigTests.json so that ``awsbackend`` 
+specifies the bucketname of a bucket you have access to based on your
+credentials, and modify ``azurebackend`` with details for your Azure account.
 
 The test suite requires additional tools, **s3cmd** and **Redis**
 installed in the environment the tests are running in.
 
--  Install `s3cmd <http://s3tools.org/download>`__
--  Install `redis <https://redis.io/download>`__ and start Redis.
--  Add localCache section to your ``config.json``:
+1. Install `s3cmd <http://s3tools.org/download>`__
 
-::
+2. Install `redis <https://redis.io/download>`__ and start Redis.
 
-    "localCache": {
+3. Add localCache section to ``config.json``:
+
+   .. code:: json
+
+      "localCache": {
         "host": REDIS_HOST,
         "port": REDIS_PORT
-    }
+	}
 
-where ``REDIS_HOST`` is your Redis instance IP address (``"127.0.0.1"``
-if your Redis is running locally) and ``REDIS_PORT`` is your Redis
-instance port (``6379`` by default)
+   where ``REDIS_HOST`` is the Redis instance IP address (``"127.0.0.1"``
+   if Redis is running locally) and ``REDIS_PORT`` is the Redis instance
+   port (``6379`` by default)
 
--  Add the following to the etc/hosts file on your machine:
+4. Add the following to the local etc/hosts file:
 
-.. code:: shell
+   .. code-block:: shell
 
-    127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
+      127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
 
--  Start the Zenko CloudServer in memory and run the functional tests:
+5. Start Zenko CloudServer in memory and run the functional tests:
 
-.. code:: shell
+   .. code-block:: shell
 
-    CI=true npm run mem_backend
-    CI=true npm run ft_test
+      $ CI=true yarn run mem_backend
+      $ CI=true yarn run ft_test
+
+.. _Configuration:
 
 Configuration
 -------------
 
-There are three configuration files for your Scality Zenko CloudServer:
+There are three configuration files for Zenko CloudServer:
 
-1. ``conf/authdata.json``, described above for authentication
+* ``conf/authdata.json``, for authentication.
 
-2. ``locationConfig.json``, to set up configuration options for
+* ``locationConfig.json``, to configure where data is saved.
 
-   where data will be saved
+* ``config.json``, for general configuration options.
 
-3. ``config.json``, for general configuration options
+.. _location-configuration:
 
 Location Configuration
 ~~~~~~~~~~~~~~~~~~~~~~
 
-You must specify at least one locationConstraint in your
-locationConfig.json (or leave as pre-configured).
+You must specify at least one locationConstraint in locationConfig.json
+(or leave it as pre-configured).
 
-You must also specify 'us-east-1' as a locationConstraint so if you only
-define one locationConstraint, that would be it. If you put a bucket to
-an unknown endpoint and do not specify a locationConstraint in the put
-bucket call, us-east-1 will be used.
+You must also specify 'us-east-1' as a locationConstraint. If you put a 
+bucket to an unknown endpoint and do not specify a locationConstraint in
+the PUT bucket call, us-east-1 is used.
 
-For instance, the following locationConstraint will save data sent to
+For instance, the following locationConstraint saves data sent to
 ``myLocationConstraint`` to the file backend:
 
 .. code:: json
 
-    "myLocationConstraint": {
-        "type": "file",
-        "legacyAwsBehavior": false,
-        "details": {}
-    },
+   "myLocationConstraint": {
+       "type": "file",
+       "legacyAwsBehavior": false,
+       "details": {}
+   },
 
-Each locationConstraint must include the ``type``,
-``legacyAwsBehavior``, and ``details`` keys. ``type`` indicates which
-backend will be used for that region. Currently, mem, file, and scality
-are the supported backends. ``legacyAwsBehavior`` indicates whether the
-region will have the same behavior as the AWS S3 'us-east-1' region. If
-the locationConstraint type is scality, ``details`` should contain
-connector information for sproxyd. If the locationConstraint type is mem
-or file, ``details`` should be empty.
+Each locationConstraint must include the ``type``, ``legacyAwsBehavior``,
+and ``details`` keys. ``type`` indicates which backend is used for that
+region. Supported backends are mem, file, and scality.``legacyAwsBehavior``
+indicates whether the region behaves the same as the AWS S3 'us-east-1' 
+region. If the locationConstraint type is ``scality``, ``details`` must 
+contain connector information for sproxyd. If the locationConstraint type
+is ``mem`` or ``file``, ``details`` must be empty.
 
-Once you have your locationConstraints in your locationConfig.json, you
-can specify a default locationConstraint for each of your endpoints.
+Once locationConstraints is set in locationConfig.json, specify a default
+locationConstraint for each endpoint.
 
 For instance, the following sets the ``localhost`` endpoint to the
 ``myLocationConstraint`` data backend defined above:
@@ -219,26 +226,24 @@ For instance, the following sets the ``localhost`` endpoint to the
          "localhost": "myLocationConstraint"
     },
 
-If you would like to use an endpoint other than localhost for your
-Scality Zenko CloudServer, that endpoint MUST be listed in your
-``restEndpoints``. Otherwise if your server is running with a:
+To use an endpoint other than localhost for Zenko CloudServer, the endpoint
+must be listed in ``restEndpoints``. Otherwise, if the server is running
+with a:
 
--  **file backend**: your default location constraint will be ``file``
-
--  **memory backend**: your default location constraint will be ``mem``
+*  **file backend**: The default location constraint is ``file``
+*  **memory backend**: The default location constraint is ``mem``
 
 Endpoints
 ~~~~~~~~~
 
-Note that our Zenko CloudServer supports both:
+The Zenko CloudServer supports endpoints that are rendered in either:
 
--  path-style: http://myhostname.com/mybucket
--  hosted-style: http://mybucket.myhostname.com
+* path style: http://myhostname.com/mybucket or
+* hosted style: http://mybucket.myhostname.com
 
-However, hosted-style requests will not hit the server if you are using
-an ip address for your host. So, make sure you are using path-style
-requests in that case. For instance, if you are using the AWS SDK for
-JavaScript, you would instantiate your client like this:
+However, if an IP address is specified for the host, hosted-style requests
+cannot reach the server. Use path-style requests in that case. For example,
+if you are using the AWS SDK for JavaScript, instantiate your client like this:
 
 .. code:: js
 
@@ -247,87 +252,99 @@ JavaScript, you would instantiate your client like this:
        s3ForcePathStyle: true,
     });
 
-Setting your own access key and secret key pairs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Setting Your Own Access and Secret Key Pairs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can set credentials for many accounts by editing
-``conf/authdata.json`` but if you want to specify one set of your own
-credentials, you can use ``SCALITY_ACCESS_KEY_ID`` and
-``SCALITY_SECRET_ACCESS_KEY`` environment variables.
+Credentials can be set for many accounts by editing ``conf/authdata.json``, 
+but use the ``SCALITY_ACCESS_KEY_ID`` and ``SCALITY_SECRET_ACCESS_KEY`` 
+environment variables to specify your own credentials.
+
+_`scality-access-key-id-and-scality-secret-access-key`
 
 SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 These variables specify authentication credentials for an account named
-"CustomAccount".
+“CustomAccount”.
 
-Note: Anything in the ``authdata.json`` file will be ignored.
+.. note:: Anything in the ``authdata.json`` file is ignored.
 
-.. code:: shell
+.. code-block:: shell
 
-    SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey npm start
+   $ SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey yarn start
 
+.. _Using_SSL:
 
-Scality with SSL
-~~~~~~~~~~~~~~~~~~~~~~
+Using SSL
+~~~~~~~~~
 
-If you wish to use https with your local Zenko CloudServer, you need to set up
-SSL certificates. Here is a simple guide of how to do it.
+To use https with your local CloudServer, you must set up
+SSL certificates. 
 
-Deploying Zenko CloudServer
-^^^^^^^^^^^^^^^^^^^
+1. Deploy CloudServer using `our DockerHub page
+   <https://hub.docker.com/r/zenko/cloudserver/>`__ (run it with a file
+   backend).
 
-First, you need to deploy **Zenko CloudServer**. This can be done very easily
-via `our **DockerHub**
-page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
-with a file backend).
+   .. Note:: If Docker is not installed locally, follow the
+      `instructions to install it for your distribution 
+      <https://docs.docker.com/engine/installation/>`__
 
-    *Note:* *- If you don't have docker installed on your machine, here
-    are the `instructions to install it for your
-    distribution <https://docs.docker.com/engine/installation/>`__*
+2. Update the CloudServer container’s config 
 
-Updating your Zenko CloudServer container's config
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   Add your certificates to your container. To do this, 
+   #. exec inside the CloudServer container. 
 
-You're going to add your certificates to your container. In order to do
-so, you need to exec inside your Zenko CloudServer container. Run a
-``$> docker ps`` and find your container's id (the corresponding image
-name should be ``scality/s3server``. Copy the corresponding container id
-(here we'll use ``894aee038c5e``, and run:
+   #. Run ``$> docker ps`` to find the container’s ID (the corresponding 
+      image name is ``scality/cloudserver``. 
+      
+   #. Copy the corresponding container ID (``894aee038c5e`` in the present
+      example), and run:
 
-.. code:: sh
+      .. code-block:: shell
 
-    $> docker exec -it 894aee038c5e bash
+         $> docker exec -it 894aee038c5e bash
 
-You're now inside your container, using an interactive terminal :)
+      This puts you inside your container, using an interactive terminal.
 
-Generate SSL key and certificates
-**********************************
+3. Generate the SSL key and certificates. The paths where the different
+   files are stored are defined after the ``-out`` option in each of the 
+   following commands.
 
-There are 5 steps to this generation. The paths where the different
-files are stored are defined after the ``-out`` option in each command
+    #. Generate a private key for your certificate signing request (CSR):
 
-.. code:: sh
+       .. code-block:: shell
 
-    # Generate a private key for your CSR
-    $> openssl genrsa -out ca.key 2048
-    # Generate a self signed certificate for your local Certificate Authority
-    $> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
+	  $> openssl genrsa -out ca.key 2048
 
-    # Generate a key for Zenko CloudServer
-    $> openssl genrsa -out test.key 2048
-    # Generate a Certificate Signing Request for S3 Server
-    $> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
-    # Generate a local-CA-signed certificate for S3 Server
-    $> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
+    #. Generate a self-signed certificate for your local certificate 
+       authority (CA):
 
-Update Zenko CloudServer ``config.json``
-**********************************
+       .. code:: shell
 
-Add a ``certFilePaths`` section to ``./config.json`` with the
-appropriate paths:
+	  $> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
 
-.. code:: json
+    #. Generate a key for the CloudServer:
+
+       .. code:: shell
+
+          $> openssl genrsa -out test.key 2048
+
+    #. Generate a CSR for CloudServer:
+
+       .. code:: shell
+
+          $> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
+
+    #. Generate a certificate for CloudServer signed by the local CA:
+
+       .. code:: shell
+
+          $> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
+
+4. Update Zenko CloudServer ``config.json``. Add a ``certFilePaths`` 
+   section to ``./config.json`` with appropriate paths:
+
+   .. code:: json
 
         "certFilePaths": {
             "key": "./test.key",
@@ -335,42 +352,36 @@ appropriate paths:
             "ca": "./ca.crt"
         }
 
-Run your container with the new config
-****************************************
+5. Run your container with the new config. 
 
-First, you need to exit your container. Simply run ``$> exit``. Then,
-you need to restart your container. Normally, a simple
-``$> docker restart s3server`` should do the trick.
+   #. Exit the container by running ``$> exit``. 
 
-Update your host config
-^^^^^^^^^^^^^^^^^^^^^^^
+   #. Restart the container with ``$> docker restart cloudserver``.
 
-Associates local IP addresses with hostname
-*******************************************
+6. Update the host configuration by adding s3.scality.test 
+   to /etc/hosts:
 
-In your ``/etc/hosts`` file on Linux, OS X, or Unix (with root
-permissions), edit the line of localhost so it looks like this:
+   .. code:: bash
 
-::
+      127.0.0.1      localhost s3.scality.test
 
-    127.0.0.1      localhost s3.scality.test
+7. Copy the local certificate authority (ca.crt in step 4) from your 
+   container. Choose the path to save this file to (in the present 
+   example, ``/root/ca.crt``), and run:
 
-Copy the local certificate authority from your container
-*********************************************************
+   .. code:: shell
 
-In the above commands, it's the file named ``ca.crt``. Choose the path
-you want to save this file at (here we chose ``/root/ca.crt``), and run
-something like:
+      $> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
 
-.. code:: sh
+   .. note:: Your container ID will be different, and your path to 
+	     ca.crt may be different.
 
-    $> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
+Test the Config
+^^^^^^^^^^^^^^^
 
-Test your config
-^^^^^^^^^^^^^^^^^
+If aws-sdk is not installed, run ``$> yarn install aws-sdk``. 
 
-If you do not have aws-sdk installed, run ``$> npm install aws-sdk``. In
-a ``test.js`` file, paste the following script:
+Paste the following script into a file named "test.js":
 
 .. code:: js
 
@@ -410,8 +421,13 @@ a ``test.js`` file, paste the following script:
         });
     });
 
-Now run that script with ``$> nodejs test.js``. If all goes well, it
-should output ``SSL is cool!``. Enjoy that added security!
+Now run this script with:
+
+.. code::
+
+   $> nodejs test.js
+ 
+On success, the script outputs ``SSL is cool!``.
 
 
 .. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg

docs/INTEGRATIONS.rst

@@ -4,479 +4,415 @@ Integrations
 High Availability
 =================
 
-`Docker swarm <https://docs.docker.com/engine/swarm/>`__ is a
-clustering tool developped by Docker and ready to use with its
-containers. It allows to start a service, which we define and use as a
-means to ensure Zenko CloudServer's continuous availability to the end user.
-Indeed, a swarm defines a manager and n workers among n+1 servers. We
-will do a basic setup in this tutorial, with just 3 servers, which
-already provides a strong service resiliency, whilst remaining easy to
-do as an individual. We will use NFS through docker to share data and
+`Docker Swarm <https://docs.docker.com/engine/swarm/>`__ is a clustering tool
+developed by Docker for use with its containers. It can be used to start
+services, which we define to ensure CloudServer's continuous availability to
+end users. A swarm defines a manager and *n* workers among *n* + 1 servers.
+
+This tutorial shows how to perform a basic setup with three servers, which
+provides strong service resiliency, while remaining easy to use and
+maintain. We will use NFS through Docker to share data and
 metadata between the different servers.
 
-You will see that the steps of this tutorial are defined as **On
-Server**, **On Clients**, **On All Machines**. This refers respectively
-to NFS Server, NFS Clients, or NFS Server and Clients. In our example,
-the IP of the Server will be **10.200.15.113**, while the IPs of the
-Clients will be **10.200.15.96 and 10.200.15.97**
+Sections are labeled **On Server**, **On Clients**, or
+**On All Machines**, referring respectively to NFS server, NFS clients, or
+NFS server and clients. In the present example, the server’s IP address is
+**10.200.15.113** and the client IP addresses are **10.200.15.96** and
+**10.200.15.97**
 
-Installing docker
------------------
+1. Install Docker (on All Machines)
 
-Any version from docker 1.12.6 onwards should work; we used Docker
-17.03.0-ce for this tutorial.
+   Docker 17.03.0-ce is used for this tutorial. Docker 1.12.6 and later will
+   likely work, but is not tested.
 
-On All Machines
-~~~~~~~~~~~~~~~
+   * On Ubuntu 14.04
+     Install Docker CE for Ubuntu as `documented at Docker
+     <https://docs.docker.com/install/linux/docker-ce/ubuntu/>`__.
+     Install the aufs dependency as recommended by Docker. The required
+     commands are:
 
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
+     .. code:: sh
 
-The docker website has `solid
-documentation <https://docs.docker.com/engine/installation/linux/ubuntu/>`__.
-We have chosen to install the aufs dependency, as recommended by Docker.
-Here are the required commands:
+      $> sudo apt-get update
+      $> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
+      $> sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
+      $> curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+      $> sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+      $> sudo apt-get update
+      $> sudo apt-get install docker-ce
+
+    * On CentOS 7
+      Install Docker CE as `documented at Docker
+      <https://docs.docker.com/install/linux/docker-ce/centos/>`__.
+      The required commands are:
+
+      .. code:: sh
+
+        $> sudo yum install -y yum-utils
+        $> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+        $> sudo yum makecache fast
+        $> sudo yum install docker-ce
+        $> sudo systemctl start docker
+
+2. Install NFS on Client(s)
+
+   NFS clients mount Docker volumes over the NFS server’s shared folders.
+   If the NFS commons are installed, manual mounts are no longer needed.
+
+   * On Ubuntu 14.04
+
+     Install the NFS commons with apt-get:
+
+     .. code:: sh
+
+       $> sudo apt-get install nfs-common
+
+   * On CentOS 7
+
+     Install the NFS utils; then start required services:
+
+     .. code:: sh
+
+      $> yum install nfs-utils
+      $> sudo systemctl enable rpcbind
+      $> sudo systemctl enable nfs-server
+      $> sudo systemctl enable nfs-lock
+      $> sudo systemctl enable nfs-idmap
+      $> sudo systemctl start rpcbind
+      $> sudo systemctl start nfs-server
+      $> sudo systemctl start nfs-lock
+      $> sudo systemctl start nfs-idmap
+
+3. Install NFS (on Server)
+
+   The NFS server hosts the data and metadata. The package(s) to install on it
+   differs from the package installed on the clients.
+
+   * On Ubuntu 14.04
+
+     Install the NFS server-specific package and the NFS commons:
+
+     .. code:: sh
+
+      $> sudo apt-get install nfs-kernel-server nfs-common
+
+   * On CentOS 7
+
+     Install the NFS utils and start the required services:
+
+     .. code:: sh
+
+       $> yum install nfs-utils
+       $> sudo systemctl enable rpcbind
+       $> sudo systemctl enable nfs-server
+       $> sudo systemctl enable nfs-lock
+       $> sudo systemctl enable nfs-idmap
+       $> sudo systemctl start rpcbind
+       $> sudo systemctl start nfs-server
+       $> sudo systemctl start nfs-lock
+       $> sudo systemctl start nfs-idmap
+
+   For both distributions:
+
+   #. Choose where shared data and metadata from the local
+      `CloudServer <http://www.zenko.io/cloudserver/>`__ shall be stored (The
+      present example uses /var/nfs/data and /var/nfs/metadata). Set permissions
+      for these folders for
+      sharing over NFS:
+
+      .. code:: sh
+
+        $> mkdir -p /var/nfs/data /var/nfs/metadata
+        $> chmod -R 777 /var/nfs/
+
+   #. The /etc/exports file configures network permissions and r-w-x permissions
+      for NFS access. Edit /etc/exports, adding the following lines:
+
+      .. code:: sh
+
+        /var/nfs/data        10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
+        /var/nfs/metadata    10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
+
+      Ubuntu applies the no\_subtree\_check option by default, so both
+      folders are declared with the same permissions, even though they’re in
+      the same tree.
+
+   #. Export this new NFS table:
+
+      .. code:: sh
+
+        $> sudo exportfs -a
+
+   #. Edit the ``MountFlags`` option in the Docker config in
+      /lib/systemd/system/docker.service to enable NFS mount from Docker volumes
+      on other machines:
+
+      .. code:: sh
+
+        MountFlags=shared
+
+   #. Restart the NFS server and Docker daemons to apply these changes.
+
+      * On Ubuntu 14.04
+
+        .. code:: sh
+
+          $> sudo service nfs-kernel-server restart
+          $> sudo service docker restart
+
+      * On CentOS 7
+
+        .. code:: sh
+
+         $> sudo systemctl restart nfs-server
+         $> sudo systemctl daemon-reload
+         $> sudo systemctl restart docker
+
+
+4. Set Up a Docker Swarm
+
+  * On all machines and distributions:
+
+    Set up the Docker volumes to be mounted to the NFS server for CloudServer’s
+    data and metadata storage. The following commands must be replicated on all
+    machines:
+
+    .. code:: sh
+
+     $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
+     $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
+
+    There is no need to ``docker exec`` these volumes to mount them: the
+    Docker Swarm manager does this when the Docker service is started.
+
+  * On a server:
+
+    To start a Docker service on a Docker Swarm cluster, initialize the cluster
+    (that is, define a manager), prompt workers/nodes to join in, and then start
+    the service.
+
+    Initialize the swarm cluster, and review its response:
+
+    .. code:: sh
+
+      $> docker swarm init --advertise-addr 10.200.15.113
+
+      Swarm initialized: current node (db2aqfu3bzfzzs9b1kfeaglmq) is now a manager.
+
+      To add a worker to this swarm, run the following command:
+
+      docker swarm join \
+      --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka \
+      10.200.15.113:2377
+
+      To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
+
+  * On clients:
+
+    Copy and paste the command provided by your Docker Swarm init. A successful
+    request/response will resemble:
+
+    .. code:: sh
+
+      $> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
+
+      This node joined a swarm as a worker.
+
+Set Up Docker Swarm on Clients on a Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Start the service on the Swarm cluster.
 
 .. code:: sh
 
-    $> sudo apt-get update
-    $> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
-    $> sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
-    $> curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-    $> sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-    $> sudo apt-get update
-    $> sudo apt-get install docker-ce
+  $> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/cloudserver
 
-On CentOS 7
-^^^^^^^^^^^
-
-The docker website has `solid
-documentation <https://docs.docker.com/engine/installation/linux/centos/>`__.
-Here are the required commands:
-
-.. code:: sh
-
-    $> sudo yum install -y yum-utils
-    $> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-    $> sudo yum makecache fast
-    $> sudo yum install docker-ce
-    $> sudo systemctl start docker
-
-Configure NFS
--------------
-
-On Clients
-~~~~~~~~~~
-
-Your NFS Clients will mount Docker volumes over your NFS Server's shared
-folders. Hence, you don't have to mount anything manually, you just have
-to install the NFS commons:
-
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
-
-Simply install the NFS commons:
-
-.. code:: sh
-
-    $> sudo apt-get install nfs-common
-
-On CentOS 7
-^^^^^^^^^^^
-
-Install the NFS utils, and then start the required services:
-
-.. code:: sh
-
-    $> yum install nfs-utils
-    $> sudo systemctl enable rpcbind
-    $> sudo systemctl enable nfs-server
-    $> sudo systemctl enable nfs-lock
-    $> sudo systemctl enable nfs-idmap
-    $> sudo systemctl start rpcbind
-    $> sudo systemctl start nfs-server
-    $> sudo systemctl start nfs-lock
-    $> sudo systemctl start nfs-idmap
-
-On Server
-~~~~~~~~~
-
-Your NFS Server will be the machine to physically host the data and
-metadata. The package(s) we will install on it is slightly different
-from the one we installed on the clients.
-
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
-
-Install the NFS server specific package and the NFS commons:
-
-.. code:: sh
-
-    $> sudo apt-get install nfs-kernel-server nfs-common
-
-On CentOS 7
-^^^^^^^^^^^
-
-Same steps as with the client: install the NFS utils and start the
-required services:
-
-.. code:: sh
-
-    $> yum install nfs-utils
-    $> sudo systemctl enable rpcbind
-    $> sudo systemctl enable nfs-server
-    $> sudo systemctl enable nfs-lock
-    $> sudo systemctl enable nfs-idmap
-    $> sudo systemctl start rpcbind
-    $> sudo systemctl start nfs-server
-    $> sudo systemctl start nfs-lock
-    $> sudo systemctl start nfs-idmap
-
-On Ubuntu 14.04 and CentOS 7
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Choose where your shared data and metadata from your local `Zenko CloudServer
-<http://www.zenko.io/cloudserver/>`__ will be stored.
-We chose to go with /var/nfs/data and /var/nfs/metadata. You also need
-to set proper sharing permissions for these folders as they'll be shared
-over NFS:
-
-.. code:: sh
-
-    $> mkdir -p /var/nfs/data /var/nfs/metadata
-    $> chmod -R 777 /var/nfs/
-
-Now you need to update your **/etc/exports** file. This is the file that
-configures network permissions and rwx permissions for NFS access. By
-default, Ubuntu applies the no\_subtree\_check option, so we declared
-both folders with the same permissions, even though they're in the same
-tree:
-
-.. code:: sh
-
-    $> sudo vim /etc/exports
-
-In this file, add the following lines:
-
-.. code:: sh
-
-    /var/nfs/data        10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
-    /var/nfs/metadata    10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
-
-Export this new NFS table:
-
-.. code:: sh
-
-    $> sudo exportfs -a
-
-Eventually, you need to allow for NFS mount from Docker volumes on other
-machines. You need to change the Docker config in
-**/lib/systemd/system/docker.service**:
-
-.. code:: sh
-
-    $> sudo vim /lib/systemd/system/docker.service
-
-In this file, change the **MountFlags** option:
-
-.. code:: sh
-
-    MountFlags=shared
-
-Now you just need to restart the NFS server and docker daemons so your
-changes apply.
-
-On Ubuntu 14.04
-^^^^^^^^^^^^^^^
-
-Restart your NFS Server and docker services:
-
-.. code:: sh
-
-    $> sudo service nfs-kernel-server restart
-    $> sudo service docker restart
-
-On CentOS 7
-^^^^^^^^^^^
-
-Restart your NFS Server and docker daemons:
-
-.. code:: sh
-
-    $> sudo systemctl restart nfs-server
-    $> sudo systemctl daemon-reload
-    $> sudo systemctl restart docker
-
-Set up your Docker Swarm service
---------------------------------
-
-On All Machines
-~~~~~~~~~~~~~~~
-
-On Ubuntu 14.04 and CentOS 7
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-We will now set up the Docker volumes that will be mounted to the NFS
-Server and serve as data and metadata storage for Zenko CloudServer. These two
-commands have to be replicated on all machines:
-
-.. code:: sh
-
-    $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
-    $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
-
-There is no need to ""docker exec" these volumes to mount them: the
-Docker Swarm manager will do it when the Docker service will be started.
-
-On Server
-^^^^^^^^^
-
-To start a Docker service on a Docker Swarm cluster, you first have to
-initialize that cluster (i.e.: define a manager), then have the
-workers/nodes join in, and then start the service. Initialize the swarm
-cluster, and look at the response:
-
-.. code:: sh
-
-    $> docker swarm init --advertise-addr 10.200.15.113
-
-    Swarm initialized: current node (db2aqfu3bzfzzs9b1kfeaglmq) is now a manager.
-
-    To add a worker to this swarm, run the following command:
-
-        docker swarm join \
-        --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka \
-        10.200.15.113:2377
-
-    To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
-
-On Clients
-^^^^^^^^^^
-
-Simply copy/paste the command provided by your docker swarm init. When
-all goes well, you'll get something like this:
-
-.. code:: sh
-
-    $> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
-
-    This node joined a swarm as a worker.
-
-On Server
-^^^^^^^^^
-
-Start the service on your swarm cluster!
-
-.. code:: sh
-
-    $> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/s3server
-
-If you run a docker service ls, you should have the following output:
+On a successful installation, ``docker service ls`` returns the following
+output:
 
 .. code:: sh
 
     $> docker service ls
     ID            NAME  MODE        REPLICAS  IMAGE
-    ocmggza412ft  s3    replicated  1/1       scality/s3server:latest
+    ocmggza412ft  s3    replicated  1/1       scality/cloudserver:latest
 
-If your service won't start, consider disabling apparmor/SELinux.
+If the service does not start, consider disabling apparmor/SELinux.
 
-Testing your High Availability S3Server
----------------------------------------
+Testing the High-Availability CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-On All Machines
-~~~~~~~~~~~~~~~
-
-On Ubuntu 14.04 and CentOS 7
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Try to find out where your Scality Zenko CloudServer is actually running using
-the **docker ps** command. It can be on any node of the swarm cluster,
-manager or worker. When you find it, you can kill it, with **docker stop
-<container id>** and you'll see it respawn on a different node of the
-swarm cluster. Now you see, if one of your servers falls, or if docker
-stops unexpectedly, your end user will still be able to access your
-local Zenko CloudServer.
+On all machines (client/server) and distributions (Ubuntu and CentOS),
+determine where CloudServer is running using ``docker ps``. CloudServer can
+operate on any node of the Swarm cluster, manager or worker. When you find
+it, you can kill it with ``docker stop <container id>``. It will respawn
+on a different node. Now, if one server falls, or if Docker stops
+unexpectedly, the end user will still be able to access your the local CloudServer.
 
 Troubleshooting
----------------
+~~~~~~~~~~~~~~~
 
-To troubleshoot the service you can run:
+To troubleshoot the service, run:
 
 .. code:: sh
 
     $> docker service ps s3docker service ps s3
     ID                         NAME      IMAGE             NODE                               DESIRED STATE  CURRENT STATE       ERROR
-    0ar81cw4lvv8chafm8pw48wbc  s3.1      scality/s3server  localhost.localdomain.localdomain  Running        Running 7 days ago
-    cvmf3j3bz8w6r4h0lf3pxo6eu   \_ s3.1  scality/s3server  localhost.localdomain.localdomain  Shutdown       Failed 7 days ago   "task: non-zero exit (137)"
+    0ar81cw4lvv8chafm8pw48wbc  s3.1      scality/cloudserver  localhost.localdomain.localdomain  Running        Running 7 days ago
+    cvmf3j3bz8w6r4h0lf3pxo6eu   \_ s3.1  scality/cloudserver  localhost.localdomain.localdomain  Shutdown       Failed 7 days ago   "task: non-zero exit (137)"
 
-If the error is truncated it is possible to have a more detailed view of
-the error by inspecting the docker task ID:
+If the error is truncated, view the error in detail by inspecting the
+Docker task ID:
 
 .. code:: sh
 
     $> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu
 
 Off you go!
------------
-
-Let us know what you use this functionality for, and if you'd like any
-specific developments around it. Or, even better: come and contribute to
-our `Github repository <https://github.com/scality/s3/>`__! We look
-forward to meeting you!
+~~~~~~~~~~~
 
+Let us know how you use this and if you'd like any specific developments 
+around it. Even better: come and contribute to our `Github repository 
+<https://github.com/scality/s3/>`__! We look forward to meeting you!
 
 S3FS
 ====
-Export your buckets as a filesystem with s3fs on top of Zenko CloudServer
+
+You can export buckets as a filesystem with s3fs on CloudServer.
 
 `s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source
-tool that allows you to mount an S3 bucket on a filesystem-like backend.
-It is available both on Debian and RedHat distributions. For this
-tutorial, we used an Ubuntu 14.04 host to deploy and use s3fs over
-Scality's Zenko CloudServer.
+tool, available both on Debian and RedHat distributions, that enables
+you to mount an S3 bucket on a filesystem-like backend. This tutorial uses
+an Ubuntu 14.04 host to deploy and use s3fs over CloudServer.
 
 Deploying Zenko CloudServer with SSL
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-First, you need to deploy **Zenko CloudServer**. This can be done very easily
-via `our DockerHub
-page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
-with a file backend).
+First, deploy CloudServer with a file backend using `our DockerHub page
+<https://hub.docker.com/r/zenko/cloudserver>`__.
 
-    *Note:* *- If you don't have docker installed on your machine, here
-    are the `instructions to install it for your
-    distribution <https://docs.docker.com/engine/installation/>`__*
+.. note::
 
-You also necessarily have to set up SSL with Zenko CloudServer to use s3fs. We
-have a nice
-`tutorial <https://s3.scality.com/v1.0/page/scality-with-ssl>`__ to help
-you do it.
+  If Docker is not installed on your machine, follow
+  `these instructions <https://docs.docker.com/engine/installation/>`__
+  to install it for your distribution.
 
-s3fs setup
-----------
+You must also set up SSL with CloudServer to use s3fs. See `Using SSL
+<./GETTING_STARTED#Using_SSL>`__ for instructions.
+
+s3fs Setup
+~~~~~~~~~~
 
 Installing s3fs
-~~~~~~~~~~~~~~~
+---------------
 
-s3fs has quite a few dependencies. As explained in their
-`README <https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation>`__,
-the following commands should install everything for Ubuntu 14.04:
+Follow the instructions in the s3fs `README
+<https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation-from-pre-built-packages>`__,
+
+Check that s3fs is properly installed. A version check should return
+a response resembling:
 
 .. code:: sh
 
-    $> sudo apt-get install automake autotools-dev g++ git libcurl4-gnutls-dev
-    $> sudo apt-get install libfuse-dev libssl-dev libxml2-dev make pkg-config
-
-Now you want to install s3fs per se:
-
-.. code:: sh
-
-    $> git clone https://github.com/s3fs-fuse/s3fs-fuse.git
-    $> cd s3fs-fuse
-    $> ./autogen.sh
-    $> ./configure
-    $> make
-    $> sudo make install
-
-Check that s3fs is properly installed by checking its version. it should
-answer as below:
-
-.. code:: sh
-
-     $> s3fs --version
+    $> s3fs --version
 
     Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL
+    Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
+    License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>
+    This is free software: you are free to change and redistribute it.
+    There is NO WARRANTY, to the extent permitted by law.
 
 Configuring s3fs
-~~~~~~~~~~~~~~~~
+----------------
 
 s3fs expects you to provide it with a password file. Our file is
 ``/etc/passwd-s3fs``. The structure for this file is
-``ACCESSKEYID:SECRETKEYID``, so, for S3Server, you can run:
+``ACCESSKEYID:SECRETKEYID``, so, for CloudServer, you can run:
 
 .. code:: sh
 
     $> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs
     $> chmod 600 /etc/passwd-s3fs
 
-Using Zenko CloudServer with s3fs
-------------------------
+Using CloudServer with s3fs
+---------------------------
 
-First, you're going to need a mountpoint; we chose ``/mnt/tests3fs``:
+1. Use /mnt/tests3fs as a mount point.
 
-.. code:: sh
+   .. code:: sh
 
     $> mkdir /mnt/tests3fs
 
-Then, you want to create a bucket on your local Zenko CloudServer; we named it
-``tests3fs``:
+2. Create a bucket on your local CloudServer. In the present example it is
+   named “tests3fs”.
 
-.. code:: sh
+   .. code:: sh
 
     $> s3cmd mb s3://tests3fs
 
-    *Note:* *- If you've never used s3cmd with our Zenko CloudServer, our README
-    provides you with a `recommended
-    config <https://github.com/scality/S3/blob/master/README.md#s3cmd>`__*
+3. Mount the bucket to your mount point with s3fs:
 
-Now you can mount your bucket to your mountpoint with s3fs:
-
-.. code:: sh
+   .. code:: sh
 
     $> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style
 
-    *If you're curious, the structure of this command is*
-    ``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``\ *, and the
-    options are mandatory and serve the following purposes:
-    * ``passwd_file``\ *: specifiy path to password file;
-    * ``url``\ *: specify the hostname used by your SSL provider;
-    * ``use_path_request_style``\ *: force path style (by default, s3fs
-    uses subdomains (DNS style)).*
+   The structure of this command is:
+   ``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``. Of these mandatory
+   options:
 
-| From now on, you can either add files to your mountpoint, or add
-  objects to your bucket, and they'll show in the other.
-| For example, let's' create two files, and then a directory with a file
-  in our mountpoint:
+   * ``passwd_file`` specifies the path to the password file.
+   * ``url`` specifies the host name used by your SSL provider.
+   * ``use_path_request_style`` forces the path style (by default,
+       s3fs uses DNS-style subdomains).
 
-.. code:: sh
+Once the bucket is mounted, files added to the mount point or
+objects added to the bucket will appear in both locations.
 
-    $> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
-    $> mkdir /mnt/tests3fs/dir1
-    $> touch /mnt/tests3fs/dir1/file3
+Example
+-------
 
-Now, I can use s3cmd to show me what is actually in S3Server:
+   Create two files, and then a directory with a file in our mount point:
 
-.. code:: sh
+   .. code:: sh
 
-    $> s3cmd ls -r s3://tests3fs
+      $> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
+      $> mkdir /mnt/tests3fs/dir1
+      $> touch /mnt/tests3fs/dir1/file3
 
-    2017-02-28 17:28         0   s3://tests3fs/dir1/
-    2017-02-28 17:29         0   s3://tests3fs/dir1/file3
-    2017-02-28 17:28         0   s3://tests3fs/file1
-    2017-02-28 17:28         0   s3://tests3fs/file2
+   Now, use s3cmd to show what is in CloudServer:
 
-Now you can enjoy a filesystem view on your local Zenko CloudServer!
+   .. code:: sh
+
+      $> s3cmd ls -r s3://tests3fs
+
+      2017-02-28 17:28         0   s3://tests3fs/dir1/
+      2017-02-28 17:29         0   s3://tests3fs/dir1/file3
+      2017-02-28 17:28         0   s3://tests3fs/file1
+      2017-02-28 17:28         0   s3://tests3fs/file2
+
+   Now you can enjoy a filesystem view on your local CloudServer.
 
 
 Duplicity
 =========
 
-How to backup your files with Zenko CloudServer.
+How to back up your files with CloudServer.
 
-Installing
------------
-
-Installing Duplicity and its dependencies
+Installing Duplicity and its Dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Second, you want to install
-`Duplicity <http://duplicity.nongnu.org/index.html>`__. You have to
-download `this
-tarball <https://code.launchpad.net/duplicity/0.7-series/0.7.11/+download/duplicity-0.7.11.tar.gz>`__,
-decompress it, and then checkout the README inside, which will give you
-a list of dependencies to install. If you're using Ubuntu 14.04, this is
-your lucky day: here is a lazy step by step install.
+To install `Duplicity <http://duplicity.nongnu.org/>`__,
+go to `this site <https://code.launchpad.net/duplicity/0.7-series>`__.
+Download the latest tarball. Decompress it and follow the instructions
+in the README.
+
+.. code:: sh
+
+   $> tar zxvf duplicity-0.7.11.tar.gz
+   $> cd duplicity-0.7.11
+   $> python setup.py install
+
+You may receive error messages indicating the need to install some or all
+of the following dependencies:
 
 .. code:: sh
 
@@ -484,30 +420,20 @@ your lucky day: here is a lazy step by step install.
     $> apt-get install python-dev python-pip python-lockfile
     $> pip install -U boto
 
-Then you want to actually install Duplicity:
+Testing the Installation
+------------------------
 
-.. code:: sh
+1. Check that CloudServer is running. Run ``$> docker ps``. You should
+   see one container named ``scality/cloudserver``. If you do not, run
+   ``$> docker start cloudserver`` and check again.
 
-    $> tar zxvf duplicity-0.7.11.tar.gz
-    $> cd duplicity-0.7.11
-    $> python setup.py install
 
-Using
-------
-
-Testing your installation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-First, we're just going to quickly check that Zenko CloudServer is actually
-running. To do so, simply run ``$> docker ps`` . You should see one
-container named ``scality/s3server``. If that is not the case, try
-``$> docker start s3server``, and check again.
-
-Secondly, as you probably know, Duplicity uses a module called **Boto**
-to send requests to S3. Boto requires a configuration file located in
-**``/etc/boto.cfg``** to have your credentials and preferences. Here is
-a minimalistic config `that you can finetune following these
-instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
+2. Duplicity uses a module called “Boto” to send requests to S3. Boto
+   requires a configuration file located in ``/etc/boto.cfg`` to store
+   your credentials and preferences. A minimal configuration
+   you can fine tune `following these instructions
+   <http://boto.cloudhackers.com/en/latest/getting_started.html>`__ is
+   shown here:
 
 ::
 
@@ -521,54 +447,51 @@ instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
     # If using SSL, unmute and provide absolute path to local CA certificate
     # ca_certificates_file = /absolute/path/to/ca.crt
 
-    *Note:* *If you want to set up SSL with Zenko CloudServer, check out our
-    `tutorial <http://link/to/SSL/tutorial>`__*
+ .. note:: To set up SSL with CloudServer, check out our `Using SSL
+	   <./GETTING_STARTED#Using_SSL>`__ in GETTING STARTED.
 
-At this point, we've met all the requirements to start running Zenko CloudServer
-as a backend to Duplicity. So we should be able to back up a local
-folder/file to local S3. Let's try with the duplicity decompressed
-folder:
+3. At this point all requirements to run CloudServer as a backend to Duplicity
+   have been met. A local folder/file should back up to the local S3.
+   Try it with the decompressed Duplicity folder:
 
 .. code:: sh
 
     $> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/"
 
-    *Note:* *Duplicity will prompt you for a symmetric encryption
-    passphrase. Save it somewhere as you will need it to recover your
-    data. Alternatively, you can also add the ``--no-encryption`` flag
-    and the data will be stored plain.*
+.. note:: Duplicity will prompt for a symmetric encryption passphrase.
+	  Save it carefully, as you will need it to recover your data.
+	  Alternatively, you can add the ``--no-encryption`` flag
+	  and the data will be stored plain.
 
-If this command is succesful, you will get an output looking like this:
+   If this command is successful, you will receive an output resembling:
 
-::
+   .. code:: sh
 
-    --------------[ Backup Statistics ]--------------
-    StartTime 1486486547.13 (Tue Feb  7 16:55:47 2017)
-    EndTime 1486486547.40 (Tue Feb  7 16:55:47 2017)
-    ElapsedTime 0.27 (0.27 seconds)
-    SourceFiles 388
-    SourceFileSize 6634529 (6.33 MB)
-    NewFiles 388
-    NewFileSize 6634529 (6.33 MB)
-    DeletedFiles 0
-    ChangedFiles 0
-    ChangedFileSize 0 (0 bytes)
-    ChangedDeltaSize 0 (0 bytes)
-    DeltaEntries 388
-    RawDeltaSize 6392865 (6.10 MB)
-    TotalDestinationSizeChange 2003677 (1.91 MB)
-    Errors 0
-    -------------------------------------------------
+      --------------[ Backup Statistics ]--------------
+      StartTime 1486486547.13 (Tue Feb  7 16:55:47 2017)
+      EndTime 1486486547.40 (Tue Feb  7 16:55:47 2017)
+      ElapsedTime 0.27 (0.27 seconds)
+      SourceFiles 388
+      SourceFileSize 6634529 (6.33 MB)
+      NewFiles 388
+      NewFileSize 6634529 (6.33 MB)
+      DeletedFiles 0
+      ChangedFiles 0
+      ChangedFileSize 0 (0 bytes)
+      ChangedDeltaSize 0 (0 bytes)
+      DeltaEntries 388
+      RawDeltaSize 6392865 (6.10 MB)
+      TotalDestinationSizeChange 2003677 (1.91 MB)
+      Errors 0
+      -------------------------------------------------
 
-Congratulations! You can now backup to your local S3 through duplicity
-:)
+Congratulations! You can now back up to your local S3 through Duplicity.
 
-Automating backups
-~~~~~~~~~~~~~~~~~~~
+Automating Backups
+------------------
 
-Now you probably want to back up your files periodically. The easiest
-way to do this is to write a bash script and add it to your crontab.
-Here is my suggestion for such a file:
+The easiest way to back up files periodically is to write a bash script
+and add it to your crontab. A suggested script follows.
 
 .. code:: sh
 
@@ -577,33 +500,33 @@ Here is my suggestion for such a file:
     # Export your passphrase so you don't have to type anything
     export PASSPHRASE="mypassphrase"
 
-    # If you want to use a GPG Key, put it here and unmute the line below
+    # To use a GPG key, put it here and uncomment the line below
     #GPG_KEY=
 
     # Define your backup bucket, with localhost specified
-    DEST="s3://127.0.0.1:8000/testbuckets3server/"
+    DEST="s3://127.0.0.1:8000/testbucketcloudserver/"
 
-    # Define the absolute path to the folder you want to backup
+    # Define the absolute path to the folder to back up
     SOURCE=/root/testfolder
 
     # Set to "full" for full backups, and "incremental" for incremental backups
-    # Warning: you have to perform one full backup befor you can perform
+    # Warning: you must perform one full backup befor you can perform
     # incremental ones on top of it
     FULL=incremental
 
-    # How long to keep backups for; if you don't want to delete old
-    # backups, keep empty; otherwise, syntax is "1Y" for one year, "1M"
-    # for one month, "1D" for one day
+    # How long to keep backups. If you don't want to delete old backups, keep
+    # this value empty; otherwise, the syntax is "1Y" for one year, "1M" for
+    # one month, "1D" for one day.
     OLDER_THAN="1Y"
 
-    # is_running checks whether duplicity is currently completing a task
+    # is_running checks whether Duplicity is currently completing a task
     is_running=$(ps -ef | grep duplicity  | grep python | wc -l)
 
-    # If duplicity is already completing a task, this will simply not run
+    # If Duplicity is already completing a task, this will not run
     if [ $is_running -eq 0 ]; then
         echo "Backup for ${SOURCE} started"
 
-        # If you want to delete backups older than a certain time, we do it here
+        # To delete backups older than a certain time, do it here
         if [ "$OLDER_THAN" != "" ]; then
             echo "Removing backups older than ${OLDER_THAN}"
             duplicity remove-older-than ${OLDER_THAN} ${DEST}
@@ -626,17 +549,17 @@ Here is my suggestion for such a file:
     # Forget the passphrase...
     unset PASSPHRASE
 
-So let's say you put this file in ``/usr/local/sbin/backup.sh.`` Next
-you want to run ``crontab -e`` and paste your configuration in the file
-that opens. If you're unfamiliar with Cron, here is a good `How
-To <https://help.ubuntu.com/community/CronHowto>`__. The folder I'm
-backing up is a folder I modify permanently during my workday, so I want
-incremental backups every 5mn from 8AM to 9PM monday to friday. Here is
-the line I will paste in my crontab:
+Put this file in ``/usr/local/sbin/backup.sh``. Run ``crontab -e`` and
+paste your configuration into the file that opens. If you're unfamiliar
+with Cron, here is a good `HowTo
+<https://help.ubuntu.com/community/CronHowto>`__. If the folder being
+backed up is a folder to be modified permanently during the work day,
+we can set incremental backups every 5 minutes from 8 AM to 9 PM Monday
+through Friday by pasting the following line into crontab:
 
-.. code:: cron
+.. code:: sh
 
     */5 8-20 * * 1-5 /usr/local/sbin/backup.sh
 
-Now I can try and add / remove files from the folder I'm backing up, and
-I will see incremental backups in my bucket.
+Adding or removing files from the folder being backed up will result in
+incremental backups in the bucket.

docs/MD_SEARCH.rst

@@ -0,0 +1,263 @@
+Metadata Search Documentation
+=============================
+
+Description
+-----------
+
+This feature enables metadata search to be performed on the metadata of objects
+stored in Zenko.
+
+Requirements
+------------
+
+* MongoDB
+
+Design
+------
+
+The Metadata Search feature expands on the existing :code:`GET Bucket` S3 API by
+enabling users to conduct metadata searches by adding the custom Zenko query
+string parameter, :code:`search`. The :code:`search` parameter is structured as a pseudo
+SQL WHERE clause, and supports basic SQL operators. For example:
+:code:`"A=1 AND B=2 OR C=3"` (complex queries can be built using nesting
+operators, :code:`(` and :code:`)`).
+
+The search process is as follows:
+
+* Zenko receives a :code:`GET` request.
+
+  .. code::
+
+    # regular getBucket request
+    GET /bucketname HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # getBucket versions request
+    GET /bucketname?versions HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # search getBucket request
+    GET /bucketname?search=key%3Dsearch-item HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+* If the request does *not* contain the :code:`search` query parameter, Zenko performs
+  a normal bucket listing and returns an XML result containing the list of
+  objects.
+* If the request *does* contain the :code:`search` query parameter, Zenko parses and
+  validates the search string.
+
+  - If the search string is invalid, Zenko returns an :code:`InvalidArgument` error.
+
+    .. code::
+
+      <?xml version=\"1.0\" encoding=\"UTF-8\"?>
+      <Error>
+        <Code>InvalidArgument</Code>
+        <Message>Invalid sql where clause sent as search query</Message>
+        <Resource></Resource>
+        <RequestId>d1d6afc64345a8e1198e</RequestId>
+      </Error>
+
+  - If the search string is valid, Zenko parses it and generates an abstract
+    syntax tree (AST). The AST is then passed to the MongoDB backend to be
+    used as the query filter for retrieving objects from a bucket that
+    satisfies the requested search conditions. Zenko parses the filtered
+    results and returns them as the response.
+
+Metadata search results have the same structure as a :code:`GET Bucket` response:
+
+.. code:: xml
+
+  <?xml version="1.0" encoding="UTF-8"?>
+  <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+      <Name>bucketname</Name>
+      <Prefix/>
+      <Marker/>
+      <MaxKeys>1000</MaxKeys>
+      <IsTruncated>false</IsTruncated>
+      <Contents>
+          <Key>objectKey</Key>
+          <LastModified>2018-04-19T18:31:49.426Z</LastModified>
+          <ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
+          <Size>0</Size>
+          <Owner>
+              <ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
+              <DisplayName>Bart</DisplayName>
+          </Owner>
+          <StorageClass>STANDARD</StorageClass>
+      </Contents>
+      <Contents>
+          ...
+      </Contents>
+  </ListBucketResult>
+
+Performing Metadata Searches with Zenko
+---------------------------------------
+
+You can perform metadata searches by:
+
++ Using the :code:`search_bucket` tool in the
+  `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository.
++ Creating a signed HTTP request to Zenko in your preferred programming
+  language.
+
+Using the S3 Tool
++++++++++++++++++
+
+After cloning the `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository
+and installing the necessary dependencies, run the following command in the S3
+project’s root directory to access the search tool:
+
+.. code::
+
+  node bin/search_bucket
+
+This generates the following output:
+
+.. code::
+
+    Usage: search_bucket [options]
+
+    Options:
+
+      -V, --version                 output the version number
+      -a, --access-key <accessKey>  Access key id
+      -k, --secret-key <secretKey>  Secret access key
+      -b, --bucket <bucket>         Name of the bucket
+      -q, --query <query>           Search query
+      -h, --host <host>             Host of the server
+      -p, --port <port>             Port of the server
+      -s                            --ssl
+      -v, --verbose
+      -h, --help                    output usage information
+
+In the following examples, Zenko Server is accessible on endpoint
+:code:`http://127.0.0.1:8000` and contains the bucket :code:`zenkobucket`.
+
+.. code::
+
+    # search for objects with metadata "blue"
+    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+        -q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
+
+    # search for objects tagged with "type=color"
+    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+        -q "tags.type=color" -h 127.0.0.1 -p 8000
+
+Coding Examples
++++++++++++++++
+
+Search requests can be also performed by making HTTP requests authenticated
+with one of the AWS Signature schemes: version 2 or version 4. \
+For more about authentication scheme, see:
+
+* https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
+* http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
+* http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+
+You can also view examples for making requests with Auth V4 in various
+languages `here <../../../examples>`__.
+
+Specifying Metadata Fields
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To search system metadata headers:
+
+.. code::
+
+    {system-metadata-key}{supported SQL op}{search value}
+    # example
+    key = blueObject
+    size > 0
+    key LIKE "blue.*"
+
+To search custom user metadata:
+
+.. code::
+
+    # metadata must be prefixed with "x-amz-meta-"
+    x-amz-meta-{user-metadata-key}{supported SQL op}{search value}
+
+    # example
+    x-amz-meta-color = blue
+    x-amz-meta-color != red
+    x-amz-meta-color LIKE "b.*"
+
+To search tags:
+
+.. code::
+
+    # tag searches must be prefixed with "tags."
+    tags.{tag-key}{supported SQL op}{search value}
+    # example
+    tags.type = color
+
+Examples queries:
+
+.. code::
+
+    # searching for objects with custom metadata "color"=blue" and are tagged
+    # "type"="color"
+
+    tags.type="color" AND  x-amz-meta-color="blue"
+
+    # searching for objects with the object key containing the substring "blue"
+    # or (custom metadata "color"=blue" and are tagged "type"="color")
+
+    key LIKE '.*blue.*' OR (x-amz-meta-color="blue" AND tags.type="color")
+
+Differences from SQL
+++++++++++++++++++++
+
+Zenko metadata search queries are similar to SQL-query :code:`WHERE` clauses, but
+differ in that:
+
+* They follow the :code:`PCRE` format
+* They do not require values with hyphens to be enclosed in
+  backticks, :code:``(`)``
+
+  .. code::
+
+        # SQL query
+        `x-amz-meta-search-item` = `ice-cream-cone`
+
+        # MD Search query
+        x-amz-meta-search-item = ice-cream-cone
+
+* Search queries do not support all SQL operators.
+
+  .. code::
+
+    # Supported SQL operators:
+    =, <, >, <=, >=, !=, AND, OR, LIKE, <>
+
+    # Unsupported SQL operators:
+    NOT, BETWEEN, IN, IS, +, -, %, ^, /, *, !
+
+Using Regular Expressions in Metadata Search
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Regular expressions in Zenko metadata search differ from SQL in the following
+ways:
+
++ Wildcards are represented with :code:`.*` instead of :code:`%`.
++ Regex patterns must be wrapped in quotes. Failure to do this can lead to
+  misinterpretation of patterns.
++ As with :code:`PCRE`, regular expressions can be entered in either the
+  :code:`/pattern/` syntax or as the pattern itself if regex options are
+  not required.
+
+Example regular expressions:
+
+.. code::
+
+    # search for strings containing word substring "helloworld"
+        ".*helloworld.*"
+        "/.*helloworld.*/"
+        "/.*helloworld.*/i"

docs/Makefile

@@ -0,0 +1,21 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Zenko
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+

docs/RELEASE.md

@@ -0,0 +1,73 @@
+# Cloudserver Release Plan
+
+## Docker Image Generation
+
+Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
+CloudServer has a few images there:
+
+* Cloudserver container image: ghcr.io/scality/cloudserver
+* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
+* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
+
+With every CI build, the CI will push images, tagging the
+content with the developer branch's short SHA-1 commit hash.
+This allows those images to be used by developers, CI builds,
+build chain and so on.
+
+Tagged versions of cloudserver will be stored in the production namespace.
+
+## How to Pull Docker Images
+
+```sh
+docker pull ghcr.io/scality/cloudserver:<commit hash>
+docker pull ghcr.io/scality/cloudserver:<tag>
+```
+
+## Release Process
+
+To release a production image:
+
+* Create a PR to bump the package version
+  Update Cloudserver's `package.json` by bumping it to the relevant next
+  version in a new PR. Per example if the last released version was
+  `8.4.7`, the next version would be `8.4.8`.
+
+```js
+{
+  "name": "cloudserver",
+  "version": "8.4.8", <--- Here
+  [...]
+}
+```
+
+* Review & merge the PR
+
+* Create the release on GitHub
+  
+  * Go the Release tab (https://github.com/scality/cloudserver/releases);
+  * Click on the `Draft new release button`;
+  * In the `tag` field, type the name of the release (`8.4.8`), and confirm
+    to create the tag on publish;
+  * Click on `Generate release notes` button to fill the fields;
+  * Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
+  * Click to `Publish the release` to create the GitHub release and git tag
+
+  Notes:
+  * the Git tag will be created automatically.
+  * this should be done as soon as the PR is merged, so that the tag
+    is put on the "version bump" commit.
+
+* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
+
+  * Branch Name: The one used for the tag earlier. In this example `development/8.4`
+  * Override Stage: 'release'
+  * Extra properties:
+    * name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
+
+* Release the release version on Jira
+
+  * Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
+  * Create a next version
+    * Name: `[next version]`, in this example `8.4.9`
+  * Click `...` and select `Release` on the recently released version (`8.4.8`)
+  * Fill in the field to move incomplete version to the next one

docs/USING_PUBLIC_CLOUDS.rst

@@ -1,10 +1,12 @@
+.. _use-public-cloud:
+
 Using Public Clouds as data backends
 ====================================
 
 Introduction
 ------------
 
-As stated in our `GETTING STARTED guide <../GETTING_STARTED/#location-configuration>`__,
+As stated in our `GETTING STARTED guide <GETTING_STARTED.html#location-configuration>`__,
 new data backends can be added by creating a region (also called location
 constraint) with the right endpoint and credentials.
 This section of the documentation shows you how to set up our currently
@@ -160,8 +162,8 @@ CloudServer.
    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
    -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
    -v ~/.aws/credentials:/root/.aws/credentials \
-   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-   -d scality/s3server
+   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 \
+   -d scality/cloudserver
 
 Testing: put an object to AWS S3 using CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -243,7 +245,7 @@ There are a few configurable options here:
   this region should behave like any other AWS S3 region (in the case of MS Azure
   hosted data, this is mostly relevant for the format of errors);
 - :code:`azureStorageEndpoint` : set to your storage account's endpoint, usually
-  :code:`https://{{storageAccountName}}.blob.core.windows.name`;
+  :code:`https://{{storageAccountName}}.blob.core.windows.net`;
 - :code:`azureContainerName` : set to an *existing container* in your MS Azure
   storage account; this is the container in which your data will be stored for
   this location constraint;
@@ -322,7 +324,7 @@ CloudServer.
    -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
    -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
    -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-   -d scality/s3server
+   -d scality/cloudserver
 
 Testing: put an object to MS Azure using CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/developers/getting-started.rst

@@ -0,0 +1,79 @@
+============================================
+Add New Backend Storage To Zenko CloudServer
+============================================
+
+This set of documents aims at bootstrapping developers with Zenko's CloudServer
+module, so they can then go on and contribute features.
+
+.. toctree::
+    :maxdepth: 2
+
+    non-s3-compatible-backend
+    s3-compatible-backend
+
+We always encourage our community to offer new extensions to Zenko,
+and new backend support is paramount to meeting more community needs.
+If that is something you want to contribute (or just do on your own
+version of the cloudserver image), this is the guid to read. Please
+make sure you follow our `Contributing Guidelines`_/.
+
+If you need help with anything, please search our `forum`_ for more
+information.
+
+Add support for a new backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently the main public cloud protocols are supported and more can
+be added. There are two main types of backend: those compatible with
+Amazon's S3 protocol and those not compatible.
+
+================= ========== ============ ===========
+Backend type      Supported  Active WIP   Not started
+================= ========== ============ ===========
+Private disk/fs       x
+AWS S3                x
+Microsoft Azure       x
+Backblaze B2                    x
+Google Cloud          x
+Openstack Swift                               x
+================= ========== ============ ===========
+
+.. important:: Should you want to request for a new backend to be
+               supported, please do so by opening a `Github issue`_,
+               and filling out the "Feature Request" section of our
+               template.
+
+To add support for a new backend support to CloudServer official
+repository, please follow these steps:
+
+- familiarize yourself with our `Contributing Guidelines`_
+- open a `Github issue`_ and fill out Feature Request form, and
+  specify you would like to contribute it yourself;
+- wait for our core team to get back to you with an answer on whether
+  we are interested in taking that contribution in (and hence
+  committing to maintaining it over time);
+- once approved, fork the repository and start your development;
+- use the `forum`_ with any question you may have during the
+  development process;
+- when you think it's ready, let us know so that we create a feature
+  branch against which we'll compare and review your code;
+- open a pull request with your changes against that dedicated feature
+  branch;
+- once that pull request gets merged, you're done.
+
+.. tip::
+
+    While we do take care of the final rebase (when we merge your feature
+    branch on the latest default branch), we do ask that you keep up to date with our latest default branch
+    until then.
+
+.. important::
+
+    If we do not approve your feature request, you may of course still
+    work on supporting a new backend: all our "no" means is that we do not
+    have the resources, as part of our core development team, to maintain
+    this feature for the moment.
+
+.. _GitHub issue: https://github.com/scality/S3/issues
+.. _Contributing Guidelines: https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md
+.. _forum: https://forum.zenko.io

docs/developers/new-backend.rst

@@ -0,0 +1,53 @@
+=================
+Add A New Backend
+=================
+
+Supporting all possible public cloud storage APIs is CloudServer's
+ultimate goal. As an open source project, contributions are welcome.
+
+The first step is to get familiar with building a custom Docker image
+for CloudServer.
+
+Build a Custom Docker Image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clone Zenko's CloudServer, install all dependencies and start the
+service:
+
+.. code-block:: shell
+
+  $ git clone https://github.com/scality/cloudserver
+  $ cd cloudserver
+  $ yarn install
+  $ yarn start
+
+.. tip::
+
+    Some optional dependencies may fail, resulting in you seeing `yarn
+    WARN` messages; these can safely be ignored.  Refer to the User
+    documentation for all available options.
+
+Build the Docker image:
+
+.. code-block:: shell
+
+     # docker build . -t
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+Push the newly created Docker image to your own hub:
+
+.. code-block:: shell
+
+     # docker push
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+.. note::
+
+    To perform this last operation, you need to be authenticated with DockerHub
+
+There are two main types of backend you could want Zenko to support:
+
+== link:S3_COMPATIBLE_BACKENDS.adoc[S3 compatible data backends]
+
+== link:NON_S3_COMPATIBLE_BACKENDS.adoc[Data backends using another protocol than the S3 protocol]
+

docs/developers/non-s3-compatible-backend.rst

@@ -0,0 +1,530 @@
+==========================================================
+Adding support for data backends not supporting the S3 API
+==========================================================
+
+These backends abstract the complexity of multiple APIs to let users
+work on a single common namespace across multiple clouds.
+
+This documents aims at introducing you to the right files in
+CloudServer (the Zenko stack's subcomponent in charge of API
+translation, among other things) to add support to your own backend of
+choice.
+
+General configuration
+~~~~~~~~~~~~~~~~~~~~~
+
+There are a number of constants and environment variables to define to support a
+new data backend; here is a list and where to find them:
+
+:file:`/constants.js`
+---------------------
+
+ * give your backend type a name, as part of the `externalBackends` object;
+ * specify whether versioning is implemented, as part of the
+   `versioningNotImplemented` object;
+
+:file:`/lib/Config.js`
+----------------------
+
+ * this is where you should put common utility functions, like the ones to parse
+   the location object from `locationConfig.json`;
+ * make sure you define environment variables (like `GCP_SERVICE_EMAIL` as we'll
+   use those internally for the CI to test against the real remote backend;
+
+:file:`/lib/data/external/{{backendName}}Client.js`
+---------------------------------------------------
+
+ * this file is where you'll instantiate your backend client; this should be a
+   class with a constructor taking the config object built in `/lib/Config.js` as
+   parameter;
+ * over time, you may need some utility functions which we've defined in the
+   folder `/api/apiUtils`, and in the file `/lib/data/external/utils`;
+
+:file:`/lib/data/external/utils.js`
+-----------------------------------
+
+ * make sure to add options for `sourceLocationConstraintType` to be equal to
+   the name you gave your backend in :file:`/constants.js`;
+
+:file:`/lib/data/external/{{BackendName}}_lib/`
+-----------------------------------------------
+
+ * this folder is where you'll put the functions needed for supporting your
+   backend; keep your files as atomic as possible;
+
+:file:`/tests/locationConfig/locationConfigTests.json`
+------------------------------------------------------
+
+ * this file is where you'll create location profiles to be used by your
+   functional tests;
+
+:file:`/lib/data/locationConstraintParser.js`
+---------------------------------------------
+
+ * this is where you'll instantiate your client if the operation the end user
+   sent effectively writes to your backend; everything happens inside the
+   function `parseLC()`; you should add a condition that executes if
+   `locationObj.type` is the name of your backend (that you defined in
+   `constants.js`), and instantiates a client of yours. See pseudocode below,
+   assuming location type name is `ztore`:
+
+
+.. code-block:: js
+   :linenos:
+   :emphasize-lines: 12
+
+    (...) //<1>
+    const ZtoreClient = require('./external/ZtoreClient');
+    const { config } = require('../Config'); //<1>
+
+    function parseLC(){ //<1>
+    (...) //<1>
+        Object.keys(config.locationConstraints).forEach(location => { //<1>
+            const locationObj = config.locationConstraints[location]; //<1>
+            (...) //<1>
+            if (locationObj.type === 'ztore' {
+                const ztoreEndpoint = config.getZtoreEndpoint(location);
+                const ztoreCredentials = config.getZtoreCredentials(location); //<2>
+                clients[location] = new ZtoreClient({
+                    ztoreEndpoint,
+                    ztoreCredentials,
+                    ztoreBucketname: locationObj.details.ztoreBucketName,
+                    bucketMatch:  locationObj.details.BucketMatch,
+                    dataStoreName: location,
+                }); //<3>
+                clients[location].clientType = 'ztore';
+            });
+            (...) //<1>
+        });
+    }
+
+
+1. Code that is already there
+2. You may need more utility functions depending on your backend specs
+3. You may have more fields required in your constructor object depending on
+   your backend specs
+
+Operation of type PUT
+~~~~~~~~~~~~~~~~~~~~~
+
+PUT routes are usually where people get started, as it's the easiest to check!
+Simply go on your remote backend console and you'll be able to see whether your
+object actually went up in the cloud...
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `put()` function is also called
+  `put()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `put(stream, size, keyContext, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+  //TODO: generate this from jsdoc
+
+ - `stream`: the stream of data you want to put in the cloud; if you're
+   unfamiliar with node.js streams, we suggest you start training, as we use
+   them a lot !
+
+ - `size`: the size of the object you're trying to put;
+
+ - `keyContext`: an object with metadata about the operation; common entries are
+   `namespace`, `buckerName`, `owner`, `cipherBundle`, and `tagging`; if these
+   are not sufficient for your integration, contact us to get architecture
+   validation before adding new entries;
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your PUT operation, and
+  then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/put/put{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+  function returning ajusted `keys[]` to your tests.
+
+Operation of type GET
+~~~~~~~~~~~~~~~~~~~~~
+
+GET routes are easy to test after PUT routes are implemented, hence why we're
+covering them second.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `get()` function is also called
+  `get()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `get(objectGetInfo, range, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+
+ - `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+
+ - `range`: the range of bytes you will get, for "get-by-range" operations (we
+   recommend you do simple GETs first, and then look at this);
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your GET operation, and
+  then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type DELETE
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+DELETE routes are easy to test after PUT routes are implemented, and they are
+similar to GET routes in our implementation, hence why we're covering them
+third.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `delete()` function is also called
+  `delete()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `delete(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your DELETE operation,
+  and then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/delete/delete{{BackendName}}js`
+-----------------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the
+          tutorial in order (that is, if you have covered the PUT operation
+          already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type HEAD
+~~~~~~~~~~~~~~~~~~~~~~
+
+HEAD routes are very similar to DELETE routes in our implementation, hence why
+we're covering them fourth.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `head()` function is also called
+  `head()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `head(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+// TODO:: generate this from jsdoc
+
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your HEAD operation,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Healthcheck
+~~~~~~~~~~~
+
+Healtchecks are used to make sure failure to write to a remote cloud is due to
+a problem on that remote cloud, an not on Zenko's side.
+This is usually done by trying to create a bucket that already exists, and
+making sure you get the expected answer.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `healthcheck()` function is called
+  `checkExternalBackend()` and it's defined in
+  :file:`/lib/data/multipleBackendGateway.js`; you will need to add your own;
+- your healtcheck function should get `location` as a parameter, which is an
+  object comprising:`
+
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/{{backendName}}_create_bucket.js`
+-------------------------------------------------------------------------------
+
+- this is where you should write the function performing the actual bucket
+  creation;
+
+:file:`/lib/data/external/{{backendName}}_lib/utils.js`
+-------------------------------------------------------
+
+- add an object named per your backend's name to the `backendHealth` dictionary,
+  with proper `response` and `time` entries;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `healthcheck` function to add your location's array, and call your
+  healthcheck; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+
+.. code-block:: js
+   :linenos:
+
+    (...) //<1>
+
+        healthcheck: (flightCheckOnStartUp, log, callback) => { //<1>
+            (...) //<1>
+            const ztoreArray = []; //<2>
+            async.each(Object.keys(clients), (location, cb) => { //<1>
+                (...) //<1>
+                } else if (client.clientType === 'ztore' {
+                    ztoreArray.push(location); //<3>
+                    return cb();
+                }
+            (...) //<1>
+            multBackendResp[location] = { code: 200, message: 'OK' }; //<1>
+            return cb();
+        }, () => { //<1>
+            async.parallel([
+                (...) //<1>
+                next => checkExternalBackend( //<4>
+                    clients, ztoreArray, 'ztore', flightCheckOnStartUp,
+                    externalBackendHealthCheckInterval, next),
+            ] (...) //<1>
+            });
+            (...) //<1>
+        });
+    }
+
+1. Code that is already there
+2. The array that will store all locations of type 'ztore'
+3. Where you add locations of type 'ztore' to the array
+4. Where you actually call the healthcheck function on all 'ztore' locations
+
+Multipart upload (MPU)
+~~~~~~~~~~~~~~~~~~~~~~
+
+This is the final part to supporting a new backend! MPU is far from
+the easiest subject, but you've come so far it shouldn't be a problem.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+You'll be creating four functions with template signatures:
+
+- `createMPU(Key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
+  cacheControl, contentDisposition, contentEncoding, log, callback)` will
+  initiate the multi part upload process; now, here, all parameters are
+  metadata headers except for:
+
+ * `Key`, the key id for the final object (collection of all parts);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `uploadPart(request, streamingV4Params, stream, size, key, uploadId, partNumber, bucketName, log, callback)`
+   will be called for each part; the parameters can be explicited as follow:
+
+ * `request`, the request object for putting the part;
+ * `streamingV4Params`, parameters for auth V4 parameters against S3;
+ * `stream`, the node.js readable stream used to put the part;
+ * `size`, the size of the part;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `partNumber`, the number of the part in this MPU (ordered);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback)` will
+   end the MPU process once all parts are uploaded; parameters can be explicited
+   as follows:
+
+ * `jsonList`, user-sent list of parts to include in final mpu object;
+ * `mdInfo`, object containing 3 keys: storedParts, mpuOverviewKey, and
+   splitter;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance:
+
+- `abortMPU(key, uploadId, bucketName, log, callback)` will handle errors, and
+  make sure that all parts that may have been uploaded will be deleted if the
+  MPU ultimately fails; the parameters are:
+
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance.
+
+:file:`/lib/api/objectPutPart.js`
+---------------------------------
+
+- you'll need to add your backend type in appropriate sections (simply look for
+  other backends already implemented).
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your MPU operations,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `createMPU` function to add your location type, and call your
+  `createMPU()`; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+.. code-block:: javascript
+   :linenos:
+
+    (...) //<1>
+        createMPU:(key, metaHeaders, bucketName, websiteRedirectHeader, //<1>
+         location, contentType, cacheControl, contentDisposition,
+         contentEncoding, log, cb) => {
+            const client = clients[location]; //<1>
+            if (client.clientType === 'aws_s3') { //<1>
+                return client.createMPU(key, metaHeaders, bucketName,
+                websiteRedirectHeader, contentType, cacheControl,
+                contentDisposition, contentEncoding, log, cb);
+            } else if (client.clientType === 'ztore') { //<2>
+                return client.createMPU(key, metaHeaders, bucketName,
+                  websiteRedirectHeader, contentType, cacheControl,
+                  contentDisposition, contentEncoding, log, cb);
+            }
+            return cb();
+        };
+    (...) //<1>
+
+1. Code that is already there
+2. Where the `createMPU()` of your client is actually called
+
+Add functional tests
+~~~~~~~~~~~~~~~~~~~~
+
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/initMPU/{{BackendName}}InitMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/listParts/{{BackendName}}ListPart.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/{{BackendName}}AbortMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/{{BackendName}}CompleteMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/{{BackendName}}UploadPart.js`
+
+Adding support in Orbit, Zenko's UI for simplified Multi Cloud Management
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can only be done by our core developers' team. Once your backend
+integration is merged, you may open a feature request on the
+`Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/developers/s3-compatible-backend.rst

@@ -0,0 +1,43 @@
+======================
+S3-Compatible Backends
+======================
+
+
+Adding Support in CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is the easiest case for backend support integration: there is nothing to do
+but configuration!  Follow the steps described in our
+:ref:`use-public-cloud` and make sure you:
+
+- set ``details.awsEndpoint`` to your storage provider endpoint;
+
+- use ``details.credentials`` and *not* ``details.credentialsProfile`` to set your
+  credentials for that S3-compatible backend.
+
+For example, if you’re using a Wasabi bucket as a backend, then your region
+definition for that backend will look something like:
+::
+
+    "wasabi-bucket-zenkobucket": {
+    "type": "aws_s3",
+    "legacyAwsBehavior": true,
+    "details": {
+    "awsEndpoint": "s3.wasabisys.com",
+    "bucketName": "zenkobucket",
+    "bucketMatch": true,
+    "credentials": {
+    "accessKey": "\\{YOUR_WASABI_ACCESS_KEY}",
+    "secretKey": "\\{YOUR_WASABI_SECRET_KEY}"
+    }
+    }
+    },
+
+Adding Support in Zenko Orbit
+#############################
+
+This can only be done by our core developpers' team. If that’s what you’re
+after, open a feature request on the `Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/index.rst

@@ -1,11 +1,12 @@
 Scality Zenko CloudServer
-==================
+=========================
 
 .. _user-docs:
 
 .. toctree::
    :maxdepth: 2
    :caption: Documentation
+   :glob:
 
    CONTRIBUTING
    GETTING_STARTED
@@ -14,3 +15,4 @@ Scality Zenko CloudServer
    DOCKER
    INTEGRATIONS
    ARCHITECTURE
+   developers/*

docs/requirements.in

@@ -0,0 +1,2 @@
+Sphinx >= 1.7.5
+recommonmark >= 0.4.0

docs/requirements.txt

@@ -0,0 +1,119 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    tox -e pip-compile
+#
+alabaster==0.7.12 \
+    --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
+    --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 \
+    # via sphinx
+babel==2.6.0 \
+    --hash=sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669 \
+    --hash=sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23 \
+    # via sphinx
+certifi==2018.10.15 \
+    --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
+    --hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+commonmark==0.5.4 \
+    --hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5 \
+    # via recommonmark
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
+    # via recommonmark, sphinx
+idna==2.7 \
+    --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
+    --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
+    # via requests
+imagesize==1.1.0 \
+    --hash=sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8 \
+    --hash=sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5 \
+    # via sphinx
+jinja2==2.10 \
+    --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
+    --hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
+    # via sphinx
+markupsafe==1.1.0 \
+    --hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \
+    --hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \
+    --hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \
+    --hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \
+    --hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \
+    --hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \
+    --hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \
+    --hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \
+    --hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \
+    --hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \
+    --hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \
+    --hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \
+    --hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \
+    --hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \
+    --hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \
+    --hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \
+    --hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \
+    --hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \
+    --hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \
+    --hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \
+    --hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \
+    --hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \
+    --hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \
+    --hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \
+    --hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \
+    --hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \
+    --hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \
+    --hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 \
+    # via jinja2
+packaging==18.0 \
+    --hash=sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807 \
+    --hash=sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9 \
+    # via sphinx
+pygments==2.2.0 \
+    --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
+    --hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc \
+    # via sphinx
+pyparsing==2.3.0 \
+    --hash=sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b \
+    --hash=sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592 \
+    # via packaging
+pytz==2018.7 \
+    --hash=sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca \
+    --hash=sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6 \
+    # via babel
+recommonmark==0.4.0 \
+    --hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d \
+    --hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7
+requests==2.20.1 \
+    --hash=sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54 \
+    --hash=sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263 \
+    # via sphinx
+six==1.11.0 \
+    --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
+    --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \
+    # via packaging, sphinx
+snowballstemmer==1.2.1 \
+    --hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 \
+    --hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \
+    # via sphinx
+sphinx==1.8.2 \
+    --hash=sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea \
+    --hash=sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64
+sphinxcontrib-websupport==1.1.0 \
+    --hash=sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd \
+    --hash=sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9 \
+    # via sphinx
+typing==3.6.6 \
+    --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \
+    --hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
+    --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
+    # via sphinx
+urllib3==1.24.1 \
+    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
+    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via requests

examples/go-md-search.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+    "fmt"
+    "time"
+    "bytes"
+    "net/http"
+    "net/url"
+    "io/ioutil"
+    "github.com/aws/aws-sdk-go/aws/credentials"
+    "github.com/aws/aws-sdk-go/aws/signer/v4"
+)
+
+func main() {
+    // Input AWS access key, secret key
+    aws_access_key_id := "accessKey1"
+    aws_secret_access_key := "verySecretKey1"
+    endpoint := "http://localhost:8000"
+    bucket_name := "bucketname"
+    searchQuery := url.QueryEscape("x-amz-meta-color=blue")
+    buf := bytes.NewBuffer([]byte{})
+
+    requestUrl := fmt.Sprintf("%s/%s?search=%s",
+        endpoint, bucket_name, searchQuery)
+
+    request, err := http.NewRequest("GET", requestUrl, buf)
+    if err != nil {
+        panic(err)
+    }
+    reader := bytes.NewReader(buf.Bytes())
+    credentials := credentials.NewStaticCredentials(aws_access_key_id,
+        aws_secret_access_key, "")
+    signer := v4.NewSigner(credentials)
+    signer.Sign(request, reader, "s3", "us-east-1", time.Now())
+    client := &http.Client{}
+    resp, err := client.Do(request)
+    if err != nil {
+        panic(err)
+    }
+    defer resp.Body.Close()
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println(string(body))
+}

examples/node-md-search.js

@@ -0,0 +1,28 @@
+const { S3 } = require('aws-sdk');
+const config = {
+    sslEnabled: false,
+    endpoint: 'http://127.0.0.1:8000',
+    signatureCache: false,
+    signatureVersion: 'v4',
+    region: 'us-east-1',
+    s3ForcePathStyle: true,
+    accessKeyId: 'accessKey1',
+    secretAccessKey: 'verySecretKey1',
+};
+const s3Client = new S3(config);
+
+const encodedSearch =
+    encodeURIComponent('x-amz-meta-color="blue"');
+const req = s3Client.listObjects({ Bucket: 'bucketname' });
+
+// the build event
+req.on('build', () => {
+    req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
+});
+req.on('success', res => {
+    process.stdout.write(`Result ${res.data}`);
+});
+req.on('error', err => {
+    process.stdout.write(`Error ${err}`);
+});
+req.send();

examples/python-md-search.py

@@ -0,0 +1,79 @@
+import datetime
+import hashlib
+import hmac
+import urllib
+# pip install requests
+import requests
+
+access_key = 'accessKey1'
+secret_key = 'verySecretKey1'
+
+method = 'GET'
+service = 's3'
+host = 'localhost:8000'
+region = 'us-east-1'
+canonical_uri = '/bucketname'
+query = 'x-amz-meta-color=blue'
+canonical_querystring = 'search=%s' % (urllib.quote(query))
+algorithm = 'AWS4-HMAC-SHA256'
+
+t = datetime.datetime.utcnow()
+amz_date = t.strftime('%Y%m%dT%H%M%SZ')
+date_stamp = t.strftime('%Y%m%d')
+
+# Key derivation functions. See:
+# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
+
+
+def sign(key, msg):
+    return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
+
+
+def getSignatureKey(key, date_stamp, regionName, serviceName):
+    kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
+    kRegion = sign(kDate, regionName)
+    kService = sign(kRegion, serviceName)
+    kSigning = sign(kService, 'aws4_request')
+    return kSigning
+
+
+payload_hash = hashlib.sha256('').hexdigest()
+
+canonical_headers = \
+    'host:{0}\nx-amz-content-sha256:{1}\nx-amz-date:{2}\n' \
+    .format(host, payload_hash, amz_date)
+
+signed_headers = 'host;x-amz-content-sha256;x-amz-date'
+
+canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
+    .format(method, canonical_uri, canonical_querystring, canonical_headers,
+            signed_headers, payload_hash)
+print(canonical_request)
+
+credential_scope = '{0}/{1}/{2}/aws4_request' \
+    .format(date_stamp, region, service)
+
+string_to_sign = '{0}\n{1}\n{2}\n{3}' \
+    .format(algorithm, amz_date, credential_scope,
+            hashlib.sha256(canonical_request).hexdigest())
+
+signing_key = getSignatureKey(secret_key, date_stamp, region, service)
+
+signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
+                     hashlib.sha256).hexdigest()
+
+authorization_header = \
+    '{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}' \
+    .format(algorithm, access_key, credential_scope, signed_headers, signature)
+
+# The 'host' header is added automatically by the Python 'requests' library.
+headers = {
+    'X-Amz-Content-Sha256': payload_hash,
+    'X-Amz-Date': amz_date,
+    'Authorization': authorization_header
+}
+
+endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
+
+r = requests.get(endpoint, headers=headers)
+print(r.text)

images/svc-base/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.scality.com/federation/nodesvc-base:7.10.6.0
+FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
 
 ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
 ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
@@ -14,8 +14,10 @@ RUN rm -f ~/.gitconfig && \
     git config --global --add safe.directory . && \
     git lfs install && \
     GIT_LFS_SKIP_SMUDGE=1 && \
+    yarn global add typescript && \
     yarn install --frozen-lockfile --production --network-concurrency 1 && \
-    yarn cache clean --all
+    yarn cache clean --all && \
+    yarn global remove typescript
 
 # run symlinking separately to avoid yarn installation errors
 # we might have to check if the symlinking is really needed!

index.js

@@ -1,3 +1,10 @@
 'use strict'; // eslint-disable-line strict
 
+require('werelogs').stderrUtils.catchAndTimestampStderr(
+    undefined,
+    // Do not exit as workers have their own listener that will exit
+    // But primary don't have another listener
+    require('cluster').isPrimary ? 1 : null,
+);
+
 require('./lib/server.js')();

lib/api/api.js

@@ -7,7 +7,8 @@ const bucketDeleteEncryption = require('./bucketDeleteEncryption');
 const bucketDeleteWebsite = require('./bucketDeleteWebsite');
 const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
 const bucketDeletePolicy = require('./bucketDeletePolicy');
-const bucketGet = require('./bucketGet');
+const bucketDeleteQuota = require('./bucketDeleteQuota');
+const { bucketGet } = require('./bucketGet');
 const bucketGetACL = require('./bucketGetACL');
 const bucketGetCors = require('./bucketGetCors');
 const bucketGetVersioning = require('./bucketGetVersioning');
@@ -17,12 +18,16 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
 const bucketGetNotification = require('./bucketGetNotification');
 const bucketGetObjectLock = require('./bucketGetObjectLock');
 const bucketGetPolicy = require('./bucketGetPolicy');
+const bucketGetQuota = require('./bucketGetQuota');
 const bucketGetEncryption = require('./bucketGetEncryption');
 const bucketHead = require('./bucketHead');
 const { bucketPut } = require('./bucketPut');
 const bucketPutACL = require('./bucketPutACL');
 const bucketPutCors = require('./bucketPutCors');
 const bucketPutVersioning = require('./bucketPutVersioning');
+const bucketPutTagging = require('./bucketPutTagging');
+const bucketDeleteTagging = require('./bucketDeleteTagging');
+const bucketGetTagging = require('./bucketGetTagging');
 const bucketPutWebsite = require('./bucketPutWebsite');
 const bucketPutReplication = require('./bucketPutReplication');
 const bucketPutLifecycle = require('./bucketPutLifecycle');
@@ -30,6 +35,7 @@ const bucketPutNotification = require('./bucketPutNotification');
 const bucketPutEncryption = require('./bucketPutEncryption');
 const bucketPutPolicy = require('./bucketPutPolicy');
 const bucketPutObjectLock = require('./bucketPutObjectLock');
+const bucketUpdateQuota = require('./bucketUpdateQuota');
 const bucketGetReplication = require('./bucketGetReplication');
 const bucketDeleteReplication = require('./bucketDeleteReplication');
 const corsPreflight = require('./corsPreflight');
@@ -37,10 +43,11 @@ const completeMultipartUpload = require('./completeMultipartUpload');
 const initiateMultipartUpload = require('./initiateMultipartUpload');
 const listMultipartUploads = require('./listMultipartUploads');
 const listParts = require('./listParts');
+const metadataSearch = require('./metadataSearch');
 const { multiObjectDelete } = require('./multiObjectDelete');
 const multipartDelete = require('./multipartDelete');
 const objectCopy = require('./objectCopy');
-const objectDelete = require('./objectDelete');
+const { objectDelete } = require('./objectDelete');
 const objectDeleteTagging = require('./objectDeleteTagging');
 const objectGet = require('./objectGet');
 const objectGetACL = require('./objectGetACL');
@@ -55,16 +62,17 @@ const objectPutTagging = require('./objectPutTagging');
 const objectPutPart = require('./objectPutPart');
 const objectPutCopyPart = require('./objectPutCopyPart');
 const objectPutRetention = require('./objectPutRetention');
+const objectRestore = require('./objectRestore');
 const prepareRequestContexts
     = require('./apiUtils/authorization/prepareRequestContexts');
 const serviceGet = require('./serviceGet');
 const vault = require('../auth/vault');
-const websiteGet = require('./websiteGet');
-const websiteHead = require('./websiteHead');
+const website = require('./website');
 const writeContinue = require('../utilities/writeContinue');
 const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
 const parseCopySource = require('./apiUtils/object/parseCopySource');
 const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
+const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
 const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
 
 const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
@@ -74,6 +82,14 @@ auth.setHandler(vault);
 /* eslint-disable no-param-reassign */
 const api = {
     callApiMethod(apiMethod, request, response, log, callback) {
+        // Attach the apiMethod method to the request, so it can used by monitoring in the server
+        // eslint-disable-next-line no-param-reassign
+        request.apiMethod = apiMethod;
+        // Array of end of API callbacks, used to perform some logic
+        // at the end of an API.
+        // eslint-disable-next-line no-param-reassign
+        request.finalizerHooks = [];
+
         const actionLog = monitoringMap[apiMethod];
         if (!actionLog &&
             apiMethod !== 'websiteGet' &&
@@ -181,21 +197,27 @@ const api = {
 
         return async.waterfall([
             next => auth.server.doAuth(
-                request, log, (err, userInfo, authorizationResults, streamingV4Params) => {
+                request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
                     if (err) {
+                        // VaultClient returns standard errors, but the route requires
+                        // Arsenal errors
+                        const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
                         log.trace('authentication error', { error: err });
-                        return next(err);
+                        return next(arsenalError);
                     }
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                 }, 's3', requestContexts),
-            (userInfo, authorizationResults, streamingV4Params, next) => {
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => {
                 const authNames = { accountName: userInfo.getAccountDisplayName() };
                 if (userInfo.isRequesterAnIAMUser()) {
                     authNames.userName = userInfo.getIAMdisplayName();
                 }
+                if (isRequesterASessionUser(userInfo)) {
+                    authNames.sessionName = userInfo.getShortid().split(':')[1];
+                }
                 log.addDefaultFields(authNames);
                 if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                 }
                 // issue 100 Continue to the client
                 writeContinue(request, response);
@@ -226,12 +248,12 @@ const api = {
                     }
                     // Convert array of post buffers into one string
                     request.post = Buffer.concat(post, postLength).toString();
-                    return next(null, userInfo, authorizationResults, streamingV4Params);
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                 });
                 return undefined;
             },
             // Tag condition keys require information from CloudServer for evaluation
-            (userInfo, authorizationResults, streamingV4Params, next) => tagConditionKeyAuth(
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
                 authorizationResults,
                 request,
                 requestContexts,
@@ -242,13 +264,14 @@ const api = {
                         log.trace('tag authentication error', { error: err });
                         return next(err);
                     }
-                    return next(null, userInfo, authResultsWithTags, streamingV4Params);
+                    return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
                 },
             ),
-        ], (err, userInfo, authorizationResults, streamingV4Params) => {
+        ], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
             if (err) {
                 return callback(err);
             }
+            request.accountQuotas = infos?.accountQuota;
             if (authorizationResults) {
                 const checkedResults = checkAuthResults(authorizationResults);
                 if (checkedResults instanceof Error) {
@@ -265,19 +288,23 @@ const api = {
                     return acc;
                 }, {});
             }
+            const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
+                    (hook, done) => hook(err, done),
+                    () => callback(err, ...results));
+
             if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
                 request._response = response;
                 return this[apiMethod](userInfo, request, streamingV4Params,
-                    log, callback, authorizationResults);
+                    log, methodCallback, authorizationResults);
             }
             if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
                 return this[apiMethod](userInfo, request, sourceBucket,
-                    sourceObject, sourceVersionId, log, callback);
+                    sourceObject, sourceVersionId, log, methodCallback);
             }
             if (apiMethod === 'objectGet') {
                 return this[apiMethod](userInfo, request, returnTagCount, log, callback);
             }
-            return this[apiMethod](userInfo, request, log, callback);
+            return this[apiMethod](userInfo, request, log, methodCallback);
         });
     },
     bucketDelete,
@@ -297,15 +324,21 @@ const api = {
     bucketPutACL,
     bucketPutCors,
     bucketPutVersioning,
+    bucketPutTagging,
+    bucketDeleteTagging,
+    bucketGetTagging,
     bucketPutWebsite,
     bucketPutReplication,
     bucketGetReplication,
     bucketDeleteReplication,
+    bucketDeleteQuota,
     bucketPutLifecycle,
+    bucketUpdateQuota,
     bucketGetLifecycle,
     bucketDeleteLifecycle,
     bucketPutPolicy,
     bucketGetPolicy,
+    bucketGetQuota,
     bucketDeletePolicy,
     bucketPutObjectLock,
     bucketPutNotification,
@@ -316,6 +349,7 @@ const api = {
     initiateMultipartUpload,
     listMultipartUploads,
     listParts,
+    metadataSearch,
     multiObjectDelete,
     multipartDelete,
     objectDelete,
@@ -334,9 +368,10 @@ const api = {
     objectPutPart,
     objectPutCopyPart,
     objectPutRetention,
+    objectRestore,
     serviceGet,
-    websiteGet,
-    websiteHead,
+    websiteGet: website,
+    websiteHead: website,
 };
 
 module.exports = api;

lib/api/apiUtils/authorization/permissionChecks.js

@@ -1,28 +1,71 @@
-const { evaluators, actionMaps, RequestContext } = require('arsenal').policies;
+const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
+const { errors } = require('arsenal');
+const { parseCIDR, isValid } = require('ipaddr.js');
 const constants = require('../../../../constants');
+const { config } = require('../../../Config');
 
 const {
-    allAuthedUsersId, bucketOwnerActions, logId, publicId,
-    assumedRoleArnResourceType, backbeatLifecycleSessionName, arrayOfAllowed,
+    allAuthedUsersId,
+    bucketOwnerActions,
+    logId,
+    publicId,
+    arrayOfAllowed,
+    assumedRoleArnResourceType,
+    backbeatLifecycleSessionName,
+    actionsToConsiderAsObjectPut,
 } = constants;
 
 // whitelist buckets to allow public read on objects
 const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
     ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
 
+function getServiceAccountProperties(canonicalID) {
+    const canonicalIDArray = canonicalID.split('/');
+    const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
+    return constants.serviceAccountProperties[serviceName];
+}
+
+function isServiceAccount(canonicalID) {
+    return getServiceAccountProperties(canonicalID) !== undefined;
+}
+
+function isRequesterASessionUser(authInfo) {
+    const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
+    return regexpAssumedRoleArn.test(authInfo.getArn());
+}
+
+function isRequesterNonAccountUser(authInfo) {
+    return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
+}
+
+/**
+ * Checks the access control for a given bucket based on the request type and user's canonical ID.
+ *
+ * @param {Bucket} bucket - The bucket to check access control for.
+ * @param {string} requestType - The list of s3 actions to check within the API call.
+ * @param {string} canonicalID - The canonical ID of the user making the request.
+ * @param {string} mainApiCall - The main API call (first item of the requestType).
+ *
+ * @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
+ */
+
 function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
     // Same logic applies on the Versioned APIs, so let's simplify it.
-    const requestTypeParsed = requestType.endsWith('Version')
-        ? requestType.slice(0, -7) : requestType;
+    let requestTypeParsed = requestType.endsWith('Version') ?
+        requestType.slice(0, 'Version'.length * -1) : requestType;
+    requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
+        'objectPut' : requestTypeParsed;
+    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
+        'objectPut' : mainApiCall;
     if (bucket.getOwner() === canonicalID) {
         return true;
     }
-    if (mainApiCall === 'objectGet') {
+    if (parsedMainApiCall === 'objectGet') {
         if (requestTypeParsed === 'objectGetTagging') {
             return true;
         }
     }
-    if (mainApiCall === 'objectPut') {
+    if (parsedMainApiCall === 'objectPut') {
         if (arrayOfAllowed.includes(requestTypeParsed)) {
             return true;
         }
@@ -104,8 +147,12 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
 function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
     isUserUnauthenticated, mainApiCall) {
     const bucketOwner = bucket.getOwner();
+    const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
+        'objectPut' : requestType;
+    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
+        'objectPut' : mainApiCall;
     // acls don't distinguish between users and accounts, so both should be allowed
-    if (bucketOwnerActions.includes(requestType)
+    if (bucketOwnerActions.includes(requestTypeParsed)
         && (bucketOwner === canonicalID)) {
         return true;
     }
@@ -114,9 +161,9 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
     }
 
     // Backward compatibility
-    if (mainApiCall === 'objectGet') {
+    if (parsedMainApiCall === 'objectGet') {
         if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
-            && requestType === 'objectGetTagging') {
+            && requestTypeParsed === 'objectGetTagging') {
             return true;
         }
     }
@@ -125,7 +172,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
         return false;
     }
 
-    if (requestType === 'objectGet' || requestType === 'objectHead') {
+    if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') {
         if (objectMD.acl.Canned === 'public-read'
             || objectMD.acl.Canned === 'public-read-write'
             || (objectMD.acl.Canned === 'authenticated-read'
@@ -151,11 +198,11 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
 
     // User is already authorized on the bucket for FULL_CONTROL or WRITE or
     // bucket has canned ACL public-read-write
-    if (requestType === 'objectPut' || requestType === 'objectDelete') {
+    if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') {
         return true;
     }
 
-    if (requestType === 'objectPutACL') {
+    if (requestTypeParsed === 'objectPutACL') {
         if ((objectMD.acl.Canned === 'bucket-owner-full-control'
             && bucketOwner === canonicalID)
             || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
@@ -171,7 +218,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
         }
     }
 
-    if (requestType === 'objectGetACL') {
+    if (requestTypeParsed === 'objectGetACL') {
         if ((objectMD.acl.Canned === 'bucket-owner-full-control'
             && bucketOwner === canonicalID)
             || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
@@ -192,7 +239,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
     const bucketAcl = bucket.getAcl();
     const allowPublicReads = publicReadBuckets.includes(bucket.getName())
         && bucketAcl.Canned === 'public-read'
-        && (requestType === 'objectGet' || requestType === 'objectHead');
+        && (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
     if (allowPublicReads) {
         return true;
     }
@@ -219,6 +266,20 @@ function _checkBucketPolicyResources(request, resource, log) {
     return evaluators.isResourceApplicable(requestContext, resource, log);
 }
 
+function _checkBucketPolicyConditions(request, conditions, log) {
+    const ip = request ? requestUtils.getClientIp(request, config) : undefined;
+    if (!conditions) {
+        return true;
+    }
+    // build request context from the request!
+    const requestContext = new RequestContext(request.headers, request.query,
+        request.bucketName, request.objectKey, ip,
+        request.connection.encrypted, request.resourceType, 's3', null, null,
+        null, null, null, null, null, null, null, null, null,
+        request.objectLockRetentionDays);
+    return evaluators.meetConditions(requestContext, conditions, log);
+}
+
 function _getAccountId(arn) {
     // account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
     return arn.substr(13, 12);
@@ -263,11 +324,11 @@ function _checkPrincipals(canonicalID, arn, principal) {
     return false;
 }
 
-function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request) {
+function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) {
     let permission = 'defaultDeny';
     // if requester is user within bucket owner account, actions should be
     // allowed unless explicitly denied (assumes allowed by IAM policy)
-    if (bucketOwner === canonicalID) {
+    if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) {
         permission = 'allow';
     }
     let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
@@ -276,12 +337,13 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l
         const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
         const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
         const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
+        const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
 
-        if (principalMatch && actionMatch && resourceMatch && s.Effect === 'Deny') {
+        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') {
             // explicit deny trumps any allows, so return immediately
             return 'explicitDeny';
         }
-        if (principalMatch && actionMatch && resourceMatch && s.Effect === 'Allow') {
+        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') {
             permission = 'allow';
         }
         copiedStatement = copiedStatement.splice(1);
@@ -292,84 +354,72 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l
 function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log,
     request, aclPermission, results, actionImplicitDenies) {
     const bucketPolicy = bucket.getBucketPolicy();
-
+    let processedResult = results[requestType];
     if (!bucketPolicy) {
-        // eslint-disable-next-line no-param-reassign
-        results[requestType] = actionImplicitDenies[requestType] === false && aclPermission;
+        processedResult = actionImplicitDenies[requestType] === false && aclPermission;
     } else {
         const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
-            bucketOwner, log, request);
+            bucketOwner, log, request, actionImplicitDenies);
 
         if (bucketPolicyPermission === 'explicitDeny') {
-            // eslint-disable-next-line no-param-reassign
-            results[requestType] = false;
+            processedResult = false;
         } else if (bucketPolicyPermission === 'allow') {
-            // eslint-disable-next-line no-param-reassign
-            results[requestType] = true;
+            processedResult = true;
         } else {
-            // eslint-disable-next-line no-param-reassign
-            results[requestType] = actionImplicitDenies[requestType] === false && aclPermission;
+            processedResult = actionImplicitDenies[requestType] === false && aclPermission;
         }
     }
-    return results[requestType];
+    return processedResult;
 }
 
-function isBucketAuthorized(bucket, requestTypes, canonicalID, authInfo, log, request, actionImplicitDenies = {}) {
-    if (!Array.isArray(requestTypes)) {
-        // eslint-disable-next-line no-param-reassign
-        requestTypes = [requestTypes];
-    }
-    if (!actionImplicitDenies) {
-        // eslint-disable-next-line no-param-reassign
-        actionImplicitDenies = {};
-    }
+function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
+    actionImplicitDeniesInput = {}, isWebsite = false) {
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
     const mainApiCall = requestTypes[0];
     const results = {};
     return requestTypes.every(_requestType => {
         // By default, all missing actions are defined as allowed from IAM, to be
         // backward compatible
-        if (actionImplicitDenies[_requestType] === undefined) {
-            // eslint-disable-next-line no-param-reassign
-            actionImplicitDenies[_requestType] = false;
-        }
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
         // Check to see if user is authorized to perform a
         // particular action on bucket based on ACLs.
         // TODO: Add IAM checks
         let requesterIsNotUser = true;
         let arn = null;
         if (authInfo) {
-            requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
+            requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
             arn = authInfo.getArn();
         }
         // if the bucket owner is an account, users should not have default access
-        if ((bucket.getOwner() === canonicalID) && requesterIsNotUser) {
+        if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
             results[_requestType] = actionImplicitDenies[_requestType] === false;
             return results[_requestType];
         }
         const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
+        // In case of error bucket access is checked with bucketGet
+        // For website, bucket policy only uses objectGet and ignores bucketGet
+        // https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
+        // bucketGet should be used to check acl but switched to objectGet for bucket policy
+        if (isWebsite && _requestType === 'bucketGet') {
+            // eslint-disable-next-line no-param-reassign
+            _requestType = 'objectGet';
+            actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
+        }
         return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
             request, aclPermission, results, actionImplicitDenies);
     });
 }
 
-function evaluateBucketPolicyWithIAM(bucket, requestTypes, canonicalID, authInfo, actionImplicitDenies = {},
+function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {},
     log, request) {
-    if (!Array.isArray(requestTypes)) {
-        // eslint-disable-next-line no-param-reassign
-        requestTypes = [requestTypes];
-    }
-    if (!actionImplicitDenies) {
-        // eslint-disable-next-line no-param-reassign
-        actionImplicitDenies = {};
-    }
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
     const results = {};
     return requestTypes.every(_requestType => {
         // By default, all missing actions are defined as allowed from IAM, to be
         // backward compatible
-        if (actionImplicitDenies[_requestType] === undefined) {
-            // eslint-disable-next-line no-param-reassign
-            actionImplicitDenies[_requestType] = false;
-        }
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
         let arn = null;
         if (authInfo) {
             arn = authInfo.getArn();
@@ -379,49 +429,45 @@ function evaluateBucketPolicyWithIAM(bucket, requestTypes, canonicalID, authInfo
     });
 }
 
-function isObjAuthorized(bucket, objectMD, requestTypes, canonicalID, authInfo, log, request,
-    actionImplicitDenies = {}) {
-    if (!Array.isArray(requestTypes)) {
-        // eslint-disable-next-line no-param-reassign
-        requestTypes = [requestTypes];
-    }
-    if (!actionImplicitDenies) {
-        // eslint-disable-next-line no-param-reassign
-        actionImplicitDenies = {};
-    }
+function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
+    actionImplicitDeniesInput = {}, isWebsite = false) {
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
     const results = {};
     const mainApiCall = requestTypes[0];
     return requestTypes.every(_requestType => {
         // By default, all missing actions are defined as allowed from IAM, to be
         // backward compatible
-        if (actionImplicitDenies[_requestType] === undefined) {
-            // eslint-disable-next-line no-param-reassign
-            actionImplicitDenies[_requestType] = false;
-        }
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
         const parsedMethodName = _requestType.endsWith('Version')
             ? _requestType.slice(0, -7) : _requestType;
         const bucketOwner = bucket.getOwner();
         if (!objectMD) {
-        // User is already authorized on the bucket for FULL_CONTROL or WRITE or
-        // bucket has canned ACL public-read-write
-            if (parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete') {
-                results[_requestType] = actionImplicitDenies[_requestType] === false;
-                return results[_requestType];
-            }
             // check bucket has read access
             // 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
-            results[_requestType] = isBucketAuthorized(bucket, 'bucketGet', canonicalID, authInfo, log, request);
+            let permission = 'bucketGet';
+            if (actionsToConsiderAsObjectPut.includes(_requestType)) {
+                permission = 'objectPut';
+            }
+            results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
+                actionImplicitDenies, isWebsite);
+            // User is already authorized on the bucket for FULL_CONTROL or WRITE or
+            // bucket has canned ACL public-read-write
+            if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
+                && results[_requestType] === false) {
+                results[_requestType] = actionImplicitDenies[_requestType] === false;
+            }
             return results[_requestType];
         }
         let requesterIsNotUser = true;
         let arn = null;
         let isUserUnauthenticated = false;
         if (authInfo) {
-            requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
+            requesterIsNotUser =  !isRequesterNonAccountUser(authInfo);
             arn = authInfo.getArn();
             isUserUnauthenticated = arn === undefined;
         }
-        if (objectMD['owner-id'] === canonicalID && requesterIsNotUser) {
+        if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
             results[_requestType] = actionImplicitDenies[_requestType] === false;
             return results[_requestType];
         }
@@ -468,6 +514,94 @@ function validatePolicyResource(bucketName, policy) {
     });
 }
 
+function checkIp(value) {
+    const errString = 'Invalid IP address in Conditions';
+
+    const values = Array.isArray(value) ? value : [value];
+
+    for (let i = 0; i < values.length; i++) {
+        // these preliminary checks are validating the provided
+        // ip address against ipaddr.js, the library we use when
+        // evaluating IP condition keys. It ensures compatibility,
+        // but additional checks are required to enforce the right
+        // notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
+        // we would accept different ip formats, which is not
+        // standard in an AWS use case.
+        try {
+            try {
+                parseCIDR(values[i]);
+            } catch (err) {
+                isValid(values[i]);
+            }
+        } catch (err) {
+            return errString;
+        }
+
+        // Apply the existing IP validation logic to each element
+        const validateIpRegex = ip => {
+            if (constants.ipv4Regex.test(ip)) {
+                return ip.split('.').every(part => parseInt(part, 10) <= 255);
+            }
+            if (constants.ipv6Regex.test(ip)) {
+                return ip.split(':').every(part => part.length <= 4);
+            }
+            return false;
+        };
+
+        if (validateIpRegex(values[i]) !== true) {
+            return errString;
+        }
+    }
+
+    // If the function hasn't returned by now, all elements are valid
+    return null;
+}
+
+// This function checks all bucket policy conditions if the values provided
+// are valid for the condition type. If not it returns a relevant Malformed policy error string
+function validatePolicyConditions(policy) {
+    const validConditions = [
+        { conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
+        { conditionKey: 's3:object-lock-remaining-retention-days' },
+    ];
+    // keys where value type does not seem to be checked by AWS:
+    // - s3:object-lock-remaining-retention-days
+
+    if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
+        return null;
+    }
+
+    // there can be multiple statements in the policy, each with a Condition enclosure
+    for (let i = 0; i < policy.Statement.length; i++) {
+        const s = policy.Statement[i];
+        if (s.Condition) {
+            const conditionOperators = Object.keys(s.Condition);
+            // there can be multiple condition operations in the Condition enclosure
+            // eslint-disable-next-line no-restricted-syntax
+            for (const conditionOperator of conditionOperators) {
+                const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
+                const conditionValue = s.Condition[conditionOperator][conditionKey];
+                const validCondition = validConditions.find(validCondition =>
+                    validCondition.conditionKey === conditionKey
+                );
+                // AWS returns does not return an error if the condition starts with 'aws:'
+                // so we reproduce this behaviour
+                if (!validCondition && !conditionKey.startsWith('aws:')) {
+                    return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
+                }
+                if (validCondition && validCondition.conditionValueTypeChecker) {
+                    const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
+                    if (conditionValueTypeError) {
+                        return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
+                    }
+                }
+            }
+        }
+    }
+    return null;
+}
+
+
 /** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
  * @param {string} arn - Amazon resource name - example:
  * arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
@@ -494,9 +628,14 @@ function isLifecycleSession(arn) {
 module.exports = {
     isBucketAuthorized,
     isObjAuthorized,
+    getServiceAccountProperties,
+    isServiceAccount,
+    isRequesterASessionUser,
+    isRequesterNonAccountUser,
     checkBucketAcls,
     checkObjectAcls,
     validatePolicyResource,
+    validatePolicyConditions,
     isLifecycleSession,
     evaluateBucketPolicyWithIAM,
 };

lib/api/apiUtils/authorization/prepareRequestContexts.js

@@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
             apiMethod, 's3');
     }
 
-    if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
+    if (apiMethod === 'bucketPut') {
         return null;
     }
 
@@ -65,7 +65,17 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
 
     const requestContexts = [];
 
-    if (apiMethodAfterVersionCheck === 'objectCopy'
+    if (apiMethod === 'multiObjectDelete') {
+        // MultiObjectDelete does not require any authorization when evaluating
+        // the API. Instead, we authorize each object passed.
+        // But in order to get any relevant information from the authorization service
+        // for example, the account quota, we must send a request context object
+        // with no `specificResource`. We expect the result to be an implicit deny.
+        // In the API, we then ignore these authorization results, and we can use
+        // any information returned, e.g., the quota.
+        const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
+          requestContexts.push(requestContextMultiObjectDelete);
+    } else if (apiMethodAfterVersionCheck === 'objectCopy'
         || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
         const objectGetAction = sourceVersionId ? 'objectGetVersion' :
           'objectGet';
@@ -147,20 +157,59 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
             generateRequestContext('objectGet');
         requestContexts.push(getObjectRequestContext);
     } else if (apiMethodAfterVersionCheck === 'objectPut') {
-        const putRequestContext =
-          generateRequestContext(apiMethodAfterVersionCheck);
-        requestContexts.push(putRequestContext);
-        // if put object (versioning) with tag set
-        if (request.headers['x-amz-tagging']) {
-            const putTaggingRequestContext =
-              generateRequestContext('objectPutTagging');
-            requestContexts.push(putTaggingRequestContext);
+        // if put object with version
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
+            const putRequestContext =
+              generateRequestContext(apiMethodAfterVersionCheck);
+            requestContexts.push(putRequestContext);
+            // if put object (versioning) with tag set
+            if (request.headers['x-amz-tagging']) {
+                const putTaggingRequestContext =
+                  generateRequestContext('objectPutTagging');
+                requestContexts.push(putTaggingRequestContext);
+            }
+            if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
+                const putLegalHoldStatusAction =
+                    generateRequestContext('objectPutLegalHold');
+                requestContexts.push(putLegalHoldStatusAction);
+            }
+            // if put object (versioning) with ACL
+            if (isHeaderAcl(request.headers)) {
+                const putAclRequestContext =
+                  generateRequestContext('objectPutACL');
+                requestContexts.push(putAclRequestContext);
+            }
+            if (request.headers['x-amz-object-lock-mode']) {
+                const putObjectLockRequestContext =
+                  generateRequestContext('objectPutRetention');
+                requestContexts.push(putObjectLockRequestContext);
+            }
+            if (request.headers['x-amz-version-id']) {
+                const putObjectVersionRequestContext =
+                    generateRequestContext('objectPutTaggingVersion');
+                requestContexts.push(putObjectVersionRequestContext);
+            }
         }
-        if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
-            const putLegalHoldStatusAction =
-                generateRequestContext('objectPutLegalHold');
-            requestContexts.push(putLegalHoldStatusAction);
+    } else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
+      apiMethodAfterVersionCheck === 'objectPutPart' ||
+      apiMethodAfterVersionCheck === 'completeMultipartUpload'
+      ) {
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
+            const putRequestContext =
+              generateRequestContext(apiMethodAfterVersionCheck);
+            requestContexts.push(putRequestContext);
         }
+
         // if put object (versioning) with ACL
         if (isHeaderAcl(request.headers)) {
             const putAclRequestContext =

lib/api/apiUtils/authorization/tagConditionKeys.js

@@ -60,6 +60,8 @@ function updateRequestContextsWithTags(request, requestContexts, apiMethod, log,
             log.trace('error processing tag condition key evaluation');
             return cb(err);
         }
+        // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
+        // eslint-disable-next-line no-restricted-syntax
         for (const rc of requestContexts) {
             rc.setNeedTagEval(true);
             if (requestTagsQuery) {

lib/api/apiUtils/bucket/bucketCreation.js

@@ -10,13 +10,15 @@ const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
 const metadata = require('../../../metadata/wrapper');
 const kms = require('../../../kms/wrapper');
 const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
+const { isServiceAccount } = require('../authorization/permissionChecks');
 
 const usersBucket = constants.usersBucket;
 const oldUsersBucket = constants.oldUsersBucket;
+const zenkoSeparator = constants.zenkoSeparator;
 const userBucketOwner = 'admin';
 
 
-function addToUsersBucket(canonicalID, bucketName, log, cb) {
+function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
     // BACKWARD: Simplify once do not have to deal with old
     // usersbucket name and old splitter
 
@@ -28,7 +30,10 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
         const splitter = usersBucketAttrs ?
             constants.splitter : constants.oldSplitter;
         let key = createKeyForUserBucket(canonicalID, splitter, bucketName);
-        const omVal = { creationDate: new Date().toJSON() };
+        const omVal = {
+            creationDate: new Date().toJSON(),
+            ingestion: bucketMD.getIngestion(),
+        };
         // If the new format usersbucket does not exist, try to put the
         // key in the old usersBucket using the old splitter.
         // Otherwise put the key in the new format usersBucket
@@ -36,7 +41,7 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
             usersBucket : oldUsersBucket;
         return metadata.putObjectMD(usersBucketBeingCalled, key,
             omVal, {}, log, err => {
-                if (err && err.is.NoSuchBucket) {
+                if (err?.is?.NoSuchBucket) {
                     // There must be no usersBucket so createBucket
                     // one using the new format
                     log.trace('users bucket does not exist, ' +
@@ -91,7 +96,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
             return callback(err);
         }
         log.trace('created bucket in metadata');
-        return addToUsersBucket(canonicalID, bucketName, log, err => {
+        return addToUsersBucket(canonicalID, bucketName, bucket, log, err => {
             if (err) {
                 return callback(err);
             }
@@ -114,7 +119,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
  */
 function cleanUpBucket(bucketMD, canonicalID, log, callback) {
     const bucketName = bucketMD.getName();
-    return addToUsersBucket(canonicalID, bucketName, log, err => {
+    return addToUsersBucket(canonicalID, bucketName, bucketMD, log, err => {
         if (err) {
             return callback(err);
         }
@@ -165,15 +170,28 @@ function createBucket(authInfo, bucketName, headers,
     const ownerDisplayName =
         authInfo.getAccountDisplayName();
     const creationDate = new Date().toJSON();
+    const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
     const headerObjectLock = headers['x-amz-bucket-object-lock-enabled'];
     const objectLockEnabled
         = headerObjectLock && headerObjectLock.toLowerCase() === 'true';
     const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName,
         creationDate, BucketInfo.currentModelVersion(), null, null, null, null,
-        null, null, null, null, null, null, null, null, objectLockEnabled);
+        null, null, null, null, null, null, null, null, null, isNFSEnabled,
+        null, null, objectLockEnabled);
+    let locationConstraintVal = null;
 
-    if (locationConstraint !== undefined) {
-        bucket.setLocationConstraint(locationConstraint);
+    if (locationConstraint) {
+        const [locationConstraintStr, ingestion] =
+            locationConstraint.split(zenkoSeparator);
+        if (locationConstraintStr) {
+            locationConstraintVal = locationConstraintStr;
+            bucket.setLocationConstraint(locationConstraintStr);
+        }
+        if (ingestion === 'ingest') {
+            bucket.enableIngestion();
+            //automatically enable versioning for ingestion buckets
+            bucket.setVersioningConfiguration({ Status: 'Enabled' });
+        }
     }
     if (objectLockEnabled) {
         // default versioning configuration AWS sets
@@ -223,7 +241,8 @@ function createBucket(authInfo, bucketName, headers,
         }
         const existingBucketMD = results.getAnyExistingBucketInfo;
         if (existingBucketMD instanceof BucketInfo &&
-            existingBucketMD.getOwner() !== canonicalID) {
+            existingBucketMD.getOwner() !== canonicalID &&
+            !isServiceAccount(canonicalID)) {
             // return existingBucketMD to collect cors headers
             return cb(errors.BucketAlreadyExists, existingBucketMD);
         }
@@ -254,7 +273,7 @@ function createBucket(authInfo, bucketName, headers,
         // error unless old AWS behavior (us-east-1)
         // Existing locationConstraint must have legacyAwsBehavior === true
         // New locationConstraint should have legacyAwsBehavior === true
-        if (isLegacyAWSBehavior(locationConstraint) &&
+        if (isLegacyAWSBehavior(locationConstraintVal) &&
             isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) {
             log.trace('returning 200 instead of 409 to mirror us-east-1');
             return cb(null, existingBucketMD);

lib/api/apiUtils/bucket/bucketDeletion.js

@@ -93,7 +93,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
                 log, (err, objectsListRes) => {
                     // If no shadow bucket ever created, no ongoing MPU's, so
                     // continue with deletion
-                    if (err && err.is.NoSuchBucket) {
+                    if (err?.is.NoSuchBucket) {
                         return next();
                     }
                     if (err) {

lib/api/apiUtils/bucket/bucketShield.js

@@ -30,6 +30,9 @@ function bucketShield(bucket, requestType) {
      // Otherwise return an error to the client
     if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
         (requestType !== 'objectPut' &&
+        requestType !== 'initiateMultipartUpload' &&
+        requestType !== 'objectPutPart' &&
+        requestType !== 'completeMultipartUpload' &&
         requestType !== 'bucketPutACL' &&
         requestType !== 'bucketDelete')) {
         return true;

lib/api/apiUtils/bucket/checkPreferredLocations.js

@@ -0,0 +1,26 @@
+const { errors } = require('arsenal');
+
+function checkPreferredLocations(location, locationConstraints, log) {
+    const retError = loc => {
+        const errMsg = 'value of the location you are attempting to set - ' +
+        `${loc} - is not listed in the locationConstraint config`;
+        log.trace(`locationConstraint is invalid - ${errMsg}`,
+          { locationConstraint: loc });
+        return errors.InvalidLocationConstraint.customizeDescription(errMsg);
+    };
+    if (typeof location === 'string' && !locationConstraints[location]) {
+        return retError(location);
+    }
+    if (typeof location === 'object') {
+        const { read, write } = location;
+        if (!locationConstraints[read]) {
+            return retError(read);
+        }
+        if (!locationConstraints[write]) {
+            return retError(write);
+        }
+    }
+    return null;
+}
+
+module.exports = checkPreferredLocations;

lib/api/apiUtils/bucket/deleteUserBucketEntry.js

@@ -11,11 +11,11 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
     metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
         // If the object representing the bucket is not in the
         // users bucket just continue
-        if (error && error.is.NoSuchKey) {
+        if (error?.is.NoSuchKey) {
             return cb(null);
         // BACKWARDS COMPATIBILITY: Remove this once no longer
         // have old user bucket format
-        } else if (error && error.NoSuchBucket) {
+        } else if (error?.is.NoSuchBucket) {
             const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
                 oldSplitter, bucketName);
             return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,

lib/api/apiUtils/bucket/getNotificationConfiguration.js

@@ -15,11 +15,21 @@ function getNotificationConfiguration(parsedXml) {
     }
     const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource));
     const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]);
-    if (!notifConfigTargets.every(t => targets.has(t))) {
-        // TODO: match the error message to AWS's response along with
-        // the request destination name in the response
-        const errDesc = 'Unable to validate the destination configuration';
-        return { error: errors.InvalidArgument.customizeDescription(errDesc) };
+    // getting invalid targets
+    const invalidTargets = [];
+    notifConfigTargets.forEach((t, i) => {
+        if (!targets.has(t)) {
+            invalidTargets.push({
+                ArgumentName: notifConfig.queueConfig[i].queueArn,
+                ArgumentValue: 'The destination queue does not exist',
+            });
+        }
+    });
+    if (invalidTargets.length > 0) {
+        const errDesc = 'Unable to validate the following destination configurations';
+        let error = errors.InvalidArgument.customizeDescription(errDesc);
+        error = error.addMetadataEntry('invalidArguments', invalidTargets);
+        return { error };
     }
     return notifConfig;
 }

lib/api/apiUtils/bucket/parseLikeExpression.js

@@ -0,0 +1,19 @@
+/**
+ * parse LIKE expressions
+ * @param {string} regex - regex pattern
+ * @return {object} MongoDB search object
+ */
+function parseLikeExpression(regex) {
+    if (typeof regex !== 'string') {
+        return null;
+    }
+    const split = regex.split('/');
+    if (split.length < 3 || split[0] !== '') {
+        return { $regex: regex };
+    }
+    const pattern = split.slice(1, split.length - 1).join('/');
+    const regexOpt = split[split.length - 1];
+    return { $regex: new RegExp(pattern), $options: regexOpt };
+}
+
+module.exports = parseLikeExpression;

lib/api/apiUtils/bucket/parseWhere.js

@@ -0,0 +1,85 @@
+const parseLikeExpression = require('./parseLikeExpression');
+
+/*
+  This code is based on code from https://github.com/olehch/sqltomongo
+  with the following license:
+
+  The MIT License (MIT)
+
+  Copyright (c) 2016 Oleh
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+/**
+ * A helper object to map SQL-like naming to MongoDB query syntax
+ */
+const exprMapper = {
+    '=': '$eq',
+    '!=': '$ne',
+    '<>': '$ne',
+    '>': '$gt',
+    '<': '$lt',
+    '>=': '$gte',
+    '<=': '$lte',
+    'LIKE': '$regex',
+};
+
+/*
+ * Parses object with WHERE clause recursively
+ * and generates MongoDB `find` query object
+ */
+function parseWhere(root) {
+    const operator = Object.keys(root)[0];
+
+    // extract leaf binary expressions
+    if (operator === 'AND') {
+        const e1 = parseWhere(root[operator][0]);
+        const e2 = parseWhere(root[operator][1]);
+
+        // eslint-disable-next-line
+        return { '$and' : [
+            e1,
+            e2,
+        ] };
+    } else if (operator === 'OR') {
+        const e1 = parseWhere(root[operator][0]);
+        const e2 = parseWhere(root[operator][1]);
+
+        // eslint-disable-next-line
+        return  { '$or' : [
+            e1,
+            e2,
+        ] };
+    }
+    const field = root[operator][0];
+    const value = root[operator][1];
+    const expr = exprMapper[operator];
+    const obj = {};
+
+    if (operator === 'LIKE') {
+        obj[`value.${field}`] = parseLikeExpression(value);
+    } else {
+        obj[`value.${field}`] = { [expr]: value };
+    }
+
+    return obj;
+}
+
+module.exports = parseWhere;

lib/api/apiUtils/bucket/validateReplicationConfig.js

@@ -0,0 +1,32 @@
+const config = require('../../../Config').config;
+
+/**
+ * Validates that the replication configuration contains a preferred
+ * read location if the bucket location is a transient source
+ *
+ * @param {object} repConfig - replication configuration
+ * @param {object} bucket - bucket metadata
+ *
+ * @return {boolean} validity of replication configuration with
+ * transient source
+ */
+function validateReplicationConfig(repConfig, bucket) {
+    const bucketLocationName = bucket.getLocationConstraint();
+    if (!repConfig || !repConfig.rules) {
+        return false;
+    }
+    const bucketLocation = config.locationConstraints[bucketLocationName];
+    if (!bucketLocation.isTransient) {
+        return true;
+    }
+    return repConfig.rules.every(rule => {
+        if (!rule.storageClass) {
+            return true;
+        }
+        const storageClasses = rule.storageClass.split(',');
+        return storageClasses.some(
+            site => site.endsWith(':preferred_read'));
+    });
+}
+
+module.exports = validateReplicationConfig;

lib/api/apiUtils/bucket/validateSearch.js

@@ -0,0 +1,96 @@
+const Parser = require('sql-where-parser');
+const { errors } = require('arsenal');
+const objModel = require('arsenal').models.ObjectMD;
+
+const BINARY_OP = 2;
+const sqlConfig = {
+    operators: [
+        {
+            '=': BINARY_OP,
+            '<': BINARY_OP,
+            '>': BINARY_OP,
+            '<>': BINARY_OP,
+            '<=': BINARY_OP,
+            '>=': BINARY_OP,
+            '!=': BINARY_OP,
+        },
+        { LIKE: BINARY_OP },
+        { AND: BINARY_OP },
+        { OR: BINARY_OP },
+    ],
+    tokenizer: {
+        shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'],
+        shouldMatch: ['"', '\'', '`'],
+        shouldDelimitBy: [' ', '\n', '\r', '\t'],
+    },
+};
+const parser = new Parser(sqlConfig);
+
+function _validateTree(whereClause, possibleAttributes) {
+    let invalidAttribute;
+
+    function _searchTree(node) {
+        if (typeof node !== 'object') {
+            invalidAttribute = node;
+        } else {
+            const operator = Object.keys(node)[0];
+            if (operator === 'AND' || operator === 'OR') {
+                _searchTree(node[operator][0]);
+                _searchTree(node[operator][1]);
+            } else {
+                const field = node[operator][0];
+                if (!field.startsWith('tags.') &&
+                    !possibleAttributes[field] &&
+                    !field.startsWith('replicationInfo.') &&
+                    !field.startsWith('x-amz-meta-')) {
+                    invalidAttribute = field;
+                }
+            }
+        }
+    }
+    _searchTree(whereClause);
+    return invalidAttribute;
+}
+
+/**
+ * validateSearchParams - validate value of ?search= in request
+ * @param  {string} searchParams - value of search params in request
+ *  which should be jsu sql where clause
+ * For metadata: x-amz-meta-color=\"blue\"
+ * For tags: tags.x-amz-meta-color=\"blue\"
+ * For replication status : replication-status=\"PENDING\"
+ * For any other attribute: `content-length`=5
+ * @return {undefined | error} undefined if validates or arsenal error if not
+ */
+function validateSearchParams(searchParams) {
+    let ast;
+    try {
+        // allow using 'replicationStatus' as search param to increase
+        // ease of use, pending metadata search rework
+        // eslint-disable-next-line no-param-reassign
+        searchParams = searchParams.replace(
+            'replication-status', 'replicationInfo.status');
+        ast = parser.parse(searchParams);
+    } catch (e) {
+        if (e) {
+            return {
+                error: errors.InvalidArgument
+                .customizeDescription('Invalid sql where clause ' +
+                'sent as search query'),
+            };
+        }
+    }
+    const possibleAttributes = objModel.getAttributes();
+    const invalidAttribute = _validateTree(ast, possibleAttributes);
+    if (invalidAttribute) {
+        return {
+            error: errors.InvalidArgument
+                .customizeDescription('Search param ' +
+                `contains unknown attribute: ${invalidAttribute}`) };
+    }
+    return {
+        ast,
+    };
+}
+
+module.exports = validateSearchParams;

lib/api/apiUtils/object/BackendInfo.js

@@ -1,233 +0,0 @@
-const { config } = require('../../../Config');
-const { legacyLocations } = require('../../../../constants.js');
-const { locationConstraints } = config;
-const escapeForXml = require('arsenal').s3middleware.escapeForXml;
-
-class BackendInfo {
-    /**
-    * Represents the info necessary to evaluate which data backend to use
-    * on a data put call.
-    * @constructor
-    * @param {string | undefined} objectLocationConstraint - location constraint
-    * for object based on user meta header
-    * @param {string | undefined } bucketLocationConstraint - location
-    * constraint for bucket based on bucket metadata
-    * @param {string} requestEndpoint - endpoint to which request was made
-    * @param {string | undefined } legacyLocationConstraint - legacy location
-    * constraint
-    */
-    constructor(objectLocationConstraint, bucketLocationConstraint,
-        requestEndpoint, legacyLocationConstraint) {
-        this._objectLocationConstraint = objectLocationConstraint;
-        this._bucketLocationConstraint = bucketLocationConstraint;
-        this._requestEndpoint = requestEndpoint;
-        this._legacyLocationConstraint = legacyLocationConstraint;
-        return this;
-    }
-
-    /**
-     * validate proposed location constraint against config
-     * @param {string | undefined} locationConstraint - value of user
-     * metadata location constraint header or bucket location constraint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if valid, false if not
-     */
-    static isValidLocationConstraint(locationConstraint, log) {
-        if (Object.keys(config.locationConstraints).
-            indexOf(locationConstraint) < 0) {
-            log.trace('proposed locationConstraint is invalid',
-                { locationConstraint });
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * validate that request endpoint is listed in the restEndpoint config
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if present, false if not
-     */
-    static isRequestEndpointPresent(requestEndpoint, log) {
-        if (Object.keys(config.restEndpoints).indexOf(requestEndpoint) < 0) {
-            log.trace('requestEndpoint does not match config restEndpoints',
-              { requestEndpoint });
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * validate that locationConstraint for request Endpoint matches
-     * one config locationConstraint
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if matches, false if not
-     */
-    static isRequestEndpointValueValid(requestEndpoint, log) {
-        if (Object.keys(config.locationConstraints).indexOf(config
-            .restEndpoints[requestEndpoint]) < 0) {
-            log.trace('the default locationConstraint for request' +
-                'Endpoint does not match any config locationConstraint',
-                { requestEndpoint });
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * validate that s3 server is running with a file or memory backend
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if running with file/mem backend, false if not
-     */
-    static isMemOrFileBackend(requestEndpoint, log) {
-        if (config.backends.data === 'mem' ||
-        config.backends.data === 'file') {
-            log.trace('use data backend for the location', {
-                dataBackend: config.backends.data,
-                method: 'isMemOrFileBackend',
-            });
-            return true;
-        }
-        return false;
-    }
-
-    /**
-     * validate requestEndpoint against config or mem/file data backend
-     * - if there is no match for the request endpoint in the config
-     * restEndpoints and data backend is set to mem or file we will use this
-     * data backend for the location.
-     * - if locationConstraint for request Endpoint does not match
-     * any config locationConstraint, we will return an error
-     * @param {string} requestEndpoint - request endpoint
-     * @param {object} log - werelogs logger
-     * @return {boolean} - true if valid, false if not
-     */
-    static isValidRequestEndpointOrBackend(requestEndpoint, log) {
-        if (!BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
-            return BackendInfo.isMemOrFileBackend(requestEndpoint, log);
-        }
-        return BackendInfo.isRequestEndpointValueValid(requestEndpoint, log);
-    }
-
-    /**
-     * validate controlling BackendInfo Parameter
-     * @param {string | undefined} objectLocationConstraint - value of user
-     * metadata location constraint header
-     * @param {string | null} bucketLocationConstraint - location
-     * constraint from bucket metadata
-     * @param {string} requestEndpoint - endpoint of request
-     * @param {object} log - werelogs logger
-     * @return {object} - location constraint validity
-     */
-    static controllingBackendParam(objectLocationConstraint,
-        bucketLocationConstraint, requestEndpoint, log) {
-        if (objectLocationConstraint) {
-            if (BackendInfo.isValidLocationConstraint(objectLocationConstraint,
-                log)) {
-                log.trace('objectLocationConstraint is valid');
-                return { isValid: true };
-            }
-            log.trace('objectLocationConstraint is invalid');
-            return { isValid: false, description: 'Object Location Error - ' +
-            `Your object location "${escapeForXml(objectLocationConstraint)}"` +
-            'is not  in your location config - Please update.' };
-        }
-        if (bucketLocationConstraint) {
-            if (BackendInfo.isValidLocationConstraint(bucketLocationConstraint,
-                log)) {
-                log.trace('bucketLocationConstraint is valid');
-                return { isValid: true };
-            }
-            log.trace('bucketLocationConstraint is invalid');
-            return { isValid: false, description: 'Bucket Location Error - ' +
-            `Your bucket location "${escapeForXml(bucketLocationConstraint)}"` +
-            ' is not in your location config - Please update.' };
-        }
-        const legacyLocationConstraint =
-        BackendInfo.getLegacyLocationConstraint();
-        if (legacyLocationConstraint) {
-            log.trace('legacy location is valid');
-            return { isValid: true, legacyLocationConstraint };
-        }
-        if (!BackendInfo.isValidRequestEndpointOrBackend(requestEndpoint,
-          log)) {
-            return { isValid: false, description: 'Endpoint Location Error - ' +
-            `Your endpoint "${requestEndpoint}" is not in restEndpoints ` +
-            'in your config OR the default location constraint for request ' +
-            `endpoint "${escapeForXml(requestEndpoint)}" does not ` +
-            'match any config locationConstraint - Please update.' };
-        }
-        if (BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
-            return { isValid: true };
-        }
-        return { isValid: true, defaultedToDataBackend: true };
-    }
-
-    /**
-    * Return legacyLocationConstraint
-    * @return {string | undefined} legacyLocationConstraint;
-    */
-    static getLegacyLocationConstraint() {
-        return legacyLocations.find(ll => locationConstraints[ll]);
-    }
-
-    /**
-    * Return objectLocationConstraint
-    * @return {string | undefined} objectLocationConstraint;
-    */
-    getObjectLocationConstraint() {
-        return this._objectLocationConstraint;
-    }
-
-    /**
-    * Return bucketLocationConstraint
-    * @return {string | undefined} bucketLocationConstraint;
-    */
-    getBucketLocationConstraint() {
-        return this._bucketLocationConstraint;
-    }
-
-    /**
-    * Return requestEndpoint
-    * @return {string} requestEndpoint;
-    */
-    getRequestEndpoint() {
-        return this._requestEndpoint;
-    }
-
-    /**
-    * Return locationConstraint that should be used with put request
-    * Order of priority is:
-    * (1) objectLocationConstraint,
-    * (2) bucketLocationConstraint,
-    * (3) legacyLocationConstraint,
-    * (4) default locationConstraint for requestEndpoint  if requestEndpoint
-    *     is listed in restEndpoints in config.json
-    * (5) default data backend
-    * @return {string} locationConstraint;
-    */
-    getControllingLocationConstraint() {
-        const objectLC = this.getObjectLocationConstraint();
-        const bucketLC = this.getBucketLocationConstraint();
-        const reqEndpoint = this.getRequestEndpoint();
-        if (objectLC) {
-            return objectLC;
-        }
-        if (bucketLC) {
-            return bucketLC;
-        }
-        if (this._legacyLocationConstraint) {
-            return this._legacyLocationConstraint;
-        }
-        if (config.restEndpoints[reqEndpoint]) {
-            return config.restEndpoints[reqEndpoint];
-        }
-        return config.backends.data;
-    }
-}
-
-module.exports = {
-    BackendInfo,
-};

lib/api/apiUtils/object/abortMultipartUpload.js

@@ -3,7 +3,7 @@ const async = require('async');
 const constants = require('../../../../constants');
 const { data } = require('../../../data/wrapper');
 const locationConstraintCheck = require('../object/locationConstraintCheck');
-const { metadataValidateBucketAndObj } =
+const { standardMetadataValidateBucketAndObj } =
     require('../../../metadata/metadataUtils');
 const services = require('../../../services');
 
@@ -14,7 +14,7 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
         bucketName,
         objectKey,
         uploadId,
-        preciseRequestType: 'multipartDelete',
+        preciseRequestType: request.apiMethods || 'multipartDelete',
         request,
     };
     // For validating the request at the destinationBucket level
@@ -22,10 +22,11 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
     // but the requestType is the more general 'objectDelete'
     const metadataValParams = Object.assign({}, metadataValMPUparams);
     metadataValParams.requestType = 'objectPut';
+    const authzIdentityResult = request ? request.actionImplicitDenies : false;
 
     async.waterfall([
         function checkDestBucketVal(next) {
-            metadataValidateBucketAndObj(metadataValParams, log,
+            standardMetadataValidateBucketAndObj(metadataValParams, authzIdentityResult, log,
                 (err, destinationBucket) => {
                     if (err) {
                         return next(err, destinationBucket);
@@ -56,9 +57,14 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
         function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket,
         next) {
             const location = mpuOverviewObj.controllingLocationConstraint;
+            const originalIdentityAuthzResults = request.actionImplicitDenies;
+            // eslint-disable-next-line no-param-reassign
+            delete request.actionImplicitDenies;
             return data.abortMPU(objectKey, uploadId, location, bucketName,
             request, destBucket, locationConstraintCheck, log,
             (err, skipDataDelete) => {
+                // eslint-disable-next-line no-param-reassign
+                request.actionImplicitDenies = originalIdentityAuthzResults;
                 if (err) {
                     return next(err, destBucket);
                 }
@@ -67,15 +73,6 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
                 return next(null, mpuBucket, destBucket, skipDataDelete);
             });
         },
-        function sendAbortPut(mpuBucket, destBucket, skipDataDelete, next) {
-            services.sendAbortMPUPut(bucketName, objectKey, uploadId, log,
-                err => {
-                    if (err) {
-                        return next(err, destBucket);
-                    }
-                    return next(null, mpuBucket, destBucket, skipDataDelete);
-                });
-        },
         function getPartLocations(mpuBucket, destBucket, skipDataDelete,
         next) {
             services.getMPUparts(mpuBucket.getName(), uploadId, log,
@@ -90,7 +87,6 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
         },
         function deleteData(mpuBucket, storedParts, destBucket,
         skipDataDelete, next) {
-            // for Azure we do not need to delete data
             if (skipDataDelete) {
                 return next(null, mpuBucket, storedParts, destBucket);
             }

lib/api/apiUtils/object/applyZenkoUserMD.js

@@ -0,0 +1,19 @@
+const { zenkoIDHeader } = require('arsenal').constants;
+
+const _config = require('../../../Config').config;
+
+/**
+ * applyZenkoUserMD - if request is within a Zenko deployment, apply user
+ * metadata called "zenko-source" to the object
+ * @param {Object} metaHeaders - user metadata object
+ * @return {undefined}
+ */
+function applyZenkoUserMD(metaHeaders) {
+    if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' &&
+        !metaHeaders[zenkoIDHeader]) {
+        // eslint-disable-next-line no-param-reassign
+        metaHeaders[zenkoIDHeader] = _config.getPublicInstanceId();
+    }
+}
+
+module.exports = applyZenkoUserMD;

lib/api/apiUtils/object/checkReadLocation.js

@@ -0,0 +1,27 @@
+/**
+ * checkReadLocation - verify that a bucket's default read location exists
+ * for a specified read data locator
+ * @param {Config} config - Config object
+ * @param {string} locationName - location constraint
+ * @param {string} objectKey - object key
+ * @param {string} bucketName - bucket name
+ * @return {Object | null} return object containing location information
+ * if location exists; otherwise, null
+ */
+function checkReadLocation(config, locationName, objectKey, bucketName) {
+    const readLocation = config.getLocationConstraint(locationName);
+    if (readLocation) {
+        const bucketMatch = readLocation.details &&
+            readLocation.details.bucketMatch;
+        const backendKey = bucketMatch ? objectKey :
+            `${bucketName}/${objectKey}`;
+        return {
+            location: locationName,
+            key: backendKey,
+            locationType: readLocation.type,
+        };
+    }
+    return null;
+}
+
+module.exports = checkReadLocation;

lib/api/apiUtils/object/coldStorage.js

@@ -0,0 +1,247 @@
+/*
+ * Code based on Yutaka Oishi (Fujifilm) contributions
+ * Date: 11 Sep 2020
+ */
+const { ObjectMDArchive } = require('arsenal').models;
+const errors = require('arsenal').errors;
+const { config } = require('../../../Config');
+const { locationConstraints } = config;
+
+const { scaledMsPerDay } = config.getTimeOptions();
+
+/**
+ * Get response header "x-amz-restore"
+ * Be called by objectHead.js
+ * @param {object} objMD - object's metadata
+ * @returns {string|undefined} x-amz-restore
+ */
+function getAmzRestoreResHeader(objMD) {
+    if (objMD.archive &&
+        objMD.archive.restoreRequestedAt &&
+        !objMD.archive.restoreCompletedAt) {
+        // Avoid race condition by relying on the `archive` MD of the object
+        // and return the right header after a RESTORE request.
+        // eslint-disable-next-line
+        return `ongoing-request="true"`;
+    }
+    if (objMD['x-amz-restore']) {
+        if (objMD['x-amz-restore']['expiry-date']) {
+            const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString();
+            // eslint-disable-next-line
+            return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
+        }
+    }
+    return undefined;
+}
+
+/**
+ * Check if restore can be done.
+ *
+ * @param {ObjectMD} objectMD - object metadata
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
+ */
+function _validateStartRestore(objectMD, log) {
+    if (objectMD.archive?.restoreCompletedAt) {
+        if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
+            // return InvalidObjectState error if the restored object is expired
+            // but restore info md of this object has not yet been cleared
+            log.debug('The restored object already expired.',
+                {
+                    archive: objectMD.archive,
+                    method: '_validateStartRestore',
+                });
+            return errors.InvalidObjectState;
+        }
+
+        // If object is already restored, no further check is needed
+        // Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have
+        // been reset.
+        return undefined;
+    }
+    const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
+    if (!isLocationCold) {
+        // return InvalidObjectState error if the object is not in cold storage,
+        // not in cold storage means either location cold flag not exists or cold flag is explicit false
+        log.debug('The bucket of the object is not in a cold storage location.',
+            {
+                isLocationCold,
+                method: '_validateStartRestore',
+            });
+        return errors.InvalidObjectState;
+    }
+    if (objectMD.archive?.restoreRequestedAt) {
+        // return RestoreAlreadyInProgress error if the object is currently being restored
+        // check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
+        log.debug('The object is currently being restored.',
+            {
+                archive: objectMD.archive,
+                method: '_validateStartRestore',
+            });
+        return errors.RestoreAlreadyInProgress;
+    }
+    return undefined;
+}
+
+/**
+ * Check if "put version id" is allowed
+ *
+ * @param {ObjectMD} objMD - object metadata
+ * @param {string} versionId - object's version id
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} - undefined if "put version id" is allowed
+ */
+function validatePutVersionId(objMD, versionId, log) {
+    if (!objMD) {
+        const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
+        log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
+        return err;
+    }
+
+    if (objMD.isDeleteMarker) {
+        log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
+        return errors.MethodNotAllowed;
+    }
+
+    const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
+    if (!isLocationCold) {
+        log.error('The object data is not stored in a cold storage location.',
+            {
+                isLocationCold,
+                dataStoreName: objMD.dataStoreName,
+                method: 'validatePutVersionId',
+            });
+        return errors.InvalidObjectState;
+    }
+
+    // make sure object archive restoration is in progress
+    // NOTE: we do not use putObjectVersion to update the restoration period.
+    if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
+        || objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
+        log.error('object archive restoration is not in progress',
+            { method: 'validatePutVersionId', versionId });
+        return errors.InvalidObjectState;
+    }
+
+    return undefined;
+}
+
+/**
+ * Check if the object is already restored, and update the expiration date accordingly:
+ * > After restoring an archived object, you can update the restoration period by reissuing the
+ * > request with a new period. Amazon S3 updates the restoration period relative to the current
+ * > time.
+ *
+ * @param {ObjectMD} objectMD - object metadata
+ * @param {object} log - werelogs logger
+ * @return {boolean} - true if the object is already restored
+ */
+function _updateObjectExpirationDate(objectMD, log) {
+    // Check if restoreCompletedAt field exists
+    // Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
+    // checked earlier in the process, so checking again here would create weird states
+    const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
+    log.debug('The restore status of the object.', {
+        isObjectAlreadyRestored,
+        method: 'isObjectAlreadyRestored'
+    });
+    if (isObjectAlreadyRestored) {
+        const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
+        expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
+
+        /* eslint-disable no-param-reassign */
+        objectMD.archive.restoreWillExpireAt = expiryDate;
+        objectMD['x-amz-restore'] = {
+            'ongoing-request': false,
+            'expiry-date': expiryDate,
+        };
+        /* eslint-enable no-param-reassign */
+    }
+    return isObjectAlreadyRestored;
+}
+
+/**
+ * update restore expiration date.
+ *
+ * @param {ObjectMD} objectMD - objectMD instance
+ * @param {object} restoreParam - restore param
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} internal error if object MD is not valid
+ *
+ */
+function _updateRestoreInfo(objectMD, restoreParam, log) {
+    if (!objectMD.archive) {
+        log.debug('objectMD.archive doesn\'t exits', {
+            objectMD,
+            method: '_updateRestoreInfo'
+        });
+        return errors.InternalError.customizeDescription('Archive metadata is missing.');
+    }
+    /* eslint-disable no-param-reassign */
+    objectMD.archive.restoreRequestedAt = new Date();
+    objectMD.archive.restoreRequestedDays = restoreParam.days;
+    objectMD.originOp = 's3:ObjectRestore:Post';
+    /* eslint-enable no-param-reassign */
+    if (!ObjectMDArchive.isValid(objectMD.archive)) {
+        log.debug('archive is not valid', {
+            archive: objectMD.archive,
+            method: '_updateRestoreInfo'
+        });
+        return errors.InternalError.customizeDescription('Invalid archive metadata.');
+    }
+    return undefined;
+}
+
+/**
+ * start to restore object.
+ * If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
+ * calculate restore expiry-date and add it to objectMD.
+ * Be called by objectRestore.js
+ *
+ * @param {ObjectMD} objectMD - objectMd instance
+ * @param {object} restoreParam - bucket name
+ * @param {object} log - werelogs logger
+ * @param {function} cb - bucket name
+ * @return {undefined}
+ *
+ */
+function startRestore(objectMD, restoreParam, log, cb) {
+    log.info('Validating if restore can be done or not.');
+    const checkResultError = _validateStartRestore(objectMD, log);
+    if (checkResultError) {
+        return cb(checkResultError);
+    }
+    log.info('Updating restore information.');
+    const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
+    if (updateResultError) {
+        return cb(updateResultError);
+    }
+    const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
+    return cb(null, isObjectAlreadyRestored);
+}
+
+/**
+ * checks if object data is available or if it's in cold storage
+ * @param {ObjectMD} objMD Object metadata
+ * @returns {ArsenalError|null} error if object data is not available
+ */
+function verifyColdObjectAvailable(objMD) {
+    // return error when object is cold
+    if (objMD.archive &&
+        // Object is in cold backend
+        (!objMD.archive.restoreRequestedAt ||
+            // Object is being restored
+            (objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
+        const err = errors.InvalidObjectState
+            .customizeDescription('The operation is not valid for the object\'s storage class');
+        return err;
+    }
+    return null;
+}
+
+module.exports = {
+    startRestore,
+    getAmzRestoreResHeader,
+    validatePutVersionId,
+    verifyColdObjectAvailable,
+};

lib/api/apiUtils/object/createAndStoreObject.js

@@ -5,49 +5,22 @@ const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
 const constants = require('../../../../constants');
 const { data } = require('../../../data/wrapper');
 const services = require('../../../services');
-const logger = require('../../../utilities/logger');
 const { dataStore } = require('./storeObject');
 const locationConstraintCheck = require('./locationConstraintCheck');
-const { versioningPreprocessing } = require('./versioning');
+const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
 const removeAWSChunked = require('./removeAWSChunked');
 const getReplicationInfo = require('./getReplicationInfo');
 const { config } = require('../../../Config');
 const validateWebsiteHeader = require('./websiteServing')
     .validateWebsiteHeader;
-const {
-    externalBackends, versioningNotImplBackends, zenkoIDHeader,
-} = constants;
+const applyZenkoUserMD = require('./applyZenkoUserMD');
+const { externalBackends, versioningNotImplBackends } = constants;
 
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
-'a versioned object to a location-constraint of type Azure.';
-
-/**
- * Retro-propagation is where S3C ingestion will re-ingest an object whose
- * request originated from Zenko.
- * To avoid this, Zenko requests which create objects/versions will be tagged
- * with a user-metadata header defined in constants.zenkoIDHeader. When
- * ingesting objects into Zenko, we can determine if this object has already
- * been created in Zenko.
- * Delete marker requests cannot specify user-metadata fields, so we instead
- * rely on checking the "user-agent" to see the origin of a request.
- * If delete marker, and user-agent came from a Zenko client, we add the
- * user-metadata field to the object metadata.
- * @param {Object} metaHeaders - user metadata object
- * @param {http.ClientRequest} request - client request with user-agent header
- * @param {Boolean} isDeleteMarker - delete marker indicator
- * @return {undefined}
- */
-function _checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker) {
-    const userAgent = request.headers['user-agent'];
-
-    if (isDeleteMarker && userAgent && userAgent.includes('Zenko')) {
-        // eslint-disable-next-line no-param-reassign
-        metaHeaders[zenkoIDHeader] = 'zenko';
-    }
-}
+'a versioned object to a location-constraint of type Azure or GCP.';
 
 function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
-    metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
+    metadataStoreParams, dataToDelete, log, requestMethod, callback) {
     services.metadataStoreObject(bucketName, dataGetInfo,
         cipherBundle, metadataStoreParams, (err, result) => {
             if (err) {
@@ -57,7 +30,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
                 const newDataStoreName = Array.isArray(dataGetInfo) ?
                     dataGetInfo[0].dataStoreName : null;
                 return data.batchDelete(dataToDelete, requestMethod,
-                    newDataStoreName, deleteLog, err => callback(err, result));
+                    newDataStoreName, log, err => callback(err, result));
             }
             return callback(null, result);
         });
@@ -77,7 +50,9 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
  * @param {(object|null)} streamingV4Params - if v4 auth, object containing
  * accessKey, signatureFromRequest, region, scopeDate, timestamp, and
  * credentialScope (to be used for streaming v4 auth if applicable)
+ * @param {(object|null)} overheadField - fields to be included in metadata overhead
  * @param {RequestLogger} log - logger instance
+ * @param {string} originOp - Origin operation
  * @param {function} callback - callback function
  * @return {undefined} and call callback with (err, result) -
  * result.contentMD5 - content md5 of new object or version
@@ -85,7 +60,10 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
  */
 function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
-        log, callback) {
+        overheadField, log, originOp, callback) {
+    const putVersionId = request.headers['x-scal-s3-version-id'];
+    const isPutVersion = putVersionId || putVersionId === '';
+
     const size = isDeleteMarker ? 0 : request.parsedContentLength;
     // although the request method may actually be 'DELETE' if creating a
     // delete marker, for our purposes we consider this to be a 'PUT'
@@ -108,9 +86,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         });
         return process.nextTick(() => callback(metaHeaders));
     }
-    // if receiving a request from Zenko for a delete marker, we place a
-    // user-metadata field on the object
-    _checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker);
+    // if the request occurs within a Zenko deployment, we place a user-metadata
+    // field on the object
+    applyZenkoUserMD(metaHeaders);
 
     log.trace('meta headers', { metaHeaders, method: 'objectPut' });
     const objectKeyContext = {
@@ -136,10 +114,26 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         size,
         headers,
         isDeleteMarker,
-        replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
+        replicationInfo: getReplicationInfo(
+            objectKey, bucketMD, false, size, null, null, authInfo),
+        overheadField,
         log,
     };
 
+    // For Azure BlobStorage API compatability
+    // If an object already exists copy/repair creation-time
+    // creation-time must remain static after an object is created
+    //  --> EVEN FOR VERSIONS <--
+    if (objMD) {
+        if (objMD['creation-time']) {
+            metadataStoreParams.creationTime = objMD['creation-time'];
+        } else {
+            // If creation-time is not set (for old objects)
+            // fall back to the last modified and store it back to the db
+            metadataStoreParams.creationTime = objMD['last-modified'];
+        }
+    }
+
     if (!isDeleteMarker) {
         metadataStoreParams.contentType = request.headers['content-type'];
         metadataStoreParams.cacheControl = request.headers['cache-control'];
@@ -149,7 +143,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             removeAWSChunked(request.headers['content-encoding']);
         metadataStoreParams.expires = request.headers.expires;
         metadataStoreParams.tagging = request.headers['x-amz-tagging'];
-        metadataStoreParams.originOp = 's3:ObjectCreated:Put';
+        metadataStoreParams.originOp = originOp;
         const defaultObjectLockConfiguration
             = bucketMD.getObjectLockConfiguration();
         if (defaultObjectLockConfiguration) {
@@ -164,7 +158,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         // eslint-disable-next-line no-param-reassign
         request.headers[constants.objectLocationConstraintHeader] =
             objMD[constants.objectLocationConstraintHeader];
-        metadataStoreParams.originOp = 's3:ObjectRemoved:DeleteMarkerCreated';
+        metadataStoreParams.originOp = originOp;
     }
 
     const backendInfoObj =
@@ -203,13 +197,41 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
     const dontSkipBackend = externalBackends;
     /* eslint-enable camelcase */
 
-    const requestLogger =
-        logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
+    const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
+    const mdOnlySize = request.headers['x-amz-meta-size'];
+
     return async.waterfall([
         function storeData(next) {
-            if (size === 0 && !dontSkipBackend[locationType]) {
-                metadataStoreParams.contentMD5 = constants.emptyFileMd5;
-                return next(null, null, null);
+            if (size === 0) {
+                if (!dontSkipBackend[locationType]) {
+                    metadataStoreParams.contentMD5 = constants.emptyFileMd5;
+                    return next(null, null, null);
+                }
+                // Handle mdOnlyHeader as a metadata only operation. If
+                // the object in question is actually 0 byte or has a body size
+                // then handle normally.
+                if (mdOnlyHeader === 'true' && mdOnlySize > 0) {
+                    log.debug('metadata only operation x-amz-meta-mdonly');
+                    const md5 = request.headers['x-amz-meta-md5chksum']
+                        ? new Buffer(request.headers['x-amz-meta-md5chksum'],
+                        'base64').toString('hex') : null;
+                    const numParts = request.headers['x-amz-meta-md5numparts'];
+                    let _md5;
+                    if (numParts === undefined) {
+                        _md5 = md5;
+                    } else {
+                        _md5 = `${md5}-${numParts}`;
+                    }
+                    const versionId = request.headers['x-amz-meta-version-id'];
+                    const dataGetInfo = {
+                        key: objectKey,
+                        dataStoreName: location,
+                        dataStoreType: locationType,
+                        dataStoreVersionId: versionId,
+                        dataStoreMD5: _md5,
+                    };
+                    return next(null, dataGetInfo, _md5);
+                }
             }
             return dataStore(objectKeyContext, cipherBundle, request, size,
                     streamingV4Params, backendInfo, log, next);
@@ -234,10 +256,19 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
                 dataGetInfoArr[0].cipheredDataKey =
                     cipherBundle.cipheredDataKey;
             }
+            if (mdOnlyHeader === 'true') {
+                metadataStoreParams.size = mdOnlySize;
+                dataGetInfoArr[0].size = mdOnlySize;
+            }
             metadataStoreParams.contentMD5 = calculatedHash;
             return next(null, dataGetInfoArr);
         },
         function getVersioningInfo(infoArr, next) {
+            // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
+            if (isPutVersion) {
+                const options = overwritingVersioning(objMD, metadataStoreParams);
+                return process.nextTick(() => next(null, options, infoArr));
+            }
             return versioningPreprocessing(bucketName, bucketMD,
                 metadataStoreParams.objectKey, objMD, log, (err, options) => {
                     if (err) {
@@ -257,11 +288,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             metadataStoreParams.versionId = options.versionId;
             metadataStoreParams.versioning = options.versioning;
             metadataStoreParams.isNull = options.isNull;
-            metadataStoreParams.nullVersionId = options.nullVersionId;
-            metadataStoreParams.nullUploadId = options.nullUploadId;
+            metadataStoreParams.deleteNullKey = options.deleteNullKey;
+            if (options.extraMD) {
+                Object.assign(metadataStoreParams, options.extraMD);
+            }
             return _storeInMDandDeleteData(bucketName, infoArr,
                 cipherBundle, metadataStoreParams,
-                options.dataToDelete, requestLogger, requestMethod, next);
+                options.dataToDelete, log, requestMethod, next);
         },
     ], callback);
 }

lib/api/apiUtils/object/deleteObject.js

@@ -0,0 +1,18 @@
+/**
+ * _bucketRequiresOplogUpdate - DELETE an object from a bucket
+ * @param {BucketInfo} bucket - bucket object
+ * @return {boolean} whether objects require oplog updates on deletion, or not
+ */
+function _bucketRequiresOplogUpdate(bucket) {
+    // Default behavior is to require an oplog update
+    if (!bucket || !bucket.getLifecycleConfiguration || !bucket.getNotificationConfiguration) {
+        return true;
+    }
+    // If the bucket has lifecycle configuration or notification configuration
+    // set, we also require an oplog update
+    return bucket.getLifecycleConfiguration() || bucket.getNotificationConfiguration();
+}
+
+module.exports = {
+    _bucketRequiresOplogUpdate,
+};

lib/api/apiUtils/object/expirationHeaders.js

@@ -4,23 +4,25 @@ const {
     LifecycleDateTime,
     LifecycleUtils,
 } = require('arsenal').s3middleware.lifecycleHelpers;
+const { config } = require('../../../Config');
 
-// moves lifecycle transition deadlines 1 day earlier, mostly for testing
-const transitionOneDayEarlier = process.env.TRANSITION_ONE_DAY_EARLIER === 'true';
-// moves lifecycle expiration deadlines 1 day earlier, mostly for testing
-const expireOneDayEarlier = process.env.EXPIRE_ONE_DAY_EARLIER === 'true';
+const {
+    expireOneDayEarlier,
+    transitionOneDayEarlier,
+    timeProgressionFactor,
+    scaledMsPerDay,
+} = config.getTimeOptions();
 
 const lifecycleDateTime = new LifecycleDateTime({
     transitionOneDayEarlier,
     expireOneDayEarlier,
+    timeProgressionFactor,
 });
 
-const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime);
-
-const oneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
+const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor);
 
 function calculateDate(objDate, expDays, datetime) {
-    return new Date(datetime.getTimestamp(objDate) + expDays * oneDay);
+    return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay));
 }
 
 function formatExpirationHeader(date, id) {

lib/api/apiUtils/object/getReplicationBackendDataLocator.js

@@ -0,0 +1,51 @@
+const { errors } = require('arsenal');
+
+/**
+ * getReplicationBackendDataLocator - compares given location constraint to
+ * replication backends
+ * @param {object} locationObj - object containing location information
+ * @param {string} locationObj.location - name of location constraint
+ * @param {string} locationObj.key - keyname of object in location constraint
+ * @param {string} locationObj.locationType - type of location constraint
+ * @param {object} replicationInfo - information about object replication
+ * @param {array} replicationInfo.backends - array containing information about
+ * each replication location
+ * @param {string} replicationInfo.backends[].site - name of replication
+ * location
+ * @param {string} replicationInfo.backends[].status - status of replication
+ * @param {string} replicationInfo.backends[].dataStoreVersionId - version id
+ * of object at replication location
+ * @return {object} res - response object
+ *     {array} [res.dataLocator] - if COMPLETED status: array
+ *                                 containing the cloud location,
+ *                                 undefined otherwise
+ *     {string} [res.status] - replication status if no error
+ *     {string} [res.reason] - reason message if PENDING/FAILED
+ *     {Error} [res.error] - defined if object is not replicated to
+ *                           location passed in locationObj
+ */
+function getReplicationBackendDataLocator(locationObj, replicationInfo) {
+    const repBackendResult = {};
+    const locMatch = replicationInfo.backends.find(
+        backend => backend.site === locationObj.location);
+    if (!locMatch) {
+        repBackendResult.error = errors.InvalidLocationConstraint.
+            customizeDescription('Object is not replicated to location ' +
+            'passed in location header');
+        return repBackendResult;
+    }
+    repBackendResult.status = locMatch.status;
+    if (['PENDING', 'FAILED'].includes(locMatch.status)) {
+        repBackendResult.reason =
+            `Object replication to specified backend is ${locMatch.status}`;
+        return repBackendResult;
+    }
+    repBackendResult.dataLocator = [{
+        key: locationObj.key,
+        dataStoreName: locationObj.location,
+        dataStoreType: locationObj.locationType,
+        dataStoreVersionId: locMatch.dataStoreVersionId }];
+    return repBackendResult;
+}
+
+module.exports = getReplicationBackendDataLocator;

lib/api/apiUtils/object/getReplicationInfo.js

@@ -1,5 +1,7 @@
 const s3config = require('../../../Config').config;
-const { isLifecycleSession } = require('../authorization/permissionChecks.js');
+const { isServiceAccount, getServiceAccountProperties } =
+      require('../authorization/permissionChecks');
+const { replicationBackends } = require('arsenal').constants;
 
 function _getBackend(objectMD, site) {
     const backends = objectMD ? objectMD.replicationInfo.backends : [];
@@ -21,25 +23,31 @@ function _getStorageClasses(rule) {
     }
     const { replicationEndpoints } = s3config;
     // If no storage class, use the given default endpoint or the sole endpoint
-    if (replicationEndpoints.length > 1) {
+    if (replicationEndpoints.length > 0) {
         const endPoint =
-            replicationEndpoints.find(endpoint => endpoint.default);
+            replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
         return [endPoint.site];
     }
-    return [replicationEndpoints[0].site];
+    return undefined;
 }
 
 function _getReplicationInfo(rule, replicationConfig, content, operationType,
-    objectMD) {
+    objectMD, bucketMD) {
     const storageTypes = [];
     const backends = [];
     const storageClasses = _getStorageClasses(rule);
+    if (!storageClasses) {
+        return undefined;
+    }
     storageClasses.forEach(storageClass => {
-        const location = s3config.locationConstraints[storageClass];
-        if (location && ['aws_s3', 'azure'].includes(location.type)) {
+        const storageClassName =
+              storageClass.endsWith(':preferred_read') ?
+              storageClass.split(':')[0] : storageClass;
+        const location = s3config.locationConstraints[storageClassName];
+        if (location && replicationBackends[location.type]) {
             storageTypes.push(location.type);
         }
-        backends.push(_getBackend(objectMD, storageClass));
+        backends.push(_getBackend(objectMD, storageClassName));
     });
     if (storageTypes.length > 0 && operationType) {
         content.push(operationType);
@@ -52,6 +60,7 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
         storageClass: storageClasses.join(','),
         role: replicationConfig.role,
         storageType: storageTypes.join(','),
+        isNFS: bucketMD.isNFS(),
     };
 }
 
@@ -65,26 +74,45 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
  * @param {string} operationType - The type of operation to replicate
  * @param {object} objectMD - The object metadata
  * @param {AuthInfo} [authInfo] - authentication info of object owner
- * @param {boolean} [isDeleteMarker] - whether creating a delete marker
  * @return {undefined}
  */
 function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
-    objectMD, authInfo, isDeleteMarker) {
+    objectMD, authInfo) {
     const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
     const config = bucketMD.getReplicationConfiguration();
-    // If bucket does not have a replication configuration, do not replicate.
-    if (config) {
-        // If delete an object due to a lifecycle action,
-        // the delete marker is not replicated to the destination buckets.
-        if (isDeleteMarker && authInfo && isLifecycleSession(authInfo.getArn())) {
-            return undefined;
-        }
 
-        const rule = config.rules.find(rule =>
-            (objKey.startsWith(rule.prefix) && rule.enabled));
-        if (rule) {
-            return _getReplicationInfo(rule, config, content, operationType,
-                objectMD);
+    // Do not replicate object in the following cases:
+    //
+    // - bucket does not have a replication configuration
+    //
+    // - replication configuration does not apply to the object
+    //   (i.e. no rule matches object prefix)
+    //
+    // - replication configuration applies to the object (i.e. a rule matches
+    //   object prefix) but the status is disabled
+    //
+    // - object owner is an internal service account like Lifecycle,
+    //   unless the account properties explicitly allow it to
+    //   replicate like MD ingestion (because we do not want to
+    //   replicate objects created from actions triggered by internal
+    //   services, by design)
+
+    if (config) {
+        let doReplicate = false;
+        if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) {
+            doReplicate = true;
+        } else {
+            const serviceAccountProps = getServiceAccountProperties(
+                authInfo.getCanonicalID());
+            doReplicate = serviceAccountProps.canReplicate;
+        }
+        if (doReplicate) {
+            const rule = config.rules.find(
+                rule => (objKey.startsWith(rule.prefix) && rule.enabled));
+            if (rule) {
+                return _getReplicationInfo(
+                    rule, config, content, operationType, objectMD, bucketMD);
+            }
         }
     }
     return undefined;

lib/api/apiUtils/object/lifecycle.js

@@ -0,0 +1,190 @@
+const { versioning } = require('arsenal');
+const versionIdUtils = versioning.VersionID;
+
+const { lifecycleListing } = require('../../../../constants');
+const { CURRENT_TYPE, NON_CURRENT_TYPE, ORPHAN_DM_TYPE } = lifecycleListing;
+
+function _makeTags(tags) {
+    const res = [];
+    Object.entries(tags).forEach(([key, value]) =>
+        res.push(
+            {
+                Key: key,
+                Value: value,
+            }
+        ));
+    return res;
+}
+
+function processCurrents(bucketName, listParams, isBucketVersioned, list) {
+    const data = {
+        Name: bucketName,
+        Prefix: listParams.prefix,
+        MaxKeys: listParams.maxKeys,
+        MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
+        IsTruncated: !!list.IsTruncated,
+        Marker: listParams.marker,
+        BeforeDate: listParams.beforeDate,
+        NextMarker: list.NextMarker,
+        Contents: [],
+    };
+
+    list.Contents.forEach(item => {
+        const v = item.value;
+
+        const content = {
+            Key: item.key,
+            LastModified: v.LastModified,
+            ETag: `"${v.ETag}"`,
+            Size: v.Size,
+            Owner: {
+                ID: v.Owner.ID,
+                DisplayName: v.Owner.DisplayName,
+            },
+            StorageClass: v.StorageClass,
+            TagSet: _makeTags(v.tags),
+            IsLatest: true, // for compatibility with AWS ListObjectVersions.
+            DataStoreName: v.dataStoreName,
+            ListType: CURRENT_TYPE,
+        };
+
+        // NOTE: The current versions listed to be lifecycle should include version id
+        // if the bucket is versioned.
+        if (isBucketVersioned) {
+            const versionId = (v.IsNull || v.VersionId === undefined) ?
+                'null' : versionIdUtils.encode(v.VersionId);
+            content.VersionId = versionId;
+        }
+
+        data.Contents.push(content);
+    });
+
+    return data;
+}
+
+function _encodeVersionId(vid) {
+    let versionId = vid;
+    if (versionId && versionId !== 'null') {
+        versionId = versionIdUtils.encode(versionId);
+    }
+    return versionId;
+}
+
+function processNonCurrents(bucketName, listParams, list) {
+    const nextVersionIdMarker = _encodeVersionId(list.NextVersionIdMarker);
+    const versionIdMarker = _encodeVersionId(listParams.versionIdMarker);
+
+    const data = {
+        Name: bucketName,
+        Prefix: listParams.prefix,
+        MaxKeys: listParams.maxKeys,
+        MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
+        IsTruncated: !!list.IsTruncated,
+        KeyMarker: listParams.keyMarker,
+        VersionIdMarker: versionIdMarker,
+        BeforeDate: listParams.beforeDate,
+        NextKeyMarker: list.NextKeyMarker,
+        NextVersionIdMarker: nextVersionIdMarker,
+        Contents: [],
+    };
+
+    list.Contents.forEach(item => {
+        const v = item.value;
+        const versionId = (v.IsNull || v.VersionId === undefined) ?
+            'null' : versionIdUtils.encode(v.VersionId);
+
+        const content = {
+            Key: item.key,
+            LastModified: v.LastModified,
+            ETag: `"${v.ETag}"`,
+            Size: v.Size,
+            Owner: {
+                ID: v.Owner.ID,
+                DisplayName: v.Owner.DisplayName,
+            },
+            StorageClass: v.StorageClass,
+            TagSet: _makeTags(v.tags),
+            staleDate: v.staleDate, // lowerCamelCase to be compatible with existing lifecycle.
+            VersionId: versionId,
+            DataStoreName: v.dataStoreName,
+            ListType: NON_CURRENT_TYPE,
+        };
+
+        data.Contents.push(content);
+    });
+
+    return data;
+}
+
+function processOrphans(bucketName, listParams, list) {
+    const data = {
+        Name: bucketName,
+        Prefix: listParams.prefix,
+        MaxKeys: listParams.maxKeys,
+        MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
+        IsTruncated: !!list.IsTruncated,
+        Marker: listParams.marker,
+        BeforeDate: listParams.beforeDate,
+        NextMarker: list.NextMarker,
+        Contents: [],
+    };
+
+    list.Contents.forEach(item => {
+        const v = item.value;
+        const versionId = (v.IsNull || v.VersionId === undefined) ?
+            'null' : versionIdUtils.encode(v.VersionId);
+        data.Contents.push({
+            Key: item.key,
+            LastModified: v.LastModified,
+            Owner: {
+                ID: v.Owner.ID,
+                DisplayName: v.Owner.DisplayName,
+            },
+            VersionId: versionId,
+            IsLatest: true, // for compatibility with AWS ListObjectVersions.
+            ListType: ORPHAN_DM_TYPE,
+        });
+    });
+
+    return data;
+}
+
+function getLocationConstraintErrorMessage(locationName) {
+    return 'value of the location you are attempting to set ' +
+        `- ${locationName} - is not listed in the locationConstraint config`;
+}
+
+/**
+ * validateMaxScannedEntries - Validates and returns the maximum scanned entries value.
+ *
+ * @param {object} params - Query parameters
+ * @param {object} config - CloudServer configuration
+ * @param {number} min - Minimum number of entries to be scanned
+ * @returns {Object} - An object indicating the validation result:
+ *   - isValid (boolean): Whether the validation is successful.
+ *   - maxScannedLifecycleListingEntries (number): The validated maximum scanned entries value if isValid is true.
+ */
+function validateMaxScannedEntries(params, config, min) {
+    let maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
+
+    if (params['max-scanned-lifecycle-listing-entries']) {
+        const maxEntriesParams = Number.parseInt(params['max-scanned-lifecycle-listing-entries'], 10);
+
+        if (Number.isNaN(maxEntriesParams) || maxEntriesParams < min ||
+            maxEntriesParams > maxScannedLifecycleListingEntries) {
+            return { isValid: false };
+        }
+
+        maxScannedLifecycleListingEntries = maxEntriesParams;
+    }
+
+    return { isValid: true, maxScannedLifecycleListingEntries };
+}
+
+module.exports = {
+    processCurrents,
+    processNonCurrents,
+    processOrphans,
+    getLocationConstraintErrorMessage,
+    validateMaxScannedEntries,
+};

lib/api/apiUtils/object/locationConstraintCheck.js

@@ -1,6 +1,7 @@
-const { errors } = require('arsenal');
+const { errors, models } = require('arsenal');
 
-const { BackendInfo } = require('./BackendInfo');
+const { BackendInfo } = models;
+const { config } = require('../../../Config');
 const constants = require('../../../../constants');
 
 /**
@@ -29,7 +30,7 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) {
     const bucketLocationConstraint = bucket.getLocationConstraint();
     const requestEndpoint = request.parsedHost;
 
-    const controllingBackend = BackendInfo.controllingBackendParam(
+    const controllingBackend = BackendInfo.controllingBackendParam(config,
         objectLocationConstraint, bucketLocationConstraint,
         requestEndpoint, log);
     if (!controllingBackend.isValid) {
@@ -39,7 +40,7 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) {
         };
         return backendInfoObj;
     }
-    const backendInfo = new BackendInfo(objectLocationConstraint,
+    const backendInfo = new BackendInfo(config, objectLocationConstraint,
         bucketLocationConstraint, requestEndpoint,
         controllingBackend.legacyLocationConstraint);
     backendInfoObj = {

lib/api/apiUtils/object/locationHeaderCheck.js

@@ -0,0 +1,37 @@
+const { errors } = require('arsenal');
+
+const { config } = require('../../../Config');
+
+/**
+ * locationHeaderCheck - compares 'x-amz-location-constraint' header
+ * to location constraints in config
+ * @param {object} headers - request headers
+ * @param {string} objectKey - key name of object
+ * @param {string} bucketName - name of bucket
+ * @return {undefined|Object} returns error, object, or undefined
+ * @return {string} return.location - name of location constraint
+ * @return {string} return.key - name of object at location constraint
+ * @return {string} - return.locationType - type of location constraint
+ */
+function locationHeaderCheck(headers, objectKey, bucketName) {
+    const location = headers['x-amz-location-constraint'];
+    if (location) {
+        const validLocation = config.locationConstraints[location];
+        if (!validLocation) {
+            return errors.InvalidLocationConstraint.customizeDescription(
+                'Invalid location constraint specified in header');
+        }
+        const bucketMatch = validLocation.details.bucketMatch;
+        const backendKey = bucketMatch ? objectKey :
+            `${bucketName}/${objectKey}`;
+        return {
+            location,
+            key: backendKey,
+            locationType: validLocation.type,
+        };
+    }
+    // no location header was passed
+    return undefined;
+}
+
+module.exports = locationHeaderCheck;

lib/api/apiUtils/object/locationKeysHaveChanged.js

@@ -22,8 +22,13 @@ function locationKeysHaveChanged(prev, curr) {
         return curr.every(v => v.key !== prev);
     }
     const keysMap = {};
-    prev.forEach(v => { keysMap[v.key] = true; });
-    return curr.every(v => !keysMap[v.key]);
+    prev.forEach(v => {
+        if (!keysMap[v.dataStoreType]) {
+            keysMap[v.dataStoreType] = {};
+        }
+        keysMap[v.dataStoreType][v.key] = true;
+    });
+    return curr.every(v => !(keysMap[v.dataStoreType] && keysMap[v.dataStoreType][v.key]));
 }
 
 module.exports = locationKeysHaveChanged;

lib/api/apiUtils/object/objectLockHelpers.js

@@ -3,7 +3,9 @@ const moment = require('moment');
 
 const { config } = require('../../../Config');
 const vault = require('../../../auth/vault');
+const { evaluateBucketPolicyWithIAM } = require('../authorization/permissionChecks');
 
+const { scaledMsPerDay } = config.getTimeOptions();
 /**
  * Calculates retain until date for the locked object version
  * @param {object} retention - includes days or years retention period
@@ -19,8 +21,9 @@ function calculateRetainUntilDate(retention) {
     const date = moment();
     // Calculate the number of days to retain the lock on the object
     const retainUntilDays = days || years * 365;
+    const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay;
     const retainUntilDate
-        = date.add(retainUntilDays, 'days');
+        = date.add(retainUntilDaysInMs, 'ms');
     return retainUntilDate.toISOString();
 }
 /**
@@ -302,7 +305,9 @@ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log,
             if (err) {
                 return cb(err);
             }
-            if (authorizationResults[0].isAllowed !== true) {
+            const explicitDenyExists = authorizationResults.some(
+                authzResult => authzResult.isAllowed === false && !authzResult.isImplicit);
+            if (explicitDenyExists) {
                 log.trace('authorization check failed for user',
                     {
                         'method': 'checkUserPolicyGovernanceBypass',
@@ -310,7 +315,25 @@ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log,
                     });
                 return cb(errors.AccessDenied);
             }
-            return cb(null);
+            // Convert authorization results into an easier to handle format
+            const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => {
+                const apiMethod = authorizationResults[idx].action;
+                // eslint-disable-next-line no-param-reassign
+                acc[apiMethod] = curr.isImplicit;
+                return acc;
+            }, {});
+
+            // Evaluate against the bucket policies
+            const areAllActionsAllowed = evaluateBucketPolicyWithIAM(
+                bucketMD,
+                Object.keys(actionImplicitDenies),
+                authInfo.getCanonicalID(),
+                authInfo,
+                actionImplicitDenies,
+                log,
+                request);
+
+            return cb(areAllActionsAllowed === true ? null : errors.AccessDenied);
         });
 }
 

lib/api/apiUtils/object/objectRestore.js

@@ -0,0 +1,172 @@
+const async = require('async');
+const { errors, s3middleware } = require('arsenal');
+
+const { allowedRestoreObjectRequestTierValues } = require('../../../../constants');
+const coldStorage = require('./coldStorage');
+const monitoring = require('../../../utilities/monitoringHandler');
+const { pushMetric } = require('../../../utapi/utilities');
+const { decodeVersionId } = require('./versioning');
+const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
+const { parseRestoreRequestXml } = s3middleware.objectRestore;
+const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
+
+/**
+ * Check if tier is supported
+ * @param {object} restoreInfo - restore information
+ * @returns {ArsenalError|undefined} return NotImplemented error if tier not support
+ */
+function checkTierSupported(restoreInfo) {
+    if (!allowedRestoreObjectRequestTierValues.includes(restoreInfo.tier)) {
+        return errors.NotImplemented;
+    }
+    return undefined;
+}
+
+/**
+ * POST Object restore process
+ *
+ * @param {MetadataWrapper} metadata - metadata wrapper
+ * @param {object} mdUtils - utility object to treat metadata
+ * @param {AuthInfo} userInfo - Instance of AuthInfo class with requester's info
+ * @param {IncomingMessage} request - request info
+ * @param {object} log - Werelogs logger
+ * @param {function} callback callback function
+ * @return {undefined}
+ */
+function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
+    const METHOD = 'objectRestore';
+
+    const { bucketName, objectKey } = request;
+
+    log.debug('processing request', { method: METHOD });
+
+    const decodedVidResult = decodeVersionId(request.query);
+    if (decodedVidResult instanceof Error) {
+        log.trace('invalid versionId query',
+            {
+                method: METHOD,
+                versionId: request.query.versionId,
+                error: decodedVidResult,
+            });
+        return process.nextTick(() => callback(decodedVidResult));
+    }
+
+    let isObjectRestored = false;
+
+    const mdValueParams = {
+        authInfo: userInfo,
+        bucketName,
+        objectKey,
+        versionId: decodedVidResult,
+        requestType: request.apiMethods || 'restoreObject',
+        /**
+         * Restoring an object might not cause any impact on
+         * the storage, if the object is already restored: in
+         * this case, the duration is extended. We disable the
+         * quota evaluation and trigger it manually.
+         */
+        checkQuota: false,
+        request,
+    };
+
+    return async.waterfall([
+            // get metadata of bucket and object
+            function validateBucketAndObject(next) {
+                return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies,
+                log, (err, bucketMD, objectMD) => {
+                    if (err) {
+                        log.trace('request authorization failed', { method: METHOD, error: err });
+                        return next(err);
+                    }
+                    // Call back error if object metadata could not be obtained
+                    if (!objectMD) {
+                        const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey;
+                        log.trace('error no object metadata found', { method: METHOD, error: err });
+                        return next(err, bucketMD);
+                    }
+                    // If object metadata is delete marker,
+                    // call back NoSuchKey or MethodNotAllowed depending on specifying versionId
+                    if (objectMD.isDeleteMarker) {
+                        let err = errors.NoSuchKey;
+                        if (decodedVidResult) {
+                            err = errors.MethodNotAllowed;
+                        }
+                        log.trace('version is a delete marker', { method: METHOD, error: err });
+                        return next(err, bucketMD, objectMD);
+                    }
+                    log.info('it acquired the object metadata.', {
+                        'method': METHOD,
+                    });
+                    return next(null, bucketMD, objectMD);
+                });
+            },
+
+            // generate restore param obj from xml of request body and check tier validity
+            function parseRequestXmlAndCheckTier(bucketMD, objectMD, next) {
+                log.trace('parsing object restore information');
+                return parseRestoreRequestXml(request.post, log, (err, restoreInfo) => {
+                    if (err) {
+                        return next(err, bucketMD, objectMD, restoreInfo);
+                    }
+                    log.info('it parsed xml of the request body.', { method: METHOD, value: restoreInfo });
+                    const checkTierResult = checkTierSupported(restoreInfo);
+                    if (checkTierResult instanceof Error) {
+                        return next(checkTierResult);
+                    }
+                    return next(null, bucketMD, objectMD, restoreInfo);
+                });
+            },
+            // start restore process
+            function startRestore(bucketMD, objectMD, restoreInfo, next) {
+                return coldStorage.startRestore(objectMD, restoreInfo, log,
+                    (err, _isObjectRestored) => {
+                        isObjectRestored = _isObjectRestored;
+                        return next(err, bucketMD, objectMD);
+                    });
+            },
+            function evaluateQuotas(bucketMD, objectMD, next) {
+                if (isObjectRestored) {
+                    return next(null, bucketMD, objectMD);
+                }
+                const actions = Array.isArray(mdValueParams.requestType) ?
+                    mdValueParams.requestType : [mdValueParams.requestType];
+                const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
+                return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
+                    false, log, err => next(err, bucketMD, objectMD));
+            },
+            function updateObjectMD(bucketMD, objectMD, next) {
+                const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
+                metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
+                    log, err => next(err, bucketMD, objectMD));
+            },
+        ],
+        (err, bucketMD) => {
+            // generate CORS response header
+            const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD);
+            if (err) {
+                log.trace('error processing request',
+                    {
+                        method: METHOD,
+                        error: err,
+                    });
+                monitoring.promMetrics(
+                    'POST', bucketName, err.code, 'restoreObject');
+                return callback(err, err.code, responseHeaders);
+            }
+            pushMetric('restoreObject', log, {
+                userInfo,
+                bucket: bucketName,
+            });
+            if (isObjectRestored) {
+                monitoring.promMetrics(
+                    'POST', bucketName, '200', 'restoreObject');
+                return callback(null, 200, responseHeaders);
+            }
+            monitoring.promMetrics(
+                'POST', bucketName, '202', 'restoreObject');
+            return callback(null, 202, responseHeaders);
+        });
+}
+
+
+module.exports = objectRestore;

lib/api/apiUtils/object/processMpuParts.js

@@ -1,241 +0,0 @@
-const { errors } = require('arsenal');
-const crypto = require('crypto');
-const constants = require('../../../../constants');
-
-/**
- * createAggregateETag - creates ETag from concatenated MPU part ETags to
- * mimic AWS
- * @param {string} concatETags - string of concatenated MPU part ETags
- * @param {array} partList - list of parts to complete MPU with
- * @return {string} aggregateETag - final complete MPU obj ETag
- */
-function createAggregateETag(concatETags, partList) {
-    // AWS documentation is unclear on what the MD5 is that it returns
-    // in the response for a complete multipart upload request.
-    // The docs state that they might or might not
-    // return the MD5 of the complete object. It appears
-    // they are returning the MD5 of the parts' MD5s so that is
-    // what we have done here. We:
-    // 1) concatenate the hex version of the
-    // individual ETags
-    // 2) convert the concatenated hex to binary
-    // 3) take the md5 of the binary
-    // 4) create the hex digest of the md5
-    // 5) add '-' plus the number of parts at the end
-
-    // Convert the concatenated hex ETags to binary
-    const bufferedHex = Buffer.from(concatETags, 'hex');
-    // Convert the buffer to a binary string
-    const binaryString = bufferedHex.toString('binary');
-    // Get the md5 of the binary string
-    const md5Hash = crypto.createHash('md5');
-    md5Hash.update(binaryString, 'binary');
-    // Get the hex digest of the md5
-    let aggregateETag = md5Hash.digest('hex');
-    // Add the number of parts at the end
-    aggregateETag = `${aggregateETag}-${partList.length}`;
-
-    return aggregateETag;
-}
-
-/**
- * generateMpuPartStorageInfo - generates info needed for storage of
- * completed MPU object
- * @param {array} filteredPartList - list of parts filtered from metadata
- * @return {object} partsInfo - contains three keys: aggregateETag,
- * dataLocations, and calculatedSize
- */
-function generateMpuPartStorageInfo(filteredPartList) {
-    // Assemble array of part locations, aggregate size
-    // and build string to create aggregate ETag
-    let calculatedSize = 0;
-    const dataLocations = [];
-    let concatETags = '';
-    const partsInfo = {};
-
-    filteredPartList.forEach((storedPart, index) => {
-        const partETagWithoutQuotes =
-            storedPart.ETag.slice(1, -1);
-        const dataStoreETag = `${index + 1}:${partETagWithoutQuotes}`;
-        concatETags += partETagWithoutQuotes;
-
-        // If part was put by a regular put part rather than a
-        // copy it is always one location.  With a put part
-        // copy, could be multiple locations so loop over array
-        // of locations.
-        for (let j = 0; j < storedPart.locations.length; j++) {
-            // If the piece has parts (was a put part object
-            // copy) each piece will have a size attribute.
-            // Otherwise, the piece was put by a regular put
-            // part and the size the of the piece is the full
-            // part size.
-            const location = storedPart.locations[j];
-            // If there is no location, move on
-            if (!location || typeof location !== 'object') {
-                continue;
-            }
-            let pieceSize = Number.parseInt(storedPart.size, 10);
-            if (location.size) {
-                pieceSize = Number.parseInt(location.size, 10);
-            }
-            const pieceRetrievalInfo = {
-                key: location.key,
-                size: pieceSize,
-                start: calculatedSize,
-                dataStoreName: location.dataStoreName,
-                dataStoreETag,
-                cryptoScheme: location.sseCryptoScheme,
-                cipheredDataKey: location.sseCipheredDataKey,
-            };
-            dataLocations.push(pieceRetrievalInfo);
-            // eslint-disable-next-line no-param-reassign
-            calculatedSize += pieceSize;
-        }
-    });
-
-    partsInfo.aggregateETag =
-        createAggregateETag(concatETags, filteredPartList);
-    partsInfo.dataLocations = dataLocations;
-    partsInfo.calculatedSize = calculatedSize;
-    return partsInfo;
-}
-
-/**
- * validateAndFilterMpuParts - validates part list sent by user and filters
- * parts stored in metadata against user part list
- * @param {array} storedParts - array of parts stored in metadata
- * @param {array} jsonList - array of parts sent by user for completion
- * @param {string} mpuOverviewKey - metadata mpu key
- * @param {string} splitter - mpu key divider
- * @param {object} log - Werelogs instance
- * @return {object} filtersPartsObj - contains 3 keys: partList, keysToDelete,
- * and extraPartLocations
- */
-function validateAndFilterMpuParts(storedParts, jsonList, mpuOverviewKey,
-splitter, log) {
-    let storedPartsCopy = [];
-    const filteredPartsObj = {};
-    filteredPartsObj.partList = [];
-
-    const keysToDelete = [];
-    storedParts.forEach(item => {
-        keysToDelete.push(item.key);
-        storedPartsCopy.push({
-            // In order to delete the part listing in the shadow
-            // bucket, need the full key
-            key: item.key,
-            ETag: `"${item.value.ETag}"`,
-            size: item.value.Size,
-            locations: Array.isArray(item.value.partLocations) ?
-                item.value.partLocations : [item.value.partLocations],
-        });
-    });
-    keysToDelete.push(mpuOverviewKey);
-
-    // Check list sent to make sure valid
-    const partLength = jsonList.Part.length;
-    // A user can put more parts than they end up including
-    // in the completed MPU but there cannot be more
-    // parts in the complete message than were already put
-    if (partLength > storedPartsCopy.length) {
-        filteredPartsObj.error = errors.InvalidPart;
-        return filteredPartsObj;
-    }
-
-    let extraParts = [];
-    const extraPartLocations = [];
-
-    for (let i = 0; i < partLength; i++) {
-        const part = jsonList.Part[i];
-        const partNumber = Number.parseInt(part.PartNumber[0], 10);
-        // If the complete list of parts sent with
-        // the complete multipart upload request is not
-        // in ascending order return an error
-        if (i > 0) {
-            const previousPartNumber =
-                Number.parseInt(jsonList.Part[i - 1].PartNumber[0], 10);
-            if (partNumber <= previousPartNumber) {
-                filteredPartsObj.error = errors.InvalidPartOrder;
-                return filteredPartsObj;
-            }
-        }
-
-        let isPartUploaded = false;
-        while (storedPartsCopy.length > 0 && !isPartUploaded) {
-            const storedPart = storedPartsCopy[0];
-            const storedPartNumber =
-                Number.parseInt(storedPart.key.split(splitter)[1], 10);
-
-            if (storedPartNumber === partNumber) {
-                isPartUploaded = true;
-                filteredPartsObj.partList.push(storedPart);
-
-                let partETag = part.ETag[0].replace(/['"]/g, '');
-                // some clients send base64, convert to hex
-                // 32 chars = 16 bytes(2 chars-per-byte) = 128 bits of
-                // MD5 hex
-                if (partETag.length !== 32) {
-                    const buffered = Buffer.from(part.ETag[0], 'base64')
-                        .toString('hex');
-                    partETag = `${buffered}`;
-                }
-                partETag = `"${partETag}"`;
-                // If list of parts sent with complete mpu request contains
-                // a part ETag that does not match the ETag for the part
-                // stored in metadata, return an error
-                if (partETag !== storedPart.ETag) {
-                    filteredPartsObj.error = errors.InvalidPart;
-                    return filteredPartsObj;
-                }
-
-                // If any part other than the last part is less than
-                // 5MB, return an error
-                const storedPartSize =
-                    Number.parseInt(storedPart.size, 10);
-                // allow smaller parts for testing
-                if (process.env.MPU_TESTING) {
-                    log.info('MPU_TESTING env variable setting',
-                        { setting: process.env.MPU_TESTING });
-                }
-                if (process.env.MPU_TESTING !== 'yes' &&
-                i < jsonList.Part.length - 1 &&
-                storedPartSize < constants.minimumAllowedPartSize) {
-                    log.debug('part too small on complete mpu');
-                    filteredPartsObj.error = errors.EntityTooSmall;
-                    return filteredPartsObj;
-                }
-
-                storedPartsCopy = storedPartsCopy.splice(1);
-            } else {
-                extraParts.push(storedPart);
-                storedPartsCopy = storedPartsCopy.splice(1);
-            }
-        }
-        if (!isPartUploaded) {
-            filteredPartsObj.error = errors.InvalidPart;
-            return filteredPartsObj;
-        }
-    }
-    extraParts = extraParts.concat(storedPartsCopy);
-    // if extra parts, need to delete the data when done with completing
-    // mpu so extract the info to delete here
-    if (extraParts.length > 0) {
-        extraParts.forEach(part => {
-            const locations = part.locations;
-            locations.forEach(location => {
-                if (!location || typeof location !== 'object') {
-                    return;
-                }
-                extraPartLocations.push(location);
-            });
-        });
-    }
-    filteredPartsObj.keysToDelete = keysToDelete;
-    filteredPartsObj.extraPartLocations = extraPartLocations;
-    return filteredPartsObj;
-}
-
-module.exports = {
-    generateMpuPartStorageInfo,
-    validateAndFilterMpuParts,
-};

lib/api/apiUtils/object/validateChecksumHeaders.js

@@ -0,0 +1,32 @@
+const { errors } = require('arsenal');
+
+const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants');
+
+function validateChecksumHeaders(headers) {
+    // If the x-amz-trailer header is present the request is using one of the
+    // trailing checksum algorithms, which are not supported.
+    if (headers['x-amz-trailer'] !== undefined) {
+        return errors.BadRequest.customizeDescription('trailing checksum is not supported');
+    }
+
+    const signatureChecksum = headers['x-amz-content-sha256'];
+    if (signatureChecksum === undefined) {
+        return null;
+    }
+
+    if (supportedSignatureChecksums.has(signatureChecksum)) {
+        return null;
+    }
+
+    // If the value is not one of the possible checksum algorithms
+    // the only other valid value is the actual sha256 checksum of the payload.
+    // Do a simple sanity check of the length to guard against future algos.
+    // If the value is an unknown algo, then it will fail checksum validation.
+    if (!unsupportedSignatureChecksums.has(signatureChecksum) && signatureChecksum.length === 64) {
+        return null;
+    }
+
+    return errors.BadRequest.customizeDescription('unsupported checksum algorithm');
+}
+
+module.exports = validateChecksumHeaders;

lib/api/apiUtils/object/versioning.js

@@ -4,13 +4,40 @@ const async = require('async');
 const metadata = require('../../../metadata/wrapper');
 const { config } = require('../../../Config');
 
+const { scaledMsPerDay } = config.getTimeOptions();
+
 const versionIdUtils = versioning.VersionID;
 // Use Arsenal function to generate a version ID used internally by metadata
 // for null versions that are created before bucket versioning is configured
 const nonVersionedObjId =
     versionIdUtils.getInfVid(config.replicationGroupId);
 
-/** decodedVidResult - decode the version id from a query object
+/** decodeVID - decode the version id
+ * @param {string} versionId - version ID
+ * @return {(Error|string|undefined)} - return Invalid Argument if decryption
+ * fails due to improper format, otherwise undefined or the decoded version id
+ */
+function decodeVID(versionId) {
+    if (versionId === 'null') {
+        return versionId;
+    }
+
+    let decoded;
+    const invalidErr = errors.InvalidArgument.customizeDescription('Invalid version id specified');
+    try {
+        decoded = versionIdUtils.decode(versionId);
+    } catch (err) {
+        return invalidErr;
+    }
+
+    if (decoded instanceof Error) {
+        return invalidErr;
+    }
+
+    return decoded;
+}
+
+/** decodeVersionId - decode the version id from a query object
  * @param {object} [reqQuery] - request query object
  * @param {string} [reqQuery.versionId] - version ID sent in request query
  * @return {(Error|string|undefined)} - return Invalid Argument if decryption
@@ -20,16 +47,7 @@ function decodeVersionId(reqQuery) {
     if (!reqQuery || !reqQuery.versionId) {
         return undefined;
     }
-    let versionId = reqQuery.versionId;
-    if (versionId === 'null') {
-        return versionId;
-    }
-    versionId = versionIdUtils.decode(versionId);
-    if (versionId instanceof Error) {
-        return errors.InvalidArgument
-            .customizeDescription('Invalid version id specified');
-    }
-    return versionId;
+    return decodeVID(reqQuery.versionId);
 }
 
 /** getVersionIdResHeader - return encrypted version ID if appropriate
@@ -40,11 +58,10 @@ function decodeVersionId(reqQuery) {
  */
 function getVersionIdResHeader(verCfg, objectMD) {
     if (verCfg) {
-        if (objectMD.isNull || (objectMD && !objectMD.versionId)) {
+        if (objectMD.isNull || !objectMD.versionId) {
             return 'null';
         }
-        return versionIdUtils.encode(objectMD.versionId,
-                                     config.versionIdEncodingType);
+        return versionIdUtils.encode(objectMD.versionId);
     }
     return undefined;
 }
@@ -62,17 +79,34 @@ function checkQueryVersionId(query) {
     return undefined;
 }
 
-function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
-    metadata.putObjectMD(bucketName, objKey, objMD, options, log, err => {
+function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb) {
+    // In compatibility mode, create null versioned keys instead of null keys
+    let versionId;
+    let nullVersionMD;
+    if (config.nullVersionCompatMode) {
+        versionId = nullVersionId;
+        nullVersionMD = Object.assign({}, objMD, {
+            versionId: nullVersionId,
+            isNull: true,
+        });
+    } else {
+        versionId = 'null';
+        nullVersionMD = Object.assign({}, objMD, {
+            versionId: nullVersionId,
+            isNull: true,
+            isNull2: true,
+        });
+    }
+    metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => {
         if (err) {
             log.debug('error from metadata storing null version as new version',
             { error: err });
         }
-        cb(err, options);
+        cb(err);
     });
 }
 
-/** get location of null version data for deletion
+/** check existence and get location of null version data for deletion
 * @param {string} bucketName - name of bucket
 * @param {string} objKey - name of object key
 * @param {object} options - metadata options for getting object MD
@@ -83,50 +117,56 @@ function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
 * @param {function} cb - callback
 * @return {undefined} - and call callback with (err, dataToDelete)
 */
-function _getNullVersionsToDelete(bucketName, objKey, options, mst, log, cb) {
+function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) {
+    const nullOptions = {};
+    if (!options.deleteData) {
+        return process.nextTick(cb, null, nullOptions);
+    }
     if (options.versionId === mst.versionId) {
-        // no need to get delete location, we already have the master's metadata
-        const dataToDelete = mst.objLocation;
-        return process.nextTick(cb, null, dataToDelete);
+        // no need to get another key as the master is the target
+        nullOptions.dataToDelete = mst.objLocation;
+        return process.nextTick(cb, null, nullOptions);
+    }
+    if (options.versionId === 'null') {
+        // deletion of the null key will be done by the main metadata
+        // PUT via this option
+        nullOptions.deleteNullKey = true;
     }
     return metadata.getObjectMD(bucketName, objKey, options, log,
         (err, versionMD) => {
             if (err) {
-                log.debug('err from metadata getting specified version', {
-                    error: err,
-                    method: '_getNullVersionsToDelete',
-                });
+                // the null key may not exist, hence it's a normal
+                // situation to have a NoSuchKey error, in which case
+                // there is nothing to delete
+                if (err.is.NoSuchKey) {
+                    log.debug('null version does not exist', {
+                        method: '_prepareNullVersionDeletion',
+                    });
+                } else {
+                    log.warn('could not get null version metadata', {
+                        error: err,
+                        method: '_prepareNullVersionDeletion',
+                    });
+                }
                 return cb(err);
             }
-            if (!versionMD.location) {
-                return cb();
+            if (versionMD.location) {
+                const dataToDelete = Array.isArray(versionMD.location) ?
+                      versionMD.location : [versionMD.location];
+                nullOptions.dataToDelete = dataToDelete;
             }
-            const dataToDelete = Array.isArray(versionMD.location) ?
-                versionMD.location : [versionMD.location];
-            return cb(null, dataToDelete);
+            return cb(null, nullOptions);
         });
 }
 
-function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
-    return _getNullVersionsToDelete(bucketName, objKey, options, mst, log,
-        (err, nullDataToDelete) => {
-            if (err) {
-                log.warn('could not find null version metadata', {
-                    error: err,
-                    method: '_deleteNullVersionMD',
-                });
-                return cb(err);
-            }
-            return metadata.deleteObjectMD(bucketName, objKey, options, log,
-                err => {
-                    if (err) {
-                        log.warn('metadata error deleting null version',
-                        { error: err, method: '_deleteNullVersionMD' });
-                        return cb(err);
-                    }
-                    return cb(null, nullDataToDelete);
-                });
-        });
+function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
+    return metadata.deleteObjectMD(bucketName, objKey, options, log, err => {
+        if (err) {
+            log.warn('metadata error deleting null versioned key',
+                     { bucketName, objKey, error: err, method: '_deleteNullVersionMD' });
+        }
+        return cb(err);
+    });
 }
 
 /**
@@ -136,73 +176,103 @@ function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
  * @param {object} mst - state of master version, as returned by
  * getMasterState()
  * @param {string} vstat - bucket versioning status: 'Enabled' or 'Suspended'
+ * @param {boolean} nullVersionCompatMode - if true, behaves in null
+ * version compatibility mode and return appropriate values: this mode
+ * does not attempt to create null keys but create null versioned keys
+ * instead
  *
  * @return {object} result object with the following attributes:
  * - {object} options: versioning-related options to pass to the
      services.metadataStoreObject() call
- * - {object} [storeOptions]: options for metadata to create a new
-     null version key, if needed
+ * - {object} [options.extraMD]: extra attributes to set in object metadata
+ * - {string} [nullVersionId]: null version key to create, if needed
  * - {object} [delOptions]: options for metadata to delete the null
      version key, if needed
  */
-function processVersioningState(mst, vstat) {
-    const options = {};
-    const storeOptions = {};
-    const delOptions = {};
-    // object does not exist or is not versioned (before versioning)
-    if (mst.versionId === undefined || mst.isNull) {
-        // versioning is suspended, overwrite existing master version
-        if (vstat === 'Suspended') {
-            options.versionId = '';
-            options.isNull = true;
-            options.dataToDelete = mst.objLocation;
-            // if null version exists, clean it up prior to put
-            if (mst.isNull) {
-                delOptions.versionId = mst.versionId;
-                if (mst.uploadId) {
-                    delOptions.replayId = mst.uploadId;
-                }
+function processVersioningState(mst, vstat, nullVersionCompatMode) {
+    const versioningSuspended = (vstat === 'Suspended');
+    const masterIsNull = mst.exists && (mst.isNull || !mst.versionId);
+
+    if (versioningSuspended) {
+        // versioning is suspended: overwrite the existing null version
+        const options = { versionId: '', isNull: true };
+        if (masterIsNull) {
+            // if the null version exists, clean it up prior to put
+            if (mst.objLocation) {
+                options.dataToDelete = mst.objLocation;
+            }
+            // backward-compat: a null version key may exist even with
+            // a null master (due to S3C-7526), if so, delete it (its
+            // data will be deleted as part of the master cleanup, so
+            // no "deleteData" param is needed)
+            //
+            // "isNull2" attribute is set in master metadata when
+            // null keys are used, which is used as an optimization to
+            // avoid having to check the versioned key since there can
+            // be no more versioned key to clean up
+            if (mst.isNull && mst.versionId && !mst.isNull2) {
+                const delOptions = { versionId: mst.versionId };
                 return { options, delOptions };
             }
             return { options };
         }
-        // versioning is enabled, create a new version
-        options.versioning = true;
-        if (mst.exists) {
-            // store master version in a new key
-            const versionId = mst.isNull ? mst.versionId : nonVersionedObjId;
-            storeOptions.versionId = versionId;
-            storeOptions.isNull = true;
-            options.nullVersionId = versionId;
-            // non-versioned (non-null) MPU objects don't have a
-            // replay ID, so don't reference their uploadId
-            if (mst.isNull && mst.uploadId) {
-                options.nullUploadId = mst.uploadId;
+        if (mst.nullVersionId) {
+            // backward-compat: delete the null versioned key and data
+            const delOptions = { versionId: mst.nullVersionId, deleteData: true };
+            if (mst.nullUploadId) {
+                delOptions.replayId = mst.nullUploadId;
             }
-            return { options, storeOptions };
-        }
-        return { options };
-    }
-    // master is versioned and is not a null version
-    const nullVersionId = mst.nullVersionId;
-    if (vstat === 'Suspended') {
-        // versioning is suspended, overwrite the existing master version
-        options.versionId = '';
-        options.isNull = true;
-        if (nullVersionId === undefined) {
-            return { options };
-        }
-        delOptions.versionId = nullVersionId;
-        if (mst.nullUploadId) {
-            delOptions.replayId = mst.nullUploadId;
+            return { options, delOptions };
         }
+        // clean up the eventual null key's location data prior to put
+
+        // NOTE: due to metadata v1 internal format, we cannot guess
+        // from the master key whether there is an associated null
+        // key, because the master key may be removed whenever the
+        // latest version becomes a delete marker. Hence we need to
+        // pessimistically try to get the null key metadata and delete
+        // it if it exists.
+        const delOptions = { versionId: 'null', deleteData: true };
         return { options, delOptions };
     }
-    // versioning is enabled, put the new version
-    options.versioning = true;
-    options.nullVersionId = nullVersionId;
-    if (mst.nullUploadId) {
-        options.nullUploadId = mst.nullUploadId;
+
+    // versioning is enabled: create a new version
+    const options = { versioning: true };
+    if (masterIsNull) {
+        // if master is a null version or a non-versioned key,
+        // copy it to a new null key
+        const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
+        if (nullVersionCompatMode) {
+            options.extraMD = {
+                nullVersionId,
+            };
+            if (mst.uploadId) {
+                options.extraMD.nullUploadId = mst.uploadId;
+            }
+            return { options, nullVersionId };
+        }
+        if (mst.isNull && !mst.isNull2) {
+            // if master null version was put with an older
+            // Cloudserver (or in compat mode), there is a
+            // possibility that it also has a null versioned key
+            // associated, so we need to delete it as we write the
+            // null key
+            const delOptions = {
+                versionId: nullVersionId,
+            };
+            return { options, nullVersionId, delOptions };
+        }
+        return { options, nullVersionId };
+    }
+    // backward-compat: keep a reference to the existing null
+    // versioned key
+    if (mst.nullVersionId) {
+        options.extraMD = {
+            nullVersionId: mst.nullVersionId,
+        };
+        if (mst.nullUploadId) {
+            options.extraMD.nullUploadId = mst.nullUploadId;
+        }
     }
     return { options };
 }
@@ -229,6 +299,7 @@ function getMasterState(objMD) {
         versionId: objMD.versionId,
         uploadId: objMD.uploadId,
         isNull: objMD.isNull,
+        isNull2: objMD.isNull2,
         nullVersionId: objMD.nullVersionId,
         nullUploadId: objMD.nullUploadId,
     };
@@ -252,9 +323,6 @@ function getMasterState(objMD) {
  *  ('' overwrites the master version)
  * options.versioning - (true/undefined) metadata instruction to create new ver
  * options.isNull - (true/undefined) whether new version is null or not
- * options.nullVersionId - if storing a null version in version history, the
- *  version id of the null version
- * options.deleteNullVersionData - whether to delete the data of the null ver
  */
 function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
     log, callback) {
@@ -266,42 +334,102 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
         return process.nextTick(callback, null, options);
     }
     // bucket is versioning configured
-    const { options, storeOptions, delOptions } =
-          processVersioningState(mst, vCfg.Status);
+    const { options, nullVersionId, delOptions } =
+          processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode);
     return async.series([
-        function storeVersion(next) {
-            if (!storeOptions) {
+        function storeNullVersionMD(next) {
+            if (!nullVersionId) {
                 return process.nextTick(next);
             }
-            const versionMD = Object.assign({}, objMD, storeOptions);
-            const params = { versionId: storeOptions.versionId };
-            return _storeNullVersionMD(bucketName, objectKey, versionMD,
-                params, log, next);
+            return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
         },
-        function deleteNullVersion(next) {
+        function prepareNullVersionDeletion(next) {
             if (!delOptions) {
                 return process.nextTick(next);
             }
-            return _deleteNullVersionMD(bucketName, objectKey, delOptions, mst,
-                log, (err, nullDataToDelete) => {
+            return _prepareNullVersionDeletion(
+                bucketName, objectKey, delOptions, mst, log,
+                (err, nullOptions) => {
                     if (err) {
-                        log.warn('unexpected error deleting null version md', {
-                            error: err,
-                            method: 'versioningPreprocessing',
-                        });
-                        // it's possible there was a concurrent request to
-                        // delete the null version, so proceed with putting a
-                        // new version
-                        if (err.is.NoSuchKey) {
-                            return next(null, options);
-                        }
-                        return next(errors.InternalError);
+                        return next(err);
                     }
-                    Object.assign(options, { dataToDelete: nullDataToDelete });
+                    Object.assign(options, nullOptions);
                     return next();
                 });
         },
-    ], err => callback(err, options));
+        function deleteNullVersionMD(next) {
+            if (delOptions &&
+                delOptions.versionId &&
+                delOptions.versionId !== 'null') {
+                // backward-compat: delete old null versioned key
+                return _deleteNullVersionMD(
+                    bucketName, objectKey, { versionId: delOptions.versionId }, log, next);
+            }
+            return process.nextTick(next);
+        },
+    ], err => {
+        // it's possible there was a prior request that deleted the
+        // null version, so proceed with putting a new version
+        if (err && err.is.NoSuchKey) {
+            return callback(null, options);
+        }
+        return callback(err, options);
+    });
+}
+
+/** Return options to pass to Metadata layer for version-specific
+ * operations with the given requested version ID
+ *
+ * @param {object} objectMD - object metadata
+ * @param {boolean} nullVersionCompatMode - if true, behaves in null
+ * version compatibility mode
+ * @return {object} options object with params:
+ * {string} [options.versionId] - specific versionId to update
+ * {boolean} [options.isNull=true|false|undefined] - if set, tells the
+ * Metadata backend if we're updating or deleting a new-style null
+ * version (stored in master or null key), or not a null version.
+ */
+function getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode) {
+    // Use the internal versionId if it is a "real" null version (not
+    // non-versioned)
+    //
+    // If the target object is non-versioned: do not specify a
+    // "versionId" attribute nor "isNull"
+    //
+    // If the target version is a null version, i.e. has the "isNull"
+    // attribute:
+    //
+    // - send the "isNull=true" param to Metadata if the version is
+    //   already a null key put by a non-compat mode Cloudserver, to
+    //   let Metadata know that the null key is to be updated or
+    //   deleted. This is the case if the "isNull2" metadata attribute
+    //   exists
+    //
+    // - otherwise, do not send the "isNull" parameter to hint
+    //   Metadata that it is a legacy null version
+    //
+    // If the target version is not a null version and is versioned:
+    //
+    // - send the "isNull=false" param to Metadata in non-compat
+    //   mode (mandatory for v1 format)
+    //
+    // - otherwise, do not send the "isNull" parameter to hint
+    //   Metadata that an existing null version may not be stored in a
+    //   null key
+    //
+    //
+    if (objectMD.versionId === undefined) {
+        return {};
+    }
+    const options = { versionId: objectMD.versionId };
+    if (objectMD.isNull) {
+        if (objectMD.isNull2) {
+            options.isNull = true;
+        }
+    } else if (!nullVersionCompatMode) {
+        options.isNull = false;
+    }
+    return options;
 }
 
 /** preprocessingVersioningDelete - return versioning information for S3 to
@@ -310,66 +438,113 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
  * @param {object} bucketMD - bucket metadata
  * @param {object} objectMD - obj metadata
  * @param {string} [reqVersionId] - specific version ID sent as part of request
- * @param {RequestLogger} log - logger instance
- * @param {function} callback - callback
- * @return {undefined} and call callback with params (err, options):
- * options.deleteData - (true/undefined) whether to delete data (if undefined
+ * @param {boolean} nullVersionCompatMode - if true, behaves in null version compatibility mode
+ * @return {object} options object with params:
+ * {boolean} [options.deleteData=true|undefined] - whether to delete data (if undefined
  *  means creating a delete marker instead)
- * options.versionId - specific versionId to delete
- * options.isNull - (true/undefined) whether version to be deleted/marked is null or not
+ * {string} [options.versionId] - specific versionId to delete
+ * {boolean} [options.isNull=true|false|undefined] - if set, tells the
+ * Metadata backend if we're deleting a new-style null version (stored
+ * in master or null key), or not a null version.
  */
-function preprocessingVersioningDelete(bucketName, bucketMD, objectMD,
-    reqVersionId, log, callback) {
-    const options = {};
-    // bucket is not versioning enabled
-    if (!bucketMD.getVersioningConfiguration()) {
+function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersionId, nullVersionCompatMode) {
+    let options = {};
+    if (bucketMD.getVersioningConfiguration() && reqVersionId) {
+        options = getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode);
+    }
+    if (!bucketMD.getVersioningConfiguration() || reqVersionId) {
+        // delete data if bucket is non-versioned or the request
+        // deletes a specific version
         options.deleteData = true;
-        return callback(null, options);
     }
-    // bucket is versioning enabled
-    if (reqVersionId && reqVersionId !== 'null') {
-        // deleting a specific version
-        options.deleteData = true;
-        options.versionId = reqVersionId;
-        if (objectMD.uploadId) {
-            options.replayId = objectMD.uploadId;
+    return options;
+}
+
+/**
+ * Keep metadatas when the object is restored from cold storage
+ * but remove the specific ones we don't want to keep
+ * @param {object} objMD - obj metadata
+ * @param {object} metadataStoreParams - custom built object containing resource details.
+ * @return {undefined}
+ */
+function restoreMetadata(objMD, metadataStoreParams) {
+    /* eslint-disable no-param-reassign */
+    const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt'];
+    // We need to keep user metadata and tags
+    Object.keys(objMD).forEach(key => {
+        if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) {
+            metadataStoreParams.metaHeaders[key] = objMD[key];
         }
-        return callback(null, options);
+    });
+
+    if (objMD['x-amz-website-redirect-location']) {
+        if (!metadataStoreParams.headers) {
+            metadataStoreParams.headers = {};
+        }
+        metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location'];
     }
-    if (reqVersionId) {
-        // deleting the 'null' version if it exists
-        if (objectMD.versionId === undefined) {
-            // object is not versioned, deleting it
-            options.deleteData = true;
-            // non-versioned (non-null) MPU objects don't have a
-            // replay ID, so don't reference their uploadId
-            return callback(null, options);
-        }
-        if (objectMD.isNull) {
-            // master is the null version
-            options.deleteData = true;
-            options.versionId = objectMD.versionId;
-            if (objectMD.uploadId) {
-                options.replayId = objectMD.uploadId;
-            }
-            return callback(null, options);
-        }
-        if (objectMD.nullVersionId) {
-            // null version exists, deleting it
-            options.deleteData = true;
-            options.versionId = objectMD.nullVersionId;
-            if (objectMD.nullUploadId) {
-                options.replayId = objectMD.nullUploadId;
-            }
-            return callback(null, options);
-        }
-        // null version does not exist, no deletion
-        // TODO check AWS behaviour for no deletion (seems having no error)
-        return callback(errors.NoSuchKey);
+
+    if (objMD.replicationInfo) {
+        metadataStoreParams.replicationInfo = objMD.replicationInfo;
     }
-    // not deleting any specific version, making a delete marker instead
-    options.isNull = true;
-    return callback(null, options);
+
+    if (objMD.legalHold) {
+        metadataStoreParams.legalHold = objMD.legalHold;
+    }
+
+    if (objMD.acl) {
+        metadataStoreParams.acl = objMD.acl;
+    }
+
+    metadataStoreParams.creationTime = objMD['creation-time'];
+    metadataStoreParams.lastModifiedDate = objMD['last-modified'];
+    metadataStoreParams.taggingCopy = objMD.tags;
+}
+
+/** overwritingVersioning - return versioning information for S3 to handle
+ * storing version metadata with a specific version id.
+ * @param {object} objMD - obj metadata
+ * @param {object} metadataStoreParams - custom built object containing resource details.
+ * @return {object} options
+ * options.versionId - specific versionId to overwrite in metadata
+ * options.isNull - (true/undefined) whether new version is null or not
+ * options.nullVersionId - if storing a null version in version history, the
+ *  version id of the null version
+ */
+function overwritingVersioning(objMD, metadataStoreParams) {
+    metadataStoreParams.updateMicroVersionId = true;
+    metadataStoreParams.amzStorageClass = objMD['x-amz-storage-class'];
+
+    // set correct originOp
+    metadataStoreParams.originOp = 's3:ObjectRestore:Completed';
+
+    // update restore
+    const days = objMD.archive?.restoreRequestedDays;
+    const now = Date.now();
+    metadataStoreParams.archive = {
+        archiveInfo: objMD.archive?.archiveInfo,
+        restoreRequestedAt: objMD.archive?.restoreRequestedAt,
+        restoreRequestedDays: objMD.archive?.restoreRequestedDays,
+        restoreCompletedAt: new Date(now),
+        restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
+    };
+
+    /* eslint-enable no-param-reassign */
+
+    const versionId = objMD.versionId || undefined;
+    const options = {
+        versionId,
+        isNull: objMD.isNull,
+    };
+    if (objMD.nullVersionId) {
+        options.extraMD = {
+            nullVersionId: objMD.nullVersionId,
+        };
+    }
+
+    restoreMetadata(objMD, metadataStoreParams);
+
+    return options;
 }
 
 module.exports = {
@@ -379,5 +554,8 @@ module.exports = {
     processVersioningState,
     getMasterState,
     versioningPreprocessing,
+    getVersionSpecificMetadataOptions,
     preprocessingVersioningDelete,
+    overwritingVersioning,
+    decodeVID,
 };

lib/api/apiUtils/object/websiteServing.js

@@ -101,8 +101,33 @@ function validateWebsiteHeader(header) {
     header.startsWith('http://') || header.startsWith('https://'));
 }
 
+/**
+ * appendWebsiteIndexDocument - append index to objectKey if necessary
+ * @param {object} request - normalized request object
+ * @param {string} indexDocumentSuffix - index document from website config
+ * @param {boolean} force - flag to force append index
+ * @return {undefined}
+ */
+function appendWebsiteIndexDocument(request, indexDocumentSuffix, force = false) {
+    const reqObjectKey = request.objectKey ? request.objectKey : '';
+    /* eslint-disable no-param-reassign */
+
+    // find index document if "directory" sent in request
+    if (reqObjectKey.endsWith('/')) {
+        request.objectKey += indexDocumentSuffix;
+    // find index document if no key provided
+    } else if (reqObjectKey === '') {
+        request.objectKey = indexDocumentSuffix;
+    // force for redirect 302 on folder without trailing / that has an index
+    } else if (force) {
+        request.objectKey += `/${indexDocumentSuffix}`;
+    }
+    /* eslint-enable no-param-reassign */
+}
+
 module.exports = {
     findRoutingRule,
     extractRedirectInfo,
     validateWebsiteHeader,
+    appendWebsiteIndexDocument,
 };

lib/api/apiUtils/quotas/quotaUtils.js

@@ -0,0 +1,314 @@
+const async = require('async');
+const { errors } = require('arsenal');
+const monitoring = require('../../../utilities/monitoringHandler');
+const {
+    actionNeedQuotaCheckCopy,
+    actionNeedQuotaCheck,
+    actionWithDataDeletion,
+} = require('arsenal').policies;
+const { config } = require('../../../Config');
+const QuotaService = require('../../../quotas/quotas');
+
+/**
+ * Process the bytes to write based on the request and object metadata
+ * @param {string} apiMethod - api method
+ * @param {BucketInfo} bucket - bucket info
+ * @param {string} versionId - version id of the object
+ * @param {number} contentLength - content length of the object
+ * @param {object} objMD - object metadata
+ * @param {object} destObjMD - destination object metadata
+ * @return {number} processed content length
+ */
+function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
+    let bytes = contentLength;
+    if (apiMethod === 'objectRestore') {
+        // object is being restored
+        bytes = Number.parseInt(objMD['content-length'], 10);
+    } else if (!bytes && objMD?.['content-length']) {
+        if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
+            if (!destObjMD || bucket.isVersioningEnabled()) {
+                // object is being copied
+                bytes = Number.parseInt(objMD['content-length'], 10);
+            } else if (!bucket.isVersioningEnabled()) {
+                // object is being copied and replaces the target
+                bytes = Number.parseInt(objMD['content-length'], 10) -
+                    Number.parseInt(destObjMD['content-length'], 10);
+            }
+        } else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
+            // object is being deleted
+            bytes = -Number.parseInt(objMD['content-length'], 10);
+        }
+    } else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
+        // object is being replaced: store the diff, if the bucket is not versioned
+        bytes = bytes - Number.parseInt(objMD['content-length'], 10);
+    }
+    return bytes || 0;
+}
+
+/**
+ * Checks if a metric is stale based on the provided parameters.
+ *
+ * @param {Object} metric - The metric object to check.
+ * @param {string} resourceType - The type of the resource.
+ * @param {string} resourceName - The name of the resource.
+ * @param {string} action - The action being performed.
+ * @param {number} inflight - The number of inflight requests.
+ * @param {Object} log - The logger object.
+ * @returns {boolean} Returns true if the metric is stale, false otherwise.
+ */
+function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
+    if (metric.date && Date.now() - new Date(metric.date).getTime() >
+        QuotaService.maxStaleness) {
+        log.warn('Stale metrics from the quota service, allowing the request', {
+            resourceType,
+            resourceName,
+            action,
+            inflight,
+        });
+        monitoring.requestWithQuotaMetricsUnavailable.inc();
+        return true;
+    }
+    return false;
+}
+
+/**
+ * Evaluates quotas for a bucket and an account and update inflight count.
+ *
+ * @param {number} bucketQuota - The quota limit for the bucket.
+ * @param {number} accountQuota - The quota limit for the account.
+ * @param {object} bucket - The bucket object.
+ * @param {object} account - The account object.
+ * @param {number} inflight - The number of inflight requests.
+ * @param {number} inflightForCheck - The number of inflight requests for checking quotas.
+ * @param {string} action - The action being performed.
+ * @param {object} log - The logger object.
+ * @param {function} callback - The callback function to be called when evaluation is complete.
+ * @returns {object} - The result of the evaluation.
+ */
+function _evaluateQuotas(
+    bucketQuota,
+    accountQuota,
+    bucket,
+    account,
+    inflight,
+    inflightForCheck,
+    action,
+    log,
+    callback,
+) {
+    let bucketQuotaExceeded = false;
+    let accountQuotaExceeded = false;
+    const creationDate = new Date(bucket.getCreationDate()).getTime();
+    return async.parallel({
+        bucketQuota: parallelDone => {
+            if (bucketQuota > 0) {
+                return QuotaService.getUtilizationMetrics('bucket',
+                    `${bucket.getName()}_${creationDate}`, null, {
+                    action,
+                    inflight,
+                }, (err, bucketMetrics) => {
+                    if (err || inflight < 0) {
+                        return parallelDone(err);
+                    }
+                    if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
+                        bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
+                        log.debug('Bucket quota exceeded', {
+                            bucket: bucket.getName(),
+                            action,
+                            inflight,
+                            quota: bucketQuota,
+                            bytesTotal: bucketMetrics.bytesTotal,
+                        });
+                        bucketQuotaExceeded = true;
+                    }
+                    return parallelDone();
+                });
+            }
+            return parallelDone();
+        },
+        accountQuota: parallelDone => {
+            if (accountQuota > 0 && account?.account) {
+                return QuotaService.getUtilizationMetrics('account',
+                    account.account, null, {
+                    action,
+                    inflight,
+                }, (err, accountMetrics) => {
+                    if (err || inflight < 0) {
+                        return parallelDone(err);
+                    }
+                    if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
+                        accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
+                        log.debug('Account quota exceeded', {
+                            accountId: account.account,
+                            action,
+                            inflight,
+                            quota: accountQuota,
+                            bytesTotal: accountMetrics.bytesTotal,
+                        });
+                        accountQuotaExceeded = true;
+                    }
+                    return parallelDone();
+                });
+            }
+            return parallelDone();
+        },
+    }, err => {
+        if (err) {
+            log.warn('Error evaluating quotas', {
+                error: err.name,
+                description: err.message,
+                isInflightDeletion: inflight < 0,
+            });
+        }
+        return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
+    });
+}
+
+/**
+ * Monitors the duration of quota evaluation for a specific API method.
+ *
+ * @param {string} apiMethod - The name of the API method being monitored.
+ * @param {string} type - The type of quota being evaluated.
+ * @param {string} code - The code associated with the quota being evaluated.
+ * @param {number} duration - The duration of the quota evaluation in nanoseconds.
+ * @returns {undefined} - Returns nothing.
+ */
+function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
+    monitoring.quotaEvaluationDuration.labels({
+        action: apiMethod,
+        type,
+        code,
+    }).observe(duration / 1e9);
+}
+
+/**
+ *
+ * @param {Request} request - request object
+ * @param {BucketInfo} bucket - bucket object
+ * @param {Account} account - account object
+ * @param {array} apiNames - action names: operations to authorize
+ * @param {string} apiMethod - the main API call
+ * @param {number} inflight - inflight bytes
+ * @param {boolean} isStorageReserved - Flag to check if the current quota, minus
+ * the incoming bytes, are under the limit.
+ * @param {Logger} log - logger
+ * @param {function} callback - callback function
+ * @returns {boolean} - true if the quota is valid, false otherwise
+ */
+function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
+    if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
+        return callback(null);
+    }
+    let type;
+    let bucketQuotaExceeded = false;
+    let accountQuotaExceeded = false;
+    let quotaEvaluationDuration;
+    const requestStartTime = process.hrtime.bigint();
+    const bucketQuota = bucket.getQuota();
+    const accountQuota = account?.quota || 0;
+    const shouldSendInflights = config.isQuotaInflightEnabled();
+
+    if (bucketQuota && accountQuota) {
+        type = 'bucket+account';
+    } else if (bucketQuota) {
+        type = 'bucket';
+    } else {
+        type = 'account';
+    }
+
+    if (actionWithDataDeletion[apiMethod]) {
+        type = 'delete';
+    }
+
+    if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
+        if (bucketQuota > 0 || accountQuota > 0) {
+            log.warn('quota is set for a bucket, but the quota service is disabled', {
+                bucketName: bucket.getName(),
+            });
+            monitoring.requestWithQuotaMetricsUnavailable.inc();
+        }
+        return callback(null);
+    }
+
+    if (isStorageReserved) {
+        // eslint-disable-next-line no-param-reassign
+        inflight = 0;
+    }
+
+    return async.forEach(apiNames, (apiName, done) => {
+        // Object copy operations first check the target object,
+        // meaning the source object, containing the current bytes,
+        // is checked second. This logic handles these APIs calls by
+        // ensuring the bytes are positives (i.e., not an object
+        // replacement).
+        if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
+            // eslint-disable-next-line no-param-reassign
+            inflight = Math.abs(inflight);
+        } else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
+            return done();
+        }
+        // When inflights are disabled, the sum of the current utilization metrics
+        // and the current bytes are compared with the quota. The current bytes
+        // are not sent to the utilization service. When inflights are enabled,
+        // the sum of the current utilization metrics only are compared with the
+        // quota. They include the current inflight bytes sent in the request.
+        let _inflights = shouldSendInflights ? inflight : undefined;
+        const inflightForCheck = shouldSendInflights ? 0 : inflight;
+        return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
+            inflightForCheck, apiName, log,
+            (err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
+                if (err) {
+                    return done(err);
+                }
+
+                bucketQuotaExceeded = _bucketQuotaExceeded;
+                accountQuotaExceeded = _accountQuotaExceeded;
+
+                // Inflights are inverted: in case of cleanup, we just re-issue
+                // the same API call.
+                if (_inflights) {
+                    _inflights = -_inflights;
+                }
+
+                request.finalizerHooks.push((errorFromAPI, _done) => {
+                    const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
+                    const quotaCleanUpStartTime = process.hrtime.bigint();
+                    // Quotas are cleaned only in case of error in the API
+                    async.waterfall([
+                        cb => {
+                            if (errorFromAPI) {
+                                return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
+                                    null, apiName, log, cb);
+                            }
+                            return cb();
+                        },
+                    ], () => {
+                        monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
+                            Number(process.hrtime.bigint() - quotaCleanUpStartTime));
+                        return _done();
+                    });
+                });
+
+                return done();
+            });
+    }, err => {
+        quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
+        if (err) {
+            log.warn('Error getting metrics from the quota service, allowing the request', {
+                error: err.name,
+                description: err.message,
+            });
+        }
+        if (!actionWithDataDeletion[apiMethod] &&
+            (bucketQuotaExceeded || accountQuotaExceeded)) {
+            return callback(errors.QuotaExceeded);
+        }
+        return callback();
+    });
+}
+
+module.exports = {
+    processBytesToWrite,
+    isMetricStale,
+    validateQuotas,
+};

lib/api/backbeat/listLifecycleCurrents.js

@@ -0,0 +1,117 @@
+const { errors } = require('arsenal');
+const constants = require('../../../constants');
+const services = require('../../services');
+const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
+const { pushMetric } = require('../../utapi/utilities');
+const monitoring = require('../../utilities/monitoringHandler');
+const { getLocationConstraintErrorMessage, processCurrents,
+    validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
+const { config } = require('../../Config');
+
+function handleResult(listParams, requestMaxKeys, authInfo,
+    bucketName, list, isBucketVersioned, log, callback) {
+    // eslint-disable-next-line no-param-reassign
+    listParams.maxKeys = requestMaxKeys;
+    const res = processCurrents(bucketName, listParams, isBucketVersioned, list);
+
+    pushMetric('listLifecycleCurrents', log, { authInfo, bucket: bucketName });
+    monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleCurrents');
+    return callback(null, res);
+}
+
+/**
+ * listLifecycleCurrents - Return list of current versions/masters in bucket
+ * @param {AuthInfo} authInfo            - Instance of AuthInfo class with
+ *                                          requester's info
+ * @param {array} locationConstraints    - array of location contraint
+ * @param {object} request               - http request object
+ * @param {function} log                 - Werelogs request logger
+ * @param {function} callback            - callback to respond to http request
+ *                                          with either error code or xml response body
+ * @return {undefined}
+ */
+function listLifecycleCurrents(authInfo, locationConstraints, request, log, callback) {
+    const params = request.query;
+    const bucketName = request.bucketName;
+
+    log.debug('processing request', { method: 'listLifecycleCurrents' });
+    const requestMaxKeys = params['max-keys'] ?
+        Number.parseInt(params['max-keys'], 10) : 1000;
+    if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listLifecycleCurrents');
+        return callback(errors.InvalidArgument);
+    }
+    const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
+
+    const minEntriesToBeScanned = 1;
+    const { isValid, maxScannedLifecycleListingEntries } =
+        validateMaxScannedEntries(params, config, minEntriesToBeScanned);
+    if (!isValid) {
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
+        return callback(errors.InvalidArgument);
+    }
+
+    const excludedDataStoreName = params['excluded-data-store-name'];
+    if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
+        const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
+        log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
+
+        return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
+    }
+
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: 'listLifecycleCurrents',
+        request,
+    };
+    const listParams = {
+        listingType: 'DelimiterCurrent',
+        maxKeys: actualMaxKeys,
+        prefix: params.prefix,
+        beforeDate: params['before-date'],
+        marker: params.marker,
+        excludedDataStoreName,
+        maxScannedLifecycleListingEntries,
+    };
+
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
+        if (err) {
+            log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'listLifecycleCurrents');
+            return callback(err, null);
+        }
+
+        const vcfg = bucket.getVersioningConfiguration();
+        const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
+
+        if (!requestMaxKeys) {
+            const emptyList = {
+                Contents: [],
+                IsTruncated: false,
+            };
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, emptyList, isBucketVersioned, log, callback);
+        }
+
+        return services.getLifecycleListing(bucketName, listParams, log,
+        (err, list) => {
+            if (err) {
+                log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'listLifecycleCurrents');
+                return callback(err, null);
+            }
+
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, list, isBucketVersioned, log, callback);
+        });
+    });
+}
+
+module.exports = {
+    listLifecycleCurrents,
+};

lib/api/backbeat/listLifecycleNonCurrents.js

@@ -0,0 +1,127 @@
+const { errors, versioning } = require('arsenal');
+const constants = require('../../../constants');
+const services = require('../../services');
+const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
+const { pushMetric } = require('../../utapi/utilities');
+const versionIdUtils = versioning.VersionID;
+const monitoring = require('../../utilities/monitoringHandler');
+const { getLocationConstraintErrorMessage, processNonCurrents,
+    validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
+const { config } = require('../../Config');
+
+function handleResult(listParams, requestMaxKeys, authInfo,
+    bucketName, list, log, callback) {
+    // eslint-disable-next-line no-param-reassign
+    listParams.maxKeys = requestMaxKeys;
+    const res = processNonCurrents(bucketName, listParams, list);
+
+    pushMetric('listLifecycleNonCurrents', log, { authInfo, bucket: bucketName });
+    monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleNonCurrents');
+    return callback(null, res);
+}
+
+/**
+ * listLifecycleNonCurrents - Return list of non-current versions in bucket
+ * @param {AuthInfo} authInfo           - Instance of AuthInfo class with
+ *                                         requester's info
+ * @param {array} locationConstraints    - array of location contraint
+ * @param {object} request              - http request object
+ * @param {function} log                - Werelogs request logger
+ * @param {function} callback           - callback to respond to http request
+ *                                          with either error code or xml response body
+ * @return {undefined}
+ */
+function listLifecycleNonCurrents(authInfo, locationConstraints, request, log, callback) {
+    const params = request.query;
+    const bucketName = request.bucketName;
+
+    log.debug('processing request', { method: 'listLifecycleNonCurrents' });
+    const requestMaxKeys = params['max-keys'] ?
+        Number.parseInt(params['max-keys'], 10) : 1000;
+    if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listLifecycleNonCurrents');
+        return callback(errors.InvalidArgument);
+    }
+    const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
+
+    // 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
+    // Two entries representing the master key and the following one representing the non-current version.
+    const minEntriesToBeScanned = 3;
+    const { isValid, maxScannedLifecycleListingEntries } =
+        validateMaxScannedEntries(params, config, minEntriesToBeScanned);
+    if (!isValid) {
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleNonCurrents');
+        return callback(errors.InvalidArgument);
+    }
+
+    const excludedDataStoreName = params['excluded-data-store-name'];
+    if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
+        const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
+        log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
+
+        return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
+    }
+
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: 'listLifecycleNonCurrents',
+        request,
+    };
+    const listParams = {
+        listingType: 'DelimiterNonCurrent',
+        maxKeys: actualMaxKeys,
+        prefix: params.prefix,
+        beforeDate: params['before-date'],
+        keyMarker: params['key-marker'],
+        excludedDataStoreName,
+        maxScannedLifecycleListingEntries,
+    };
+
+    listParams.versionIdMarker = params['version-id-marker'] ?
+        versionIdUtils.decode(params['version-id-marker']) : undefined;
+
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
+        if (err) {
+            log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'listLifecycleNonCurrents');
+            return callback(err, null);
+        }
+
+        const vcfg = bucket.getVersioningConfiguration();
+        const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
+        if (!isBucketVersioned) {
+            log.debug('bucket is not versioned');
+            return callback(errors.InvalidRequest.customizeDescription(
+                'bucket is not versioned'), null);
+        }
+
+        if (!requestMaxKeys) {
+            const emptyList = {
+                Contents: [],
+                IsTruncated: false,
+            };
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, emptyList, log, callback);
+        }
+
+        return services.getLifecycleListing(bucketName, listParams, log,
+        (err, list) => {
+            if (err) {
+                log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'listLifecycleNonCurrents');
+                return callback(err, null);
+            }
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, list, log, callback);
+        });
+    });
+}
+
+module.exports = {
+    listLifecycleNonCurrents,
+};

lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js

@@ -0,0 +1,112 @@
+const { errors } = require('arsenal');
+const constants = require('../../../constants');
+const services = require('../../services');
+const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
+const { pushMetric } = require('../../utapi/utilities');
+const monitoring = require('../../utilities/monitoringHandler');
+const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
+const { config } = require('../../Config');
+
+function handleResult(listParams, requestMaxKeys, authInfo,
+    bucketName, list, log, callback) {
+    // eslint-disable-next-line no-param-reassign
+    listParams.maxKeys = requestMaxKeys;
+    const res = processOrphans(bucketName, listParams, list);
+
+    pushMetric('listLifecycleOrphanDeleteMarkers', log, { authInfo, bucket: bucketName });
+    monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleOrphanDeleteMarkers');
+    return callback(null, res);
+}
+
+/**
+ * listLifecycleOrphanDeleteMarkers - Return list of expired object delete marker in bucket
+ * @param {AuthInfo} authInfo            - Instance of AuthInfo class with
+ *                                         requester's info
+ * @param {array} locationConstraints    - array of location contraint
+ * @param {object} request               - http request object
+ * @param {function} log                 - Werelogs request logger
+ * @param {function} callback            - callback to respond to http request
+ *                                          with either error code or xml response body
+ * @return {undefined}
+ */
+function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request, log, callback) {
+    const params = request.query;
+    const bucketName = request.bucketName;
+
+    log.debug('processing request', { method: 'listLifecycleOrphanDeleteMarkers' });
+    const requestMaxKeys = params['max-keys'] ?
+        Number.parseInt(params['max-keys'], 10) : 1000;
+    if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
+        return callback(errors.InvalidArgument);
+    }
+    const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
+
+    // 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
+    // Two entries representing the master key and the following one representing the non-current version.
+    const minEntriesToBeScanned = 3;
+    const { isValid, maxScannedLifecycleListingEntries } =
+        validateMaxScannedEntries(params, config, minEntriesToBeScanned);
+    if (!isValid) {
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
+        return callback(errors.InvalidArgument);
+    }
+
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: 'listLifecycleOrphanDeleteMarkers',
+        request,
+    };
+    const listParams = {
+        listingType: 'DelimiterOrphanDeleteMarker',
+        maxKeys: actualMaxKeys,
+        prefix: params.prefix,
+        beforeDate: params['before-date'],
+        marker: params.marker,
+        maxScannedLifecycleListingEntries,
+    };
+
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
+        if (err) {
+            log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
+            return callback(err, null);
+        }
+
+        const vcfg = bucket.getVersioningConfiguration();
+        const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
+        if (!isBucketVersioned) {
+            log.debug('bucket is not versioned or suspended');
+            return callback(errors.InvalidRequest.customizeDescription(
+                'bucket is not versioned'), null);
+        }
+
+        if (!requestMaxKeys) {
+            const emptyList = {
+                Contents: [],
+                IsTruncated: false,
+            };
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, emptyList, log, callback);
+        }
+
+        return services.getLifecycleListing(bucketName, listParams, log,
+        (err, list) => {
+            if (err) {
+                log.debug('error processing request', { error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
+                return callback(err, null);
+            }
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, list, log, callback);
+        });
+    });
+}
+
+module.exports = {
+    listLifecycleOrphanDeleteMarkers,
+};

lib/api/bucketDelete.js

@@ -2,8 +2,9 @@ const { errors } = require('arsenal');
 
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const deleteBucket = require('./apiUtils/bucket/bucketDeletion');
-const { metadataValidateBucketNew } = require('../metadata/metadataUtils');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
 
 /**
  * bucketDelete - DELETE bucket (currently supports only non-versioned buckets)
@@ -20,6 +21,8 @@ function bucketDelete(authInfo, request, log, cb) {
 
     if (authInfo.isRequesterPublicUser()) {
         log.debug('operation not available for public user');
+        monitoring.promMetrics(
+            'DELETE', request.bucketName, 403, 'deleteBucket');
         return cb(errors.AccessDenied);
     }
     const bucketName = request.bucketName;
@@ -27,17 +30,19 @@ function bucketDelete(authInfo, request, log, cb) {
     const metadataValParams = {
         authInfo,
         bucketName,
-        requestType: request.apiMethods || 'bucketDelete',
+        requestType: 'bucketDelete',
         request,
     };
 
-    return metadataValidateBucketNew(metadataValParams, request.actionImplicitDenies, log,
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
         (err, bucketMD) => {
             const corsHeaders = collectCorsHeaders(request.headers.origin,
                 request.method, bucketMD);
             if (err) {
                 log.debug('error processing request',
                     { method: 'metadataValidateBucket', error: err });
+                monitoring.promMetrics(
+                    'DELETE', bucketName, err.code, 'deleteBucket');
                 return cb(err, corsHeaders);
             }
             log.trace('passed checks',
@@ -45,12 +50,16 @@ function bucketDelete(authInfo, request, log, cb) {
             return deleteBucket(authInfo, bucketMD, bucketName,
                 authInfo.getCanonicalID(), request, log, err => {
                     if (err) {
+                        monitoring.promMetrics(
+                            'DELETE', bucketName, err.code, 'deleteBucket');
                         return cb(err, corsHeaders);
                     }
                     pushMetric('deleteBucket', log, {
                         authInfo,
                         bucket: bucketName,
                     });
+                    monitoring.promMetrics(
+                        'DELETE', bucketName, '204', 'deleteBucket');
                     return cb(null, corsHeaders);
                 });
         });