.dockerignore

@@ -0,0 +1,9 @@
+node_modules
+localData/*
+localMetadata/*
+# Keep the .git/HEAD file in order to properly report version
+.git/objects
+.github
+.tox
+coverage
+.DS_Store

.eslintrc

@@ -0,0 +1,54 @@
+{
+    "extends": "scality",
+    "plugins": [
+        "mocha"
+    ],
+    "rules": {
+        "import/extensions": "off",
+        "lines-around-directive": "off",
+        "no-underscore-dangle": "off",
+        "indent": "off",
+        "object-curly-newline": "off",
+        "operator-linebreak": "off",
+        "function-paren-newline": "off",
+        "import/newline-after-import": "off",
+        "prefer-destructuring": "off",
+        "implicit-arrow-linebreak": "off",
+        "no-bitwise": "off",
+        "dot-location": "off",
+        "comma-dangle": "off",
+        "no-undef-init": "off",
+        "global-require": "off",
+        "import/no-dynamic-require": "off",
+        "class-methods-use-this": "off",
+        "no-plusplus": "off",
+        "no-else-return": "off",
+        "object-property-newline": "off",
+        "import/order": "off",
+        "no-continue": "off",
+        "no-tabs": "off",
+        "lines-between-class-members": "off",
+        "prefer-spread": "off",
+        "no-lonely-if": "off",
+        "no-useless-escape": "off",
+        "no-restricted-globals": "off",
+        "no-buffer-constructor": "off",
+        "import/no-extraneous-dependencies": "off",
+        "space-unary-ops": "off",
+        "no-useless-return": "off",
+        "no-unexpected-multiline": "off",
+        "no-mixed-operators": "off",
+        "newline-per-chained-call": "off",
+        "operator-assignment": "off",
+        "spaced-comment": "off",
+        "comma-style": "off",
+        "no-restricted-properties": "off",
+        "new-parens": "off",
+        "no-multi-spaces": "off",
+        "quote-props": "off",
+        "mocha/no-exclusive-tests": "error",
+    },
+    "parserOptions": {
+        "ecmaVersion": 2020
+    }
+}

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,87 @@
+# General support information
+
+GitHub Issues are **reserved** for actionable bug reports (including
+documentation inaccuracies), and feature requests.
+**All questions** (regarding configuration, use cases, performance, community,
+events, setup and usage recommendations, among other things) should be asked on
+the **[Zenko Forum](http://forum.zenko.io/)**.
+
+> Questions opened as GitHub issues will systematically be closed, and moved to
+> the [Zenko Forum](http://forum.zenko.io/).
+
+--------------------------------------------------------------------------------
+
+## Avoiding duplicates
+
+When reporting a new issue/requesting a feature, make sure that we do not have
+any duplicates already open:
+
+- search the issue list for this repository (use the search bar, select
+  "Issues" on the left pane after searching);
+- if there is a duplicate, please do not open your issue, and add a comment
+  to the existing issue instead.
+
+--------------------------------------------------------------------------------
+
+## Bug report information
+
+(delete this section (everything between the lines) if you're not reporting a bug
+but requesting a feature)
+
+### Description
+
+Briefly describe the problem you are having in a few paragraphs.
+
+### Steps to reproduce the issue
+
+Please provide steps to reproduce, including full log output
+
+### Actual result
+
+Describe the results you received
+
+### Expected result
+
+Describe the results you expected
+
+### Additional information
+
+- Node.js version,
+- Docker version,
+- yarn version,
+- distribution/OS,
+- optional: anything else you deem helpful to us.
+
+--------------------------------------------------------------------------------
+
+## Feature Request
+
+(delete this section (everything between the lines) if you're not requesting
+a feature but reporting a bug)
+
+### Proposal
+
+Describe the feature
+
+### Current behavior
+
+What currently happens
+
+### Desired behavior
+
+What you would like to happen
+
+### Use case
+
+Please provide use cases for changing the current behavior
+
+### Additional information
+
+- Is this request for your company? Y/N
+  - If Y: Company name:
+  - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N
+- Are you willing to contribute this feature yourself?
+- Position/Title:
+- How did you hear about us?
+
+--------------------------------------------------------------------------------

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,28 @@
+# Pull request template
+
+## Description
+
+### Motivation and context
+
+Why is this change required? What problem does it solve?
+
+### Related issues
+
+Please use the following link syntaxes #600 to reference issues in the
+current repository
+
+## Checklist
+
+### Add tests to cover the changes
+
+New tests added or existing tests modified to cover all changes
+
+### Code conforms with the [style guide](https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md#coding-style-guidelines)
+
+### Sign your work
+
+In order to contribute to the project, you must sign your work
+https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md#sign-your-work
+
+Thank you again for contributing! We will try to test and integrate the change
+as soon as we can.

.github/actions/setup-ci/action.yaml

@@ -0,0 +1,43 @@
+---
+name: "Setup CI environment"
+description: "Setup Cloudserver CI environment"
+
+runs:
+  using: composite
+  steps:
+    - name: Setup etc/hosts
+      shell: bash
+      run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
+    - name: Setup Credentials
+      shell: bash
+      run: bash .github/scripts/credentials.bash
+    - name: Setup job artifacts directory
+      shell: bash
+      run: |-
+        set -exu;
+        mkdir -p /tmp/artifacts/${JOB_NAME}/;
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '16'
+        cache: 'yarn'
+    - name: install dependencies
+      shell: bash
+      run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
+    - uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Setup python2 test environment
+      shell: bash
+      run: |
+        sudo apt-get install -y libdigest-hmac-perl
+        pip install 's3cmd==2.3.0'
+    - name: fix sproxyd.conf permissions
+      shell: bash
+      run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
+    - name: ensure fuse kernel module is loaded (for sproxyd)
+      shell: bash
+      run: sudo modprobe fuse

.github/ceph/Dockerfile

@@ -0,0 +1,25 @@
+FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
+
+ENV CEPH_DAEMON demo
+ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
+
+ENV CEPH_DEMO_UID zenko
+ENV CEPH_DEMO_ACCESS_KEY accessKey1
+ENV CEPH_DEMO_SECRET_KEY verySecretKey1
+ENV CEPH_DEMO_BUCKET zenkobucket
+
+ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
+ENV MON_IP 0.0.0.0
+ENV NETWORK_AUTO_DETECT 4
+ENV RGW_CIVETWEB_PORT 8001
+
+RUN rm /etc/yum.repos.d/tcmu-runner.repo
+
+ADD ./entrypoint-wrapper.sh /
+RUN chmod +x /entrypoint-wrapper.sh && \
+    yum install -y python-pip && \
+    yum clean all && \
+    pip install awscli && \
+    rm -rf /root/.cache/pip
+
+ENTRYPOINT [ "/entrypoint-wrapper.sh" ]

.github/ceph/entrypoint-wrapper.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+touch /artifacts/ceph.log
+mkfifo /tmp/entrypoint_output
+# We run this in the background so that we can tail the RGW log after init,
+# because entrypoint.sh never returns
+
+# The next line will be needed when ceph builds 3.2.2 so I'll leave it here
+# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
+
+bash /entrypoint.sh > /tmp/entrypoint_output &
+entrypoint_pid="$!"
+while read -r line; do
+    echo $line
+    # When we find this line server has started
+    if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
+        break
+    fi
+done < /tmp/entrypoint_output
+
+# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
+s3cmd mb s3://cephbucket s3://cephbucket2
+
+mkdir /root/.aws
+cat > /root/.aws/credentials <<EOF
+[default]
+aws_access_key_id = accessKey1
+aws_secret_access_key = verySecretKey1
+EOF
+
+# Enable versioning on them
+for bucket in cephbucket cephbucket2; do
+    echo "Enabling versiong for $bucket"
+    aws --endpoint http://127.0.0.1:8001 s3api  put-bucket-versioning --bucket $bucket --versioning Status=Enabled
+done
+tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
+wait $entrypoint_pid

.github/ceph/wait_for_ceph.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# This script is needed because RADOS Gateway
+# will open the port before beginning to serve traffic
+# causing wait_for_local_port.bash to exit immediately
+
+echo 'Waiting for ceph'
+while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
+    sleep 1
+    echo -n "."
+done

.github/dependabot.yml

@@ -0,0 +1,10 @@
+---
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: "/"
+    schedule:
+      interval: daily
+      time: "13:00"
+    open-pull-requests-limit: 10
+    target-branch: "development/7.4"

.github/docker/creds.env

@@ -0,0 +1,36 @@
+azurebackend_AZURE_STORAGE_ACCESS_KEY
+azurebackend_AZURE_STORAGE_ACCOUNT_NAME
+azurebackend_AZURE_STORAGE_ENDPOINT
+azurebackend2_AZURE_STORAGE_ACCESS_KEY
+azurebackend2_AZURE_STORAGE_ACCOUNT_NAME
+azurebackend2_AZURE_STORAGE_ENDPOINT
+azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY
+azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME
+azurebackendmismatch_AZURE_STORAGE_ENDPOINT
+azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY
+azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME
+azurenonexistcontainer_AZURE_STORAGE_ENDPOINT
+azuretest_AZURE_BLOB_ENDPOINT
+b2backend_B2_ACCOUNT_ID
+b2backend_B2_STORAGE_ACCESS_KEY
+GOOGLE_SERVICE_EMAIL
+GOOGLE_SERVICE_KEY
+AWS_S3_BACKEND_ACCESS_KEY
+AWS_S3_BACKEND_SECRET_KEY
+AWS_S3_BACKEND_ACCESS_KEY_2
+AWS_S3_BACKEND_SECRET_KEY_2
+AWS_GCP_BACKEND_ACCESS_KEY
+AWS_GCP_BACKEND_SECRET_KEY
+AWS_GCP_BACKEND_ACCESS_KEY_2
+AWS_GCP_BACKEND_SECRET_KEY_2
+b2backend_B2_STORAGE_ENDPOINT
+gcpbackend2_GCP_SERVICE_EMAIL
+gcpbackend2_GCP_SERVICE_KEY
+gcpbackend2_GCP_SERVICE_KEYFILE
+gcpbackend_GCP_SERVICE_EMAIL
+gcpbackend_GCP_SERVICE_KEY
+gcpbackendmismatch_GCP_SERVICE_EMAIL
+gcpbackendmismatch_GCP_SERVICE_KEY
+gcpbackend_GCP_SERVICE_KEYFILE
+gcpbackendmismatch_GCP_SERVICE_KEYFILE
+gcpbackendnoproxy_GCP_SERVICE_KEYFILE

.github/docker/docker-compose.yaml

@@ -0,0 +1,92 @@
+services:
+  cloudserver:
+    image: ${CLOUDSERVER_IMAGE}
+    command: sh -c "yarn start > /artifacts/s3.log"
+    network_mode: "host"
+    volumes:
+      - /tmp/ssl:/ssl
+      - /tmp/ssl-kmip:/ssl-kmip
+      - ${HOME}/.aws/credentials:/root/.aws/credentials
+      - /tmp/artifacts/${JOB_NAME}:/artifacts
+    environment:
+      - CI=true
+      - ENABLE_LOCAL_CACHE=true
+      - REDIS_HOST=0.0.0.0
+      - REDIS_PORT=6379
+      - REPORT_TOKEN=report-token-1
+      - REMOTE_MANAGEMENT_DISABLE=1
+      - HEALTHCHECKS_ALLOWFROM=0.0.0.0/0
+      - DATA_HOST=0.0.0.0
+      - METADATA_HOST=0.0.0.0
+      - S3BACKEND
+      - S3DATA
+      - S3METADATA
+      - MPU_TESTING
+      - S3VAULT
+      - S3_LOCATION_FILE
+      - ENABLE_UTAPI_V2
+      - BUCKET_DENY_FILTER
+      - S3KMS
+      - S3KMIP_PORT
+      - S3KMIP_HOSTS
+      - S3KMIP-COMPOUND_CREATE
+      - S3KMIP_BUCKET_ATTRIBUTE_NAME
+      - S3KMIP_PIPELINE_DEPTH
+      - S3KMIP_KEY
+      - S3KMIP_CERT
+      - S3KMIP_CA
+      - MONGODB_HOSTS=0.0.0.0:27018
+      - MONGODB_RS=rs0
+      - DEFAULT_BUCKET_KEY_FORMAT
+      - METADATA_MAX_CACHED_BUCKETS
+      - ENABLE_NULL_VERSION_COMPAT_MODE
+      - SCUBA_HOST
+      - SCUBA_PORT
+      - SCUBA_HEALTHCHECK_FREQUENCY
+      - S3QUOTA
+      - QUOTA_ENABLE_INFLIGHTS
+    env_file:
+      - creds.env
+    depends_on:
+      - redis
+    extra_hosts:
+      - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
+      - "pykmip.local:127.0.0.1"
+  redis:
+    image: redis:alpine
+    network_mode: "host"
+  squid:
+    network_mode: "host"
+    profiles: ['ci-proxy']
+    image: scality/ci-squid
+    command: >-
+      sh -c 'mkdir -p /ssl &&
+            openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
+              -subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
+              -keyout /ssl/myca.pem  -out /ssl/myca.pem &&
+            cp /ssl/myca.pem /ssl/CA.pem &&
+            squid -f /etc/squid/squid.conf -N -z &&
+            squid -f /etc/squid/squid.conf -NYCd 1'
+    volumes:
+      - /tmp/ssl:/ssl
+  pykmip:
+    network_mode: "host"
+    profiles: ['pykmip']
+    image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
+    volumes:
+      - /tmp/artifacts/${JOB_NAME}:/artifacts
+  mongo:
+    network_mode: "host"
+    profiles: ['mongo', 'ceph']
+    image: ${MONGODB_IMAGE}
+  ceph: 
+    network_mode: "host"
+    profiles: ['ceph']
+    image: ghcr.io/scality/cloudserver/ci-ceph 
+  sproxyd:
+    network_mode: "host"
+    profiles: ['sproxyd']
+    image: sproxyd-standalone
+    build: ./sproxyd
+    user: 0:0
+    privileged: yes

.github/docker/mongodb/Dockerfile

@@ -0,0 +1,28 @@
+FROM mongo:5.0.21
+
+ENV USER=scality \
+    HOME_DIR=/home/scality \
+    CONF_DIR=/conf \
+    DATA_DIR=/data
+
+# Set up directories and permissions
+RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
+    mkdir /logs; \
+    adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
+
+# Set up environment variables and directories for scality user
+RUN mkdir ${CONF_DIR} && \
+    chown -R ${USER} ${CONF_DIR} && \
+    chown -R ${USER} ${DATA_DIR}
+
+# copy the mongo config file
+COPY /conf/mongod.conf /conf/mongod.conf
+COPY /conf/mongo-run.sh /conf/mongo-run.sh
+COPY /conf/initReplicaSet /conf/initReplicaSet.js
+
+EXPOSE 27017/tcp
+EXPOSE 27018
+
+# Set up CMD
+ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
+CMD ["bash", "/conf/mongo-run.sh"]

.github/docker/mongodb/conf/initReplicaSet

@@ -0,0 +1,4 @@
+rs.initiate({
+    _id: "rs0",
+    members: [{ _id: 0, host: "127.0.0.1:27018" }]
+});

.github/docker/mongodb/conf/mongo-run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -exo pipefail
+
+init_RS() {
+  sleep 5
+  mongo --port 27018 /conf/initReplicaSet.js
+}
+init_RS &
+
+mongod --bind_ip_all --config=/conf/mongod.conf

.github/docker/mongodb/conf/mongod.conf

@@ -0,0 +1,15 @@
+storage:
+   journal:
+      enabled: true
+   engine: wiredTiger
+   dbPath: "/data/db"
+processManagement:
+   fork: false
+net:
+   port: 27018
+   bindIp: 0.0.0.0
+replication:
+   replSetName: "rs0"
+   enableMajorityReadConcern: true
+security:
+    authorization: disabled

.github/docker/sproxyd/Dockerfile

@@ -0,0 +1,3 @@
+FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
+ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
+RUN chown root:root /conf/sproxyd0.conf

.github/docker/sproxyd/conf/fastcgi_params

@@ -0,0 +1,26 @@
+fastcgi_param  QUERY_STRING       $query_string;
+fastcgi_param  REQUEST_METHOD     $request_method;
+fastcgi_param  CONTENT_TYPE       $content_type;
+fastcgi_param  CONTENT_LENGTH     $content_length;
+
+#fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;
+fastcgi_param  SCRIPT_NAME        /var/www;
+fastcgi_param  PATH_INFO          $document_uri;
+
+fastcgi_param  REQUEST_URI        $request_uri;
+fastcgi_param  DOCUMENT_URI       $document_uri;
+fastcgi_param  DOCUMENT_ROOT      $document_root;
+fastcgi_param  SERVER_PROTOCOL    $server_protocol;
+fastcgi_param  HTTPS              $https if_not_empty;
+
+fastcgi_param  GATEWAY_INTERFACE  CGI/1.1;
+fastcgi_param  SERVER_SOFTWARE    nginx/$nginx_version;
+
+fastcgi_param  REMOTE_ADDR        $remote_addr;
+fastcgi_param  REMOTE_PORT        $remote_port;
+fastcgi_param  SERVER_ADDR        $server_addr;
+fastcgi_param  SERVER_PORT        $server_port;
+fastcgi_param  SERVER_NAME        $server_name;
+
+# PHP only, required if PHP was built with --enable-force-cgi-redirect
+fastcgi_param  REDIRECT_STATUS    200;

.github/docker/sproxyd/conf/nginx.conf

@@ -0,0 +1,88 @@
+worker_processes 1;
+error_log  /logs/error.log;
+user root root;
+events {
+    worker_connections  1000;
+    reuse_port on;
+    multi_accept on;
+}
+worker_rlimit_nofile    20000;
+http {
+    root /var/www/;
+    upstream sproxyds {
+        least_conn;
+        keepalive 40;
+        server 127.0.0.1:20000;
+    }
+    server {
+        client_max_body_size 0;
+        client_body_timeout 150;
+        client_header_timeout 150;
+        postpone_output 0;
+        client_body_postpone_size 0;
+        keepalive_requests 1100;
+        keepalive_timeout 300s;
+        server_tokens off;
+        default_type application/octet-stream;
+        gzip off;
+        tcp_nodelay on;
+        tcp_nopush  on;
+        sendfile on;
+        listen 81;
+        server_name localhost;
+        rewrite ^/arc/(.*)$ /dc1/$1 permanent;
+        location ~* ^/proxy/(.*)$ {
+            rewrite ^/proxy/(.*)$ /$1 last;
+        }
+        allow 127.0.0.1;
+
+        deny all;
+        set $usermd '-';
+        set $sentusermd '-';
+    	set $elapsed_ms '-';
+    	set $now '-';
+    	log_by_lua '
+    		if not(ngx.var.http_x_scal_usermd == nil) and  string.len(ngx.var.http_x_scal_usermd) > 2 then
+    			ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
+    		end
+                    if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
+                            ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
+                    end
+    		local elapsed_ms = tonumber(ngx.var.request_time)
+    		if not (  elapsed_ms == nil) then
+    			elapsed_ms = elapsed_ms * 1000
+    			ngx.var.elapsed_ms = tostring(elapsed_ms)
+    		end
+    		local time = tonumber(ngx.var.msec) * 1000
+    		ngx.var.now = time
+    	';
+        log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
+                       '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
+                       '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
+		       '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
+		       '"contentType":"$content_type","s3Address":"$remote_addr",'
+		       '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
+                       '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
+		       '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
+		       '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
+                       '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
+                       '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
+        access_log /dev/stdout irm;
+    	error_log  /dev/stdout error;
+        location / {
+            proxy_request_buffering off;
+            fastcgi_request_buffering off;
+            fastcgi_no_cache 1;
+            fastcgi_cache_bypass 1;
+            fastcgi_buffering off;
+            fastcgi_ignore_client_abort on;
+            fastcgi_keep_conn on;
+            include fastcgi_params;
+            fastcgi_pass sproxyds;
+            fastcgi_next_upstream error timeout;
+            fastcgi_send_timeout 285s;
+            fastcgi_read_timeout 285s;
+        }
+    }
+}
+

.github/docker/sproxyd/conf/sproxyd0.conf

@@ -0,0 +1,12 @@
+{
+    "general": {
+        "ring": "DATA",
+        "port": 20000,
+        "syslog_facility": "local0"
+    },
+    "ring_driver:0": {
+        "alias": "dc1",
+        "type": "local",
+        "queue_path": "/tmp/ring-objs"
+    },
+}

.github/docker/sproxyd/conf/supervisord.conf

@@ -0,0 +1,43 @@
+[supervisord]
+nodaemon = true
+loglevel = info
+logfile  = %(ENV_LOG_DIR)s/supervisord.log
+pidfile  = %(ENV_SUP_RUN_DIR)s/supervisord.pid
+logfile_maxbytes = 20MB
+logfile_backups = 2
+
+[unix_http_server]
+file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[program:nginx]
+directory=%(ENV_SUP_RUN_DIR)s
+command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart=true
+autostart=true
+user=root
+
+[program:sproxyd]
+directory=%(ENV_SUP_RUN_DIR)s
+process_name=%(program_name)s-%(process_num)s
+numprocs=1
+numprocs_start=0
+command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+redirect_stderr=true
+autorestart=true
+autostart=true
+user=root

.github/pykmip/Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.10-alpine
+
+RUN apk add --no-cache \
+        libressl && \
+    apk add --no-cache --virtual .build-deps \
+        python3-dev \
+        libffi-dev \
+        libressl-dev \
+        sqlite-dev \
+        build-base \
+        curl
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN pip3 install -U pip && \
+    pip3 install pykmip requests && \
+    apk del .build-deps && \
+    mkdir /pykmip
+
+
+ADD ./bin /usr/local/bin
+ADD ./certs /ssl
+ADD policy.json /etc/pykmip/policies/policy.json
+ADD server.conf /etc/pykmip/server.conf
+ADD docker-entrypoint.sh /
+RUN chmod +x /docker-entrypoint.sh
+
+ENTRYPOINT ["/docker-entrypoint.sh"]

.github/pykmip/bin/create_certificates.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+
+from cryptography import x509
+from cryptography.hazmat import backends
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+import datetime
+import argparse
+import sys
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        prog=sys.argv[0],
+        description='Tool to generate a x509 CA root, server and client certs')
+    parser.add_argument('-c', '--common-name', action='store',
+                        default='localhost',
+                        help='Set the common name for the server-side cert')
+    return parser.parse_args()
+
+
+def create_rsa_private_key(key_size=2048, public_exponent=65537):
+    private_key = rsa.generate_private_key(
+        public_exponent=public_exponent,
+        key_size=key_size,
+        backend=backends.default_backend()
+    )
+    return private_key
+
+
+def create_self_signed_certificate(subject_name,
+                                   private_key,
+                                   days_valid=36500):
+    subject = x509.Name([
+        x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),
+        x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name)
+    ])
+    certificate = x509.CertificateBuilder().subject_name(
+        subject
+    ).issuer_name(
+        subject
+    ).public_key(
+        private_key.public_key()
+    ).serial_number(
+        x509.random_serial_number()
+    ).not_valid_before(
+        datetime.datetime.utcnow()
+    ).not_valid_after(
+        datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
+    ).add_extension(
+        x509.BasicConstraints(True, None),
+        critical=True
+    ).sign(private_key, hashes.SHA256(), backends.default_backend())
+
+    return certificate
+
+
+def create_certificate(subject_name,
+                       private_key,
+                       signing_certificate,
+                       signing_key,
+                       days_valid=36500,
+                       client_auth=False):
+    subject = x509.Name([
+        x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),
+        x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name)
+    ])
+    builder = x509.CertificateBuilder().subject_name(
+        subject
+    ).issuer_name(
+        signing_certificate.subject
+    ).public_key(
+        private_key.public_key()
+    ).serial_number(
+        x509.random_serial_number()
+    ).not_valid_before(
+        datetime.datetime.utcnow()
+    ).not_valid_after(
+        datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
+    )
+
+    if client_auth:
+        builder = builder.add_extension(
+            x509.ExtendedKeyUsage([x509.ExtendedKeyUsageOID.CLIENT_AUTH]),
+            critical=True
+        )
+
+    certificate = builder.sign(
+        signing_key,
+        hashes.SHA256(),
+        backends.default_backend()
+    )
+    return certificate
+
+
+def main(common_name):
+    root_key = create_rsa_private_key()
+    root_certificate = create_self_signed_certificate(
+        u"Root CA",
+        root_key
+    )
+
+    server_key = create_rsa_private_key()
+    server_certificate = create_certificate(
+        common_name,
+        server_key,
+        root_certificate,
+        root_key
+    )
+
+    john_doe_client_key = create_rsa_private_key()
+    john_doe_client_certificate = create_certificate(
+        u"John Doe",
+        john_doe_client_key,
+        root_certificate,
+        root_key,
+        client_auth=True
+    )
+
+    with open("certs/kmip-ca.pem", "wb") as f:
+        f.write(
+            root_certificate.public_bytes(
+                serialization.Encoding.PEM
+            )
+        )
+    with open("certs/kmip-key.pem", "wb") as f:
+        f.write(server_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption()
+        ))
+    with open("certs/kmip-cert.pem", "wb") as f:
+        f.write(
+            server_certificate.public_bytes(
+                serialization.Encoding.PEM
+            )
+        )
+    with open("certs/kmip-client-key.pem", "wb") as f:
+        f.write(john_doe_client_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption()
+        ))
+    with open("certs/kmip-client-cert.pem", "wb") as f:
+        f.write(
+            john_doe_client_certificate.public_bytes(
+                serialization.Encoding.PEM
+            )
+        )
+
+
+if __name__ == '__main__':
+    args = get_args()
+    main(args.common_name)

.github/pykmip/bin/run_server.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2016 The Johns Hopkins University/Applied Physics Laboratory
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging  # noqa: E402
+
+logging.basicConfig(level=logging.DEBUG)
+
+from kmip.services.server import server  # noqa: E402
+
+if __name__ == '__main__':
+    print('Starting PyKMIP server on 0.0.0.0:5696')
+    server.main()

.github/pykmip/certs/kmip-ca.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC6zCCAdOgAwIBAgIUPIpMY95b4HjKAk+FyydZApAEFskwDQYJKoZIhvcNAQEL
+BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
+MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJDEQMA4GA1UECgwHU2NhbGl0eTEQ
+MA4GA1UEAwwHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AKqLFEsWtfRTxnoZrQe63tq+rQnVgninHMahRmXkzyjK/uNhoKnIh8bXdTC/eCZ6
+FBROqBYNL0TJb0HDv1FzcZS1UCUldRqTlvr6wZb0pfrp40fvztsqQgAh1t/Blg5i
+Zv5+ESSlNs5rWbFTxtq+FbMW/ERYTrVfnMkBiLg4Gq0HwID9a5jvJatzrrno2s1m
+OfZCT3HaE3tMZ6vvYuoamvLNdvdH+9KeTmBCursfNejt0rSGjIqfi6DvFJSayydQ
+is5DMSTbCLGdKQmA85VfEQmlQ8v0232WDSd6gVfp2tthDEDHnCbgWkEd1vsTyS85
+ubdt5v4CWGOWV+mu3bf8xM0CAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAQEARTjc2zV/ol1/LsSzZy6l1R0uFBmR2KumH+Se1Yq2vKpY
+Dv6xmrvmjOUr5RBO77nRhIgdcQA+LyAg8ii2Dfzc8r1RTD+j1bYOxESXctBOBcXM
+Chy6FEBydR6m7S8qQyL+caJWO1WZWp2tapcm6sUG1oRVznWtK1/SHKIzOBwsmJ07
+79KsCJ6wf9tzD05EDTI2QhAObE9/thy+zc8l8cmv9A6p3jKkx9rwXUttSUqTn0CW
+w45bgKg6+DDcrhZ+MATbzuTfhuA4NFUTzK7KeX9sMuOV03Zs8SA3VhAOXmu063M3
+0f9X7P/0RmGTTp7GGCqEINcZdbLh3k7CpFb2Ox998Q==
+-----END CERTIFICATE-----

.github/pykmip/certs/kmip-cert.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC2zCCAcOgAwIBAgIUIlE8UAkqQ+6mbJDtrt9kkmi8aJYwDQYJKoZIhvcNAQEL
+BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
+MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowKTEQMA4GA1UECgwHU2NhbGl0eTEV
+MBMGA1UEAwwMcHlrbWlwLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAtxr7pq/lnzVeZz4z52Yc3DeaPqjNfRSyW5cPUlT7ABXFb7+tja7K2C7u
+DYVK+Q+2yJCQwYJY47aKJB++ewam9t2V8Xy0Z8S+0I2ImCwuyeihaD/f6uJZRzms
+ycdECH22BA6tCPlQLnlboRiZzI6rcIvXAbUMvLvFm3nyYIs9qidExRnfyMjISknM
+V+83LT5QW4IcHgKYqzdz2ZmOnk+f4wmMmitcivTdIZCL8Z0cxr7BJlOh5JZ/V5uj
+WUXeNa+ttW0RKKBlg9T+wj0JvwoJBPZTmsMAy3tI9tjLg3DwGYKsflbFeU2tebXI
+gncGFZ/dFxj331GGtq3kz1PzAUYf2wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB1
+8HgJ0fu6/pCrDxAm90eESFjmaTFyTN8q00zhq4Cb3zAT9KMWzAygkZ9n4ZFgELPo
+7kBE2H6RcDdoBmjVYd8HnBloDdYzYbncKgt5YBvxRaMSF4/l65BM8wjatyXErqnH
+QLLTRe5AuF0/F0KtPeDQ2JFVu8dZ35W3fyKGPRsEdVOSCTHROmqpGhZCpscyUP4W
+Hb0dBTESQ9mQHw14OCaaahARd0X5WdcA/E+m0fpGqj1rQCXS+PrRcSLe1E1hqPlK
+q/hXSXD5nybwipktELvJCbB7l4HmJr2pIpldeR5+ef68Cs8hqs6DRlsJX9sK2ng+
+TFe5v6SCarqZ9kFvr6Yp
+-----END CERTIFICATE-----

.github/pykmip/certs/kmip-client-cert.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC8zCCAdugAwIBAgIUBs6nVXQXhrFbClub3aSLg72/DiYwDQYJKoZIhvcNAQEL
+BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
+MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJTEQMA4GA1UECgwHU2NhbGl0eTER
+MA8GA1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQC6neSYoBoWh/i2mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4L
+jLLRsPGt9KkJlUhHGWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jt
+UDnQUwshj+hRFe0iKAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsH
+PVju9yZBYzYhwAMyYFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFip
+yR2Fh3WGSwWh45HgMT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+p
+SMLm0T4iNxedQWBtDM7ts4EjAgMBAAGjGjAYMBYGA1UdJQEB/wQMMAoGCCsGAQUF
+BwMCMA0GCSqGSIb3DQEBCwUAA4IBAQCMi9HEhZc5jHJMj18Wq00fZy4O9XtjCe0J
+nntW9tzi3rTQcQWKA7i9uVdDoCg+gMFVxWMvV7luFEUc/VYV1v8hFfbIFygzFsZY
+xwv4GQaIwbsgzD+oziia53w0FSuNL0uE0MeKvrt3yzHxCxylHyl+TQd/UdAtAo+k
+RL1sI0mBZx5qo6d1J7ZMCxzAGaT7KjnJvziFr/UbfSNnwDsxsUwGaI1ZeAxJN8DI
+zTrg3f3lrrmHcauEgKnuQwIqaMZR6veG6RkjtcYSlJYID1irkE6njs7+wivOAkzt
+fBt/0PD76FmAI0VArgU/zDB8dGyYzrq39W749LuEfm1TPmlnUtDr
+-----END CERTIFICATE-----

.github/pykmip/certs/kmip-client-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6neSYoBoWh/i2
+mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4LjLLRsPGt9KkJlUhH
+GWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jtUDnQUwshj+hRFe0i
+KAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsHPVju9yZBYzYhwAMy
+YFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFipyR2Fh3WGSwWh45Hg
+MT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+pSMLm0T4iNxedQWBt
+DM7ts4EjAgMBAAECggEANNXdUeUKXdSzcycPV/ea/c+0XFcy8e9B46lfQTpTqQOx
+xD8GbWD1L/gdk6baJgT43+ukEWdSsJbmdtLXti29Ta8OF2VtIDhIbCVtvs3dq3zt
+vrvugsiVDr8nkP306qOrKrNIVIFE+igmEmSaXsu/h/33ladxeeV9/s2DC7NOOjWN
+Mu4KYr5BBbu3qAavdzbrcz7Sch+GzsYqK/pBounCTQu3o9E4TSUcmcsasWmtHN3u
+e6G2UjObdzEW7J0wWvvtJ0wHQUVRueHfqwqKf0dymcZ3xOlx3ZPhKPz5n4F1UGUt
+RQaNazqs5SzZpUgDuPw4k8h/aCHK21Yexw/l4+O9KQKBgQD1WZSRK54zFoExBQgt
+OZSBNZW3Ibti5lSiF0M0g+66yNZSWfPuABEH0tu5CXopdPDXo4kW8NLGEqQStWTX
+RGK0DE9buEL3eebOfjIdS2IZ3t3dX3lMypplVCj4HzAgITlweSH1LLTyAtaaOpwa
+jksqfcn5Zw+XGkyc6GBBVaZetQKBgQDCt6Xf/g26+zjvHscjdzsfBhnYvTOrr6+F
+xqFFxOEOocGr+mL7UTAs+a9m/6lOWhlagk+m+TIZNL8o3IN7KFTYxPYPxTiewgVE
+rIm3JBmPxRiPn01P3HrtjaqfzsXF30j3ele7ix5OxieZq4vsW7ZXP3GZE34a08Ov
+12sE1DlvdwKBgQDzpYQOLhyqazzcqzyVfMrnDYmiFVN7QXTmiudobWRUBUIhAcdl
+oJdJB7K/rJOuO704x+RJ7dnCbZyWH6EGzZifaGIemXuXO21jvpqR0NyZCGOXhUp2
+YfS1j8AntwEZxyS9du2sBjui4gKvomiHTquChOxgSmKHEcznPTTpbN8MyQKBgF5F
+LVCZniolkLXsL7tS8VOez4qoZ0i6wP7CYLf3joJX+/z4N023S9yqcaorItvlMRsp
+tciAIyoi6F2vDRTmPNXJ3dtav4PVKVnLMs1w89MwOCjoljSQ6Q7zpGTEZenbpWbz
+W2BYBS9cLjXu4MpoyInLFINo9YeleLs8TvrCiKAXAoGBANsduqLnlUW/f5zDb5Fe
+SB51+KhBjsVIeYmU+8xtur9Z7IxZXK28wpoEsm7LmX7Va5dERjI+tItBiJ5+Unu1
+Xs2ljDg35ARKHs0dWBJGpbnZg4dbT6xpIL4YMPXm1Zu++PgRpxPIMn646xqd8GlH
+bavm6Km/fXNG58xus+EeLpV5
+-----END PRIVATE KEY-----

.github/pykmip/certs/kmip-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC3Gvumr+WfNV5n
+PjPnZhzcN5o+qM19FLJblw9SVPsAFcVvv62NrsrYLu4NhUr5D7bIkJDBgljjtook
+H757Bqb23ZXxfLRnxL7QjYiYLC7J6KFoP9/q4llHOazJx0QIfbYEDq0I+VAueVuh
+GJnMjqtwi9cBtQy8u8WbefJgiz2qJ0TFGd/IyMhKScxX7zctPlBbghweApirN3PZ
+mY6eT5/jCYyaK1yK9N0hkIvxnRzGvsEmU6Hkln9Xm6NZRd41r621bREooGWD1P7C
+PQm/CgkE9lOawwDLe0j22MuDcPAZgqx+VsV5Ta15tciCdwYVn90XGPffUYa2reTP
+U/MBRh/bAgMBAAECggEABCvcMcbuDztzBB0Zp5re63Fk1SqZS9Et4wJE+hYvhaf5
+UHtoY8LoohYnnC0+MQBXpKgOdCoZBk8BRKNofnr/UL5pjQ/POFH2GuAujXDsO/NN
+wgc6fapcaE/7DLm6ZgsfG2aOMJclaXmgScI6trtFUpIM+t/6A06vyMP1bpeddwPW
+Fqu7NvpDiEcTRUGd+z1JooYgUhGgC7peYUx5+9zqFrwoDBKxnUOnz3BkDsXBy3qm
+65Vu0BSjuJzf6vVMpNGUHY6JXjopVNWku+JAX0wD+iikOd5sziNVdIj1fnZ+IHIf
+7G5h5owHpvSGzJFQ18/g5VHtJdCm+4WQSnbSJRsCAQKBgQDu4IH8yspyeH44fhoS
+PAp/OtILqSP+Da0zAp2LbhrOgyzyuSTdEAYyptqjqHS6QkB1Bu1H44FS0BYUxRXc
+iu2e9AndiLVCGngsE7TpA/ZVLN1B0LEZEHjM6p4d6zZM6iveKVnPAOkTWTBAgzCt
+b31nj4jL8PdlPKQil1AMrOlRAQKBgQDEOwshzIdr2Iy6B/n4CuBViEtwnbAd5f/c
+atA9bcfF8kCahokJsI4eCCLgBwDZpYKD+v0AwOBlacF6t6TX+vdlJsi5EP7uxZ22
+ILsuWqVm/0H77PACuckc5/qLZoGGC81l0DhnpoeMEb6r/TKOo5xAK1gxdlwNNrq+
+nP1zdZnU2wKBgBAS92xFUR4m0YeHpMV5WNN658t1FEDyNqdqE6PgQtmGpi2nG73s
+aB5cb/X3TfOCpce6MZlWy8sAyZuYL4Jprte1YDySCHBsS43bvZ64b4kHvdPB8UjY
+fOh9GSq2Oy8tysnmSm7NhuGQbNjKeyoQiIXBeNkQW/VqATl6qR5RPFoBAoGACNqV
+JQBCd/Y8W0Ry3eM3vgQ5SyqCQMcY5UwYez0Rz3efvJknY72InAhH8o2+VxOlsOjJ
+M5iAR3MfHLdeg7Q6J2E5m0gOCJ34ALi3WV8TqXMI+iH1rlnNnjVFU7bbTz4HFXnw
+oZSc9w/x53a0KkVtjmOmRg0OGDaI9ILG2MfMmhMCgYB8ZqJtX8qZ2TqKU3XdLZ4z
+T2N7xMFuKohWP420r5jKm3Xw85IC+y1SUTB9XGcL79r2eJzmzmdKQ3A3sf3oyUH3
+RdYWxtKcZ5PAE8hVRtn1ETZqUgxASGOUn/6w0npkYSOXPU5bc0W6RSLkjES0i+c3
+fv3OMNI8qpmQhEjpHHQS1g==
+-----END PRIVATE KEY-----

.github/pykmip/docker-entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+python3 /usr/local/bin/run_server.py 2>&1 | tee -a /artifacts/pykmip.log

.github/pykmip/policy.json

@@ -0,0 +1,168 @@
+{
+    "example": {
+        "preset": {
+            "CERTIFICATE": {
+                "LOCATE": "ALLOW_ALL",
+                "CHECK": "ALLOW_ALL",
+                "GET": "ALLOW_ALL",
+                "GET_ATTRIBUTES": "ALLOW_ALL",
+                "GET_ATTRIBUTE_LIST": "ALLOW_ALL",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_ALL",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            },
+            "SYMMETRIC_KEY": {
+                "REKEY": "ALLOW_OWNER",
+                "REKEY_KEY_PAIR": "ALLOW_OWNER",
+                "DERIVE_KEY": "ALLOW_OWNER",
+                "LOCATE": "ALLOW_OWNER",
+                "CHECK": "ALLOW_OWNER",
+                "GET": "ALLOW_OWNER",
+                "GET_ATTRIBUTES": "ALLOW_OWNER",
+                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_OWNER",
+                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            },
+            "PUBLIC_KEY": {
+                "LOCATE": "ALLOW_ALL",
+                "CHECK": "ALLOW_ALL",
+                "GET": "ALLOW_ALL",
+                "GET_ATTRIBUTES": "ALLOW_ALL",
+                "GET_ATTRIBUTE_LIST": "ALLOW_ALL",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_ALL",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            },
+            "PRIVATE_KEY": {
+                "REKEY": "ALLOW_OWNER",
+                "REKEY_KEY_PAIR": "ALLOW_OWNER",
+                "DERIVE_KEY": "ALLOW_OWNER",
+                "LOCATE": "ALLOW_OWNER",
+                "CHECK": "ALLOW_OWNER",
+                "GET": "ALLOW_OWNER",
+                "GET_ATTRIBUTES": "ALLOW_OWNER",
+                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_OWNER",
+                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            },
+            "SPLIT_KEY": {
+                "REKEY": "ALLOW_OWNER",
+                "REKEY_KEY_PAIR": "ALLOW_OWNER",
+                "DERIVE_KEY": "ALLOW_OWNER",
+                "LOCATE": "ALLOW_OWNER",
+                "CHECK": "ALLOW_OWNER",
+                "GET": "ALLOW_OWNER",
+                "GET_ATTRIBUTES": "ALLOW_OWNER",
+                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_OWNER",
+                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            },
+            "TEMPLATE": {
+                "LOCATE": "ALLOW_OWNER",
+                "GET": "ALLOW_OWNER",
+                "GET_ATTRIBUTES": "ALLOW_OWNER",
+                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER"
+            },
+            "SECRET_DATA": {
+                "REKEY": "ALLOW_OWNER",
+                "REKEY_KEY_PAIR": "ALLOW_OWNER",
+                "DERIVE_KEY": "ALLOW_OWNER",
+                "LOCATE": "ALLOW_OWNER",
+                "CHECK": "ALLOW_OWNER",
+                "GET": "ALLOW_OWNER",
+                "GET_ATTRIBUTES": "ALLOW_OWNER",
+                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_OWNER",
+                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            },
+            "OPAQUE_DATA": {
+                "REKEY": "ALLOW_OWNER",
+                "REKEY_KEY_PAIR": "ALLOW_OWNER",
+                "DERIVE_KEY": "ALLOW_OWNER",
+                "LOCATE": "ALLOW_OWNER",
+                "CHECK": "ALLOW_OWNER",
+                "GET": "ALLOW_OWNER",
+                "GET_ATTRIBUTES": "ALLOW_OWNER",
+                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_OWNER",
+                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            },
+            "PGP_KEY": {
+                "REKEY": "ALLOW_OWNER",
+                "REKEY_KEY_PAIR": "ALLOW_OWNER",
+                "DERIVE_KEY": "ALLOW_OWNER",
+                "LOCATE": "ALLOW_OWNER",
+                "CHECK": "ALLOW_OWNER",
+                "GET": "ALLOW_OWNER",
+                "GET_ATTRIBUTES": "ALLOW_OWNER",
+                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
+                "ADD_ATTRIBUTE": "ALLOW_OWNER",
+                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
+                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
+                "OBTAIN_LEASE": "ALLOW_OWNER",
+                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
+                "ACTIVATE": "ALLOW_OWNER",
+                "REVOKE": "ALLOW_OWNER",
+                "DESTROY": "ALLOW_OWNER",
+                "ARCHIVE": "ALLOW_OWNER",
+                "RECOVER": "ALLOW_OWNER"
+            }
+        }
+    }
+}

.github/pykmip/server.conf

@@ -0,0 +1,15 @@
+[server]
+hostname=0.0.0.0
+port=5696
+certificate_path=/ssl/kmip-cert.pem
+key_path=/ssl/kmip-key.pem
+ca_path=/ssl/kmip-ca.pem
+auth_suite=TLS1.2
+policy_path=/etc/pykmip/policies
+enable_tls_client_auth=True
+database_path=/pykmip/pykmip.db
+tls_cipher_suites=
+    TLS_RSA_WITH_AES_128_CBC_SHA256
+    TLS_RSA_WITH_AES_256_CBC_SHA256
+    TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384
+logging_level=DEBUG

.github/scripts/credentials.bash

@@ -0,0 +1,20 @@
+#!/bin/bash -x
+set -x #echo on
+set -e #exit at the first error
+
+mkdir -p $HOME/.aws
+
+cat >>$HOME/.aws/credentials <<EOF
+[default]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
+[default_2]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
+[google]
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY
+[google_2]
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY_2
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY_2
+EOF

.github/workflows/alerts.yaml

@@ -0,0 +1,35 @@
+name: Test alerts
+
+on:
+  push:
+    branches-ignore:
+      - 'development/**'
+      - 'q/*/**'
+
+jobs:
+  run-alert-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tests:
+          - name: 1 minute interval tests
+            file: monitoring/alerts.test.yaml
+          
+          - name: 10 seconds interval tests
+            file: monitoring/alerts.10s.test.yaml
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Render and test ${{ matrix.tests.name }}
+        uses: scality/action-prom-render-test@1.0.3
+        with:
+          alert_file_path: monitoring/alerts.yaml
+          test_file_path: ${{ matrix.tests.file }}
+          alert_inputs: |
+            namespace=zenko
+            service=artesca-data-connector-s3api-metrics
+            reportJob=artesca-data-ops-report-handler
+            replicas=3
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql.yaml

@@ -0,0 +1,25 @@
+---
+name: codeQL
+
+on:
+  push:
+    branches: [w/**, q/*]
+  pull_request:
+    branches: [development/*, stabilization/*, hotfix/*]
+  workflow_dispatch:
+
+jobs:
+  analyze:
+    name: Static analysis with CodeQL
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: javascript, python, ruby
+
+      - name: Build and analyze
+        uses: github/codeql-action/analyze@v3

.github/workflows/dependency-review.yaml

@@ -0,0 +1,16 @@
+---
+name: dependency review
+
+on:
+  pull_request:
+    branches: [development/*, stabilization/*, hotfix/*]
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v4
+
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v4

.github/workflows/migrate.yaml

@@ -1,40 +0,0 @@
----
-name: migrate
-
-on:
-  push:
-    branches-ignore:
-    - development/**
-    - q/*/**
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: Sync ${{ github.event.repository.name }}
-      run: |
-        docker run --rm quay.io/skopeo/stable:v1.15.0 sync \
-            --src docker --dest docker --all --preserve-digests --retry-times 5 \
-            --src-creds ${{ secrets.REGISTRY_LOGIN }}:${{ secrets.REGISTRY_PASSWORD }} \
-            --dest-creds ${{ github.repository_owner }}:${{ github.token }} \
-            registry.scality.com/${{ github.event.repository.name }}/${{ github.event.repository.name }} \
-            ghcr.io/scality/
-
-    - name: Sync ${{ github.event.repository.name }}-dashboards
-      run: |
-        docker run --rm quay.io/skopeo/stable:v1.15.0 sync \
-            --src docker --dest docker --all --preserve-digests --retry-times 5 \
-            --src-creds ${{ secrets.REGISTRY_LOGIN }}:${{ secrets.REGISTRY_PASSWORD }} \
-            --dest-creds ${{ github.repository_owner }}:${{ github.token }} \
-            registry.scality.com/${{ github.event.repository.name }}/${{ github.event.repository.name }}-dashboards \
-            ghcr.io/scality/${{ github.event.repository.name }}
-
-    - name: Sync pykmip
-      run: |
-        docker run --rm quay.io/skopeo/stable:v1.15.0 sync \
-            --src docker --dest docker --all --preserve-digests --retry-times 5 \
-            --src-creds ${{ secrets.REGISTRY_LOGIN }}:${{ secrets.REGISTRY_PASSWORD }} \
-            --dest-creds ${{ github.repository_owner }}:${{ github.token }} \
-            registry.scality.com/${{ github.event.repository.name }}-dev/pykmip \
-            ghcr.io/scality/${{ github.event.repository.name }}

.github/workflows/release.yaml

@@ -0,0 +1,80 @@
+---
+name: release
+run-name: release ${{ inputs.tag }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to be released'
+        required: true
+
+env:
+  PROJECT_NAME: ${{ github.event.repository.name }}
+
+jobs:
+  build-federation-image:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push image for federation
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          file: images/svc-base/Dockerfile
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
+          cache-from: type=gha,scope=federation
+          cache-to: type=gha,mode=max,scope=federation
+
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildk
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Push dashboards into the production namespace
+        run: |
+          oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
+            dashboard.json:application/grafana-dashboard+json \
+            alerts.yaml:application/prometheus-alerts+yaml
+        working-directory: monitoring
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Create Release
+        uses: softprops/action-gh-release@v2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          name: Release ${{ github.event.inputs.tag }}
+          tag_name: ${{ github.event.inputs.tag }}
+          generate_release_notes: true
+          target_commitish: ${{ github.sha }}

.github/workflows/tests.yaml

@@ -0,0 +1,533 @@
+---
+name: tests
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches-ignore:
+      - 'development/**'
+      - 'q/*/**'
+
+env:
+  # Secrets
+  azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
+  azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
+  azurebackend_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
+  azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
+  azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
+  azurebackend2_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
+  azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
+  azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
+  azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
+  azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
+  azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
+  azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
+  azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
+  b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
+  b2backend_B2_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
+  GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
+  GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
+  AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
+  AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
+  AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
+  AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
+  AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
+  AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
+  AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
+  AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
+  b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
+  gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
+  gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
+  gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
+  gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
+  gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
+    ${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
+  gcpbackendmismatch_GCP_SERVICE_KEY: >-
+    ${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
+  gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  # Configs
+  ENABLE_LOCAL_CACHE: "true"
+  REPORT_TOKEN: "report-token-1"
+  REMOTE_MANAGEMENT_DISABLE: "1"
+  # https://github.com/git-lfs/git-lfs/issues/5749
+  GIT_CLONE_PROTECTION_ACTIVE: 'false'
+jobs:
+  linting-coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '16'
+        cache: yarn
+    - name: install dependencies
+      run: yarn install --frozen-lockfile --network-concurrency 1
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+    - name: Install python deps
+      run: pip install flake8
+    - name: Lint Javascript
+      run: yarn run --silent lint -- --max-warnings 0
+    - name: Lint Markdown
+      run: yarn run --silent lint_md
+    - name: Lint python
+      run: flake8 $(git ls-files "*.py")
+    - name: Lint Yaml
+      run: yamllint -c yamllint.yml $(git ls-files "*.yml")
+    - name: Unit Coverage
+      run: |
+        set -ex
+        mkdir -p $CIRCLE_TEST_REPORTS/unit
+        yarn test
+        yarn run test_legacy_location
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+        CIRCLE_TEST_REPORTS: /tmp
+        CIRCLE_ARTIFACTS: /tmp
+        CI_REPORTS: /tmp
+    - name: Unit Coverage logs
+      run: find /tmp/unit -exec cat {} \;
+    - name: preparing junit files for upload
+      run: |
+        mkdir -p artifacts/junit
+        find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
+      if: always()
+    - name: Upload files to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: artifacts
+      if: always()
+
+  build:
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push cloudserver image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          provenance: false
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.sha }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=cloudserver
+          cache-to: type=gha,mode=max,scope=cloudserver
+      - name: Build and push pykmip image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .github/pykmip
+          tags: |
+            ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=pykmip
+          cache-to: type=gha,mode=max,scope=pykmip
+      - name: Build and push MongoDB
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .github/docker/mongodb
+          tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+          cache-from: type=gha,scope=mongodb
+          cache-to: type=gha,mode=max,scope=mongodb
+
+  multiple-backend:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      S3BACKEND: mem
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      S3DATA: multiple
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Login to Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile sproxyd up -d
+      working-directory: .github/docker
+    - name: Run multiple backend test
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        bash wait_for_local_port.bash 81 40
+        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+ 
+  mongo-v0-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v0
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  mongo-v1-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v1
+      METADATA_MAX_CACHED_BUCKETS: 1
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;        
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+        yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  file-ft-tests:
+    strategy:
+      matrix:
+        include:
+          - job-name: file-ft-tests
+    name: ${{ matrix.job-name }}
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: file
+      S3VAULT: mem
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      MPU_TESTING: "yes"
+      JOB_NAME: ${{ matrix.job-name }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup matrix job artifacts directory
+      shell: bash
+      run: |
+        set -exu
+        mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
+    - name: Setup CI services
+      run: docker compose up -d
+      working-directory: .github/docker
+    - name: Run file ft tests
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  utapi-v2-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      ENABLE_UTAPI_V2: t
+      S3BACKEND: mem
+      BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose up -d
+      working-directory: .github/docker
+    - name: Run file utapi v2 tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  quota-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        inflights:
+          - name: "With Inflights"
+            value: "true"
+          - name: "Without Inflights"
+            value: "false"
+    env:
+      S3METADATA: mongodb
+      S3BACKEND: mem
+      S3QUOTA: scuba
+      QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
+      SCUBA_HOST: localhost
+      SCUBA_PORT: 8100
+      SCUBA_HEALTHCHECK_FREQUENCY: 100
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run quota tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  kmip-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: file
+      S3VAULT: mem
+      MPU_TESTING: "yes" 
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Copy KMIP certs
+      run: cp -r ./certs /tmp/ssl-kmip
+      working-directory: .github/pykmip
+    - name: Setup CI services
+      run: docker compose --profile pykmip up -d
+      working-directory: .github/docker
+    - name: Run file KMIP tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        bash wait_for_local_port.bash 5696 40
+        yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  ceph-backend-test:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      S3DATA: multiple
+      S3KMS: file
+      CI_CEPH: 'true'
+      MPU_TESTING: "yes" 
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Login to GitHub Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '2.5.9'
+    - name: Install Ruby dependencies
+      run: |
+        gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
+    - name: Install Java dependencies
+      run: |
+        sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
+    - name: Setup CI services
+      run: docker compose --profile ceph up -d
+      working-directory: .github/docker
+      env: 
+        S3METADATA: mongodb
+    - name: Run Ceph multiple backend tests
+      run: |-
+        set -ex -o pipefail;
+        bash .github/ceph/wait_for_ceph.sh
+        bash wait_for_local_port.bash 27018 40
+        bash wait_for_local_port.bash 8000 40
+        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+        S3METADATA: mem
+    - name: Run Java tests
+      run: |-
+        set -ex -o pipefail;
+        mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
+      working-directory: tests/functional/jaws
+    - name: Run Ruby tests
+      run: |-
+        set -ex -o pipefail;
+        rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
+      working-directory: tests/functional/fog
+    - name: Run Javascript AWS SDK tests
+      run: |-
+        set -ex -o pipefail;
+        yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
+        yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
+        S3BACKEND: file
+        S3VAULT: mem
+        S3METADATA: mongodb
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()

.gitignore

@@ -0,0 +1,35 @@
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Sphinx build dir
+_build
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+yarn.lock
+.tox
+
+# Junit directory
+junit

CONTRIBUTING.md

@@ -0,0 +1,5 @@
+# Contributing rules
+
+Please follow the
+[Contributing Guidelines](
+https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).

DESIGN.md

@@ -0,0 +1,63 @@
+# S3 connector
+
+S3 handles the requests coming from S3 clients. It is based on the
+current amazon-S3 documentation.
+
+## Implementation
+
+### Architecture
+
+S3 consists of multiple daemons listening RESTfully to http requests
+on a single port. We then route the requests depending on the HTTP verbs
+used by the request. Parsing the header allows us to determine the request
+type. From there, authentication will be confirmed
+by the Vault module using v2 or v4 authentication depending on whether the
+client sent a v2 or v4 authentication header. Applicable bucket and object
+metadata will be pulled from the metadata backend, IM-Metadata to check proper
+authorization (ACL's, IAM and bucket policies). If a user has been
+authenticated and is authorized to write/read data, we then proceed to
+send the write/read request to our storage backend (RING or IM-Data), before
+sending a response to the request sender. Any problem that arises during
+the handling of the request due to client error or system error will result
+in an error being returned to the client that follows S3's error specifications.
+
+The multi-daemon architecture allows us to restart daemons on the fly in case
+of any crash without interrupting the service. The daemon handles
+remaining requests even in case of an error, stopping listening while another
+daemon is spawned to handle future requests in its stead.
+
+![Arch](res/architecture.png)
+
+### API specifications
+
+Right now, the following operations are implemented:
+
+- PutBucket
+- GetBucket
+- HeadBucket
+- DeleteBucket
+- PutBucketACL
+- GetBucketACL
+- PutBucketCors
+- DeleteBucketCors
+- GetBucketCors
+- PutObject
+- PutObject - Copy
+- GetObject
+- HeadObject
+- DeleteObject
+- Multi-Object Delete
+- PutObjectACL
+- GetObjectACL
+- Multipart Upload
+- Upload Part
+- Upload Part - Copy
+- GetService
+- Put Bucket Website
+- Get Bucket Website
+- Delete Bucket Website
+- Put Bucket Versioning
+- Get Bucket Versioning
+- v2 Authentication
+- v4 Authentication (Transferring Payload in a Single Chunk)
+- v4 Authentication (Transferring Payload in Multiple Chunks)

Dockerfile

@@ -0,0 +1,60 @@
+ARG NODE_VERSION=16.20-bullseye-slim
+
+FROM node:${NODE_VERSION} as builder
+
+WORKDIR /usr/src/app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        curl \
+        git \
+        gnupg2 \
+        jq \
+        python3 \
+        ssh \
+        wget \
+        libffi-dev \
+        zlib1g-dev \
+    && apt-get clean \
+    && mkdir -p /root/ssh \
+    && ssh-keyscan -H github.com > /root/ssh/known_hosts
+
+ENV PYTHON=python3
+COPY package.json yarn.lock /usr/src/app/
+RUN npm install typescript -g
+RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
+
+################################################################################
+FROM node:${NODE_VERSION}
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        jq \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV NO_PROXY localhost,127.0.0.1
+ENV no_proxy localhost,127.0.0.1
+
+EXPOSE 8000
+EXPOSE 8002
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        jq \
+        tini \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /usr/src/app
+
+# Keep the .git directory in order to properly report version
+COPY . /usr/src/app
+COPY --from=builder /usr/src/app/node_modules ./node_modules/
+
+
+VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
+
+ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
+
+CMD [ "yarn", "start" ]

DockerfileMem

@@ -0,0 +1,22 @@
+FROM node:6-slim
+MAINTAINER Giorgio Regni <gr@scality.com>
+
+WORKDIR /usr/src/app
+
+COPY . /usr/src/app
+
+RUN apt-get update \
+    && apt-get install -y jq python git build-essential --no-install-recommends \
+    && yarn install --production \
+    && apt-get autoremove --purge -y python git build-essential \
+    && rm -rf /var/lib/apt/lists/* \
+    && yarn cache clean \
+    && rm -rf ~/.node-gyp \
+    && rm -rf /tmp/yarn-*
+
+ENV S3BACKEND mem
+
+ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
+CMD [ "yarn", "start" ]
+
+EXPOSE 8000

Healthchecks.md

@@ -0,0 +1,78 @@
+# S3 Healthcheck
+
+Scality S3 exposes a healthcheck route `/live` on the port used
+for the metrics (defaults to port 8002) which returns a
+response with HTTP code
+
+- 200 OK
+
+ Server is up and running
+
+- 500 Internal Server error
+
+ Server is experiencing an Internal Error
+
+- 400 Bad Request
+
+ Bad Request due to unsupported HTTP methods
+
+- 403 Forbidden
+
+ Request is not allowed due to IP restriction
+
+## Stats
+
+The healthcheck route's successful response (200 OK) is appended with
+additional statistics in the request body indicating the number of requests
+performed, number of 500 errors occurred over the time interval
+specified in the response.
+
+A sample response would look something like
+
+```json
+{
+    "requests": 5000,
+    "500s": 2,
+    "sampleDuration": 30
+}
+```
+
+## Redis schema
+
+The goal is to return stats for the set interval, i.e., if interval is 30
+seconds, return stats only for the last 30 seconds.
+
+The stats use simple keys with INCR command for every new push. Each key is
+appended with a normalized unix timestamp, as the idea is to store the stats in
+5 second interval(default but configurable) keys. A default TTL of 30
+seconds is associated with each key, this way any keys older than the TTL are
+automatically removed.
+
+When a stats query is received, the results for the prior 30 seconds will be
+returned. This is accomplished by retrieving the 6 keys that represent the 6
+five-second intervals. As Redis does not have a performant RANGE query, the
+list of keys are built manually as follows
+
+* Take current timestamp
+
+* Build each key by subtracting the interval from the timestamp (5 seconds)
+
+* Total keys for each metric (total requests, 500s etc.) is TTL / interval
+  30/5  = 6
+
+Note: When Redis is queried, results from non-existent keys are set to 0.
+
+## Configuration
+
+To gather stats, S3 uses a local Redis instance as a temporary
+datastore. By adding the following config to `config.json`, stats
+will be recorded in Redis.
+
+```json
+{
+    "localCache": {
+        "host": "127.0.0.1",
+        "port": 6379
+    }
+}
+```

LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2015-2017 Scality
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,81 @@
+# Zenko CloudServer with Vitastor Backend
+
+![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
+
+## Overview
+
+CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
+object storage server that is part of [Zenko](https://www.zenko.io),
+Scality’s Open Source Multi-Cloud Data Controller.
+
+CloudServer provides a single AWS S3 API interface to access multiple
+backend data storage both on-premise or public in the cloud.
+
+This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
+backend support.
+
+## Quick Start with Vitastor
+
+Vitastor Backend is in experimental status, however you can already try to
+run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
+it works too 😊.
+
+Installation instructions:
+
+### Install Vitastor
+
+Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
+
+### Install Zenko with Vitastor Backend
+
+- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
+- Install dependencies: `npm install --omit dev` or just `npm install`
+- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
+- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
+  You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
+- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
+
+### Install and Configure MongoDB
+
+Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
+
+### Setup Zenko
+
+- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
+- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
+- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
+- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
+- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
+  and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
+  access keys, but it's not published, so let's use a file for now.
+- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
+  You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
+  in this file.
+
+Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
+instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
+
+### Start Zenko
+
+Start the S3 server with: `node index.js`
+
+If you use default settings, Zenko CloudServer starts on port 8000.
+The default access key is `accessKey1` with a secret key of `verySecretKey1`.
+
+Now you can access your S3 with `s3cmd` or `geesefs`:
+
+```
+s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
+```
+
+```
+AWS_ACCESS_KEY_ID=accessKey1 \
+    AWS_SECRET_ACCESS_KEY=verySecretKey1 \
+    geesefs --endpoint http://localhost:8000 testbucket mountdir
+```
+
+# Author & License
+
+- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
+- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
+  (a "network copyleft" license based on AGPL/SSPL, but worded in a better way)

TESTING.md

@@ -0,0 +1,77 @@
+# S3 Protocol Test Plan
+
+## Unit Tests
+
+### Architecture
+
+- s3 running on a server with a testing framework such as Mocha with
+  an assertion library.
+
+### Features tested
+
+- Authentication
+  - Building signature
+  - Checking timestamp
+  - Canonicalization
+  - Error Handling
+
+- Bucket Metadata API
+  - GET, PUT, DELETE Bucket Metadata
+
+- s3 API
+  - GET Service
+  - GET, PUT, DELETE, HEAD Object
+  - GET, PUT, DELETE, HEAD Bucket
+  - ACL's
+  - Bucket Policies
+  - Lifecycle
+  - Range requests
+  - Multi-part upload
+
+- Routes
+  - GET, PUT, PUTRAW, DELETE, HEAD for objects and buckets
+
+## Functional Tests
+
+### Architecture
+
+- A Docker instance running S3 and a Docker instance running certain S3
+  clients (Node SDK, S3cmd and AWS S3API CLI).
+
+### Features tested
+
+- Same as unit tests plus concurrent access
+- Compliance with S3 clients
+- http, https, ipv4, ipv6, hosting of website in bucket
+- ssl integration
+- Accuracy of s3 request logging
+
+## Integration Tests
+
+### Architecture
+
+- Several s3 Docker instances, Metadata Docker instances,
+  Data Docker instances and several Vault Docker instances.
+
+### Features tested
+
+- Ability to access the same buckets through different s3 connectors.
+- Creating and destroying access and secret keys.
+- Stopping and restarting Docker instances of s3, Vault, Metadata and Data
+  under load.
+- Load balancing between s3 instances?
+- Throughput of the system increases as we add s3 instances.
+- Operation time as we add disk (the more disk, the faster the system should be
+  per operation).
+- Large buckets (1 billion objects in a single bucket).
+- Large objects (10 terabytes).
+- End to end streaming ability for large objects (put from a client through
+  s3 to Data and get from Data to s3 out to a
+  client).
+- End to end range requests.
+- Load testing during extended period of time to observe side effects (i.e.
+  compaction of metadata).
+- Deleting a large number of objects at once and observe effect on data
+  (recovery of free space).
+- Failures of metadata under load.
+- Effect at s3 level of data and metadata being out of sync.

_config.yml

@@ -0,0 +1,2 @@
+---
+theme: jekyll-theme-modernist

authdata.json.example

@@ -0,0 +1,56 @@
+{
+    "accounts": [{
+        "name": "Bart",
+        "email": "sampleaccount1@sampling.com",
+        "arn": "arn:aws:iam::123456789012:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
+        "shortid": "123456789012",
+        "keys": [{
+            "access": "accessKey1",
+            "secret": "verySecretKey1"
+        }]
+    }, {
+        "name": "Lisa",
+        "email": "sampleaccount2@sampling.com",
+        "arn": "arn:aws:iam::123456789013:root",
+        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
+        "shortid": "123456789013",
+        "keys": [{
+            "access": "accessKey2",
+            "secret": "verySecretKey2"
+        }]
+    },
+    {
+        "name": "Clueso",
+        "email": "inspector@clueso.info",
+        "arn": "arn:aws:iam::123456789014:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/clueso",
+        "shortid": "123456789014",
+        "keys": [{
+            "access": "cluesoKey1",
+            "secret": "cluesoSecretKey1"
+        }]
+    },
+    {
+        "name": "Replication",
+        "email": "inspector@replication.info",
+        "arn": "arn:aws:iam::123456789015:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/replication",
+        "shortid": "123456789015",
+        "keys": [{
+            "access": "replicationKey1",
+            "secret": "replicationSecretKey1"
+        }]
+    },
+    {
+        "name": "Lifecycle",
+        "email": "inspector@lifecycle.info",
+        "arn": "arn:aws:iam::123456789016:root",
+        "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle",
+        "shortid": "123456789016",
+        "keys": [{
+            "access": "lifecycleKey1",
+            "secret": "lifecycleSecretKey1"
+        }]
+    }]
+}

bin/create_bucket_with_NFS_enabled.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict'; // eslint-disable-line strict
+
+require('../lib/nfs/utilities.js').createBucketWithNFSEnabled();

bin/create_encrypted_bucket.js

@@ -0,0 +1,5 @@
+#!/bin/sh
+// 2>/dev/null ; exec "$(which nodejs || which node)" "$0" "$@"
+'use strict'; // eslint-disable-line strict
+
+require('../lib/kms/utilities.js').createEncryptedBucket();

bin/list_bucket_metrics.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict'; // eslint-disable-line strict
+
+require('../lib/utapi/utilities.js').listMetrics('buckets');

bin/list_metrics.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict'; // eslint-disable-line strict
+
+require('../lib/utapi/utilities.js').listMetrics();

bin/search_bucket.js

@@ -0,0 +1,108 @@
+#!/bin/sh
+// 2>/dev/null ; exec "$(which nodejs 2>/dev/null || which node)" "$0" "$@"
+'use strict'; // eslint-disable-line strict
+
+const { auth } = require('arsenal');
+const commander = require('commander');
+
+const http = require('http');
+const https = require('https');
+const logger = require('../lib/utilities/logger');
+
+function _performSearch(host,
+                        port,
+                        bucketName,
+                        query,
+                        listVersions,
+                        accessKey,
+                        secretKey,
+                        sessionToken,
+                        verbose, ssl) {
+    const escapedSearch = encodeURIComponent(query);
+    const options = {
+        host,
+        port,
+        method: 'GET',
+        path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`,
+        headers: {
+            'Content-Length': 0,
+        },
+        rejectUnauthorized: false,
+        versions: '',
+    };
+    if (sessionToken) {
+        options.headers['x-amz-security-token'] = sessionToken;
+    }
+    const transport = ssl ? https : http;
+    const request = transport.request(options, response => {
+        if (verbose) {
+            logger.info('response status code', {
+                statusCode: response.statusCode,
+            });
+            logger.info('response headers', { headers: response.headers });
+        }
+        const body = [];
+        response.setEncoding('utf8');
+        response.on('data', chunk => body.push(chunk));
+        response.on('end', () => {
+            if (response.statusCode >= 200 && response.statusCode < 300) {
+                logger.info('Success');
+                process.stdout.write(body.join(''));
+                process.exit(0);
+            } else {
+                logger.error('request failed with HTTP Status ', {
+                    statusCode: response.statusCode,
+                    body: body.join(''),
+                });
+                process.exit(1);
+            }
+        });
+    });
+    // generateV4Headers exepects request object with path that does not
+    // include query
+    request.path = `/${bucketName}`;
+    const requestData = listVersions ? { search: query, versions: '' } : { search: query };
+    auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3');
+    request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`;
+    if (verbose) {
+        logger.info('request headers', { headers: request._headers });
+    }
+    request.end();
+}
+
+/**
+ * This function is used as a binary to send a request to S3 to perform a
+ * search on the objects in a bucket
+ *
+ * @return {undefined}
+ */
+function searchBucket() {
+    // TODO: Include other bucket listing possible query params?
+    commander
+        .version('0.0.1')
+        .option('-a, --access-key <accessKey>', 'Access key id')
+        .option('-k, --secret-key <secretKey>', 'Secret access key')
+        .option('-t, --session-token <sessionToken>', 'Session token')
+        .option('-b, --bucket <bucket>', 'Name of the bucket')
+        .option('-q, --query <query>', 'Search query')
+        .option('-h, --host <host>', 'Host of the server')
+        .option('-p, --port <port>', 'Port of the server')
+        .option('-s', '--ssl', 'Enable ssl')
+        .option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
+            'otherwise only list the latest version')
+        .option('-v, --verbose')
+        .parse(process.argv);
+    const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
+        commander;
+
+    if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
+        logger.error('missing parameter');
+        commander.outputHelp();
+        process.exit(1);
+    }
+
+    _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose,
+        ssl);
+}
+
+searchBucket();

config.json.example

@@ -0,0 +1,143 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "metricsPort": 8002,
+    "metricsListenOn": [],
+    "replicationGroupId": "RG001",
+    "workers": 4,
+    "restEndpoints": {
+        "localhost": "us-east-1",
+        "127.0.0.1": "us-east-1",
+        "cloudserver-front": "us-east-1",
+        "s3.docker.test": "us-east-1",
+        "127.0.0.2": "us-east-1",
+        "s3.amazonaws.com": "us-east-1",
+        "zenko-cloudserver-replicator": "us-east-1",
+        "lb": "us-east-1"
+    },
+    "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
+                        "s3-website.us-east-2.amazonaws.com",
+                        "s3-website-us-west-1.amazonaws.com",
+                        "s3-website-us-west-2.amazonaws.com",
+                        "s3-website.ap-south-1.amazonaws.com",
+                        "s3-website.ap-northeast-2.amazonaws.com",
+                        "s3-website-ap-southeast-1.amazonaws.com",
+                        "s3-website-ap-southeast-2.amazonaws.com",
+                        "s3-website-ap-northeast-1.amazonaws.com",
+                        "s3-website.eu-central-1.amazonaws.com",
+                        "s3-website-eu-west-1.amazonaws.com",
+                        "s3-website-sa-east-1.amazonaws.com",
+                        "s3-website.localhost",
+                        "s3-website.scality.test",
+                        "zenkoazuretest.blob.core.windows.net"],
+    "replicationEndpoints": [{
+        "site": "zenko",
+        "servers": ["127.0.0.1:8000"],
+        "default": true
+    }, {
+        "site": "us-east-2",
+        "type": "aws_s3"
+    }],
+    "backbeat": {
+        "host": "localhost",
+        "port": 8900
+    },
+    "workflowEngineOperator": {
+        "host": "localhost",
+        "port": 3001
+    },
+    "cdmi": {
+        "host": "localhost",
+        "port": 81,
+        "path": "/dewpoint",
+        "readonly": true
+    },
+    "bucketd": {
+        "bootstrap": ["localhost:9000"]
+    },
+    "vaultd": {
+        "host": "localhost",
+        "port": 8500
+    },
+    "clusters": 1,
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "metadataClient": {
+        "host": "localhost",
+        "port": 9990
+    },
+    "dataClient": {
+        "host": "localhost",
+        "port": 9991
+    },
+    "pfsClient": {
+        "host": "localhost",
+        "port": 9992
+    },
+    "metadataDaemon": {
+        "bindAddress": "localhost",
+        "port": 9990
+    },
+    "dataDaemon": {
+        "bindAddress": "localhost",
+        "port": 9991
+    },
+    "pfsDaemon": {
+        "bindAddress": "localhost",
+        "port": 9992
+    },
+    "recordLog": {
+        "enabled": true,
+        "recordLogName": "s3-recordlog"
+    },
+    "mongodb": {
+        "replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
+        "writeConcern": "majority",
+        "replicaSet": "rs0",
+        "readPreference": "primary",
+        "database": "metadata"
+    },
+    "authdata": "authdata.json",
+    "backends": {
+        "auth": "file",
+        "data": "file",
+        "metadata": "mongodb",
+        "kms": "file",
+        "quota": "none"
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
+    },
+    "requests": {
+        "viaProxy": false,
+        "trustedProxyCIDRs": [],
+        "extractClientIPFromHeader": ""
+    },
+    "bucketNotificationDestinations": [
+        {
+            "resource": "target1",
+            "type": "dummy",
+            "host": "localhost:6000"
+        }
+    ]
+}

config.json.vitastor

@@ -0,0 +1,71 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "metricsPort": 8002,
+    "metricsListenOn": [],
+    "replicationGroupId": "RG001",
+    "restEndpoints": {
+        "localhost": "STANDARD",
+        "127.0.0.1": "STANDARD",
+        "yourhostname.ru": "STANDARD"
+    },
+    "websiteEndpoints": [
+        "static.yourhostname.ru"
+    ],
+    "replicationEndpoints": [ {
+        "site": "zenko",
+        "servers": ["127.0.0.1:8000"],
+        "default": true
+    } ],
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "backends": {
+        "metadata": "mongodb"
+    },
+    "mongodb": {
+        "replicaSetHosts": "127.0.0.1:27017",
+        "writeConcern": "majority",
+        "replicaSet": "rs0",
+        "readPreference": "primary",
+        "database": "s3",
+        "authCredentials": {
+            "username": "s3",
+            "password": ""
+        }
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
+    },
+    "requests": {
+        "viaProxy": false,
+        "trustedProxyCIDRs": [],
+        "extractClientIPFromHeader": ""
+    },
+    "bucketNotificationDestinations": [
+        {
+            "resource": "target1",
+            "type": "dummy",
+            "host": "localhost:6000"
+        }
+    ]
+}

constants.js

@@ -0,0 +1,248 @@
+const crypto = require('crypto');
+
+const constants = {
+    /*
+     * Splitter is used to build the object name for the overview of a
+     * multipart upload and to build the object names for each part of a
+     * multipart upload.  These objects with large names are then stored in
+     * metadata in a "shadow bucket" to a real bucket.  The shadow bucket
+     * contains all ongoing multipart uploads.  We include in the object
+     * name some of the info we might need to pull about an open multipart
+     * upload or about an individual part with each piece of info separated
+     * by the splitter.  We can then extract each piece of info by splitting
+     * the object name string with this splitter.
+     * For instance, assuming a splitter of '...!*!',
+     * the name of the upload overview would be:
+     *   overview...!*!objectKey...!*!uploadId
+     * For instance, the name of a part would be:
+     *   uploadId...!*!partNumber
+     *
+     * The sequence of characters used in the splitter should not occur
+     * elsewhere in the pieces of info to avoid splitting where not
+     * intended.
+     *
+     * Splitter is also used in adding bucketnames to the
+     * namespacerusersbucket.  The object names added to the
+     * namespaceusersbucket are of the form:
+     * canonicalID...!*!bucketname
+     */
+
+    splitter: '..|..',
+    // BACKWARD: This line will be removed when removing backward compatibility
+    oldSplitter: 'splitterfornow',
+    usersBucket: 'users..bucket',
+    oldUsersBucket: 'namespaceusersbucket',
+    // MPU Bucket Prefix is used to create the name of the shadow
+    // bucket used for multipart uploads.  There is one shadow mpu
+    // bucket per bucket and its name is the mpuBucketPrefix followed
+    // by the name of the final destination bucket for the object
+    // once the multipart upload is complete.
+    mpuBucketPrefix: 'mpuShadowBucket',
+    blacklistedPrefixes: { bucket: [], object: [] },
+    // GCP Object Tagging Prefix
+    gcpTaggingPrefix: 'aws-tag-',
+    // PublicId is used as the canonicalID for a request that contains
+    // no authentication information.  Requestor can access
+    // only public resources
+    publicId: 'http://acs.amazonaws.com/groups/global/AllUsers',
+    // All Authenticated Users is an ACL group.
+    allAuthedUsersId: 'http://acs.amazonaws.com/groups/' +
+        'global/AuthenticatedUsers',
+    // LogId is used for the AWS logger to write the logs
+    // to the destination bucket.  This style of logging is
+    // to be implemented later but the logId is used in the
+    // ACLs.
+    logId: 'http://acs.amazonaws.com/groups/s3/LogDelivery',
+    emptyFileMd5: 'd41d8cd98f00b204e9800998ecf8427e',
+
+    // Number of sub-directories for file backend
+    folderHash: 3511, // Prime number
+    // AWS only returns 1000 on a listing
+    // http://docs.aws.amazon.com/AmazonS3/latest/API/
+    //      RESTBucketGET.html#RESTBucketGET-requests
+    listingHardLimit: 1000,
+
+    // AWS sets a minimum size limit for parts except for the last part.
+    // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
+    minimumAllowedPartSize: 5242880,
+
+    // AWS sets a maximum total parts limit
+    // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
+    maximumAllowedPartCount: 10000,
+
+    gcpMaximumAllowedPartCount: 1024,
+
+    // Max size on put part or copy part is 5GB. For functional
+    // testing use 110 MB as max
+    maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :
+        5368709120,
+
+    // Max size allowed in a single put object request is 5GB
+    // https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
+    maximumAllowedUploadSize: 5368709120,
+
+    // AWS states max size for user-defined metadata (x-amz-meta- headers) is
+    // 2 KB: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
+    // In testing, AWS seems to allow up to 88 more bytes, so we do the same.
+    maximumMetaHeadersSize: 2136,
+
+    // Maximum HTTP headers size allowed
+    maxHttpHeadersSize: 14122,
+
+    // hex digest of sha256 hash of empty string:
+    emptyStringHash: crypto.createHash('sha256')
+        .update('', 'binary').digest('hex'),
+
+    // Queries supported by AWS that we do not currently support.
+    // Non-bucket queries
+    unsupportedQueries: [
+        'accelerate',
+        'analytics',
+        'inventory',
+        'logging',
+        'metrics',
+        'policyStatus',
+        'publicAccessBlock',
+        'requestPayment',
+        'torrent',
+    ],
+
+    // Headers supported by AWS that we do not currently support.
+    unsupportedHeaders: [
+        'x-amz-server-side-encryption-customer-algorithm',
+        'x-amz-server-side-encryption-context',
+        'x-amz-server-side-encryption-customer-key',
+        'x-amz-server-side-encryption-customer-key-md5',
+    ],
+
+    // user metadata header to set object locationConstraint
+    objectLocationConstraintHeader: 'x-amz-storage-class',
+    lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
+    legacyLocations: ['sproxyd', 'legacy'],
+    // declare here all existing service accounts and their properties
+    // (if any, otherwise an empty object)
+    serviceAccountProperties: {
+        replication: {},
+        lifecycle: {},
+        gc: {},
+        'md-ingestion': {
+            canReplicate: true,
+        },
+    },
+    /* eslint-disable camelcase */
+    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
+    // some of the available data backends  (if called directly rather
+    // than through the multiple backend gateway) need a key provided
+    // as a string as first parameter of the get/delete methods.
+    clientsRequireStringKey: { sproxyd: true, cdmi: true },
+    // healthcheck default call from nginx is every 2 seconds
+    // for external backends, don't call unless at least 1 minute
+    // (60,000 milliseconds) since last call
+    externalBackendHealthCheckInterval: 60000,
+    versioningNotImplBackends: { azure: true, gcp: true },
+    mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
+    skipBatchDeleteBackends: { azure: true, gcp: true },
+    s3HandledBackends: { azure: true, gcp: true },
+    hasCopyPartBackends: { aws_s3: true, gcp: true },
+    /* eslint-enable camelcase */
+    mpuMDStoredOnS3Backend: { azure: true },
+    azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
+    base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
+        '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
+    productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
+    // location constraint delimiter
+    zenkoSeparator: ':',
+    // user metadata applied on zenko objects
+    zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
+    bucketOwnerActions: [
+        'bucketDeleteCors',
+        'bucketDeleteLifecycle',
+        'bucketDeletePolicy',
+        'bucketDeleteReplication',
+        'bucketDeleteWebsite',
+        'bucketGetCors',
+        'bucketGetLifecycle',
+        'bucketGetLocation',
+        'bucketGetPolicy',
+        'bucketGetReplication',
+        'bucketGetVersioning',
+        'bucketGetWebsite',
+        'bucketPutCors',
+        'bucketPutLifecycle',
+        'bucketPutPolicy',
+        'bucketPutReplication',
+        'bucketPutVersioning',
+        'bucketPutWebsite',
+        'objectDeleteTagging',
+        'objectGetTagging',
+        'objectPutTagging',
+        'objectPutLegalHold',
+        'objectPutRetention',
+    ],
+    // response header to be sent when there are invalid
+    // user metadata in the object's metadata
+    invalidObjectUserMetadataHeader: 'x-amz-missing-meta',
+    // Bucket specific queries supported by AWS that we do not currently support
+    // these queries may or may not be supported at object level
+    unsupportedBucketQueries: [
+    ],
+    suppressedUtapiEventFields: [
+        'object',
+        'location',
+        'versionId',
+    ],
+    allowedUtapiEventFilterFields: [
+        'operationId',
+        'location',
+        'account',
+        'user',
+        'bucket',
+    ],
+    arrayOfAllowed: [
+        'objectPutTagging',
+        'objectPutLegalHold',
+        'objectPutRetention',
+    ],
+    allowedUtapiEventFilterStates: ['allow', 'deny'],
+    allowedRestoreObjectRequestTierValues: ['Standard'],
+    lifecycleListing: {
+        CURRENT_TYPE: 'current',
+        NON_CURRENT_TYPE: 'noncurrent',
+        ORPHAN_DM_TYPE: 'orphan',
+    },
+    multiObjectDeleteConcurrency: 50,
+    maxScannedLifecycleListingEntries: 10000,
+    overheadField: [
+        'content-length',
+        'owner-id',
+        'versionId',
+        'isNull',
+        'isDeleteMarker',
+    ],
+    unsupportedSignatureChecksums: new Set([
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
+        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
+        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
+        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
+    ]),
+    supportedSignatureChecksums: new Set([
+        'UNSIGNED-PAYLOAD',
+        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
+    ]),
+    ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
+    ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
+    // The AWS assumed Role resource type
+    assumedRoleArnResourceType: 'assumed-role',
+    // Session name of the backbeat lifecycle assumed role session.
+    backbeatLifecycleSessionName: 'backbeat-lifecycle',
+    actionsToConsiderAsObjectPut: [
+        'initiateMultipartUpload',
+        'objectPutPart',
+        'completeMultipartUpload',
+    ],
+    // if requester is not bucket owner, bucket policy actions should be denied with
+    // MethodNotAllowed error
+    onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
+};
+
+module.exports = constants;

dataserver.js

@@ -0,0 +1,39 @@
+'use strict'; // eslint-disable-line strict
+
+const arsenal = require('arsenal');
+const { config } = require('./lib/Config.js');
+const logger = require('./lib/utilities/logger');
+
+process.on('uncaughtException', err => {
+    logger.fatal('caught error', {
+        error: err.message,
+        stack: err.stack,
+        workerId: this.worker ? this.worker.id : undefined,
+        workerPid: this.worker ? this.worker.process.pid : undefined,
+    });
+    process.exit(1);
+});
+
+if (config.backends.data === 'file' ||
+    (config.backends.data === 'multiple' &&
+     config.backends.metadata !== 'scality')) {
+    const dataServer = new arsenal.network.rest.RESTServer({
+        bindAddress: config.dataDaemon.bindAddress,
+        port: config.dataDaemon.port,
+        dataStore: new arsenal.storage.data.file.DataFileStore({
+            dataPath: config.dataDaemon.dataPath,
+            log: config.log,
+            noSync: config.dataDaemon.noSync,
+            noCache: config.dataDaemon.noCache,
+        }),
+        log: config.log,
+    });
+    dataServer.setup(err => {
+        if (err) {
+            logger.error('Error initializing REST data server',
+                         { error: err });
+            return;
+        }
+        dataServer.start();
+    });
+}

docker-entrypoint.sh

@@ -0,0 +1,220 @@
+#!/bin/bash
+
+# set -e stops the execution of a script if a command or pipeline has an error
+set -e
+
+# modifying config.json
+JQ_FILTERS_CONFIG="."
+
+# ENDPOINT var can accept comma separated values
+# for multiple endpoint locations
+if [[ "$ENDPOINT" ]]; then
+    IFS="," read -ra HOST_NAMES <<< "$ENDPOINT"
+    for host in "${HOST_NAMES[@]}"; do
+        JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$host\"]=\"us-east-1\""
+    done
+    echo "Host name has been modified to ${HOST_NAMES[@]}"
+    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with ${HOST_NAMES[@]}"
+fi
+
+if [[ "$LOG_LEVEL" ]]; then
+    if [[ "$LOG_LEVEL" == "info" || "$LOG_LEVEL" == "debug" || "$LOG_LEVEL" == "trace" ]]; then
+        JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .log.logLevel=\"$LOG_LEVEL\""
+        echo "Log level has been modified to $LOG_LEVEL"
+    else
+        echo "The log level you provided is incorrect (info/debug/trace)"
+    fi
+fi
+
+if [[ "$SSL" && "$HOST_NAMES" ]]; then
+# This condition makes sure that the certificates are not generated twice. (for docker restart)
+if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
+# Compute config for utapi tests
+cat >>req.cfg <<EOF
+[req]
+distinguished_name = req_distinguished_name
+prompt = no
+req_extensions = s3_req
+
+[req_distinguished_name]
+CN = ${HOST_NAMES[0]}
+
+[s3_req]
+subjectAltName = @alt_names
+extendedKeyUsage = serverAuth, clientAuth
+
+[alt_names]
+DNS.1 = *.${HOST_NAMES[0]}
+DNS.2 = ${HOST_NAMES[0]}
+
+EOF
+
+## Generate SSL key and certificates
+# Generate a private key for your CSR
+openssl genrsa -out ca.key 2048
+# Generate a self signed certificate for your local Certificate Authority
+openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=S3 CA Server"
+# Generate a key for S3 Server
+openssl genrsa -out server.key 2048
+# Generate a Certificate Signing Request for S3 Server
+openssl req -new -key server.key -out server.csr -config req.cfg
+# Generate a local-CA-signed certificate for S3 Server
+openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 -sha256 -extfile req.cfg -extensions s3_req
+fi
+## Update S3Server config.json
+# This condition makes sure that certFilePaths section is not added twice. (for docker restart)
+if ! grep -q "certFilePaths" ./config.json; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .certFilePaths= { \"key\": \".\/server.key\", \"cert\": \".\/server.crt\", \"ca\": \".\/ca.crt\" }"
+fi
+fi
+
+if [[ "$LISTEN_ADDR" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
+fi
+
+if [[ "$REPLICATION_GROUP_ID" ]] ; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
+fi
+
+if [[ "$DATA_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
+fi
+
+if [[ "$METADATA_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
+fi
+
+if [[ "$PFSD_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
+fi
+
+if [[ "$MONGODB_HOSTS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
+fi
+
+if [[ "$MONGODB_RS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSet=\"$MONGODB_RS\""
+fi
+
+if [[ "$MONGODB_DATABASE" ]]; then
+   JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.database=\"$MONGODB_DATABASE\""
+fi
+
+if [ -z "$REDIS_HA_NAME" ]; then
+    REDIS_HA_NAME='mymaster'
+fi
+
+if [[ "$REDIS_SENTINELS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.name=\"$REDIS_HA_NAME\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.sentinels=\"$REDIS_SENTINELS\""
+elif [[ "$REDIS_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.host=\"$REDIS_HOST\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=6379"
+fi
+
+if [[ "$REDIS_PORT" ]] && [[ ! "$REDIS_SENTINELS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=$REDIS_PORT"
+fi
+
+if [[ "$REDIS_SENTINELS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.name=\"$REDIS_HA_NAME\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.sentinels=\"$REDIS_SENTINELS\""
+elif [[ "$REDIS_HA_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.host=\"$REDIS_HA_HOST\""
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=6379"
+fi
+
+if [[ "$REDIS_HA_PORT" ]] && [[ ! "$REDIS_SENTINELS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=$REDIS_HA_PORT"
+fi
+
+if [[ "$RECORDLOG_ENABLED" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true"
+fi
+
+if [[ "$STORAGE_LIMIT_ENABLED" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.metrics[.utapi.metrics | length]=\"location\""
+fi
+
+if [[ "$CRR_METRICS_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\""
+fi
+
+if [[ "$CRR_METRICS_PORT" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
+fi
+
+if [[ "$WE_OPERATOR_HOST" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
+fi
+
+if [[ "$WE_OPERATOR_PORT" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
+fi
+
+if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
+fi
+
+# external backends http(s) agent config
+
+# AWS
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
+#GCP
+if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
+fi
+
+if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
+fi
+
+if [[ -n "$BUCKET_DENY_FILTER" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
+fi
+
+if [[ "$TESTING_MODE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
+fi
+
+if [[ $JQ_FILTERS_CONFIG != "." ]]; then
+    jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
+    mv config.json.tmp config.json
+fi
+
+if test -v INITIAL_INSTANCE_ID && test -v S3METADATAPATH && ! test -f ${S3METADATAPATH}/uuid ; then
+    echo -n ${INITIAL_INSTANCE_ID} > ${S3METADATAPATH}/uuid
+fi
+
+# s3 secret credentials for Zenko
+if [ -r /run/secrets/s3-credentials ] ; then
+    . /run/secrets/s3-credentials
+fi
+
+exec "$@"

docs/ARCHITECTURE.rst

@@ -0,0 +1,993 @@
+.. role:: raw-latex(raw)
+   :format: latex
+..
+
+Architecture
+++++++++++++
+
+Versioning
+==========
+
+This document describes Zenko CloudServer's support for the AWS S3 Bucket
+Versioning feature.
+
+AWS S3 Bucket Versioning
+------------------------
+
+See AWS documentation for a description of the Bucket Versioning
+feature:
+
+-  `Bucket
+   Versioning <http://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html>`__
+-  `Object
+   Versioning <http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html>`__
+
+This document assumes familiarity with the details of Bucket Versioning,
+including null versions and delete markers, described in the above
+links.
+
+Implementation of Bucket Versioning in Zenko CloudServer
+--------------------------------------------------------
+
+Overview of Metadata and API Component Roles
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each version of an object is stored as a separate key in metadata. The
+S3 API interacts with the metadata backend to store, retrieve, and
+delete version metadata.
+
+The implementation of versioning within the metadata backend is naive.
+The metadata backend does not evaluate any information about bucket or
+version state (whether versioning is enabled or suspended, and whether a
+version is a null version or delete marker). The S3 front-end API
+manages the logic regarding versioning information, and sends
+instructions to metadata to handle the basic CRUD operations for version
+metadata.
+
+The role of the S3 API can be broken down into the following:
+
+-  put and delete version data
+-  store extra information about a version, such as whether it is a
+   delete marker or null version, in the object's metadata
+-  send instructions to metadata backend to store, retrieve, update and
+   delete version metadata based on bucket versioning state and version
+   metadata
+-  encode version ID information to return in responses to requests, and
+   decode version IDs sent in requests
+
+The implementation of Bucket Versioning in S3 is described in this
+document in two main parts. The first section, `"Implementation of
+Bucket Versioning in
+Metadata" <#implementation-of-bucket-versioning-in-metadata>`__,
+describes the way versions are stored in metadata, and the metadata
+options for manipulating version metadata.
+
+The second section, `"Implementation of Bucket Versioning in
+API" <#implementation-of-bucket-versioning-in-api>`__, describes the way
+the metadata options are used in the API within S3 actions to create new
+versions, update their metadata, and delete them. The management of null
+versions and creation of delete markers is also described in this
+section.
+
+Implementation of Bucket Versioning in Metadata
+-----------------------------------------------
+
+As mentioned above, each version of an object is stored as a separate
+key in metadata. We use version identifiers as the suffix for the keys
+of the object versions, and a special version (the `"Master
+Version" <#master-version>`__) to represent the latest version.
+
+An example of what the metadata keys might look like for an object
+``foo/bar`` with three versions (with `.` representing a null character):
+
++------------------------------------------------------+
+| key                                                  |
++======================================================+
+| foo/bar                                              |
++------------------------------------------------------+
+| foo/bar.098506163554375999999PARIS 0.a430a1f85c6ec   |
++------------------------------------------------------+
+| foo/bar.098506163554373999999PARIS 0.41b510cd0fdf8   |
++------------------------------------------------------+
+| foo/bar.098506163554373999998PARIS 0.f9b82c166f695   |
++------------------------------------------------------+
+
+The most recent version created is represented above in the key
+``foo/bar`` and is the master version. This special version is described
+further in the section `"Master Version" <#master-version>`__.
+
+Version ID and Metadata Key Format
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The version ID is generated by the metadata backend, and encoded in a
+hexadecimal string format by S3 before sending a response to a request.
+S3 also decodes the hexadecimal string received from a request before
+sending to metadata to retrieve a particular version.
+
+The format of a ``version_id`` is: ``ts`` ``rep_group_id`` ``seq_id``
+where:
+
+-  ``ts``: is the combination of epoch and an increasing number
+-  ``rep_group_id``: is the name of deployment(s) considered one unit
+   used for replication
+-  ``seq_id``: is a unique value based on metadata information.
+
+The format of a key in metadata for a version is:
+
+``object_name separator version_id`` where:
+
+-  ``object_name``: is the key of the object in metadata
+-  ``separator``: we use the ``null`` character (``0x00`` or ``\0``) as
+   the separator between the ``object_name`` and the ``version_id`` of a
+   key
+-  ``version_id``: is the version identifier; this encodes the ordering
+   information in the format described above as metadata orders keys
+   alphabetically
+
+An example of a key in metadata:
+``foo\01234567890000777PARIS 1234.123456`` indicating that this specific
+version of ``foo`` was the ``000777``\ th entry created during the epoch
+``1234567890`` in the replication group ``PARIS`` with ``1234.123456``
+as ``seq_id``.
+
+Master Version
+~~~~~~~~~~~~~~
+
+We store a copy of the latest version of an object's metadata using
+``object_name`` as the key; this version is called the master version.
+The master version of each object facilitates the standard GET
+operation, which would otherwise need to scan among the list of versions
+of an object for its latest version.
+
+The following table shows the layout of all versions of ``foo`` in the
+first example stored in the metadata (with dot ``.`` representing the
+null separator):
+
++----------+---------+
+| key      | value   |
++==========+=========+
+| foo      | B       |
++----------+---------+
+| foo.v2   | B       |
++----------+---------+
+| foo.v1   | A       |
++----------+---------+
+
+Metadata Versioning Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Zenko CloudServer sends instructions to the metadata engine about whether to
+create a new version or overwrite, retrieve, or delete a specific
+version by sending values for special options in PUT, GET, or DELETE
+calls to metadata. The metadata engine can also list versions in the
+database, which is used by Zenko CloudServer to list object versions.
+
+These only describe the basic CRUD operations that the metadata engine
+can handle. How these options are used by the S3 API to generate and
+update versions is described more comprehensively in `"Implementation of
+Bucket Versioning in
+API" <#implementation-of-bucket-versioning-in-api>`__.
+
+Note: all operations (PUT and DELETE) that generate a new version of an
+object will return the ``version_id`` of the new version to the API.
+
+PUT
+^^^
+
+-  no options: original PUT operation, will update the master version
+-  ``versioning: true`` create a new version of the object, then update
+   the master version with this version.
+-  ``versionId: <versionId>`` create or update a specific version (for updating
+   version's ACL or tags, or remote updates in geo-replication)
+
+   *  if the version identified by ``versionId`` happens to be the latest
+      version, the master version will be updated as well
+   *  if the master version is not as recent as the version identified by
+      ``versionId``, as may happen with cross-region replication, the master
+      will be updated as well
+   *  note that with ``versionId`` set to an empty string ``''``, it will
+      overwrite the master version only (same as no options, but the master
+      version will have a ``versionId`` property set in its metadata like
+      any other version). The ``versionId`` will never be exposed to an
+      external user, but setting this internal-only ``versionID`` enables
+      Zenko CloudServer to find this version later if it is no longer the master.
+      This option of ``versionId`` set to ``''`` is used for creating null
+      versions once versioning has been suspended, which is discussed in
+      `"Null Version Management" <#null-version-management>`__.
+
+In general, only one option is used at a time. When ``versionId`` and
+``versioning`` are both set, only the ``versionId`` option will have an effect.
+
+DELETE
+^^^^^^
+
+-  no options: original DELETE operation, will delete the master version
+-  ``versionId: <versionId>`` delete a specific version
+
+A deletion targeting the latest version of an object has to:
+
+-  delete the specified version identified by ``versionId``
+-  replace the master version with a version that is a placeholder for
+   deletion
+
+    -  this version contains a special keyword, 'isPHD', to indicate the
+         master version was deleted and needs to be updated
+
+-  initiate a repair operation to update the value of the master
+   version:
+
+    -  involves listing the versions of the object and get the latest
+       version to replace the placeholder delete version
+    -  if no more versions exist, metadata deletes the master version,
+       removing the key from metadata
+
+Note: all of this happens in metadata before responding to the front-end api,
+and only when the metadata engine is instructed by Zenko CloudServer to delete
+a specific version or the master version.
+See section `"Delete Markers" <#delete-markers>`__ for a description of what
+happens when a Delete Object request is sent to the S3 API.
+
+GET
+^^^
+
+-  no options: original GET operation, will get the master version
+-  ``versionId: <versionId>`` retrieve a specific version
+
+The implementation of a GET operation does not change compared to the
+standard version. A standard GET without versioning information would
+get the master version of a key. A version-specific GET would retrieve
+the specific version identified by the key for that version.
+
+LIST
+^^^^
+
+For a standard LIST on a bucket, metadata iterates through the keys by
+using the separator (``\0``, represented by ``.`` in examples) as an
+extra delimiter. For a listing of all versions of a bucket, there is no
+change compared to the original listing function. Instead, the API
+component returns all the keys in a List Objects call and filters for
+just the keys of the master versions in a List Object Versions call.
+
+For example, a standard LIST operation against the keys in a table below
+would return from metadata the list of
+``[ foo/bar, bar, qux/quz, quz ]``.
+
++--------------+
+| key          |
++==============+
+| foo/bar      |
++--------------+
+| foo/bar.v2   |
++--------------+
+| foo/bar.v1   |
++--------------+
+| bar          |
++--------------+
+| qux/quz      |
++--------------+
+| qux/quz.v2   |
++--------------+
+| qux/quz.v1   |
++--------------+
+| quz          |
++--------------+
+| quz.v2       |
++--------------+
+| quz.v1       |
++--------------+
+
+Implementation of Bucket Versioning in API
+------------------------------------------
+
+Object Metadata Versioning Attributes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To access all the information needed to properly handle all cases that
+may exist in versioned operations, the API stores certain
+versioning-related information in the metadata attributes of each
+version's object metadata.
+
+These are the versioning-related metadata properties:
+
+-  ``isNull``: whether the version being stored is a null version.
+-  ``nullVersionId``: the unencoded version ID of the latest null
+   version that existed before storing a non-null version.
+-  ``isDeleteMarker``: whether the version being stored is a delete
+   marker.
+
+The metadata engine also sets one additional metadata property when
+creating the version.
+
+-  ``versionId``: the unencoded version ID of the version being stored.
+
+Null versions and delete markers are described in further detail in
+their own subsections.
+
+Creation of New Versions
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+When versioning is enabled in a bucket, APIs which normally result in
+the creation of objects, such as Put Object, Complete Multipart Upload
+and Copy Object, will generate new versions of objects.
+
+Zenko CloudServer creates a new version and updates the master version using the
+``versioning: true`` option in PUT calls to the metadata engine. As an
+example, when two consecutive Put Object requests are sent to the Zenko
+CloudServer for a versioning-enabled bucket with the same key names, there
+are two corresponding metadata PUT calls with the ``versioning`` option
+set to true.
+
+The PUT calls to metadata and resulting keys are shown below:
+
+(1) PUT foo (first put), versioning: ``true``
+
++----------+---------+
+| key      | value   |
++==========+=========+
+| foo      | A       |
++----------+---------+
+| foo.v1   | A       |
++----------+---------+
+
+(2) PUT foo (second put), versioning: ``true``
+
++----------+---------+
+| key      | value   |
++==========+=========+
+| foo      | B       |
++----------+---------+
+| foo.v2   | B       |
++----------+---------+
+| foo.v1   | A       |
++----------+---------+
+
+Null Version Management
+^^^^^^^^^^^^^^^^^^^^^^^
+
+In a bucket without versioning, or when versioning is suspended, putting
+an object with the same name twice should result in the previous object
+being overwritten. This is managed with null versions.
+
+Only one null version should exist at any given time, and it is
+identified in Zenko CloudServer requests and responses with the version
+id "null".
+
+Case 1: Putting Null Versions
+'''''''''''''''''''''''''''''
+
+With respect to metadata, since the null version is overwritten by
+subsequent null versions, the null version is initially stored in the
+master key alone, as opposed to being stored in the master key and a new
+version. Zenko CloudServer checks if versioning is suspended or has never been
+configured, and sets the ``versionId`` option to ``''`` in PUT calls to
+the metadata engine when creating a new null version.
+
+If the master version is a null version, Zenko CloudServer also sends a DELETE
+call to metadata prior to the PUT, in order to clean up any pre-existing null
+versions which may, in certain edge cases, have been stored as a separate
+version. [1]_
+
+The tables below summarize the calls to metadata and the resulting keys if
+we put an object 'foo' twice, when versioning has not been enabled or is
+suspended.
+
+(1) PUT foo (first put), versionId: ``''``
+
++--------------+---------+
+| key          | value   |
++==============+=========+
+| foo (null)   | A       |
++--------------+---------+
+
+(2A) DELETE foo (clean-up delete before second put),
+versionId: ``<version id of master version>``
+
++--------------+---------+
+| key          | value   |
++==============+=========+
+|              |         |
++--------------+---------+
+
+(2B) PUT foo (second put), versionId: ``''``
+
++--------------+---------+
+| key          | value   |
++==============+=========+
+| foo (null)   | B       |
++--------------+---------+
+
+The S3 API also sets the ``isNull`` attribute to ``true`` in the version
+metadata before storing the metadata for these null versions.
+
+.. [1]  Some examples of these cases are: (1) when there is a null version
+        that is the second-to-latest version, and the latest version has been
+        deleted, causing metadata to repair the master value with the value of
+        the null version and (2) when putting object tag or ACL on a null
+        version that is the master version, as explained in `"Behavior of
+        Object-Targeting APIs" <#behavior-of-object-targeting-apis>`__.
+
+Case 2: Preserving Existing Null Versions in Versioning-Enabled Bucket
+''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+Null versions are preserved when new non-null versions are created after
+versioning has been enabled or re-enabled.
+
+If the master version is the null version, the S3 API preserves the
+current null version by storing it as a new key ``(3A)`` in a separate
+PUT call to metadata, prior to overwriting the master version ``(3B)``.
+This implies the null version may not necessarily be the latest or
+master version.
+
+To determine whether the master version is a null version, the S3 API
+checks if the master version's ``isNull`` property is set to ``true``,
+or if the ``versionId`` attribute of the master version is undefined
+(indicating it is a null version that was put before bucket versioning
+was configured).
+
+Continuing the example from Case 1, if we enabled versioning and put
+another object, the calls to metadata and resulting keys would resemble
+the following:
+
+(3A) PUT foo, versionId: ``<versionId of master version>`` if defined or
+``<non-versioned object id>``
+
++-----------------+---------+
+| key             | value   |
++=================+=========+
+| foo             | B       |
++-----------------+---------+
+| foo.v1 (null)   | B       |
++-----------------+---------+
+
+(3B) PUT foo, versioning: ``true``
+
++-----------------+---------+
+| key             | value   |
++=================+=========+
+| foo             | C       |
++-----------------+---------+
+| foo.v2          | C       |
++-----------------+---------+
+| foo.v1 (null)   | B       |
++-----------------+---------+
+
+To prevent issues with concurrent requests, Zenko CloudServer ensures the null
+version is stored with the same version ID by using ``versionId`` option.
+Zenko CloudServer sets the ``versionId`` option to the master version's
+``versionId`` metadata attribute value during the PUT. This creates a new
+version with the same version ID of the existing null master version.
+
+The null version's ``versionId`` attribute may be undefined because it
+was generated before the bucket versioning was configured. In that case,
+a version ID is generated using the max epoch and sequence values
+possible so that the null version will be properly ordered as the last
+entry in a metadata listing. This value ("non-versioned object id") is
+used in the PUT call with the ``versionId`` option.
+
+Case 3: Overwriting a Null Version That is Not Latest Version
+'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+Normally when versioning is suspended, Zenko CloudServer uses the
+``versionId: ''`` option in a PUT to metadata to create a null version.
+This also overwrites an existing null version if it is the master version.
+
+However, if there is a null version that is not the latest version,
+Zenko CloudServer cannot rely on the ``versionId: ''`` option will not
+overwrite the existing null version. Instead, before creating a new null
+version, the Zenko CloudServer API must send a separate DELETE call to metadata
+specifying the version id of the current null version for delete.
+
+To do this, when storing a null version (3A above) before storing a new
+non-null version, Zenko CloudServer records the version's ID in the
+``nullVersionId`` attribute of the non-null version. For steps 3A and 3B above,
+these are the values stored in the ``nullVersionId`` of each version's metadata:
+
+(3A) PUT foo, versioning: ``true``
+
++-----------------+---------+-----------------------+
+| key             | value   | value.nullVersionId   |
++=================+=========+=======================+
+| foo             | B       | undefined             |
++-----------------+---------+-----------------------+
+| foo.v1 (null)   | B       | undefined             |
++-----------------+---------+-----------------------+
+
+(3B) PUT foo, versioning: ``true``
+
++-----------------+---------+-----------------------+
+| key             | value   | value.nullVersionId   |
++=================+=========+=======================+
+| foo             | C       | v1                    |
++-----------------+---------+-----------------------+
+| foo.v2          | C       | v1                    |
++-----------------+---------+-----------------------+
+| foo.v1 (null)   | B       | undefined             |
++-----------------+---------+-----------------------+
+
+If defined, the ``nullVersionId`` of the master version is used with the
+``versionId`` option in a DELETE call to metadata if a Put Object
+request is received when versioning is suspended in a bucket.
+
+(4A) DELETE foo, versionId: ``<nullVersionId of master version>`` (v1)
+
++----------+---------+
+| key      | value   |
++==========+=========+
+| foo      | C       |
++----------+---------+
+| foo.v2   | C       |
++----------+---------+
+
+Then the master version is overwritten with the new null version:
+
+(4B) PUT foo, versionId: ``''``
+
++--------------+---------+
+| key          | value   |
++==============+=========+
+| foo (null)   | D       |
++--------------+---------+
+| foo.v2       | C       |
++--------------+---------+
+
+The ``nullVersionId`` attribute is also used to retrieve the correct
+version when the version ID "null" is specified in certain object-level
+APIs, described further in the section `"Null Version
+Mapping" <#null-version-mapping>`__.
+
+Specifying Versions in APIs for Putting Versions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Since Zenko CloudServer does not allow an overwrite of existing version data,
+Put Object, Complete Multipart Upload and Copy Object return
+``400 InvalidArgument`` if a specific version ID is specified in the
+request query, e.g. for a ``PUT /foo?versionId=v1`` request.
+
+PUT Example
+~~~~~~~~~~~
+
+When Zenko CloudServer receives a request to PUT an object:
+
+-  It checks first if versioning has been configured
+-  If it has not been configured, Zenko CloudServer proceeds to puts the new
+   data, puts the metadata by overwriting the master version, and proceeds to
+   delete any pre-existing data
+
+If versioning has been configured, Zenko CloudServer checks the following:
+
+Versioning Enabled
+^^^^^^^^^^^^^^^^^^
+
+If versioning is enabled and there is existing object metadata:
+
+-  If the master version is a null version (``isNull: true``) or has no
+   version ID (put before versioning was configured):
+
+   -  store the null version metadata as a new version
+   -  create a new version and overwrite the master version
+
+      -  set ``nullVersionId``: version ID of the null version that was
+         stored
+
+If versioning is enabled and the master version is not null; or there is
+no existing object metadata:
+
+-  create a new version and store it, and overwrite the master version
+
+Versioning Suspended
+^^^^^^^^^^^^^^^^^^^^
+
+If versioning is suspended and there is existing object metadata:
+
+-  If the master version has no version ID:
+
+   -  overwrite the master version with the new metadata (PUT ``versionId: ''``)
+   -  delete previous object data
+
+- If the master version is a null version:
+
+   -  delete the null version using the `versionId` metadata attribute of the
+      master version (PUT ``versionId: <versionId of master object MD>``)
+   -  put a new null version (PUT ``versionId: ''``)
+
+-  If master is not a null version and ``nullVersionId`` is defined in
+   the object’s metadata:
+
+   -  delete the current null version metadata and data
+   -  overwrite the master version with the new metadata
+
+If there is no existing object metadata, create the new null version as
+the master version.
+
+In each of the above cases, set ``isNull`` metadata attribute to true
+when creating the new null version.
+
+Behavior of Object-Targeting APIs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+API methods which can target existing objects or versions, such as Get
+Object, Head Object, Get Object ACL, Put Object ACL, Copy Object and
+Copy Part, will perform the action on the latest version of an object if
+no version ID is specified in the request query or relevant request
+header (``x-amz-copy-source-version-id`` for Copy Object and Copy Part
+APIs).
+
+Two exceptions are the Delete Object and Multi-Object Delete APIs, which
+will instead attempt to create delete markers, described in the
+following section, if no version ID is specified.
+
+No versioning options are necessary to retrieve the latest version from
+metadata, since the master version is stored in a key with the name of
+the object. However, when updating the latest version, such as with the
+Put Object ACL API, Zenko CloudServer sets the ``versionId`` option in the
+PUT call to metadata to the value stored in the object metadata's ``versionId``
+attribute. This is done in order to update the metadata both in the
+master version and the version itself, if it is not a null version. [2]_
+
+When a version id is specified in the request query for these APIs, e.g.
+``GET /foo?versionId=v1``, Zenko CloudServer will attempt to decode the version
+ID and perform the action on the appropriate version. To do so, the API sets
+the value of the ``versionId`` option to the decoded version ID in the
+metadata call.
+
+Delete Markers
+^^^^^^^^^^^^^^
+
+If versioning has not been configured for a bucket, the Delete Object
+and Multi-Object Delete APIs behave as their standard APIs.
+
+If versioning has been configured, Zenko CloudServer deletes object or version
+data only if a specific version ID is provided in the request query, e.g.
+``DELETE /foo?versionId=v1``.
+
+If no version ID is provided, S3 creates a delete marker by creating a
+0-byte version with the metadata attribute ``isDeleteMarker: true``. The
+S3 API will return a ``404 NoSuchKey`` error in response to requests
+getting or heading an object whose latest version is a delete maker.
+
+To restore a previous version as the latest version of an object, the
+delete marker must be deleted, by the same process as deleting any other
+version.
+
+The response varies when targeting an object whose latest version is a
+delete marker for other object-level APIs that can target existing
+objects and versions, without specifying the version ID.
+
+-  Get Object, Head Object, Get Object ACL, Object Copy and Copy Part
+   return ``404 NoSuchKey``.
+-  Put Object ACL and Put Object Tagging return
+   ``405 MethodNotAllowed``.
+
+These APIs respond to requests specifying the version ID of a delete
+marker with the error ``405 MethodNotAllowed``, in general. Copy Part
+and Copy Object respond with ``400 Invalid Request``.
+
+See section `"Delete Example" <#delete-example>`__ for a summary.
+
+Null Version Mapping
+^^^^^^^^^^^^^^^^^^^^
+
+When the null version is specified in a request with the version ID
+"null", the S3 API must use the ``nullVersionId`` stored in the latest
+version to retrieve the current null version, if the null version is not
+the latest version.
+
+Thus, getting the null version is a two step process:
+
+1. Get the latest version of the object from metadata. If the latest
+   version's ``isNull`` property is ``true``, then use the latest
+   version's metadata. Otherwise,
+2. Get the null version of the object from metadata, using the internal
+   version ID of the current null version stored in the latest version's
+   ``nullVersionId`` metadata attribute.
+
+DELETE Example
+~~~~~~~~~~~~~~
+
+The following steps are used in the delete logic for delete marker
+creation:
+
+-  If versioning has not been configured: attempt to delete the object
+-  If request is version-specific delete request: attempt to delete the
+   version
+-  otherwise, if not a version-specific delete request and versioning
+   has been configured:
+
+   -  create a new 0-byte content-length version
+   -  in version's metadata, set a 'isDeleteMarker' property to true
+
+-  Return the version ID of any version deleted or any delete marker
+   created
+-  Set response header ``x-amz-delete-marker`` to true if a delete
+   marker was deleted or created
+
+The Multi-Object Delete API follows the same logic for each of the
+objects or versions listed in an xml request. Note that a delete request
+can result in the creation of a deletion marker even if the object
+requested to delete does not exist in the first place.
+
+Object-level APIs which can target existing objects and versions perform
+the following checks regarding delete markers:
+
+-  If not a version-specific request and versioning has been configured,
+   check the metadata of the latest version
+-  If the 'isDeleteMarker' property is set to true, return
+   ``404 NoSuchKey`` or ``405 MethodNotAllowed``
+-  If it is a version-specific request, check the object metadata of the
+   requested version
+-  If the ``isDeleteMarker`` property is set to true, return
+   ``405 MethodNotAllowed`` or ``400 InvalidRequest``
+
+.. [2]  If it is a null version, this call will overwrite the null version
+        if it is stored in its own key (``foo\0<versionId>``). If the null
+        version is stored only in the master version, this call will both
+        overwrite the master version *and* create a new key
+        (``foo\0<versionId>``), resulting in the edge case referred to by the
+        previous footnote [1]_.
+
+Data-metadata daemon Architecture and Operational guide
+=======================================================
+
+This document presents the architecture of the data-metadata daemon
+(dmd) used for the community edition of Zenko CloudServer. It also provides a
+guide on how to operate it.
+
+The dmd is responsible for storing and retrieving Zenko CloudServer data and
+metadata, and is accessed by Zenko CloudServer connectors through socket.io
+(metadata) and REST (data) APIs.
+
+It has been designed such that more than one Zenko CloudServer connector can
+access the same buckets by communicating with the dmd. It also means that
+the dmd can be hosted on a separate container or machine.
+
+Operation
+---------
+
+Startup
+~~~~~~~
+
+The simplest deployment is still to launch with yarn start, this will
+start one instance of the Zenko CloudServer connector and will listen on the
+locally bound dmd ports 9990 and 9991 (by default, see below).
+
+The dmd can be started independently from the Zenko CloudServer by running this
+command in the Zenko CloudServer directory:
+
+::
+
+   yarn run start_dmd
+
+This will open two ports:
+
+- one is based on socket.io and is used for metadata transfers (9990 by
+  default)
+
+- the other is a REST interface used for data transfers (9991 by
+  default)
+
+Then, one or more instances of Zenko CloudServer without the dmd can be started
+elsewhere with:
+
+.. code:: sh
+
+   yarn run start_s3server
+
+Configuration
+~~~~~~~~~~~~~
+
+Most configuration happens in ``config.json`` for Zenko CloudServer, local
+storage paths can be changed where the dmd is started using environment
+variables, like before: ``S3DATAPATH`` and ``S3METADATAPATH``.
+
+In ``config.json``, the following sections are used to configure access
+to the dmd through separate configuration of the data and metadata
+access:
+
+::
+
+   "metadataClient": {
+       "host": "localhost",
+       "port": 9990
+   },
+   "dataClient": {
+       "host": "localhost",
+       "port": 9991
+   },
+
+To run a remote dmd, you have to do the following:
+
+- change both ``"host"`` attributes to the IP or host name where the
+  dmd is run.
+
+- Modify the ``"bindAddress"`` attributes in ``"metadataDaemon"`` and
+  ``"dataDaemon"`` sections where the dmd is run to accept remote
+  connections (e.g. ``"::"``)
+
+Architecture
+------------
+
+This section gives a bit more insight on how it works internally.
+
+.. figure:: ./images/data_metadata_daemon_arch.png
+  :alt: Architecture diagram
+
+  ./images/data\_metadata\_daemon\_arch.png
+
+Metadata on socket.io
+~~~~~~~~~~~~~~~~~~~~~
+
+This communication is based on an RPC system based on socket.io events
+sent by Zenko CloudServerconnectors, received by the DMD and acknowledged back
+to the Zenko CloudServer connector.
+
+The actual payload sent through socket.io is a JSON-serialized form of
+the RPC call name and parameters, along with some additional information
+like the request UIDs, and the sub-level information, sent as object
+attributes in the JSON request.
+
+With introduction of versioning support, the updates are now gathered in
+the dmd for some number of milliseconds max, before being batched as a
+single write to the database. This is done server-side, so the API is
+meant to send individual updates.
+
+Four RPC commands are available to clients: ``put``, ``get``, ``del``
+and ``createReadStream``. They more or less map the parameters accepted
+by the corresponding calls in the LevelUp implementation of LevelDB.
+They differ in the following:
+
+- The ``sync`` option is ignored (under the hood, puts are gathered
+  into batches which have their ``sync`` property enforced when they
+  are committed to the storage)
+
+- Some additional versioning-specific options are supported
+
+- ``createReadStream`` becomes asynchronous, takes an additional
+  callback argument and returns the stream in the second callback
+  parameter
+
+Debugging the socket.io exchanges can be achieved by running the daemon
+with ``DEBUG='socket.io*'`` environment variable set.
+
+One parameter controls the timeout value after which RPC commands sent
+end with a timeout error, it can be changed either:
+
+- via the ``DEFAULT_CALL_TIMEOUT_MS`` option in
+  ``lib/network/rpc/rpc.js``
+
+- or in the constructor call of the ``MetadataFileClient`` object (in
+  ``lib/metadata/bucketfile/backend.js`` as ``callTimeoutMs``.
+
+Default value is 30000.
+
+A specific implementation deals with streams, currently used for listing
+a bucket. Streams emit ``"stream-data"`` events that pack one or more
+items in the listing, and a special ``“stream-end”`` event when done.
+Flow control is achieved by allowing a certain number of “in flight”
+packets that have not received an ack yet (5 by default). Two options
+can tune the behavior (for better throughput or getting it more robust
+on weak networks), they have to be set in ``mdserver.js`` file directly,
+as there is no support in ``config.json`` for now for those options:
+
+- ``streamMaxPendingAck``: max number of pending ack events not yet
+  received (default is 5)
+
+- ``streamAckTimeoutMs``: timeout for receiving an ack after an output
+  stream packet is sent to the client (default is 5000)
+
+Data exchange through the REST data port
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Data is read and written with REST semantic.
+
+The web server recognizes a base path in the URL of ``/DataFile`` to be
+a request to the data storage service.
+
+PUT
+^^^
+
+A PUT on ``/DataFile`` URL and contents passed in the request body will
+write a new object to the storage.
+
+On success, a ``201 Created`` response is returned and the new URL to
+the object is returned via the ``Location`` header (e.g.
+``Location: /DataFile/50165db76eecea293abfd31103746dadb73a2074``). The
+raw key can then be extracted simply by removing the leading
+``/DataFile`` service information from the returned URL.
+
+GET
+^^^
+
+A GET is simply issued with REST semantic, e.g.:
+
+::
+
+   GET /DataFile/50165db76eecea293abfd31103746dadb73a2074 HTTP/1.1
+
+A GET request can ask for a specific range. Range support is complete
+except for multiple byte ranges.
+
+DELETE
+^^^^^^
+
+DELETE is similar to GET, except that a ``204 No Content`` response is
+returned on success.
+
+
+Listing
+=======
+
+Listing Types
+-------------
+
+We use three different types of metadata listing for various operations.
+Here are the scenarios we use each for:
+
+- 'Delimiter' - when no versions are possible in the bucket since it is
+  an internally-used only bucket which is not exposed to a user.
+  Namely,
+
+1. to list objects in the "user's bucket" to respond to a GET SERVICE
+   request and
+2. to do internal listings on an MPU shadow bucket to complete multipart
+   upload operations.
+
+- 'DelimiterVersion' - to list all versions in a bucket
+- 'DelimiterMaster' - to list just the master versions of objects in a
+  bucket
+
+Algorithms
+----------
+
+The algorithms for each listing type can be found in the open-source
+`scality/Arsenal <https://github.com/scality/Arsenal>`__ repository, in
+`lib/algos/list <https://github.com/scality/Arsenal/tree/master/lib/algos/list>`__.
+
+Encryption
+===========
+
+With CloudServer, there are two possible methods of at-rest encryption.
+(1) We offer bucket level encryption where Scality CloudServer itself handles at-rest
+encryption for any object that is in an 'encrypted' bucket, regardless of what
+the location-constraint for the data is and
+(2) If the location-constraint specified for the data is of type AWS,
+you can choose to use AWS server side encryption.
+
+Note: bucket level encryption is not available on the standard AWS
+S3 protocol, so normal AWS S3 clients will not provide the option to send a
+header when creating a bucket. We have created a simple tool to enable you
+to easily create an encrypted bucket.
+
+Example:
+--------
+
+Creating encrypted bucket using our encrypted bucket tool in the bin directory
+
+.. code:: shell
+
+    ./create_encrypted_bucket.js -a accessKey1 -k verySecretKey1 -b bucketname -h localhost -p 8000
+
+
+AWS backend
+------------
+
+With real AWS S3 as a location-constraint, you have to configure the
+location-constraint as follows
+
+.. code:: json
+
+    "awsbackend": {
+        "type": "aws_s3",
+        "legacyAwsBehavior": true,
+        "details": {
+            "serverSideEncryption": true,
+            ...
+        }
+    },
+
+Then, every time an object is put to that data location, we pass the following
+header to AWS: ``x-amz-server-side-encryption: AES256``
+
+Note: due to these options, it is possible to configure encryption by both
+CloudServer and AWS S3 (if you put an object to a CloudServer bucket which has
+the encryption flag AND the location-constraint for the data is AWS S3 with
+serverSideEncryption set to true).

docs/BUCKET_POLICIES.md

@@ -0,0 +1,146 @@
+# Bucket Policy Documentation
+
+## Description
+
+Bucket policy is a method of controlling access to a user's account at the
+resource level.
+There are three associated APIs:
+
+- PUT Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTpolicy.html)
+- GET Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETpolicy.html)
+- DELETE Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEpolicy.html)
+
+More information on bucket policies in general can be found at
+https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html.
+
+## Requirements
+
+To prevent loss of access to a bucket, the root owner of a bucket will always
+be able to perform any of the three bucket policy-related operations, even
+if permission is explicitly denied.
+All other users must have permission to perform the desired operation.
+
+## Design
+
+On a PUTBucketPolicy request, the user provides a policy in JSON format.
+The policy is evaluated against our policy schema in Arsenal and, once
+validated, is stored as part of the bucket's metadata.
+On a GETBucketPolicy request, the policy is retrieved from the bucket's
+metadata.
+On a DELETEBucketPolicy request, the policy is deleted from the bucket's
+metadata.
+
+All other APIs are updated to check if a bucket policy is attached to the bucket
+the request is made on. If there is a policy, user authorization to perform
+the requested action is checked.
+
+### Differences Between Bucket and IAM Policies
+
+IAM policies are attached to an IAM identity and define what actions that
+identity is allowed to or denied from doing on what resource.
+Bucket policies attach only to buckets and define what actions are allowed or
+denied for which principles on that bucket. Permissions specified in a bucket
+policy apply to all objects in that bucket unless otherwise specified.
+
+Besides their attachment origins, the main structural difference between
+IAM policy and bucket policy is the requirement of a "Principal" element in
+bucket policies. This field is redundant in IAM policies.
+
+### Policy Validation
+
+For general guidelines for bucket policy structure, see examples here:
+https://docs.aws.amazon.com/AmazonS3/latest/dev//example-bucket-policies.html.
+
+Each bucket policy statement object requires at least four keys:
+"Effect", "Principle", "Resource", and "Action".
+
+"Effect" defines the effect of the policy and can have a string value of either
+"Allow" or "Deny".
+"Resource" defines to which bucket or list of buckets a policy is attached.
+An object within the bucket is also a valid resource. The element value can be
+either a single bucket or object ARN string or an array of ARNs.
+"Action" lists which action(s) the policy controls. Its value can also be either
+a string or array of S3 APIs. Each action is the API name prepended by "s3:".
+"Principle" specifies which user(s) are granted or denied access to the bucket
+resource. Its value can be a string or an object containing an array of users.
+Valid users can be identified with an account ARN, account id, or user ARN.
+
+There are also two optional bucket policy statement keys: Sid and Condition.
+
+"Sid" stands for "statement id". If this key is not included, one will be
+generated for the statement.
+"Condition" lists the condition under which a statement will take affect.
+The possibilities are as follows:
+
+- ArnEquals
+- ArnEqualsIfExists
+- ArnLike
+- ArnLikeIfExists
+- ArnNotEquals
+- ArnNotEqualsIfExists
+- ArnNotLike
+- ArnNotLikeIfExists
+- BinaryEquals
+- BinaryEqualsIfExists
+- BinaryNotEquals
+- BinaryNotEqualsIfExists
+- Bool
+- BoolIfExists
+- DateEquals
+- DateEqualsIfExists
+- DateGreaterThan
+- DateGreaterThanEquals
+- DateGreaterThanEqualsIfExists
+- DateGreaterThanIfExists
+- DateLessThan
+- DateLessThanEquals
+- DateLessThanEqualsIfExists
+- DateLessThanIfExists
+- DateNotEquals
+- DateNotEqualsIfExists
+- IpAddress
+- IpAddressIfExists
+- NotIpAddress
+- NotIpAddressIfExists
+- Null
+- NumericEquals
+- NumericEqualsIfExists
+- NumericGreaterThan
+- NumericGreaterThanEquals
+- NumericGreaterThanEqualsIfExists
+- NumericGreaterThanIfExists
+- NumericLessThan
+- NumericLessThanEquals
+- NumericLessThanEqualsIfExists
+- NumericLessThanIfExists
+- NumericNotEquals
+- NumericNotEqualsIfExists
+- StringEquals
+- StringEqualsIfExists
+- StringEqualsIgnoreCase
+- StringEqualsIgnoreCaseIfExists
+- StringLike
+- StringLikeIfExists
+- StringNotEquals
+- StringNotEqualsIfExists
+- StringNotEqualsIgnoreCase
+- StringNotEqualsIgnoreCaseIfExists
+- StringNotLike
+- StringNotLikeIfExists
+
+The value of the Condition key will be an object containing the desired
+condition name as that key. The value of inner object can be a string, boolean,
+number, or object, depending on the condition.
+
+## Authorization with Multiple Access Control Mechanisms
+
+In the case where multiple access control mechanisms (such as IAM policies,
+bucket policies, and ACLs) refer to the same resource, the principle of
+least-privilege is applied. Unless an action is explicitly allowed, access will
+by default be denied. An explicit DENY in any policy will trump another
+policy's ALLOW for an action. The request will only be allowed if at least one
+policy specifies an ALLOW, and there is no overriding DENY.
+
+The following diagram illustrates this logic:
+
+![Access_Control_Authorization_Chart](./images/access_control_authorization.png)

docs/CLIENTS.rst

@@ -0,0 +1,345 @@
+Clients
+=========
+
+List of applications that have been tested with Zenko CloudServer.
+
+GUI
+~~~
+
+`Cyberduck <https://cyberduck.io/?l=en>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+-  https://www.youtube.com/watch?v=-n2MCt4ukUg
+-  https://www.youtube.com/watch?v=IyXHcu4uqgU
+
+`Cloud Explorer <https://www.linux-toys.com/?p=945>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+-  https://www.youtube.com/watch?v=2hhtBtmBSxE
+
+`CloudBerry Lab <http://www.cloudberrylab.com>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+-  https://youtu.be/IjIx8g\_o0gY
+
+Command Line Tools
+~~~~~~~~~~~~~~~~~~
+
+`s3curl <https://github.com/rtdp/s3curl>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+https://github.com/scality/S3/blob/master/tests/functional/s3curl/s3curl.pl
+
+`aws-cli <http://docs.aws.amazon.com/cli/latest/reference/>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``~/.aws/credentials`` on Linux, OS X, or Unix or
+``C:\Users\USERNAME\.aws\credentials`` on Windows
+
+.. code:: shell
+
+    [default]
+    aws_access_key_id = accessKey1
+    aws_secret_access_key = verySecretKey1
+
+``~/.aws/config`` on Linux, OS X, or Unix or
+``C:\Users\USERNAME\.aws\config`` on Windows
+
+.. code:: shell
+
+    [default]
+    region = us-east-1
+
+Note: ``us-east-1`` is the default region, but you can specify any
+region.
+
+See all buckets:
+
+.. code:: shell
+
+    aws s3 ls --endpoint-url=http://localhost:8000
+
+Create bucket:
+
+.. code:: shell
+
+    aws --endpoint-url=http://localhost:8000 s3 mb s3://mybucket
+
+`s3cmd <http://s3tools.org/s3cmd>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If using s3cmd as a client to S3 be aware that v4 signature format is
+buggy in s3cmd versions < 1.6.1.
+
+``~/.s3cfg`` on Linux, OS X, or Unix or ``C:\Users\USERNAME\.s3cfg`` on
+Windows
+
+.. code:: shell
+
+    [default]
+    access_key = accessKey1
+    secret_key = verySecretKey1
+    host_base = localhost:8000
+    host_bucket = %(bucket).localhost:8000
+    signature_v2 = False
+    use_https = False
+
+See all buckets:
+
+.. code:: shell
+
+    s3cmd ls
+
+`rclone <http://rclone.org/s3/>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``~/.rclone.conf`` on Linux, OS X, or Unix or
+``C:\Users\USERNAME\.rclone.conf`` on Windows
+
+.. code:: shell
+
+    [remote]
+    type = s3
+    env_auth = false
+    access_key_id = accessKey1
+    secret_access_key = verySecretKey1
+    region = other-v2-signature
+    endpoint = http://localhost:8000
+    location_constraint =
+    acl = private
+    server_side_encryption =
+    storage_class =
+
+See all buckets:
+
+.. code:: shell
+
+    rclone lsd remote:
+
+JavaScript
+~~~~~~~~~~
+
+`AWS JavaScript SDK <http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: javascript
+
+    const AWS = require('aws-sdk');
+
+    const s3 = new AWS.S3({
+        accessKeyId: 'accessKey1',
+        secretAccessKey: 'verySecretKey1',
+        endpoint: 'localhost:8000',
+        sslEnabled: false,
+        s3ForcePathStyle: true,
+    });
+
+JAVA
+~~~~
+
+`AWS JAVA SDK <http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Client.html>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: java
+
+    import com.amazonaws.auth.AWSCredentials;
+    import com.amazonaws.auth.BasicAWSCredentials;
+    import com.amazonaws.services.s3.AmazonS3;
+    import com.amazonaws.services.s3.AmazonS3Client;
+    import com.amazonaws.services.s3.S3ClientOptions;
+    import com.amazonaws.services.s3.model.Bucket;
+
+    public class S3 {
+
+        public static void main(String[] args) {
+
+            AWSCredentials credentials = new BasicAWSCredentials("accessKey1",
+            "verySecretKey1");
+
+            // Create a client connection based on credentials
+            AmazonS3 s3client = new AmazonS3Client(credentials);
+            s3client.setEndpoint("http://localhost:8000");
+            // Using path-style requests
+            // (deprecated) s3client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+            s3client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
+
+            // Create bucket
+            String bucketName = "javabucket";
+            s3client.createBucket(bucketName);
+
+            // List off all buckets
+            for (Bucket bucket : s3client.listBuckets()) {
+                System.out.println(" - " + bucket.getName());
+            }
+        }
+    }
+
+Ruby
+~~~~
+
+`AWS SDK for Ruby - Version 2 <http://docs.aws.amazon.com/sdkforruby/api/>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: ruby
+
+    require 'aws-sdk'
+
+    s3 = Aws::S3::Client.new(
+      :access_key_id => 'accessKey1',
+      :secret_access_key => 'verySecretKey1',
+      :endpoint => 'http://localhost:8000',
+      :force_path_style => true
+    )
+
+    resp = s3.list_buckets
+
+`fog <http://fog.io/storage/>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: ruby
+
+    require "fog"
+
+    connection = Fog::Storage.new(
+    {
+        :provider => "AWS",
+        :aws_access_key_id => 'accessKey1',
+        :aws_secret_access_key => 'verySecretKey1',
+        :endpoint => 'http://localhost:8000',
+        :path_style => true,
+        :scheme => 'http',
+    })
+
+Python
+~~~~~~
+
+`boto2 <http://boto.cloudhackers.com/en/latest/ref/s3.html>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: python
+
+    import boto
+    from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+
+
+    connection = S3Connection(
+        aws_access_key_id='accessKey1',
+        aws_secret_access_key='verySecretKey1',
+        is_secure=False,
+        port=8000,
+        calling_format=OrdinaryCallingFormat(),
+        host='localhost'
+    )
+
+    connection.create_bucket('mybucket')
+
+`boto3 <http://boto3.readthedocs.io/en/latest/index.html>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Client integration
+
+.. code:: python
+
+    import boto3
+
+    client = boto3.client(
+        's3',
+        aws_access_key_id='accessKey1',
+        aws_secret_access_key='verySecretKey1',
+        endpoint_url='http://localhost:8000'
+    )
+
+    lists = client.list_buckets()
+
+Full integration (with object mapping)
+
+.. code:: python
+
+    import os
+
+    from botocore.utils import fix_s3_host
+    import boto3
+
+    os.environ['AWS_ACCESS_KEY_ID'] = "accessKey1"
+    os.environ['AWS_SECRET_ACCESS_KEY'] = "verySecretKey1"
+
+    s3 = boto3.resource(service_name='s3', endpoint_url='http://localhost:8000')
+    s3.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
+
+    for bucket in s3.buckets.all():
+        print(bucket.name)
+
+PHP
+~~~
+
+Should force path-style requests even though v3 advertises it does by default.
+
+`AWS PHP SDK v3 <https://docs.aws.amazon.com/aws-sdk-php/v3/guide>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: php
+
+    use Aws\S3\S3Client;
+
+    $client = S3Client::factory([
+        'region'  => 'us-east-1',
+        'version'   => 'latest',
+        'endpoint' => 'http://localhost:8000',
+        'use_path_style_endpoint' => true,
+        'credentials' => [
+             'key'    => 'accessKey1',
+             'secret' => 'verySecretKey1'
+        ]
+    ]);
+
+    $client->createBucket(array(
+        'Bucket' => 'bucketphp',
+    ));
+
+Go
+~~
+
+`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: go
+
+   package main
+
+   import (
+      "context"
+      "fmt"
+      "log"
+      "os"
+      "time"
+
+      "github.com/aws/aws-sdk-go/aws"
+      "github.com/aws/aws-sdk-go/aws/endpoints"
+      "github.com/aws/aws-sdk-go/aws/session"
+      "github.com/aws/aws-sdk-go/service/s3"
+   )
+
+   func main() {
+      os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
+      os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
+      endpoint := "http://localhost:8000"
+      timeout := time.Duration(10) * time.Second
+      sess := session.Must(session.NewSession())
+
+      // Create a context with a timeout that will abort the upload if it takes
+      // more than the passed in timeout.
+      ctx, cancel := context.WithTimeout(context.Background(), timeout)
+      defer cancel()
+
+      svc := s3.New(sess, &aws.Config{
+         Region:   aws.String(endpoints.UsEast1RegionID),
+         Endpoint: &endpoint,
+      })
+
+      out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
+      if err != nil {
+         log.Fatal(err)
+      } else {
+         fmt.Println(out)
+      }
+   }

docs/CONTRIBUTING.rst

@@ -0,0 +1,24 @@
+Contributing
+============
+
+Need help?
+----------
+We're always glad to help out. Simply open a
+`GitHub issue <https://github.com/scality/S3/issues>`__ and we'll give you
+insight. If what you want is not available, and if you're willing to help us
+out, we'll be happy to welcome you in the team, whether for a small fix or for
+a larger feature development. Thanks for your interest!
+
+Got an idea? Get started!
+-------------------------
+In order to contribute, please follow the `Contributing
+Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__.
+If anything is unclear to you, reach out to us on
+`forum <https://forum.zenko.io/>`__ or via a GitHub issue.
+
+Don't write code? There are other ways to help!
+-----------------------------------------------
+We're always eager to learn about our users' stories. If you can't contribute
+code, but would love to help us, please shoot us an email at zenko@scality.com,
+and tell us what our software enables you to do! Thanks for your time!
+

docs/DOCKER.rst

@@ -0,0 +1,371 @@
+Docker
+======
+
+.. _environment-variables:
+
+Environment Variables
+---------------------
+
+S3DATA
+~~~~~~
+
+S3DATA=multiple
+^^^^^^^^^^^^^^^
+
+This variable enables running CloudServer with multiple data backends, defined
+as regions.
+
+For multiple data backends, a custom locationConfig.json file is required.
+This file enables you to set custom regions. You must provide associated 
+rest_endpoints for each custom region in config.json.
+
+`Learn more about multiple-backend configurations <GETTING_STARTED.html#location-configuration>`__
+
+If you are using Scality RING endpoints, refer to your customer documentation.
+
+Running CloudServer with an AWS S3-Hosted Backend
+"""""""""""""""""""""""""""""""""""""""""""""""""
+
+To run CloudServer with an S3 AWS backend, add a new section to the 
+``locationConfig.json`` file with the ``aws_s3`` location type:
+
+.. code:: json
+
+    (...)
+    "awsbackend": {
+        "type": "aws_s3",
+        "details": {
+            "awsEndpoint": "s3.amazonaws.com",
+            "bucketName": "yourawss3bucket",
+            "bucketMatch": true,
+            "credentialsProfile": "aws_hosted_profile"
+        }
+    }
+    (...)
+
+Edit your AWS credentials file to enable your preferred command-line tool.
+This file must mention credentials for all backends in use. You can use 
+several profiles if multiple profiles are configured.
+
+.. code:: json
+
+    [default]
+    aws_access_key_id=accessKey1
+    aws_secret_access_key=verySecretKey1
+    [aws_hosted_profile]
+    aws_access_key_id={{YOUR_ACCESS_KEY}}
+    aws_secret_access_key={{YOUR_SECRET_KEY}}
+
+As with locationConfig.json, the AWS credentials file must be mounted at 
+run time: ``-v ~/.aws/credentials:/root/.aws/credentials`` on Unix-like 
+systems (Linux, OS X, etc.), or 
+``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows
+
+.. note:: One account cannot copy to another account with a source and
+   destination on real AWS unless the account associated with the 
+   accessKey/secretKey pairs used for the destination bucket has source 
+   bucket access privileges. To enable this, update ACLs directly on AWS.
+
+S3BACKEND
+~~~~~~~~~
+
+S3BACKEND=file
+^^^^^^^^^^^^^^
+
+For stored file data to persist, you must mount Docker volumes
+for both data and metadata. See :ref:`In Production with a Docker-Hosted CloudServer <in-production-w-a-Docker-hosted-cloudserver>`
+
+S3BACKEND=mem
+^^^^^^^^^^^^^
+
+This is ideal for testing: no data remains after the container is shut down.
+
+ENDPOINT
+~~~~~~~~
+
+This variable specifies the endpoint. To direct CloudServer requests to 
+new.host.com, for example, specify the endpoint with:
+
+.. code-block:: shell
+
+    $ docker run -d --name cloudserver -p 8000:8000 -e ENDPOINT=new.host.com zenko/cloudserver
+
+.. note:: On Unix-like systems (Linux, OS X, etc.) edit /etc/hosts
+   to associate 127.0.0.1 with new.host.com.
+
+REMOTE_MANAGEMENT_DISABLE
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CloudServer is a part of `Zenko <https://www.zenko.io/>`__. When you run CloudServer standalone it will still try to connect to Orbit by default (browser-based graphical user interface for Zenko).
+
+Setting this variable to true(1) will default to accessKey1 and verySecretKey1 for credentials and disable the automatic Orbit management:
+
+.. code-block:: shell
+
+    $ docker run -d --name cloudserver -p 8000:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
+
+SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These variables specify authentication credentials for an account named
+“CustomAccount”.
+
+Set account credentials for multiple accounts by editing conf/authdata.json
+(see below for further details). To specify one set for personal use, set these 
+environment variables:
+
+.. code-block:: shell
+
+   $ docker run -d --name cloudserver -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey \
+   -e SCALITY_SECRET_ACCESS_KEY=newSecretKey zenko/cloudserver
+
+.. note:: This takes precedence over the contents of the authdata.json 
+	  file. The authdata.json file is ignored. 
+
+.. note:: The ACCESS_KEY and SECRET_KEY environment variables are 
+	  deprecated.
+
+LOG\_LEVEL
+~~~~~~~~~~
+
+This variable changes the log level. There are three levels: info, debug, 
+and trace. The default is info. Debug provides more detailed logs, and trace
+provides the most detailed logs.
+
+.. code-block:: shell
+
+    $ docker run -d --name cloudserver -p 8000:8000 -e LOG_LEVEL=trace zenko/cloudserver
+
+SSL
+~~~
+
+Set true, this variable runs CloudServer with SSL.
+
+If SSL is set true: 
+
+* The ENDPOINT environment variable must also be specified.
+
+* On Unix-like systems (Linux, OS X, etc.), 127.0.0.1 must be associated with
+  <YOUR_ENDPOINT> in /etc/hosts.
+
+   .. Warning:: Self-signed certs with a CA generated within the container are 
+      suitable for testing purposes only. Clients cannot trust them, and they may
+      disappear altogether on a container upgrade. The best security practice for 
+      production environments is to use an extra container, such as 
+      haproxy/nginx/stunnel, for SSL/TLS termination and to pull certificates
+      from a mounted volume, limiting what an exploit on either component
+      can expose. 
+
+.. code:: shell
+
+     $ docker run -d --name cloudserver -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT> \
+     zenko/cloudserver
+
+  For more information about using ClousdServer with SSL, see `Using SSL <GETTING_STARTED.html#Using SSL>`__
+
+LISTEN\_ADDR
+~~~~~~~~~~~~
+
+This variable causes CloudServer and its data and metadata components to 
+listen on the specified address. This allows starting the data or metadata 
+servers as standalone services, for example.
+
+.. code:: shell
+
+    docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0
+    scality/s3server yarn run start_dataserver
+
+
+DATA\_HOST and METADATA\_HOST
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These variables configure the data and metadata servers to use,
+usually when they are running on another host and only starting the stateless
+Zenko CloudServer.
+
+.. code:: shell
+
+    $ docker run -d --name cloudserver -e DATA_HOST=cloudserver-data \
+    -e METADATA_HOST=cloudserver-metadata zenko/cloudserver yarn run start_s3server
+
+REDIS\_HOST
+~~~~~~~~~~~
+
+Use this variable to connect to the redis cache server on another host than
+localhost.
+
+.. code:: shell
+
+    $ docker run -d --name cloudserver -p 8000:8000 \
+    -e REDIS_HOST=my-redis-server.example.com zenko/cloudserver
+
+REDIS\_PORT
+~~~~~~~~~~~
+
+Use this variable to connect to the Redis cache server on a port other 
+than the default 6379.
+
+.. code:: shell
+
+    $ docker run -d --name cloudserver -p 8000:8000 \
+    -e REDIS_PORT=6379 zenko/cloudserver
+
+.. _tunables-and-setup-tips:
+
+Tunables and Setup Tips
+-----------------------
+
+Using Docker Volumes
+~~~~~~~~~~~~~~~~~~~~
+
+CloudServer runs with a file backend by default, meaning that data is 
+stored inside the CloudServer’s Docker container.
+
+For data and metadata to persist, data and metadata must be hosted in Docker 
+volumes outside the CloudServer’s Docker container. Otherwise, the data
+and metadata are destroyed when the container is erased.
+
+.. code-block:: shell
+
+    $ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata \
+    -p 8000:8000 ­-d zenko/cloudserver
+
+This command mounts the ./data host directory to the container
+at /usr/src/app/localData and the ./metadata host directory to
+the container at /usr/src/app/localMetaData. 
+
+.. tip:: These host directories can be mounted to any accessible mount 
+   point, such as /mnt/data and /mnt/metadata, for example.
+
+Adding, Modifying, or Deleting Accounts or Credentials
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. Create a customized authdata.json file locally based on /conf/authdata.json.
+
+2. Use `Docker volumes <https://docs.docker.com/storage/volumes/>`__
+   to override the default ``authdata.json`` through a Docker file mapping.
+
+For example:
+
+.. code-block:: shell
+
+    $ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d \
+    zenko/cloudserver
+
+Specifying a Host Name
+~~~~~~~~~~~~~~~~~~~~~~
+
+To specify a host name (for example, s3.domain.name), provide your own
+`config.json <https://github.com/scality/cloudserver/blob/master/config.json>`__
+file using `Docker volumes <https://docs.docker.com/storage/volumes/>`__.
+
+First, add a new key-value pair to the restEndpoints section of your
+config.json. Make the key the host name you want, and the value the default 
+location\_constraint for this endpoint.
+
+For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one
+of the ``location_constraints`` listed in your locationConfig.json file
+`here <https://github.com/scality/S3/blob/master/locationConfig.json>`__.
+
+For more information about location configuration, see:
+`GETTING STARTED <GETTING_STARTED.html#location-configuration>`__
+
+.. code:: json
+
+    "restEndpoints": {
+        "localhost": "file",
+        "127.0.0.1": "file",
+        ...
+        "cloudserver.example.com": "us-east-1"
+    },
+
+Next, run CloudServer using a `Docker volume 
+<https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
+
+.. code-block:: shell
+
+    $ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d zenko/cloudserver
+
+The local ``config.json`` file overrides the default one through a Docker 
+file mapping.
+
+Running as an Unprivileged User
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CloudServer runs as root by default.
+
+To change this, modify the dockerfile and specify a user before the 
+entry point.
+
+The user must exist within the container, and must own the 
+/usr/src/app directory for CloudServer to run.
+
+For example, the following dockerfile lines can be modified:
+
+.. code-block:: shell
+
+    ...
+    && groupadd -r -g 1001 scality \
+    && useradd -u 1001 -g 1001 -d /usr/src/app -r scality \
+    && chown -R scality:scality /usr/src/app
+
+    ...
+
+    USER scality
+    ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
+
+.. _continuous-integration-with-docker-hosted-cloudserver:
+
+Continuous Integration with a Docker-Hosted CloudServer
+-------------------------------------------------------
+
+When you start the Docker CloudServer image, you can adjust the
+configuration of the CloudServer instance by passing one or more
+environment variables on the ``docker run`` command line.
+
+
+To run CloudServer for CI with custom locations (one in-memory, 
+one hosted on AWS), and custom credentials mounted:
+
+.. code-block:: shell
+
+   $ docker run --name CloudServer -p 8000:8000 \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
+   -v ~/.aws/credentials:/root/.aws/credentials \
+   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
+
+To run CloudServer for CI with custom locations, (one in-memory, one
+hosted on AWS, and one file), and custom credentials `set as environment 
+variables <GETTING_STARTED.html#scality-access-key-id-and-scality-secret-access-key>`__):
+
+.. code-block:: shell
+
+   $ docker run --name CloudServer -p 8000:8000 \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v ~/.aws/credentials:/root/.aws/credentials \
+   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -e SCALITY_ACCESS_KEY_ID=accessKey1 \
+   -e SCALITY_SECRET_ACCESS_KEY=verySecretKey1 \
+   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
+
+.. _in-production-w-a-Docker-hosted-cloudserver:
+
+In Production with a Docker-Hosted CloudServer
+----------------------------------------------
+
+Because data must persist in production settings, CloudServer offers
+multiple-backend capabilities. This requires a custom endpoint 
+and custom credentials for local storage.
+
+Customize these with:
+
+.. code-block:: shell
+
+   $ docker run -d --name CloudServer \
+   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
+   -v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple \
+   -e ENDPOINT=custom.endpoint.com \
+   -p 8000:8000 ­-d zenko/cloudserver \

docs/GETTING_STARTED.rst

@@ -0,0 +1,436 @@
+Getting Started
+===============
+
+.. figure:: ../res/scality-cloudserver-logo.png
+   :alt: Zenko CloudServer logo
+
+
+Dependencies
+------------
+
+Building and running the Scality Zenko CloudServer requires node.js 10.x and
+yarn v1.17.x. Up-to-date versions can be found at
+`Nodesource <https://github.com/nodesource/distributions>`__.
+
+Installation
+------------
+
+1. Clone the source code
+
+   .. code-block:: shell
+
+      $ git clone https://github.com/scality/cloudserver.git
+
+2. Go to the cloudserver directory and use yarn to install the js dependencies.
+
+   .. code-block:: shell
+
+      $ cd cloudserver
+      $ yarn install
+
+Running CloudServer with a File Backend
+---------------------------------------
+
+.. code-block:: shell
+
+   $ yarn start
+
+This starts a Zenko CloudServer on port 8000. Two additional ports, 9990 
+and 9991, are also open locally for internal transfer of metadata and 
+data, respectively.
+
+The default access key is accessKey1. The secret key is verySecretKey1.
+
+By default, metadata files are saved in the localMetadata directory and 
+data files are saved in the localData directory in the local ./cloudserver 
+directory. These directories are pre-created within the repository. To 
+save data or metadata in different locations, you must specify them using 
+absolute paths. Thus, when starting the server:
+
+.. code-block:: shell
+
+   $ mkdir -m 700 $(pwd)/myFavoriteDataPath
+   $ mkdir -m 700 $(pwd)/myFavoriteMetadataPath
+   $ export S3DATAPATH="$(pwd)/myFavoriteDataPath"
+   $ export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
+   $ yarn start
+
+Running CloudServer with Multiple Data Backends
+-----------------------------------------------
+
+.. code-block:: shell
+
+   $ export S3DATA='multiple'
+   $ yarn start
+
+This starts a Zenko CloudServer on port 8000. 
+
+The default access key is accessKey1. The secret key is verySecretKey1.
+
+With multiple backends, you can choose where each object is saved by setting
+the following header with a location constraint in a PUT request:
+
+.. code-block:: shell
+
+    'x-amz-meta-scal-location-constraint':'myLocationConstraint'
+
+If no header is sent with a PUT object request, the bucket’s location
+constraint determines where the data is saved. If the bucket has no
+location constraint, the endpoint of the PUT request determines location.
+
+See the Configuration_ section to set location constraints.
+
+Run CloudServer with an In-Memory Backend
+-----------------------------------------
+
+.. code-block:: shell
+
+   $ yarn run mem_backend
+
+This starts a Zenko CloudServer on port 8000. 
+
+The default access key is accessKey1. The secret key is verySecretKey1.
+
+Run CloudServer with Vault User Management
+------------------------------------------
+
+.. code:: shell
+
+   export S3VAULT=vault
+   yarn start
+
+Note: Vault is proprietary and must be accessed separately.
+This starts a Zenko CloudServer using Vault for user management.
+
+Run CloudServer for Continuous Integration Testing or in Production with Docker
+-------------------------------------------------------------------------------
+
+Run Cloudserver with `DOCKER <DOCKER.html>`__
+
+Testing
+~~~~~~~
+
+Run unit tests with the command:
+
+.. code-block:: shell
+
+   $ yarn test
+
+Run multiple-backend unit tests with:
+
+.. code-block:: shell
+
+   $ CI=true S3DATA=multiple yarn start
+   $ yarn run multiple_backend_test
+
+Run the linter with:
+
+.. code-block:: shell
+
+   $ yarn run lint
+
+Running Functional Tests Locally
+--------------------------------
+
+To pass AWS and Azure backend tests locally, modify 
+tests/locationConfig/locationConfigTests.json so that ``awsbackend`` 
+specifies the bucketname of a bucket you have access to based on your
+credentials, and modify ``azurebackend`` with details for your Azure account.
+
+The test suite requires additional tools, **s3cmd** and **Redis**
+installed in the environment the tests are running in.
+
+1. Install `s3cmd <http://s3tools.org/download>`__
+
+2. Install `redis <https://redis.io/download>`__ and start Redis.
+
+3. Add localCache section to ``config.json``:
+
+   .. code:: json
+
+      "localCache": {
+        "host": REDIS_HOST,
+        "port": REDIS_PORT
+	}
+
+   where ``REDIS_HOST`` is the Redis instance IP address (``"127.0.0.1"``
+   if Redis is running locally) and ``REDIS_PORT`` is the Redis instance
+   port (``6379`` by default)
+
+4. Add the following to the local etc/hosts file:
+
+   .. code-block:: shell
+
+      127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
+
+5. Start Zenko CloudServer in memory and run the functional tests:
+
+   .. code-block:: shell
+
+      $ CI=true yarn run mem_backend
+      $ CI=true yarn run ft_test
+
+.. _Configuration:
+
+Configuration
+-------------
+
+There are three configuration files for Zenko CloudServer:
+
+* ``conf/authdata.json``, for authentication.
+
+* ``locationConfig.json``, to configure where data is saved.
+
+* ``config.json``, for general configuration options.
+
+.. _location-configuration:
+
+Location Configuration
+~~~~~~~~~~~~~~~~~~~~~~
+
+You must specify at least one locationConstraint in locationConfig.json
+(or leave it as pre-configured).
+
+You must also specify 'us-east-1' as a locationConstraint. If you put a 
+bucket to an unknown endpoint and do not specify a locationConstraint in
+the PUT bucket call, us-east-1 is used.
+
+For instance, the following locationConstraint saves data sent to
+``myLocationConstraint`` to the file backend:
+
+.. code:: json
+
+   "myLocationConstraint": {
+       "type": "file",
+       "legacyAwsBehavior": false,
+       "details": {}
+   },
+
+Each locationConstraint must include the ``type``, ``legacyAwsBehavior``,
+and ``details`` keys. ``type`` indicates which backend is used for that
+region. Supported backends are mem, file, and scality.``legacyAwsBehavior``
+indicates whether the region behaves the same as the AWS S3 'us-east-1' 
+region. If the locationConstraint type is ``scality``, ``details`` must 
+contain connector information for sproxyd. If the locationConstraint type
+is ``mem`` or ``file``, ``details`` must be empty.
+
+Once locationConstraints is set in locationConfig.json, specify a default
+locationConstraint for each endpoint.
+
+For instance, the following sets the ``localhost`` endpoint to the
+``myLocationConstraint`` data backend defined above:
+
+.. code:: json
+
+    "restEndpoints": {
+         "localhost": "myLocationConstraint"
+    },
+
+To use an endpoint other than localhost for Zenko CloudServer, the endpoint
+must be listed in ``restEndpoints``. Otherwise, if the server is running
+with a:
+
+*  **file backend**: The default location constraint is ``file``
+*  **memory backend**: The default location constraint is ``mem``
+
+Endpoints
+~~~~~~~~~
+
+The Zenko CloudServer supports endpoints that are rendered in either:
+
+* path style: http://myhostname.com/mybucket or
+* hosted style: http://mybucket.myhostname.com
+
+However, if an IP address is specified for the host, hosted-style requests
+cannot reach the server. Use path-style requests in that case. For example,
+if you are using the AWS SDK for JavaScript, instantiate your client like this:
+
+.. code:: js
+
+    const s3 = new aws.S3({
+       endpoint: 'http://127.0.0.1:8000',
+       s3ForcePathStyle: true,
+    });
+
+Setting Your Own Access and Secret Key Pairs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Credentials can be set for many accounts by editing ``conf/authdata.json``, 
+but use the ``SCALITY_ACCESS_KEY_ID`` and ``SCALITY_SECRET_ACCESS_KEY`` 
+environment variables to specify your own credentials.
+
+_`scality-access-key-id-and-scality-secret-access-key`
+
+SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These variables specify authentication credentials for an account named
+“CustomAccount”.
+
+.. note:: Anything in the ``authdata.json`` file is ignored.
+
+.. code-block:: shell
+
+   $ SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey yarn start
+
+.. _Using_SSL:
+
+Using SSL
+~~~~~~~~~
+
+To use https with your local CloudServer, you must set up
+SSL certificates. 
+
+1. Deploy CloudServer using `our DockerHub page
+   <https://hub.docker.com/r/zenko/cloudserver/>`__ (run it with a file
+   backend).
+
+   .. Note:: If Docker is not installed locally, follow the
+      `instructions to install it for your distribution 
+      <https://docs.docker.com/engine/installation/>`__
+
+2. Update the CloudServer container’s config 
+
+   Add your certificates to your container. To do this, 
+   #. exec inside the CloudServer container. 
+
+   #. Run ``$> docker ps`` to find the container’s ID (the corresponding 
+      image name is ``scality/cloudserver``. 
+      
+   #. Copy the corresponding container ID (``894aee038c5e`` in the present
+      example), and run:
+
+      .. code-block:: shell
+
+         $> docker exec -it 894aee038c5e bash
+
+      This puts you inside your container, using an interactive terminal.
+
+3. Generate the SSL key and certificates. The paths where the different
+   files are stored are defined after the ``-out`` option in each of the 
+   following commands.
+
+    #. Generate a private key for your certificate signing request (CSR):
+
+       .. code-block:: shell
+
+	  $> openssl genrsa -out ca.key 2048
+
+    #. Generate a self-signed certificate for your local certificate 
+       authority (CA):
+
+       .. code:: shell
+
+	  $> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
+
+    #. Generate a key for the CloudServer:
+
+       .. code:: shell
+
+          $> openssl genrsa -out test.key 2048
+
+    #. Generate a CSR for CloudServer:
+
+       .. code:: shell
+
+          $> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
+
+    #. Generate a certificate for CloudServer signed by the local CA:
+
+       .. code:: shell
+
+          $> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
+
+4. Update Zenko CloudServer ``config.json``. Add a ``certFilePaths`` 
+   section to ``./config.json`` with appropriate paths:
+
+   .. code:: json
+
+        "certFilePaths": {
+            "key": "./test.key",
+            "cert": "./test.crt",
+            "ca": "./ca.crt"
+        }
+
+5. Run your container with the new config. 
+
+   #. Exit the container by running ``$> exit``. 
+
+   #. Restart the container with ``$> docker restart cloudserver``.
+
+6. Update the host configuration by adding s3.scality.test 
+   to /etc/hosts:
+
+   .. code:: bash
+
+      127.0.0.1      localhost s3.scality.test
+
+7. Copy the local certificate authority (ca.crt in step 4) from your 
+   container. Choose the path to save this file to (in the present 
+   example, ``/root/ca.crt``), and run:
+
+   .. code:: shell
+
+      $> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
+
+   .. note:: Your container ID will be different, and your path to 
+	     ca.crt may be different.
+
+Test the Config
+^^^^^^^^^^^^^^^
+
+If aws-sdk is not installed, run ``$> yarn install aws-sdk``. 
+
+Paste the following script into a file named "test.js":
+
+.. code:: js
+
+    const AWS = require('aws-sdk');
+    const fs = require('fs');
+    const https = require('https');
+
+    const httpOptions = {
+        agent: new https.Agent({
+            // path on your host of the self-signed certificate
+            ca: fs.readFileSync('./ca.crt', 'ascii'),
+        }),
+    };
+
+    const s3 = new AWS.S3({
+        httpOptions,
+        accessKeyId: 'accessKey1',
+        secretAccessKey: 'verySecretKey1',
+        // The endpoint must be s3.scality.test, else SSL will not work
+        endpoint: 'https://s3.scality.test:8000',
+        sslEnabled: true,
+        // With this setup, you must use path-style bucket access
+        s3ForcePathStyle: true,
+    });
+
+    const bucket = 'cocoriko';
+
+    s3.createBucket({ Bucket: bucket }, err => {
+        if (err) {
+            return console.log('err createBucket', err);
+        }
+        return s3.deleteBucket({ Bucket: bucket }, err => {
+            if (err) {
+                return console.log('err deleteBucket', err);
+            }
+            return console.log('SSL is cool!');
+        });
+    });
+
+Now run this script with:
+
+.. code::
+
+   $> nodejs test.js
+ 
+On success, the script outputs ``SSL is cool!``.
+
+
+.. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg
+   :target: https://circleci.com/gh/scality/S3
+.. |Scality CI| image:: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
+   :target: http://ci.ironmann.io/gh/scality/S3

docs/GET_BUCKET_V2.md

@@ -0,0 +1,69 @@
+# Get Bucket Version 2 Documentation
+
+## Description
+
+This feature implements version 2 of the GET Bucket (List Objects)
+operation, following AWS specifications
+(see https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html).
+
+## Requirements
+
+The user must have READ access to the bucket.
+
+## Design
+
+### Request
+
+The `delimiter`, `encoding-type`, `max-keys`, and `prefix` request parameters
+from GET Bucket v1 remain unchanged.
+In order to specify v2, the parameter `list-type` must be included and
+set to `2`.
+The `marker` v1 parameter's functionality has been split in two and replaced by
+`start-after` and `continuation-token` in v2. The `start-after` parameter is
+a specific object key after which the API will return key names. It is only
+valid in the first GET request. If both the `start-after` and
+`continuation-token` parameters are included in a request, the API will
+ignore the `start-after` parameter in favor of the `continuation-token`.
+If the GET Bucket v2 response is truncated, a `NextContinuationToken` will
+also be included. To list the next set of objects, the `NextContinuationToken`
+can be used as the `continuation-token` in the next request. The continuation
+token is an obfuscated string of 57 characters that CloudServer understands and
+interprets.
+By default, the v2 response does not include object owner information. To
+include owner information like the default v1 response, use the `fetch-owner`
+request parameter set to `true`.
+
+### Response
+
+The GET Bucket v1 and v2 responses are largely the same, with only a few changes.
+The `NextMarker` v1 parameter has been replaced by the
+`NextContinuationToken`. The `NextContinuationToken` is included with any
+truncated response, even if no delimiter is sent in the request. Its value is an
+obfuscated string that can be passed at the `continuation-token` in the next
+request, which will be interpreted by CloudServer.
+The `KeyCounter` parameter is returned in every response. Its value is the
+number of keys included in the response. It is always less than or equal to
+the `MaxKeys` value.
+If the `start-after` or `continuation-token` parameter is used in the
+request, it is also included in the response.
+By default, the v2 response does not include object owner information, unlike
+the v1 response. See the `Request` section for including it.
+
+### Continuation Token
+
+An example continuation token:
+
+```
+NextContinuationToken: '1bunC4s+crlZNAAbKUGBLyajJUQKp22TOdUR6/01snxD2cZtjJD0ugA=='
+```
+
+In order to generate a comparable token, CloudServer uses base64 encoding to
+obfuscate the key name of the next object to be listed.
+Encoded continuation tokens are similarly decoded in order for listing to
+continue from the correct object.
+
+## Performing Get Bucket V2 Operation
+
+When performing the GET Bucket V2 operation, if the request is built manually,
+the parameter `list-type` must be included and set to `2`.
+Using the AWS cli client, the command becomes `list-objects-v2`.

docs/INTEGRATIONS.rst

@@ -0,0 +1,565 @@
+Integrations
+++++++++++++
+
+High Availability
+=================
+
+`Docker Swarm <https://docs.docker.com/engine/swarm/>`__ is a clustering tool
+developed by Docker for use with its containers. It can be used to start
+services, which we define to ensure CloudServer's continuous availability to
+end users. A swarm defines a manager and *n* workers among *n* + 1 servers.
+
+This tutorial shows how to perform a basic setup with three servers, which
+provides strong service resiliency, while remaining easy to use and
+maintain. We will use NFS through Docker to share data and
+metadata between the different servers.
+
+Sections are labeled **On Server**, **On Clients**, or
+**On All Machines**, referring respectively to NFS server, NFS clients, or
+NFS server and clients. In the present example, the server’s IP address is
+**10.200.15.113** and the client IP addresses are **10.200.15.96** and
+**10.200.15.97**
+
+1. Install Docker (on All Machines)
+
+   Docker 17.03.0-ce is used for this tutorial. Docker 1.12.6 and later will
+   likely work, but is not tested.
+
+   * On Ubuntu 14.04
+     Install Docker CE for Ubuntu as `documented at Docker
+     <https://docs.docker.com/install/linux/docker-ce/ubuntu/>`__.
+     Install the aufs dependency as recommended by Docker. The required
+     commands are:
+
+     .. code:: sh
+
+      $> sudo apt-get update
+      $> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
+      $> sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
+      $> curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+      $> sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+      $> sudo apt-get update
+      $> sudo apt-get install docker-ce
+
+    * On CentOS 7
+      Install Docker CE as `documented at Docker
+      <https://docs.docker.com/install/linux/docker-ce/centos/>`__.
+      The required commands are:
+
+      .. code:: sh
+
+        $> sudo yum install -y yum-utils
+        $> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+        $> sudo yum makecache fast
+        $> sudo yum install docker-ce
+        $> sudo systemctl start docker
+
+2. Install NFS on Client(s)
+
+   NFS clients mount Docker volumes over the NFS server’s shared folders.
+   If the NFS commons are installed, manual mounts are no longer needed.
+
+   * On Ubuntu 14.04
+
+     Install the NFS commons with apt-get:
+
+     .. code:: sh
+
+       $> sudo apt-get install nfs-common
+
+   * On CentOS 7
+
+     Install the NFS utils; then start required services:
+
+     .. code:: sh
+
+      $> yum install nfs-utils
+      $> sudo systemctl enable rpcbind
+      $> sudo systemctl enable nfs-server
+      $> sudo systemctl enable nfs-lock
+      $> sudo systemctl enable nfs-idmap
+      $> sudo systemctl start rpcbind
+      $> sudo systemctl start nfs-server
+      $> sudo systemctl start nfs-lock
+      $> sudo systemctl start nfs-idmap
+
+3. Install NFS (on Server)
+
+   The NFS server hosts the data and metadata. The package(s) to install on it
+   differs from the package installed on the clients.
+
+   * On Ubuntu 14.04
+
+     Install the NFS server-specific package and the NFS commons:
+
+     .. code:: sh
+
+      $> sudo apt-get install nfs-kernel-server nfs-common
+
+   * On CentOS 7
+
+     Install the NFS utils and start the required services:
+
+     .. code:: sh
+
+       $> yum install nfs-utils
+       $> sudo systemctl enable rpcbind
+       $> sudo systemctl enable nfs-server
+       $> sudo systemctl enable nfs-lock
+       $> sudo systemctl enable nfs-idmap
+       $> sudo systemctl start rpcbind
+       $> sudo systemctl start nfs-server
+       $> sudo systemctl start nfs-lock
+       $> sudo systemctl start nfs-idmap
+
+   For both distributions:
+
+   #. Choose where shared data and metadata from the local
+      `CloudServer <http://www.zenko.io/cloudserver/>`__ shall be stored (The
+      present example uses /var/nfs/data and /var/nfs/metadata). Set permissions
+      for these folders for
+      sharing over NFS:
+
+      .. code:: sh
+
+        $> mkdir -p /var/nfs/data /var/nfs/metadata
+        $> chmod -R 777 /var/nfs/
+
+   #. The /etc/exports file configures network permissions and r-w-x permissions
+      for NFS access. Edit /etc/exports, adding the following lines:
+
+      .. code:: sh
+
+        /var/nfs/data        10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
+        /var/nfs/metadata    10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
+
+      Ubuntu applies the no\_subtree\_check option by default, so both
+      folders are declared with the same permissions, even though they’re in
+      the same tree.
+
+   #. Export this new NFS table:
+
+      .. code:: sh
+
+        $> sudo exportfs -a
+
+   #. Edit the ``MountFlags`` option in the Docker config in
+      /lib/systemd/system/docker.service to enable NFS mount from Docker volumes
+      on other machines:
+
+      .. code:: sh
+
+        MountFlags=shared
+
+   #. Restart the NFS server and Docker daemons to apply these changes.
+
+      * On Ubuntu 14.04
+
+        .. code:: sh
+
+          $> sudo service nfs-kernel-server restart
+          $> sudo service docker restart
+
+      * On CentOS 7
+
+        .. code:: sh
+
+         $> sudo systemctl restart nfs-server
+         $> sudo systemctl daemon-reload
+         $> sudo systemctl restart docker
+
+
+4. Set Up a Docker Swarm
+
+  * On all machines and distributions:
+
+    Set up the Docker volumes to be mounted to the NFS server for CloudServer’s
+    data and metadata storage. The following commands must be replicated on all
+    machines:
+
+    .. code:: sh
+
+     $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
+     $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
+
+    There is no need to ``docker exec`` these volumes to mount them: the
+    Docker Swarm manager does this when the Docker service is started.
+
+  * On a server:
+
+    To start a Docker service on a Docker Swarm cluster, initialize the cluster
+    (that is, define a manager), prompt workers/nodes to join in, and then start
+    the service.
+
+    Initialize the swarm cluster, and review its response:
+
+    .. code:: sh
+
+      $> docker swarm init --advertise-addr 10.200.15.113
+
+      Swarm initialized: current node (db2aqfu3bzfzzs9b1kfeaglmq) is now a manager.
+
+      To add a worker to this swarm, run the following command:
+
+      docker swarm join \
+      --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka \
+      10.200.15.113:2377
+
+      To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
+
+  * On clients:
+
+    Copy and paste the command provided by your Docker Swarm init. A successful
+    request/response will resemble:
+
+    .. code:: sh
+
+      $> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
+
+      This node joined a swarm as a worker.
+
+Set Up Docker Swarm on Clients on a Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Start the service on the Swarm cluster.
+
+.. code:: sh
+
+  $> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/cloudserver
+
+On a successful installation, ``docker service ls`` returns the following
+output:
+
+.. code:: sh
+
+    $> docker service ls
+    ID            NAME  MODE        REPLICAS  IMAGE
+    ocmggza412ft  s3    replicated  1/1       scality/cloudserver:latest
+
+If the service does not start, consider disabling apparmor/SELinux.
+
+Testing the High-Availability CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On all machines (client/server) and distributions (Ubuntu and CentOS),
+determine where CloudServer is running using ``docker ps``. CloudServer can
+operate on any node of the Swarm cluster, manager or worker. When you find
+it, you can kill it with ``docker stop <container id>``. It will respawn
+on a different node. Now, if one server falls, or if Docker stops
+unexpectedly, the end user will still be able to access your the local CloudServer.
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+To troubleshoot the service, run:
+
+.. code:: sh
+
+    $> docker service ps s3docker service ps s3
+    ID                         NAME      IMAGE             NODE                               DESIRED STATE  CURRENT STATE       ERROR
+    0ar81cw4lvv8chafm8pw48wbc  s3.1      scality/cloudserver  localhost.localdomain.localdomain  Running        Running 7 days ago
+    cvmf3j3bz8w6r4h0lf3pxo6eu   \_ s3.1  scality/cloudserver  localhost.localdomain.localdomain  Shutdown       Failed 7 days ago   "task: non-zero exit (137)"
+
+If the error is truncated, view the error in detail by inspecting the
+Docker task ID:
+
+.. code:: sh
+
+    $> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu
+
+Off you go!
+~~~~~~~~~~~
+
+Let us know how you use this and if you'd like any specific developments 
+around it. Even better: come and contribute to our `Github repository 
+<https://github.com/scality/s3/>`__! We look forward to meeting you!
+
+S3FS
+====
+
+You can export buckets as a filesystem with s3fs on CloudServer.
+
+`s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source
+tool, available both on Debian and RedHat distributions, that enables
+you to mount an S3 bucket on a filesystem-like backend. This tutorial uses
+an Ubuntu 14.04 host to deploy and use s3fs over CloudServer.
+
+Deploying Zenko CloudServer with SSL
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+First, deploy CloudServer with a file backend using `our DockerHub page
+<https://hub.docker.com/r/zenko/cloudserver>`__.
+
+.. note::
+
+  If Docker is not installed on your machine, follow
+  `these instructions <https://docs.docker.com/engine/installation/>`__
+  to install it for your distribution.
+
+You must also set up SSL with CloudServer to use s3fs. See `Using SSL
+<./GETTING_STARTED#Using_SSL>`__ for instructions.
+
+s3fs Setup
+~~~~~~~~~~
+
+Installing s3fs
+---------------
+
+Follow the instructions in the s3fs `README
+<https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation-from-pre-built-packages>`__,
+
+Check that s3fs is properly installed. A version check should return
+a response resembling:
+
+.. code:: sh
+
+    $> s3fs --version
+
+    Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL
+    Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
+    License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>
+    This is free software: you are free to change and redistribute it.
+    There is NO WARRANTY, to the extent permitted by law.
+
+Configuring s3fs
+----------------
+
+s3fs expects you to provide it with a password file. Our file is
+``/etc/passwd-s3fs``. The structure for this file is
+``ACCESSKEYID:SECRETKEYID``, so, for CloudServer, you can run:
+
+.. code:: sh
+
+    $> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs
+    $> chmod 600 /etc/passwd-s3fs
+
+Using CloudServer with s3fs
+---------------------------
+
+1. Use /mnt/tests3fs as a mount point.
+
+   .. code:: sh
+
+    $> mkdir /mnt/tests3fs
+
+2. Create a bucket on your local CloudServer. In the present example it is
+   named “tests3fs”.
+
+   .. code:: sh
+
+    $> s3cmd mb s3://tests3fs
+
+3. Mount the bucket to your mount point with s3fs:
+
+   .. code:: sh
+
+    $> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style
+
+   The structure of this command is:
+   ``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``. Of these mandatory
+   options:
+
+   * ``passwd_file`` specifies the path to the password file.
+   * ``url`` specifies the host name used by your SSL provider.
+   * ``use_path_request_style`` forces the path style (by default,
+       s3fs uses DNS-style subdomains).
+
+Once the bucket is mounted, files added to the mount point or
+objects added to the bucket will appear in both locations.
+
+Example
+-------
+
+   Create two files, and then a directory with a file in our mount point:
+
+   .. code:: sh
+
+      $> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
+      $> mkdir /mnt/tests3fs/dir1
+      $> touch /mnt/tests3fs/dir1/file3
+
+   Now, use s3cmd to show what is in CloudServer:
+
+   .. code:: sh
+
+      $> s3cmd ls -r s3://tests3fs
+
+      2017-02-28 17:28         0   s3://tests3fs/dir1/
+      2017-02-28 17:29         0   s3://tests3fs/dir1/file3
+      2017-02-28 17:28         0   s3://tests3fs/file1
+      2017-02-28 17:28         0   s3://tests3fs/file2
+
+   Now you can enjoy a filesystem view on your local CloudServer.
+
+
+Duplicity
+=========
+
+How to back up your files with CloudServer.
+
+Installing Duplicity and its Dependencies
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To install `Duplicity <http://duplicity.nongnu.org/>`__,
+go to `this site <https://code.launchpad.net/duplicity/0.7-series>`__.
+Download the latest tarball. Decompress it and follow the instructions
+in the README.
+
+.. code:: sh
+
+   $> tar zxvf duplicity-0.7.11.tar.gz
+   $> cd duplicity-0.7.11
+   $> python setup.py install
+
+You may receive error messages indicating the need to install some or all
+of the following dependencies:
+
+.. code:: sh
+
+    $> apt-get install librsync-dev gnupg
+    $> apt-get install python-dev python-pip python-lockfile
+    $> pip install -U boto
+
+Testing the Installation
+------------------------
+
+1. Check that CloudServer is running. Run ``$> docker ps``. You should
+   see one container named ``scality/cloudserver``. If you do not, run
+   ``$> docker start cloudserver`` and check again.
+
+
+2. Duplicity uses a module called “Boto” to send requests to S3. Boto
+   requires a configuration file located in ``/etc/boto.cfg`` to store
+   your credentials and preferences. A minimal configuration
+   you can fine tune `following these instructions
+   <http://boto.cloudhackers.com/en/latest/getting_started.html>`__ is
+   shown here:
+
+::
+
+    [Credentials]
+    aws_access_key_id = accessKey1
+    aws_secret_access_key = verySecretKey1
+
+    [Boto]
+    # If using SSL, set to True
+    is_secure = False
+    # If using SSL, unmute and provide absolute path to local CA certificate
+    # ca_certificates_file = /absolute/path/to/ca.crt
+
+ .. note:: To set up SSL with CloudServer, check out our `Using SSL
+	   <./GETTING_STARTED#Using_SSL>`__ in GETTING STARTED.
+
+3. At this point all requirements to run CloudServer as a backend to Duplicity
+   have been met. A local folder/file should back up to the local S3.
+   Try it with the decompressed Duplicity folder:
+
+.. code:: sh
+
+    $> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/"
+
+.. note:: Duplicity will prompt for a symmetric encryption passphrase.
+	  Save it carefully, as you will need it to recover your data.
+	  Alternatively, you can add the ``--no-encryption`` flag
+	  and the data will be stored plain.
+
+   If this command is successful, you will receive an output resembling:
+
+   .. code:: sh
+
+      --------------[ Backup Statistics ]--------------
+      StartTime 1486486547.13 (Tue Feb  7 16:55:47 2017)
+      EndTime 1486486547.40 (Tue Feb  7 16:55:47 2017)
+      ElapsedTime 0.27 (0.27 seconds)
+      SourceFiles 388
+      SourceFileSize 6634529 (6.33 MB)
+      NewFiles 388
+      NewFileSize 6634529 (6.33 MB)
+      DeletedFiles 0
+      ChangedFiles 0
+      ChangedFileSize 0 (0 bytes)
+      ChangedDeltaSize 0 (0 bytes)
+      DeltaEntries 388
+      RawDeltaSize 6392865 (6.10 MB)
+      TotalDestinationSizeChange 2003677 (1.91 MB)
+      Errors 0
+      -------------------------------------------------
+
+Congratulations! You can now back up to your local S3 through Duplicity.
+
+Automating Backups
+------------------
+
+The easiest way to back up files periodically is to write a bash script
+and add it to your crontab. A suggested script follows.
+
+.. code:: sh
+
+    #!/bin/bash
+
+    # Export your passphrase so you don't have to type anything
+    export PASSPHRASE="mypassphrase"
+
+    # To use a GPG key, put it here and uncomment the line below
+    #GPG_KEY=
+
+    # Define your backup bucket, with localhost specified
+    DEST="s3://127.0.0.1:8000/testbucketcloudserver/"
+
+    # Define the absolute path to the folder to back up
+    SOURCE=/root/testfolder
+
+    # Set to "full" for full backups, and "incremental" for incremental backups
+    # Warning: you must perform one full backup befor you can perform
+    # incremental ones on top of it
+    FULL=incremental
+
+    # How long to keep backups. If you don't want to delete old backups, keep
+    # this value empty; otherwise, the syntax is "1Y" for one year, "1M" for
+    # one month, "1D" for one day.
+    OLDER_THAN="1Y"
+
+    # is_running checks whether Duplicity is currently completing a task
+    is_running=$(ps -ef | grep duplicity  | grep python | wc -l)
+
+    # If Duplicity is already completing a task, this will not run
+    if [ $is_running -eq 0 ]; then
+        echo "Backup for ${SOURCE} started"
+
+        # To delete backups older than a certain time, do it here
+        if [ "$OLDER_THAN" != "" ]; then
+            echo "Removing backups older than ${OLDER_THAN}"
+            duplicity remove-older-than ${OLDER_THAN} ${DEST}
+        fi
+
+        # This is where the actual backup takes place
+        echo "Backing up ${SOURCE}..."
+        duplicity ${FULL} \
+            ${SOURCE} ${DEST}
+            # If you're using GPG, paste this in the command above
+            # --encrypt-key=${GPG_KEY} --sign-key=${GPG_KEY} \
+            # If you want to exclude a subfolder/file, put it below and
+            # paste this
+            # in the command above
+            # --exclude=/${SOURCE}/path_to_exclude \
+
+        echo "Backup for ${SOURCE} complete"
+        echo "------------------------------------"
+    fi
+    # Forget the passphrase...
+    unset PASSPHRASE
+
+Put this file in ``/usr/local/sbin/backup.sh``. Run ``crontab -e`` and
+paste your configuration into the file that opens. If you're unfamiliar
+with Cron, here is a good `HowTo
+<https://help.ubuntu.com/community/CronHowto>`__. If the folder being
+backed up is a folder to be modified permanently during the work day,
+we can set incremental backups every 5 minutes from 8 AM to 9 PM Monday
+through Friday by pasting the following line into crontab:
+
+.. code:: sh
+
+    */5 8-20 * * 1-5 /usr/local/sbin/backup.sh
+
+Adding or removing files from the folder being backed up will result in
+incremental backups in the bucket.

docs/MD_SEARCH.rst

@@ -0,0 +1,263 @@
+Metadata Search Documentation
+=============================
+
+Description
+-----------
+
+This feature enables metadata search to be performed on the metadata of objects
+stored in Zenko.
+
+Requirements
+------------
+
+* MongoDB
+
+Design
+------
+
+The Metadata Search feature expands on the existing :code:`GET Bucket` S3 API by
+enabling users to conduct metadata searches by adding the custom Zenko query
+string parameter, :code:`search`. The :code:`search` parameter is structured as a pseudo
+SQL WHERE clause, and supports basic SQL operators. For example:
+:code:`"A=1 AND B=2 OR C=3"` (complex queries can be built using nesting
+operators, :code:`(` and :code:`)`).
+
+The search process is as follows:
+
+* Zenko receives a :code:`GET` request.
+
+  .. code::
+
+    # regular getBucket request
+    GET /bucketname HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # getBucket versions request
+    GET /bucketname?versions HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+    # search getBucket request
+    GET /bucketname?search=key%3Dsearch-item HTTP/1.1
+    Host: 127.0.0.1:8000
+    Date: Wed, 18 Oct 2018 17:50:00 GMT
+    Authorization: authorization string
+
+* If the request does *not* contain the :code:`search` query parameter, Zenko performs
+  a normal bucket listing and returns an XML result containing the list of
+  objects.
+* If the request *does* contain the :code:`search` query parameter, Zenko parses and
+  validates the search string.
+
+  - If the search string is invalid, Zenko returns an :code:`InvalidArgument` error.
+
+    .. code::
+
+      <?xml version=\"1.0\" encoding=\"UTF-8\"?>
+      <Error>
+        <Code>InvalidArgument</Code>
+        <Message>Invalid sql where clause sent as search query</Message>
+        <Resource></Resource>
+        <RequestId>d1d6afc64345a8e1198e</RequestId>
+      </Error>
+
+  - If the search string is valid, Zenko parses it and generates an abstract
+    syntax tree (AST). The AST is then passed to the MongoDB backend to be
+    used as the query filter for retrieving objects from a bucket that
+    satisfies the requested search conditions. Zenko parses the filtered
+    results and returns them as the response.
+
+Metadata search results have the same structure as a :code:`GET Bucket` response:
+
+.. code:: xml
+
+  <?xml version="1.0" encoding="UTF-8"?>
+  <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+      <Name>bucketname</Name>
+      <Prefix/>
+      <Marker/>
+      <MaxKeys>1000</MaxKeys>
+      <IsTruncated>false</IsTruncated>
+      <Contents>
+          <Key>objectKey</Key>
+          <LastModified>2018-04-19T18:31:49.426Z</LastModified>
+          <ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
+          <Size>0</Size>
+          <Owner>
+              <ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
+              <DisplayName>Bart</DisplayName>
+          </Owner>
+          <StorageClass>STANDARD</StorageClass>
+      </Contents>
+      <Contents>
+          ...
+      </Contents>
+  </ListBucketResult>
+
+Performing Metadata Searches with Zenko
+---------------------------------------
+
+You can perform metadata searches by:
+
++ Using the :code:`search_bucket` tool in the
+  `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository.
++ Creating a signed HTTP request to Zenko in your preferred programming
+  language.
+
+Using the S3 Tool
++++++++++++++++++
+
+After cloning the `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository
+and installing the necessary dependencies, run the following command in the S3
+project’s root directory to access the search tool:
+
+.. code::
+
+  node bin/search_bucket
+
+This generates the following output:
+
+.. code::
+
+    Usage: search_bucket [options]
+
+    Options:
+
+      -V, --version                 output the version number
+      -a, --access-key <accessKey>  Access key id
+      -k, --secret-key <secretKey>  Secret access key
+      -b, --bucket <bucket>         Name of the bucket
+      -q, --query <query>           Search query
+      -h, --host <host>             Host of the server
+      -p, --port <port>             Port of the server
+      -s                            --ssl
+      -v, --verbose
+      -h, --help                    output usage information
+
+In the following examples, Zenko Server is accessible on endpoint
+:code:`http://127.0.0.1:8000` and contains the bucket :code:`zenkobucket`.
+
+.. code::
+
+    # search for objects with metadata "blue"
+    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+        -q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
+
+    # search for objects tagged with "type=color"
+    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
+        -q "tags.type=color" -h 127.0.0.1 -p 8000
+
+Coding Examples
++++++++++++++++
+
+Search requests can be also performed by making HTTP requests authenticated
+with one of the AWS Signature schemes: version 2 or version 4. \
+For more about authentication scheme, see:
+
+* https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
+* http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
+* http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+
+You can also view examples for making requests with Auth V4 in various
+languages `here <../../../examples>`__.
+
+Specifying Metadata Fields
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To search system metadata headers:
+
+.. code::
+
+    {system-metadata-key}{supported SQL op}{search value}
+    # example
+    key = blueObject
+    size > 0
+    key LIKE "blue.*"
+
+To search custom user metadata:
+
+.. code::
+
+    # metadata must be prefixed with "x-amz-meta-"
+    x-amz-meta-{user-metadata-key}{supported SQL op}{search value}
+
+    # example
+    x-amz-meta-color = blue
+    x-amz-meta-color != red
+    x-amz-meta-color LIKE "b.*"
+
+To search tags:
+
+.. code::
+
+    # tag searches must be prefixed with "tags."
+    tags.{tag-key}{supported SQL op}{search value}
+    # example
+    tags.type = color
+
+Examples queries:
+
+.. code::
+
+    # searching for objects with custom metadata "color"=blue" and are tagged
+    # "type"="color"
+
+    tags.type="color" AND  x-amz-meta-color="blue"
+
+    # searching for objects with the object key containing the substring "blue"
+    # or (custom metadata "color"=blue" and are tagged "type"="color")
+
+    key LIKE '.*blue.*' OR (x-amz-meta-color="blue" AND tags.type="color")
+
+Differences from SQL
+++++++++++++++++++++
+
+Zenko metadata search queries are similar to SQL-query :code:`WHERE` clauses, but
+differ in that:
+
+* They follow the :code:`PCRE` format
+* They do not require values with hyphens to be enclosed in
+  backticks, :code:``(`)``
+
+  .. code::
+
+        # SQL query
+        `x-amz-meta-search-item` = `ice-cream-cone`
+
+        # MD Search query
+        x-amz-meta-search-item = ice-cream-cone
+
+* Search queries do not support all SQL operators.
+
+  .. code::
+
+    # Supported SQL operators:
+    =, <, >, <=, >=, !=, AND, OR, LIKE, <>
+
+    # Unsupported SQL operators:
+    NOT, BETWEEN, IN, IS, +, -, %, ^, /, *, !
+
+Using Regular Expressions in Metadata Search
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Regular expressions in Zenko metadata search differ from SQL in the following
+ways:
+
++ Wildcards are represented with :code:`.*` instead of :code:`%`.
++ Regex patterns must be wrapped in quotes. Failure to do this can lead to
+  misinterpretation of patterns.
++ As with :code:`PCRE`, regular expressions can be entered in either the
+  :code:`/pattern/` syntax or as the pattern itself if regex options are
+  not required.
+
+Example regular expressions:
+
+.. code::
+
+    # search for strings containing word substring "helloworld"
+        ".*helloworld.*"
+        "/.*helloworld.*/"
+        "/.*helloworld.*/i"

docs/Makefile

@@ -0,0 +1,21 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Zenko
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+

docs/OBJECT_LOCK_TEST_PLAN.md

@@ -0,0 +1,161 @@
+# Object Lock Feature Test Plan
+
+## Feature Component Description
+
+Implementing Object Lock will introduce six new APIs:
+
+- putObjectLockConfiguration
+- getObjectLockConfiguration
+- putObjectRetention
+- getObjectRetention
+- putObjectLegalHold
+- getObjectLegalHold
+
+Along with these APIs, putBucket, putObject, deleteObject, and multiObjectDelete
+be affected. In Arsenal, both the BucketInfo and ObjectMD models will be
+updated. Bucket policy and IAM policy permissions will be updated to include
+the new API actions.
+
+## Functional Tests
+
+### putBucket tests
+
+- passing option to enable object lock updates bucket metadata and enables
+    bucket versioning
+
+### putBucketVersioning tests
+
+- suspending versioning on bucket with object lock enabled returns error
+
+### putObject tests
+
+- putting retention configuration on object should be allowed
+- putting invalid retention configuration returns error
+
+### getObject tests
+
+- getting object with retention information should include retention information
+
+### copyObject tests
+
+- copying object with retention information should include retention information
+
+### initiateMultipartUpload tests
+
+- mpu object initiated with retention information should include retention
+    information
+
+### putObjectLockConfiguration tests
+
+- putting configuration as non-bucket-owner user returns AccessDenied error
+- disabling object lock on bucket created with object lock returns error
+- enabling object lock on bucket created without object lock returns
+    InvalidBucketState error
+- enabling object lock with token on bucket created without object lock succeeds
+- putting valid object lock configuration when bucket does not have object
+    lock enabled returns error (InvalidRequest?)
+- putting valid object lock configuration updates bucket metadata
+- putting invalid object lock configuration returns error
+    - ObjectLockEnabled !== "Enabled"
+    - Rule object doesn't contain DefaultRetention key
+    - Mode !== "GOVERNANCE" or "COMPLIANCE"
+    - Days are not an integer
+    - Years are not an integer
+
+### getObjectLockConfiguration tests
+
+- getting configuration as non-bucket-owner user returns AccessDenied error
+- getting configuration when none is set returns
+    ObjectLockConfigurationNotFoundError error
+- getting configuration returns correct object lock configuration for bucket
+
+### putObjectRetention
+
+- putting retention as non-bucket-owner user returns AccessDenied error
+- putting retention on object in bucket without object lock enabled returns
+    InvalidRequest error
+- putting valid retention period updates object metadata
+
+### getObjectRetention
+
+- getting retention as non-bucket-owner user returns AccessDenied error
+- getting retention when none is set returns NoSuchObjectLockConfiguration
+    error
+- getting retention returns correct object retention period
+
+### putObjectLegalHold
+
+- putting legal hold as non-bucket-owner user returns AccessDenied error
+- putting legal hold on object in bucket without object lock enabled returns
+        InvalidRequest error
+- putting valid legal hold updates object metadata
+
+### getObjectLegalHold
+
+- getting legal hold as non-bucket-owner user returns AccessDenied error
+- getting legal hold when none is set returns NoSuchObjectLockConfiguration
+    error
+- getting legal hold returns correct object legal hold
+
+## End to End Tests
+
+### Scenarios
+
+- Create bucket with object lock enabled. Put object. Put object lock
+    configuration. Put another object.
+    - Ensure object put before configuration does not have retention period set
+    - Ensure object put after configuration does have retention period set
+
+- Create bucket without object lock. Put object. Enable object lock with token
+    and put object lock configuration. Put another object.
+    - Ensure object put before configuration does not have retention period set
+    - Ensure object put after configuration does have retention period set
+
+- Create bucket with object lock enabled and put configuration with COMPLIANCE
+    mode. Put object.
+    - Ensure object cannot be deleted (returns AccessDenied error).
+    - Ensure object cannot be overwritten.
+
+- Create bucket with object lock enabled and put configuration with GOVERNANCE
+    mode. Put object.
+    - Ensure user without permission cannot delete object
+    - Ensure user without permission cannot overwrite object
+    - Ensure user with permission can delete object
+    - Ensure user with permission can overwrite object
+    - Ensure user with permission can lengthen retention period
+    - Ensure user with permission cannot shorten retention period
+
+- Create bucket with object lock enabled and put configuration. Edit bucket
+    metadata so retention period is expired. Put object.
+    - Ensure object can be deleted.
+    - Ensure object can be overwritten.
+
+- Create bucket with object lock enabled and put configuration. Edit bucket
+    metadata so retention period is expired. Put object. Put new retention
+    period on object.
+    - Ensure object cannot be deleted.
+    - Ensure object cannot be overwritten.
+
+- Create bucket with object locked enabled and put configuration. Put object.
+    Edit object metadata so retention period is past expiration.
+    - Ensure object can be deleted.
+    - Ensure object can be overwritten.
+
+- Create bucket with object lock enabled and put configuration. Edit bucket
+    metadata so retention period is expired. Put object. Put legal hold
+    on object.
+    - Ensure object cannot be deleted.
+    - Ensure object cannot be overwritten.
+
+- Create bucket with object lock enabled and put configuration. Put object.
+    Check object retention. Change bucket object lock configuration.
+    - Ensure object retention period has not changed with bucket configuration.
+
+- Create bucket with object lock enabled. Put object with legal hold.
+    - Ensure object cannot be deleted.
+    - Ensure object cannot be overwritten.
+
+- Create bucket with object lock enabled. Put object with legal hold. Remove
+    legal hold.
+    - Ensure object can be deleted.
+    - Ensure object can be overwritten.

docs/RELEASE.md

@@ -0,0 +1,73 @@
+# Cloudserver Release Plan
+
+## Docker Image Generation
+
+Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
+CloudServer has a few images there:
+
+* Cloudserver container image: ghcr.io/scality/cloudserver
+* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
+* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
+
+With every CI build, the CI will push images, tagging the
+content with the developer branch's short SHA-1 commit hash.
+This allows those images to be used by developers, CI builds,
+build chain and so on.
+
+Tagged versions of cloudserver will be stored in the production namespace.
+
+## How to Pull Docker Images
+
+```sh
+docker pull ghcr.io/scality/cloudserver:<commit hash>
+docker pull ghcr.io/scality/cloudserver:<tag>
+```
+
+## Release Process
+
+To release a production image:
+
+* Create a PR to bump the package version
+  Update Cloudserver's `package.json` by bumping it to the relevant next
+  version in a new PR. Per example if the last released version was
+  `8.4.7`, the next version would be `8.4.8`.
+
+```js
+{
+  "name": "cloudserver",
+  "version": "8.4.8", <--- Here
+  [...]
+}
+```
+
+* Review & merge the PR
+
+* Create the release on GitHub
+  
+  * Go the Release tab (https://github.com/scality/cloudserver/releases);
+  * Click on the `Draft new release button`;
+  * In the `tag` field, type the name of the release (`8.4.8`), and confirm
+    to create the tag on publish;
+  * Click on `Generate release notes` button to fill the fields;
+  * Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
+  * Click to `Publish the release` to create the GitHub release and git tag
+
+  Notes:
+  * the Git tag will be created automatically.
+  * this should be done as soon as the PR is merged, so that the tag
+    is put on the "version bump" commit.
+
+* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
+
+  * Branch Name: The one used for the tag earlier. In this example `development/8.4`
+  * Override Stage: 'release'
+  * Extra properties:
+    * name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
+
+* Release the release version on Jira
+
+  * Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
+  * Create a next version
+    * Name: `[next version]`, in this example `8.4.9`
+  * Click `...` and select `Release` on the recently released version (`8.4.8`)
+  * Fill in the field to move incomplete version to the next one

docs/USING_PUBLIC_CLOUDS.rst

@@ -0,0 +1,398 @@
+.. _use-public-cloud:
+
+Using Public Clouds as data backends
+====================================
+
+Introduction
+------------
+
+As stated in our `GETTING STARTED guide <GETTING_STARTED.html#location-configuration>`__,
+new data backends can be added by creating a region (also called location
+constraint) with the right endpoint and credentials.
+This section of the documentation shows you how to set up our currently
+supported public cloud backends:
+
+- `Amazon S3 <#aws-s3-as-a-data-backend>`__ ;
+- `Microsoft Azure <#microsoft-azure-as-a-data-backend>`__ .
+
+For each public cloud backend, you will have to edit your CloudServer
+:code:`locationConfig.json` and do a few setup steps on the applicable public
+cloud backend.
+
+AWS S3 as a data backend
+------------------------
+
+From the AWS S3 Console (or any AWS S3 CLI tool)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create a bucket where you will host your data for this new location constraint.
+This bucket must have versioning enabled:
+
+- This is an option you may choose to activate at step 2 of Bucket Creation in
+  the Console;
+- With AWS CLI, use :code:`put-bucket-versioning` from the :code:`s3api`
+  commands on your bucket of choice;
+- Using other tools, please refer to your tool's documentation.
+
+In this example, our bucket will be named ``zenkobucket`` and has versioning
+enabled.
+
+From the CloudServer repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+locationConfig.json
+^^^^^^^^^^^^^^^^^^^
+
+Edit this file to add a new location constraint. This location constraint will
+contain the information for the AWS S3 bucket to which you will be writing your
+data whenever you create a CloudServer bucket in this location.
+There are a few configurable options here:
+
+- :code:`type` : set to :code:`aws_s3` to indicate this location constraint is
+  writing data to AWS S3;
+- :code:`legacyAwsBehavior` : set to :code:`true` to indicate this region should
+  behave like AWS S3 :code:`us-east-1` region, set to :code:`false` to indicate
+  this region should behave like any other AWS S3 region;
+- :code:`bucketName` : set to an *existing bucket* in your AWS S3 Account; this
+  is the bucket in which your data will be stored for this location constraint;
+- :code:`awsEndpoint` : set to your bucket's endpoint, usually :code:`s3.amazonaws.com`;
+- :code:`bucketMatch` : set to :code:`true` if you want your object name to be the
+  same in your local bucket and your AWS S3 bucket; set to :code:`false` if you
+  want your object name to be of the form :code:`{{localBucketName}}/{{objectname}}`
+  in your AWS S3 hosted bucket;
+- :code:`credentialsProfile` and :code:`credentials` are two ways to provide
+  your AWS S3 credentials for that bucket, *use only one of them* :
+
+  - :code:`credentialsProfile` : set to the profile name allowing you to access
+    your AWS S3 bucket from your :code:`~/.aws/credentials` file;
+  - :code:`credentials` : set the two fields inside the object (:code:`accessKey`
+    and :code:`secretKey`) to their respective values from your AWS credentials.
+
+.. code:: json
+
+    (...)
+    "aws-test": {
+        "type": "aws_s3",
+        "legacyAwsBehavior": true,
+        "details": {
+            "awsEndpoint": "s3.amazonaws.com",
+            "bucketName": "zenkobucket",
+            "bucketMatch": true,
+            "credentialsProfile": "zenko"
+        }
+    },
+    (...)
+
+.. code:: json
+
+    (...)
+    "aws-test": {
+        "type": "aws_s3",
+        "legacyAwsBehavior": true,
+        "details": {
+            "awsEndpoint": "s3.amazonaws.com",
+            "bucketName": "zenkobucket",
+            "bucketMatch": true,
+            "credentials": {
+                "accessKey": "WHDBFKILOSDDVF78NPMQ",
+                "secretKey": "87hdfGCvDS+YYzefKLnjjZEYstOIuIjs/2X72eET"
+            }
+        }
+    },
+    (...)
+
+.. WARNING::
+   If you set :code:`bucketMatch` to :code:`true`, we strongly advise that you
+   only have one local bucket per AWS S3 location.
+   Without :code:`bucketMatch` set to :code:`false`, your object names in your
+   AWS S3 bucket will not be prefixed with your Cloud Server bucket name. This
+   means that if you put an object :code:`foo` to your CloudServer bucket
+   :code:`zenko1` and you then put a different :code:`foo` to your CloudServer
+   bucket :code:`zenko2` and both :code:`zenko1` and :code:`zenko2` point to the
+   same AWS bucket, the second :code:`foo` will overwrite the first :code:`foo`.
+
+~/.aws/credentials
+^^^^^^^^^^^^^^^^^^
+
+.. TIP::
+   If you explicitly set your :code:`accessKey` and :code:`secretKey` in the 
+   :code:`credentials` object of your :code:`aws_s3` location in your
+   :code:`locationConfig.json` file, you may skip this section
+
+Make sure your :code:`~/.aws/credentials` file has a profile matching the one
+defined in your :code:`locationConfig.json`. Following our previous example, it
+would look like:
+
+
+.. code:: shell
+
+    [zenko]
+    aws_access_key_id=WHDBFKILOSDDVF78NPMQ
+    aws_secret_access_key=87hdfGCvDS+YYzefKLnjjZEYstOIuIjs/2X72eET
+
+Start the server with the ability to write to AWS S3
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Inside the repository, once all the files have been edited, you should be able
+to start the server and start writing data to AWS S3 through CloudServer.
+
+.. code:: shell
+
+   # Start the server locally
+   $> S3DATA=multiple yarn start
+
+Run the server as a docker container with the ability to write to AWS S3
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. TIP::
+   If you set the :code:`credentials` object in your
+   :code:`locationConfig.json` file, you don't need to mount your
+   :code:`.aws/credentials` file
+
+Mount all the files that have been edited to override defaults, and do a
+standard Docker run; then you can start writing data to AWS S3 through
+CloudServer.
+
+.. code:: shell
+
+   # Start the server in a Docker container
+   $> sudo docker run -d --name CloudServer \
+   -v $(pwd)/data:/usr/src/app/localData \
+   -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
+   -v ~/.aws/credentials:/root/.aws/credentials \
+   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 \
+   -d scality/cloudserver
+
+Testing: put an object to AWS S3 using CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to start testing pushing to AWS S3, you will need to create a local
+bucket in the AWS S3 location constraint - this local bucket will only store the
+metadata locally, while both the data and any user metadata (:code:`x-amz-meta`
+headers sent with a PUT object, and tags) will be stored on AWS S3.
+This example is based on all our previous steps.
+
+.. code:: shell
+
+   # Create a local bucket storing data in AWS S3
+   $> s3cmd --host=127.0.0.1:8000 mb s3://zenkobucket --region=aws-test
+   # Put an object to AWS S3, and store the metadata locally
+   $> s3cmd --host=127.0.0.1:8000 put /etc/hosts s3://zenkobucket/testput
+    upload: '/etc/hosts' -> 's3://zenkobucket/testput'  [1 of 1]
+     330 of 330   100% in    0s   380.87 B/s  done
+   # List locally to check you have the metadata
+   $> s3cmd --host=127.0.0.1:8000 ls s3://zenkobucket
+    2017-10-23 10:26       330   s3://zenkobucket/testput
+
+Then, from the AWS Console, if you go into your bucket, you should see your
+newly uploaded object:
+
+.. figure:: ../res/aws-console-successful-put.png
+   :alt: AWS S3 Console upload example
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+Make sure your :code:`~/.s3cfg` file has credentials matching your local
+CloudServer credentials defined in :code:`conf/authdata.json`. By default, the
+access key is :code:`accessKey1` and the secret key is :code:`verySecretKey1`.
+For more informations, refer to our template `~/.s3cfg <./CLIENTS/#s3cmd>`__ .
+
+Pre-existing objects in your AWS S3 hosted bucket can unfortunately not be
+accessed by CloudServer at this time.
+
+Make sure versioning is enabled in your remote AWS S3 hosted bucket. To check,
+using the AWS Console, click on your bucket name, then on "Properties" at the
+top, and then you should see something like this:
+
+.. figure:: ../res/aws-console-versioning-enabled.png
+   :alt: AWS Console showing versioning enabled
+
+Microsoft Azure as a data backend
+---------------------------------
+
+From the MS Azure Console
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+From your Storage Account dashboard, create a container where you will host your
+data for this new location constraint.
+
+You will also need to get one of your Storage Account Access Keys, and to
+provide it to CloudServer.
+This can be found from your Storage Account dashboard, under "Settings, then
+"Access keys".
+
+In this example, our container will be named ``zenkontainer``, and will belong
+to the ``zenkomeetups`` Storage Account.
+
+From the CloudServer repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+locationConfig.json
+^^^^^^^^^^^^^^^^^^^
+
+Edit this file to add a new location constraint. This location constraint will
+contain the information for the MS Azure container to which you will be writing
+your data whenever you create a CloudServer bucket in this location.
+There are a few configurable options here:
+
+- :code:`type` : set to :code:`azure` to indicate this location constraint is
+  writing data to MS Azure;
+- :code:`legacyAwsBehavior` : set to :code:`true` to indicate this region should
+  behave like AWS S3 :code:`us-east-1` region, set to :code:`false` to indicate
+  this region should behave like any other AWS S3 region (in the case of MS Azure
+  hosted data, this is mostly relevant for the format of errors);
+- :code:`azureStorageEndpoint` : set to your storage account's endpoint, usually
+  :code:`https://{{storageAccountName}}.blob.core.windows.net`;
+- :code:`azureContainerName` : set to an *existing container* in your MS Azure
+  storage account; this is the container in which your data will be stored for
+  this location constraint;
+- :code:`bucketMatch` : set to :code:`true` if you want your object name to be
+  the same in your local bucket and your MS Azure container; set to
+  :code:`false` if you want your object name to be of the form
+  :code:`{{localBucketName}}/{{objectname}}` in your MS Azure container ;
+- :code:`azureStorageAccountName` : the MS Azure Storage Account to which your
+  container belongs;
+- :code:`azureStorageAccessKey` : one of the Access Keys associated to the above
+  defined MS Azure Storage Account.
+
+.. code:: json
+
+    (...)
+    "azure-test": {
+	"type": "azure",
+        "legacyAwsBehavior": false,
+        "details": {
+          "azureStorageEndpoint": "https://zenkomeetups.blob.core.windows.net/",
+	  "bucketMatch": true,
+          "azureContainerName": "zenkontainer",
+	  "azureStorageAccountName": "zenkomeetups",
+	  "azureStorageAccessKey": "auhyDo8izbuU4aZGdhxnWh0ODKFP3IWjsN1UfFaoqFbnYzPj9bxeCVAzTIcgzdgqomDKx6QS+8ov8PYCON0Nxw=="
+	}
+    },
+    (...)
+
+.. WARNING::
+   If you set :code:`bucketMatch` to :code:`true`, we strongly advise that you
+   only have one local bucket per MS Azure location.
+   Without :code:`bucketMatch` set to :code:`false`, your object names in your
+   MS Azure container will not be prefixed with your Cloud Server bucket name.
+   This means that if you put an object :code:`foo` to your CloudServer bucket
+   :code:`zenko1` and you then put a different :code:`foo` to your CloudServer
+   bucket :code:`zenko2` and both :code:`zenko1` and :code:`zenko2` point to the
+   same MS Azure container, the second :code:`foo` will overwrite the first
+   :code:`foo`.
+
+.. TIP::
+   You may export environment variables to **override** some of your
+   :code:`locationConfig.json` variable ; the syntax for them is
+   :code:`{{region-name}}_{{ENV_VAR_NAME}}`; currently, the available variables
+   are those shown below, with the values used in the current example:
+
+   .. code:: shell
+
+      $> export azure-test_AZURE_STORAGE_ACCOUNT_NAME="zenkomeetups"
+      $> export azure-test_AZURE_STORAGE_ACCESS_KEY="auhyDo8izbuU4aZGdhxnWh0ODKFP3IWjsN1UfFaoqFbnYzPj9bxeCVAzTIcgzdgqomDKx6QS+8ov8PYCON0Nxw=="
+      $> export azure-test_AZURE_STORAGE_ENDPOINT="https://zenkomeetups.blob.core.windows.net/"
+
+Start the server with the ability to write to MS Azure
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Inside the repository, once all the files have been edited, you should be able
+to start the server and start writing data to MS Azure through CloudServer.
+
+.. code:: shell
+
+   # Start the server locally
+   $> S3DATA=multiple yarn start
+
+Run the server as a docker container with the ability to write to MS Azure
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Mount all the files that have been edited to override defaults, and do a
+standard Docker run; then you can start writing data to MS Azure through
+CloudServer.
+
+.. code:: shell
+
+   # Start the server in a Docker container
+   $> sudo docker run -d --name CloudServer \
+   -v $(pwd)/data:/usr/src/app/localData \
+   -v $(pwd)/metadata:/usr/src/app/localMetadata \
+   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
+   -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
+   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
+   -d scality/cloudserver
+
+Testing: put an object to MS Azure using CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to start testing pushing to MS Azure, you will need to create a local
+bucket in the MS Azure region - this local bucket will only store the metadata
+locally, while both the data and any user metadata (:code:`x-amz-meta` headers
+sent with a PUT object, and tags) will be stored on MS Azure.
+This example is based on all our previous steps.
+
+.. code:: shell
+
+   # Create a local bucket storing data in MS Azure
+   $> s3cmd --host=127.0.0.1:8000 mb s3://zenkontainer --region=azure-test
+   # Put an object to MS Azure, and store the metadata locally
+   $> s3cmd --host=127.0.0.1:8000 put /etc/hosts s3://zenkontainer/testput
+    upload: '/etc/hosts' -> 's3://zenkontainer/testput'  [1 of 1]
+     330 of 330   100% in    0s   380.87 B/s  done
+   # List locally to check you have the metadata
+   $> s3cmd --host=127.0.0.1:8000 ls s3://zenkobucket
+    2017-10-24 14:38       330   s3://zenkontainer/testput
+
+Then, from the MS Azure Console, if you go into your container, you should see
+your newly uploaded object:
+
+.. figure:: ../res/azure-console-successful-put.png
+   :alt: MS Azure Console upload example
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+Make sure your :code:`~/.s3cfg` file has credentials matching your local
+CloudServer credentials defined in :code:`conf/authdata.json`. By default, the
+access key is :code:`accessKey1` and the secret key is :code:`verySecretKey1`.
+For more informations, refer to our template `~/.s3cfg <./CLIENTS/#s3cmd>`__ .
+
+Pre-existing objects in your MS Azure container can unfortunately not be
+accessed by CloudServer at this time.
+
+For any data backend
+--------------------
+
+From the CloudServer repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+config.json
+^^^^^^^^^^^
+
+.. IMPORTANT::
+   You only need to follow this section if you want to define a given location
+   as the default for a specific endpoint
+
+Edit the :code:`restEndpoint` section of your :code:`config.json` file to add
+an endpoint definition matching the location you want to use as a default for an
+endpoint to this specific endpoint.
+In this example, we'll make :code:`custom-location` our default location for the
+endpoint :code:`zenkotos3.com`:
+
+.. code:: json
+
+    (...)
+    "restEndpoints": {
+        "localhost": "us-east-1",
+        "127.0.0.1": "us-east-1",
+        "cloudserver-front": "us-east-1",
+        "s3.docker.test": "us-east-1",
+        "127.0.0.2": "us-east-1",
+        "zenkotos3.com": "custom-location"
+    },
+    (...)
+

docs/conf.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+#
+# Zope docs documentation build configuration file, created by
+# sphinx-quickstart on Fri Feb 20 16:22:03 2009.
+#
+# This file is execfile()d with the current directory set to its containing
+# dir.
+#
+# The contents of this file are pickled, so don't put values in the namespace
+# that aren't pickleable (module imports are okay, they're removed
+# automatically).
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# import sys
+# import os
+
+# If your extensions are in another directory, add it here. If the directory
+# is relative to the documentation root, use os.path.abspath to make it
+# absolute, like shown here.
+# sys.path.append(os.path.abspath('.'))
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+# source_encoding = 'utf-8'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'scality-zenko-cloudserver'
+copyright = u'Apache License Version 2.0, 2004 http://www.apache.org/licenses/'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '7.0.0'
+# The full version, including alpha/beta/rc tags.
+release = '7.0.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+# unused_docs = []
+
+# List of directories, relative to source directory, that shouldn't be searched
+# for source files.
+exclude_trees = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for
+# all documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# Options for HTML output
+# -----------------------
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+html_style = 'css/default.css'
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+# html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+html_logo = '../res/scality-cloudserver-logo.png'
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+# html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_use_modindex = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+# html_copy_source = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = ''
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ZenkoCloudServerdoc'

docs/developers/getting-started.rst

@@ -0,0 +1,79 @@
+============================================
+Add New Backend Storage To Zenko CloudServer
+============================================
+
+This set of documents aims at bootstrapping developers with Zenko's CloudServer
+module, so they can then go on and contribute features.
+
+.. toctree::
+    :maxdepth: 2
+
+    non-s3-compatible-backend
+    s3-compatible-backend
+
+We always encourage our community to offer new extensions to Zenko,
+and new backend support is paramount to meeting more community needs.
+If that is something you want to contribute (or just do on your own
+version of the cloudserver image), this is the guid to read. Please
+make sure you follow our `Contributing Guidelines`_/.
+
+If you need help with anything, please search our `forum`_ for more
+information.
+
+Add support for a new backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently the main public cloud protocols are supported and more can
+be added. There are two main types of backend: those compatible with
+Amazon's S3 protocol and those not compatible.
+
+================= ========== ============ ===========
+Backend type      Supported  Active WIP   Not started
+================= ========== ============ ===========
+Private disk/fs       x
+AWS S3                x
+Microsoft Azure       x
+Backblaze B2                    x
+Google Cloud          x
+Openstack Swift                               x
+================= ========== ============ ===========
+
+.. important:: Should you want to request for a new backend to be
+               supported, please do so by opening a `Github issue`_,
+               and filling out the "Feature Request" section of our
+               template.
+
+To add support for a new backend support to CloudServer official
+repository, please follow these steps:
+
+- familiarize yourself with our `Contributing Guidelines`_
+- open a `Github issue`_ and fill out Feature Request form, and
+  specify you would like to contribute it yourself;
+- wait for our core team to get back to you with an answer on whether
+  we are interested in taking that contribution in (and hence
+  committing to maintaining it over time);
+- once approved, fork the repository and start your development;
+- use the `forum`_ with any question you may have during the
+  development process;
+- when you think it's ready, let us know so that we create a feature
+  branch against which we'll compare and review your code;
+- open a pull request with your changes against that dedicated feature
+  branch;
+- once that pull request gets merged, you're done.
+
+.. tip::
+
+    While we do take care of the final rebase (when we merge your feature
+    branch on the latest default branch), we do ask that you keep up to date with our latest default branch
+    until then.
+
+.. important::
+
+    If we do not approve your feature request, you may of course still
+    work on supporting a new backend: all our "no" means is that we do not
+    have the resources, as part of our core development team, to maintain
+    this feature for the moment.
+
+.. _GitHub issue: https://github.com/scality/S3/issues
+.. _Contributing Guidelines: https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md
+.. _forum: https://forum.zenko.io

docs/developers/new-backend.rst

@@ -0,0 +1,53 @@
+=================
+Add A New Backend
+=================
+
+Supporting all possible public cloud storage APIs is CloudServer's
+ultimate goal. As an open source project, contributions are welcome.
+
+The first step is to get familiar with building a custom Docker image
+for CloudServer.
+
+Build a Custom Docker Image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clone Zenko's CloudServer, install all dependencies and start the
+service:
+
+.. code-block:: shell
+
+  $ git clone https://github.com/scality/cloudserver
+  $ cd cloudserver
+  $ yarn install
+  $ yarn start
+
+.. tip::
+
+    Some optional dependencies may fail, resulting in you seeing `yarn
+    WARN` messages; these can safely be ignored.  Refer to the User
+    documentation for all available options.
+
+Build the Docker image:
+
+.. code-block:: shell
+
+     # docker build . -t
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+Push the newly created Docker image to your own hub:
+
+.. code-block:: shell
+
+     # docker push
+     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
+
+.. note::
+
+    To perform this last operation, you need to be authenticated with DockerHub
+
+There are two main types of backend you could want Zenko to support:
+
+== link:S3_COMPATIBLE_BACKENDS.adoc[S3 compatible data backends]
+
+== link:NON_S3_COMPATIBLE_BACKENDS.adoc[Data backends using another protocol than the S3 protocol]
+

docs/developers/non-s3-compatible-backend.rst

@@ -0,0 +1,530 @@
+==========================================================
+Adding support for data backends not supporting the S3 API
+==========================================================
+
+These backends abstract the complexity of multiple APIs to let users
+work on a single common namespace across multiple clouds.
+
+This documents aims at introducing you to the right files in
+CloudServer (the Zenko stack's subcomponent in charge of API
+translation, among other things) to add support to your own backend of
+choice.
+
+General configuration
+~~~~~~~~~~~~~~~~~~~~~
+
+There are a number of constants and environment variables to define to support a
+new data backend; here is a list and where to find them:
+
+:file:`/constants.js`
+---------------------
+
+ * give your backend type a name, as part of the `externalBackends` object;
+ * specify whether versioning is implemented, as part of the
+   `versioningNotImplemented` object;
+
+:file:`/lib/Config.js`
+----------------------
+
+ * this is where you should put common utility functions, like the ones to parse
+   the location object from `locationConfig.json`;
+ * make sure you define environment variables (like `GCP_SERVICE_EMAIL` as we'll
+   use those internally for the CI to test against the real remote backend;
+
+:file:`/lib/data/external/{{backendName}}Client.js`
+---------------------------------------------------
+
+ * this file is where you'll instantiate your backend client; this should be a
+   class with a constructor taking the config object built in `/lib/Config.js` as
+   parameter;
+ * over time, you may need some utility functions which we've defined in the
+   folder `/api/apiUtils`, and in the file `/lib/data/external/utils`;
+
+:file:`/lib/data/external/utils.js`
+-----------------------------------
+
+ * make sure to add options for `sourceLocationConstraintType` to be equal to
+   the name you gave your backend in :file:`/constants.js`;
+
+:file:`/lib/data/external/{{BackendName}}_lib/`
+-----------------------------------------------
+
+ * this folder is where you'll put the functions needed for supporting your
+   backend; keep your files as atomic as possible;
+
+:file:`/tests/locationConfig/locationConfigTests.json`
+------------------------------------------------------
+
+ * this file is where you'll create location profiles to be used by your
+   functional tests;
+
+:file:`/lib/data/locationConstraintParser.js`
+---------------------------------------------
+
+ * this is where you'll instantiate your client if the operation the end user
+   sent effectively writes to your backend; everything happens inside the
+   function `parseLC()`; you should add a condition that executes if
+   `locationObj.type` is the name of your backend (that you defined in
+   `constants.js`), and instantiates a client of yours. See pseudocode below,
+   assuming location type name is `ztore`:
+
+
+.. code-block:: js
+   :linenos:
+   :emphasize-lines: 12
+
+    (...) //<1>
+    const ZtoreClient = require('./external/ZtoreClient');
+    const { config } = require('../Config'); //<1>
+
+    function parseLC(){ //<1>
+    (...) //<1>
+        Object.keys(config.locationConstraints).forEach(location => { //<1>
+            const locationObj = config.locationConstraints[location]; //<1>
+            (...) //<1>
+            if (locationObj.type === 'ztore' {
+                const ztoreEndpoint = config.getZtoreEndpoint(location);
+                const ztoreCredentials = config.getZtoreCredentials(location); //<2>
+                clients[location] = new ZtoreClient({
+                    ztoreEndpoint,
+                    ztoreCredentials,
+                    ztoreBucketname: locationObj.details.ztoreBucketName,
+                    bucketMatch:  locationObj.details.BucketMatch,
+                    dataStoreName: location,
+                }); //<3>
+                clients[location].clientType = 'ztore';
+            });
+            (...) //<1>
+        });
+    }
+
+
+1. Code that is already there
+2. You may need more utility functions depending on your backend specs
+3. You may have more fields required in your constructor object depending on
+   your backend specs
+
+Operation of type PUT
+~~~~~~~~~~~~~~~~~~~~~
+
+PUT routes are usually where people get started, as it's the easiest to check!
+Simply go on your remote backend console and you'll be able to see whether your
+object actually went up in the cloud...
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `put()` function is also called
+  `put()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `put(stream, size, keyContext, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+  //TODO: generate this from jsdoc
+
+ - `stream`: the stream of data you want to put in the cloud; if you're
+   unfamiliar with node.js streams, we suggest you start training, as we use
+   them a lot !
+
+ - `size`: the size of the object you're trying to put;
+
+ - `keyContext`: an object with metadata about the operation; common entries are
+   `namespace`, `buckerName`, `owner`, `cipherBundle`, and `tagging`; if these
+   are not sufficient for your integration, contact us to get architecture
+   validation before adding new entries;
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your PUT operation, and
+  then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/put/put{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+  function returning ajusted `keys[]` to your tests.
+
+Operation of type GET
+~~~~~~~~~~~~~~~~~~~~~
+
+GET routes are easy to test after PUT routes are implemented, hence why we're
+covering them second.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `get()` function is also called
+  `get()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `get(objectGetInfo, range, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+
+ - `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+
+ - `range`: the range of bytes you will get, for "get-by-range" operations (we
+   recommend you do simple GETs first, and then look at this);
+
+ - `reqUids`: the request unique ID used for logging;
+
+ - `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your GET operation, and
+  then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type DELETE
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+DELETE routes are easy to test after PUT routes are implemented, and they are
+similar to GET routes in our implementation, hence why we're covering them
+third.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `delete()` function is also called
+  `delete()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `delete(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+//TODO: generate this from jsdoc
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your DELETE operation,
+  and then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/delete/delete{{BackendName}}js`
+-----------------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the
+          tutorial in order (that is, if you have covered the PUT operation
+          already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Operation of type HEAD
+~~~~~~~~~~~~~~~~~~~~~~
+
+HEAD routes are very similar to DELETE routes in our implementation, hence why
+we're covering them fourth.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `head()` function is also called
+  `head()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
+- define a function with signature like
+  `head(objectGetInfo, reqUids, callback)`; this is worth exploring a
+  bit more as these parameters are the same for all backends:
+
+// TODO:: generate this from jsdoc
+
+ * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
+   data store, and `client`, the data store name;
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your HEAD operation,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
+-----------------------------------------------------------------------------------
+
+- every contribution should come with thorough functional tests, showing
+  nominal context gives expected behaviour, and error cases are handled in a way
+  that is standard with the backend (including error messages and code);
+- the ideal setup is if you simulate your backend locally, so as not to be
+  subjected to network flakiness in the CI; however, we know there might not be
+  mockups available for every client; if that is the case of your backend, you
+  may test against the "real" endpoint of your data backend;
+
+:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
+-------------------------------------------------------------------
+
+.. note:: You should need this section if you have followed the tutorial in order
+          (that is, if you have covered the PUT operation already)
+
+- where you'll define a constant for your backend location matching your
+  :file:`/tests/locationConfig/locationConfigTests.json`
+- depending on your backend, the sample `keys[]` and associated made up objects
+  may not work for you (if your backend's key format is different, for example);
+  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
+
+Healthcheck
+~~~~~~~~~~~
+
+Healtchecks are used to make sure failure to write to a remote cloud is due to
+a problem on that remote cloud, an not on Zenko's side.
+This is usually done by trying to create a bucket that already exists, and
+making sure you get the expected answer.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+- the function that is going to call your `healthcheck()` function is called
+  `checkExternalBackend()` and it's defined in
+  :file:`/lib/data/multipleBackendGateway.js`; you will need to add your own;
+- your healtcheck function should get `location` as a parameter, which is an
+  object comprising:`
+
+ * `reqUids`: the request unique ID used for logging;
+ * `callback`: your function's callback (should handle errors);
+
+:file:`/lib/data/external/{{backendName}}_lib/{{backendName}}_create_bucket.js`
+-------------------------------------------------------------------------------
+
+- this is where you should write the function performing the actual bucket
+  creation;
+
+:file:`/lib/data/external/{{backendName}}_lib/utils.js`
+-------------------------------------------------------
+
+- add an object named per your backend's name to the `backendHealth` dictionary,
+  with proper `response` and `time` entries;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `healthcheck` function to add your location's array, and call your
+  healthcheck; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+
+.. code-block:: js
+   :linenos:
+
+    (...) //<1>
+
+        healthcheck: (flightCheckOnStartUp, log, callback) => { //<1>
+            (...) //<1>
+            const ztoreArray = []; //<2>
+            async.each(Object.keys(clients), (location, cb) => { //<1>
+                (...) //<1>
+                } else if (client.clientType === 'ztore' {
+                    ztoreArray.push(location); //<3>
+                    return cb();
+                }
+            (...) //<1>
+            multBackendResp[location] = { code: 200, message: 'OK' }; //<1>
+            return cb();
+        }, () => { //<1>
+            async.parallel([
+                (...) //<1>
+                next => checkExternalBackend( //<4>
+                    clients, ztoreArray, 'ztore', flightCheckOnStartUp,
+                    externalBackendHealthCheckInterval, next),
+            ] (...) //<1>
+            });
+            (...) //<1>
+        });
+    }
+
+1. Code that is already there
+2. The array that will store all locations of type 'ztore'
+3. Where you add locations of type 'ztore' to the array
+4. Where you actually call the healthcheck function on all 'ztore' locations
+
+Multipart upload (MPU)
+~~~~~~~~~~~~~~~~~~~~~~
+
+This is the final part to supporting a new backend! MPU is far from
+the easiest subject, but you've come so far it shouldn't be a problem.
+
+These are the files you'll need to edit:
+
+:file:`/lib/data/external/{{BackendName}}Client.js`
+---------------------------------------------------
+
+You'll be creating four functions with template signatures:
+
+- `createMPU(Key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
+  cacheControl, contentDisposition, contentEncoding, log, callback)` will
+  initiate the multi part upload process; now, here, all parameters are
+  metadata headers except for:
+
+ * `Key`, the key id for the final object (collection of all parts);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `uploadPart(request, streamingV4Params, stream, size, key, uploadId, partNumber, bucketName, log, callback)`
+   will be called for each part; the parameters can be explicited as follow:
+
+ * `request`, the request object for putting the part;
+ * `streamingV4Params`, parameters for auth V4 parameters against S3;
+ * `stream`, the node.js readable stream used to put the part;
+ * `size`, the size of the part;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `partNumber`, the number of the part in this MPU (ordered);
+ * `bucketName`, the name of the bucket to which we will do an MPU;
+ * `log`, the logger;
+
+- `completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback)` will
+   end the MPU process once all parts are uploaded; parameters can be explicited
+   as follows:
+
+ * `jsonList`, user-sent list of parts to include in final mpu object;
+ * `mdInfo`, object containing 3 keys: storedParts, mpuOverviewKey, and
+   splitter;
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance:
+
+- `abortMPU(key, uploadId, bucketName, log, callback)` will handle errors, and
+  make sure that all parts that may have been uploaded will be deleted if the
+  MPU ultimately fails; the parameters are:
+
+ * `key`, the key of the object;
+ * `uploadId`, multipart upload id string;
+ * `bucketName`, name of bucket;
+ * `log`, logger instance.
+
+:file:`/lib/api/objectPutPart.js`
+---------------------------------
+
+- you'll need to add your backend type in appropriate sections (simply look for
+  other backends already implemented).
+
+:file:`/lib/data/external/{{backendName}}_lib/`
+-----------------------------------------------
+
+- this is where you should put all utility functions for your MPU operations,
+  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
+  your code clean;
+
+:file:`lib/data/multipleBackendGateway.js`
+------------------------------------------
+
+- edit the `createMPU` function to add your location type, and call your
+  `createMPU()`; see pseudocode below for a sample implementation, provided your
+  backend name is `ztore`
+
+.. code-block:: javascript
+   :linenos:
+
+    (...) //<1>
+        createMPU:(key, metaHeaders, bucketName, websiteRedirectHeader, //<1>
+         location, contentType, cacheControl, contentDisposition,
+         contentEncoding, log, cb) => {
+            const client = clients[location]; //<1>
+            if (client.clientType === 'aws_s3') { //<1>
+                return client.createMPU(key, metaHeaders, bucketName,
+                websiteRedirectHeader, contentType, cacheControl,
+                contentDisposition, contentEncoding, log, cb);
+            } else if (client.clientType === 'ztore') { //<2>
+                return client.createMPU(key, metaHeaders, bucketName,
+                  websiteRedirectHeader, contentType, cacheControl,
+                  contentDisposition, contentEncoding, log, cb);
+            }
+            return cb();
+        };
+    (...) //<1>
+
+1. Code that is already there
+2. Where the `createMPU()` of your client is actually called
+
+Add functional tests
+~~~~~~~~~~~~~~~~~~~~
+
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/initMPU/{{BackendName}}InitMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/listParts/{{BackendName}}ListPart.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/{{BackendName}}AbortMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/{{BackendName}}CompleteMPU.js`
+* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/{{BackendName}}UploadPart.js`
+
+Adding support in Orbit, Zenko's UI for simplified Multi Cloud Management
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can only be done by our core developers' team. Once your backend
+integration is merged, you may open a feature request on the
+`Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/developers/s3-compatible-backend.rst

@@ -0,0 +1,43 @@
+======================
+S3-Compatible Backends
+======================
+
+
+Adding Support in CloudServer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is the easiest case for backend support integration: there is nothing to do
+but configuration!  Follow the steps described in our
+:ref:`use-public-cloud` and make sure you:
+
+- set ``details.awsEndpoint`` to your storage provider endpoint;
+
+- use ``details.credentials`` and *not* ``details.credentialsProfile`` to set your
+  credentials for that S3-compatible backend.
+
+For example, if you’re using a Wasabi bucket as a backend, then your region
+definition for that backend will look something like:
+::
+
+    "wasabi-bucket-zenkobucket": {
+    "type": "aws_s3",
+    "legacyAwsBehavior": true,
+    "details": {
+    "awsEndpoint": "s3.wasabisys.com",
+    "bucketName": "zenkobucket",
+    "bucketMatch": true,
+    "credentials": {
+    "accessKey": "\\{YOUR_WASABI_ACCESS_KEY}",
+    "secretKey": "\\{YOUR_WASABI_SECRET_KEY}"
+    }
+    }
+    },
+
+Adding Support in Zenko Orbit
+#############################
+
+This can only be done by our core developpers' team. If that’s what you’re
+after, open a feature request on the `Zenko repository`_, and we will
+get back to you after we evaluate feasability and maintainability.
+
+.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new

docs/index.rst

@@ -0,0 +1,18 @@
+Scality Zenko CloudServer
+=========================
+
+.. _user-docs:
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Documentation
+   :glob:
+
+   CONTRIBUTING
+   GETTING_STARTED
+   USING_PUBLIC_CLOUDS
+   CLIENTS
+   DOCKER
+   INTEGRATIONS
+   ARCHITECTURE
+   developers/*

docs/mkdocs.yml

@@ -0,0 +1,5 @@
+---
+# http://www.mkdocs.org/user-guide/configuration/
+# https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes
+
+site_name: Scality Zenko CloudServer documentation

docs/requirements.in

@@ -0,0 +1,2 @@
+Sphinx >= 1.7.5
+recommonmark >= 0.4.0

docs/requirements.txt

@@ -0,0 +1,119 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    tox -e pip-compile
+#
+alabaster==0.7.12 \
+    --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
+    --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 \
+    # via sphinx
+babel==2.6.0 \
+    --hash=sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669 \
+    --hash=sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23 \
+    # via sphinx
+certifi==2018.10.15 \
+    --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
+    --hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+commonmark==0.5.4 \
+    --hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5 \
+    # via recommonmark
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
+    # via recommonmark, sphinx
+idna==2.7 \
+    --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
+    --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
+    # via requests
+imagesize==1.1.0 \
+    --hash=sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8 \
+    --hash=sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5 \
+    # via sphinx
+jinja2==2.10 \
+    --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
+    --hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
+    # via sphinx
+markupsafe==1.1.0 \
+    --hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \
+    --hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \
+    --hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \
+    --hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \
+    --hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \
+    --hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \
+    --hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \
+    --hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \
+    --hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \
+    --hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \
+    --hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \
+    --hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \
+    --hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \
+    --hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \
+    --hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \
+    --hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \
+    --hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \
+    --hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \
+    --hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \
+    --hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \
+    --hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \
+    --hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \
+    --hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \
+    --hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \
+    --hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \
+    --hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \
+    --hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \
+    --hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 \
+    # via jinja2
+packaging==18.0 \
+    --hash=sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807 \
+    --hash=sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9 \
+    # via sphinx
+pygments==2.2.0 \
+    --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
+    --hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc \
+    # via sphinx
+pyparsing==2.3.0 \
+    --hash=sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b \
+    --hash=sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592 \
+    # via packaging
+pytz==2018.7 \
+    --hash=sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca \
+    --hash=sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6 \
+    # via babel
+recommonmark==0.4.0 \
+    --hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d \
+    --hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7
+requests==2.20.1 \
+    --hash=sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54 \
+    --hash=sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263 \
+    # via sphinx
+six==1.11.0 \
+    --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
+    --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \
+    # via packaging, sphinx
+snowballstemmer==1.2.1 \
+    --hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 \
+    --hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \
+    # via sphinx
+sphinx==1.8.2 \
+    --hash=sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea \
+    --hash=sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64
+sphinxcontrib-websupport==1.1.0 \
+    --hash=sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd \
+    --hash=sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9 \
+    # via sphinx
+typing==3.6.6 \
+    --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \
+    --hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
+    --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
+    # via sphinx
+urllib3==1.24.1 \
+    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
+    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via requests

examples/go-md-search.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+    "fmt"
+    "time"
+    "bytes"
+    "net/http"
+    "net/url"
+    "io/ioutil"
+    "github.com/aws/aws-sdk-go/aws/credentials"
+    "github.com/aws/aws-sdk-go/aws/signer/v4"
+)
+
+func main() {
+    // Input AWS access key, secret key
+    aws_access_key_id := "accessKey1"
+    aws_secret_access_key := "verySecretKey1"
+    endpoint := "http://localhost:8000"
+    bucket_name := "bucketname"
+    searchQuery := url.QueryEscape("x-amz-meta-color=blue")
+    buf := bytes.NewBuffer([]byte{})
+
+    requestUrl := fmt.Sprintf("%s/%s?search=%s",
+        endpoint, bucket_name, searchQuery)
+
+    request, err := http.NewRequest("GET", requestUrl, buf)
+    if err != nil {
+        panic(err)
+    }
+    reader := bytes.NewReader(buf.Bytes())
+    credentials := credentials.NewStaticCredentials(aws_access_key_id,
+        aws_secret_access_key, "")
+    signer := v4.NewSigner(credentials)
+    signer.Sign(request, reader, "s3", "us-east-1", time.Now())
+    client := &http.Client{}
+    resp, err := client.Do(request)
+    if err != nil {
+        panic(err)
+    }
+    defer resp.Body.Close()
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println(string(body))
+}

examples/node-md-search.js

@@ -0,0 +1,28 @@
+const { S3 } = require('aws-sdk');
+const config = {
+    sslEnabled: false,
+    endpoint: 'http://127.0.0.1:8000',
+    signatureCache: false,
+    signatureVersion: 'v4',
+    region: 'us-east-1',
+    s3ForcePathStyle: true,
+    accessKeyId: 'accessKey1',
+    secretAccessKey: 'verySecretKey1',
+};
+const s3Client = new S3(config);
+
+const encodedSearch =
+    encodeURIComponent('x-amz-meta-color="blue"');
+const req = s3Client.listObjects({ Bucket: 'bucketname' });
+
+// the build event
+req.on('build', () => {
+    req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
+});
+req.on('success', res => {
+    process.stdout.write(`Result ${res.data}`);
+});
+req.on('error', err => {
+    process.stdout.write(`Error ${err}`);
+});
+req.send();

examples/python-md-search.py

@@ -0,0 +1,79 @@
+import datetime
+import hashlib
+import hmac
+import urllib
+# pip install requests
+import requests
+
+access_key = 'accessKey1'
+secret_key = 'verySecretKey1'
+
+method = 'GET'
+service = 's3'
+host = 'localhost:8000'
+region = 'us-east-1'
+canonical_uri = '/bucketname'
+query = 'x-amz-meta-color=blue'
+canonical_querystring = 'search=%s' % (urllib.quote(query))
+algorithm = 'AWS4-HMAC-SHA256'
+
+t = datetime.datetime.utcnow()
+amz_date = t.strftime('%Y%m%dT%H%M%SZ')
+date_stamp = t.strftime('%Y%m%d')
+
+# Key derivation functions. See:
+# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
+
+
+def sign(key, msg):
+    return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
+
+
+def getSignatureKey(key, date_stamp, regionName, serviceName):
+    kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
+    kRegion = sign(kDate, regionName)
+    kService = sign(kRegion, serviceName)
+    kSigning = sign(kService, 'aws4_request')
+    return kSigning
+
+
+payload_hash = hashlib.sha256('').hexdigest()
+
+canonical_headers = \
+    'host:{0}\nx-amz-content-sha256:{1}\nx-amz-date:{2}\n' \
+    .format(host, payload_hash, amz_date)
+
+signed_headers = 'host;x-amz-content-sha256;x-amz-date'
+
+canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
+    .format(method, canonical_uri, canonical_querystring, canonical_headers,
+            signed_headers, payload_hash)
+print(canonical_request)
+
+credential_scope = '{0}/{1}/{2}/aws4_request' \
+    .format(date_stamp, region, service)
+
+string_to_sign = '{0}\n{1}\n{2}\n{3}' \
+    .format(algorithm, amz_date, credential_scope,
+            hashlib.sha256(canonical_request).hexdigest())
+
+signing_key = getSignatureKey(secret_key, date_stamp, region, service)
+
+signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
+                     hashlib.sha256).hexdigest()
+
+authorization_header = \
+    '{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}' \
+    .format(algorithm, access_key, credential_scope, signed_headers, signature)
+
+# The 'host' header is added automatically by the Python 'requests' library.
+headers = {
+    'X-Amz-Content-Sha256': payload_hash,
+    'X-Amz-Date': amz_date,
+    'Authorization': authorization_header
+}
+
+endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
+
+r = requests.get(endpoint, headers=headers)
+print(r.text)

images/svc-base/Dockerfile

@@ -0,0 +1,28 @@
+FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
+
+ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
+ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
+
+COPY . ${HOME_DIR}/s3
+RUN chown -R ${USER} ${HOME_DIR}
+RUN pip3 install redis===3.5.3 requests==2.27.1 && \
+    apt-get install -y git-lfs
+
+USER ${USER}
+WORKDIR ${HOME_DIR}/s3
+RUN rm -f ~/.gitconfig && \
+    git config --global --add safe.directory . && \
+    git lfs install && \
+    GIT_LFS_SKIP_SMUDGE=1 && \
+    yarn global add typescript && \
+    yarn install --frozen-lockfile --production --network-concurrency 1 && \
+    yarn cache clean --all && \
+    yarn global remove typescript
+
+# run symlinking separately to avoid yarn installation errors
+# we might have to check if the symlinking is really needed!
+RUN ln -sf /scality-kms node_modules
+
+EXPOSE 8000
+
+CMD bash -c "source ${CONF_DIR}/env && export && supervisord -c ${CONF_DIR}/supervisord.conf"

index.js

@@ -0,0 +1,10 @@
+'use strict'; // eslint-disable-line strict
+
+require('werelogs').stderrUtils.catchAndTimestampStderr(
+    undefined,
+    // Do not exit as workers have their own listener that will exit
+    // But primary don't have another listener
+    require('cluster').isPrimary ? 1 : null,
+);
+
+require('./lib/server.js')();

lib/api/api.js

@@ -0,0 +1,377 @@
+const { auth, errors, policies } = require('arsenal');
+const async = require('async');
+
+const bucketDelete = require('./bucketDelete');
+const bucketDeleteCors = require('./bucketDeleteCors');
+const bucketDeleteEncryption = require('./bucketDeleteEncryption');
+const bucketDeleteWebsite = require('./bucketDeleteWebsite');
+const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
+const bucketDeletePolicy = require('./bucketDeletePolicy');
+const bucketDeleteQuota = require('./bucketDeleteQuota');
+const { bucketGet } = require('./bucketGet');
+const bucketGetACL = require('./bucketGetACL');
+const bucketGetCors = require('./bucketGetCors');
+const bucketGetVersioning = require('./bucketGetVersioning');
+const bucketGetWebsite = require('./bucketGetWebsite');
+const bucketGetLocation = require('./bucketGetLocation');
+const bucketGetLifecycle = require('./bucketGetLifecycle');
+const bucketGetNotification = require('./bucketGetNotification');
+const bucketGetObjectLock = require('./bucketGetObjectLock');
+const bucketGetPolicy = require('./bucketGetPolicy');
+const bucketGetQuota = require('./bucketGetQuota');
+const bucketGetEncryption = require('./bucketGetEncryption');
+const bucketHead = require('./bucketHead');
+const { bucketPut } = require('./bucketPut');
+const bucketPutACL = require('./bucketPutACL');
+const bucketPutCors = require('./bucketPutCors');
+const bucketPutVersioning = require('./bucketPutVersioning');
+const bucketPutTagging = require('./bucketPutTagging');
+const bucketDeleteTagging = require('./bucketDeleteTagging');
+const bucketGetTagging = require('./bucketGetTagging');
+const bucketPutWebsite = require('./bucketPutWebsite');
+const bucketPutReplication = require('./bucketPutReplication');
+const bucketPutLifecycle = require('./bucketPutLifecycle');
+const bucketPutNotification = require('./bucketPutNotification');
+const bucketPutEncryption = require('./bucketPutEncryption');
+const bucketPutPolicy = require('./bucketPutPolicy');
+const bucketPutObjectLock = require('./bucketPutObjectLock');
+const bucketUpdateQuota = require('./bucketUpdateQuota');
+const bucketGetReplication = require('./bucketGetReplication');
+const bucketDeleteReplication = require('./bucketDeleteReplication');
+const corsPreflight = require('./corsPreflight');
+const completeMultipartUpload = require('./completeMultipartUpload');
+const initiateMultipartUpload = require('./initiateMultipartUpload');
+const listMultipartUploads = require('./listMultipartUploads');
+const listParts = require('./listParts');
+const metadataSearch = require('./metadataSearch');
+const { multiObjectDelete } = require('./multiObjectDelete');
+const multipartDelete = require('./multipartDelete');
+const objectCopy = require('./objectCopy');
+const { objectDelete } = require('./objectDelete');
+const objectDeleteTagging = require('./objectDeleteTagging');
+const objectGet = require('./objectGet');
+const objectGetACL = require('./objectGetACL');
+const objectGetLegalHold = require('./objectGetLegalHold');
+const objectGetRetention = require('./objectGetRetention');
+const objectGetTagging = require('./objectGetTagging');
+const objectHead = require('./objectHead');
+const objectPut = require('./objectPut');
+const objectPutACL = require('./objectPutACL');
+const objectPutLegalHold = require('./objectPutLegalHold');
+const objectPutTagging = require('./objectPutTagging');
+const objectPutPart = require('./objectPutPart');
+const objectPutCopyPart = require('./objectPutCopyPart');
+const objectPutRetention = require('./objectPutRetention');
+const objectRestore = require('./objectRestore');
+const prepareRequestContexts
+    = require('./apiUtils/authorization/prepareRequestContexts');
+const serviceGet = require('./serviceGet');
+const vault = require('../auth/vault');
+const website = require('./website');
+const writeContinue = require('../utilities/writeContinue');
+const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
+const parseCopySource = require('./apiUtils/object/parseCopySource');
+const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
+const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
+const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
+
+const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
+
+auth.setHandler(vault);
+
+/* eslint-disable no-param-reassign */
+const api = {
+    callApiMethod(apiMethod, request, response, log, callback) {
+        // Attach the apiMethod method to the request, so it can used by monitoring in the server
+        // eslint-disable-next-line no-param-reassign
+        request.apiMethod = apiMethod;
+        // Array of end of API callbacks, used to perform some logic
+        // at the end of an API.
+        // eslint-disable-next-line no-param-reassign
+        request.finalizerHooks = [];
+
+        const actionLog = monitoringMap[apiMethod];
+        if (!actionLog &&
+            apiMethod !== 'websiteGet' &&
+            apiMethod !== 'websiteHead' &&
+            apiMethod !== 'corsPreflight') {
+            log.error('callApiMethod(): No actionLog for this api method', {
+                apiMethod,
+            });
+        }
+        log.addDefaultFields({
+            service: 's3',
+            action: actionLog,
+            bucketName: request.bucketName,
+        });
+        if (request.objectKey) {
+            log.addDefaultFields({
+                objectKey: request.objectKey,
+            });
+        }
+        let returnTagCount = true;
+
+        const validationRes = validateQueryAndHeaders(request, log);
+        if (validationRes.error) {
+            log.debug('request query / header validation failed', {
+                error: validationRes.error,
+                method: 'api.callApiMethod',
+            });
+            return process.nextTick(callback, validationRes.error);
+        }
+
+        // no need to check auth on website or cors preflight requests
+        if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
+        apiMethod === 'corsPreflight') {
+            request.actionImplicitDenies = false;
+            return this[apiMethod](request, log, callback);
+        }
+
+        const { sourceBucket, sourceObject, sourceVersionId, parsingError } =
+            parseCopySource(apiMethod, request.headers['x-amz-copy-source']);
+        if (parsingError) {
+            log.debug('error parsing copy source', {
+                error: parsingError,
+            });
+            return process.nextTick(callback, parsingError);
+        }
+
+        const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers);
+        if (httpHeadersSizeError) {
+            log.debug('http header size limit exceeded', {
+                error: httpHeadersSizeError,
+            });
+            return process.nextTick(callback, httpHeadersSizeError);
+        }
+
+        const requestContexts = prepareRequestContexts(apiMethod, request,
+            sourceBucket, sourceObject, sourceVersionId);
+        // Extract all the _apiMethods and store them in an array
+        const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : [];
+        // Attach the names to the current request
+        // eslint-disable-next-line no-param-reassign
+        request.apiMethods = apiMethods;
+
+        function checkAuthResults(authResults) {
+            let returnTagCount = true;
+            const isImplicitDeny = {};
+            let isOnlyImplicitDeny = true;
+            if (apiMethod === 'objectGet') {
+                // first item checks s3:GetObject(Version) action
+                if (!authResults[0].isAllowed && !authResults[0].isImplicit) {
+                    log.trace('get object authorization denial from Vault');
+                    return errors.AccessDenied;
+                }
+                // TODO add support for returnTagCount in the bucket policy
+                // checks
+                isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
+                // second item checks s3:GetObject(Version)Tagging action
+                if (!authResults[1].isAllowed) {
+                    log.trace('get tagging authorization denial ' +
+                    'from Vault');
+                    returnTagCount = false;
+                }
+            } else {
+                for (let i = 0; i < authResults.length; i++) {
+                    isImplicitDeny[authResults[i].action] = true;
+                    if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
+                        // Any explicit deny rejects the current API call
+                        log.trace('authorization denial from Vault');
+                        return errors.AccessDenied;
+                    }
+                    if (authResults[i].isAllowed) {
+                        // If the action is allowed, the result is not implicit
+                        // Deny.
+                        isImplicitDeny[authResults[i].action] = false;
+                        isOnlyImplicitDeny = false;
+                    }
+                }
+            }
+            // These two APIs cannot use ACLs or Bucket Policies, hence, any
+            // implicit deny from vault must be treated as an explicit deny.
+            if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
+                return errors.AccessDenied;
+            }
+            return { returnTagCount, isImplicitDeny };
+        }
+
+        return async.waterfall([
+            next => auth.server.doAuth(
+                request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
+                    if (err) {
+                        // VaultClient returns standard errors, but the route requires
+                        // Arsenal errors
+                        const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
+                        log.trace('authentication error', { error: err });
+                        return next(arsenalError);
+                    }
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                }, 's3', requestContexts),
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => {
+                const authNames = { accountName: userInfo.getAccountDisplayName() };
+                if (userInfo.isRequesterAnIAMUser()) {
+                    authNames.userName = userInfo.getIAMdisplayName();
+                }
+                if (isRequesterASessionUser(userInfo)) {
+                    authNames.sessionName = userInfo.getShortid().split(':')[1];
+                }
+                log.addDefaultFields(authNames);
+                if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                }
+                // issue 100 Continue to the client
+                writeContinue(request, response);
+                const MAX_POST_LENGTH = request.method === 'POST' ?
+                      1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
+                const post = [];
+                let postLength = 0;
+                request.on('data', chunk => {
+                    postLength += chunk.length;
+                    // Sanity check on post length
+                    if (postLength <= MAX_POST_LENGTH) {
+                        post.push(chunk);
+                    }
+                });
+
+                request.on('error', err => {
+                    log.trace('error receiving request', {
+                        error: err,
+                    });
+                    return next(errors.InternalError);
+                });
+
+                request.on('end', () => {
+                    if (postLength > MAX_POST_LENGTH) {
+                        log.error('body length is too long for request type',
+                                  { postLength });
+                        return next(errors.InvalidRequest);
+                    }
+                    // Convert array of post buffers into one string
+                    request.post = Buffer.concat(post, postLength).toString();
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                });
+                return undefined;
+            },
+            // Tag condition keys require information from CloudServer for evaluation
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
+                authorizationResults,
+                request,
+                requestContexts,
+                apiMethod,
+                log,
+                (err, authResultsWithTags) => {
+                    if (err) {
+                        log.trace('tag authentication error', { error: err });
+                        return next(err);
+                    }
+                    return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
+                },
+            ),
+        ], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
+            if (err) {
+                return callback(err);
+            }
+            request.accountQuotas = infos?.accountQuota;
+            if (authorizationResults) {
+                const checkedResults = checkAuthResults(authorizationResults);
+                if (checkedResults instanceof Error) {
+                    return callback(checkedResults);
+                }
+                returnTagCount = checkedResults.returnTagCount;
+                request.actionImplicitDenies = checkedResults.isImplicitDeny;
+            } else {
+                // create an object of keys apiMethods with all values to false:
+                // for backward compatibility, all apiMethods are allowed by default
+                // thus it is explicitly allowed, so implicit deny is false
+                request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
+                    acc[curr] = false;
+                    return acc;
+                }, {});
+            }
+            const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
+                    (hook, done) => hook(err, done),
+                    () => callback(err, ...results));
+
+            if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
+                request._response = response;
+                return this[apiMethod](userInfo, request, streamingV4Params,
+                    log, methodCallback, authorizationResults);
+            }
+            if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
+                return this[apiMethod](userInfo, request, sourceBucket,
+                    sourceObject, sourceVersionId, log, methodCallback);
+            }
+            if (apiMethod === 'objectGet') {
+                return this[apiMethod](userInfo, request, returnTagCount, log, callback);
+            }
+            return this[apiMethod](userInfo, request, log, methodCallback);
+        });
+    },
+    bucketDelete,
+    bucketDeleteCors,
+    bucketDeleteEncryption,
+    bucketDeleteWebsite,
+    bucketGet,
+    bucketGetACL,
+    bucketGetCors,
+    bucketGetObjectLock,
+    bucketGetVersioning,
+    bucketGetWebsite,
+    bucketGetLocation,
+    bucketGetEncryption,
+    bucketHead,
+    bucketPut,
+    bucketPutACL,
+    bucketPutCors,
+    bucketPutVersioning,
+    bucketPutTagging,
+    bucketDeleteTagging,
+    bucketGetTagging,
+    bucketPutWebsite,
+    bucketPutReplication,
+    bucketGetReplication,
+    bucketDeleteReplication,
+    bucketDeleteQuota,
+    bucketPutLifecycle,
+    bucketUpdateQuota,
+    bucketGetLifecycle,
+    bucketDeleteLifecycle,
+    bucketPutPolicy,
+    bucketGetPolicy,
+    bucketGetQuota,
+    bucketDeletePolicy,
+    bucketPutObjectLock,
+    bucketPutNotification,
+    bucketGetNotification,
+    bucketPutEncryption,
+    corsPreflight,
+    completeMultipartUpload,
+    initiateMultipartUpload,
+    listMultipartUploads,
+    listParts,
+    metadataSearch,
+    multiObjectDelete,
+    multipartDelete,
+    objectDelete,
+    objectDeleteTagging,
+    objectGet,
+    objectGetACL,
+    objectGetLegalHold,
+    objectGetRetention,
+    objectGetTagging,
+    objectCopy,
+    objectHead,
+    objectPut,
+    objectPutACL,
+    objectPutLegalHold,
+    objectPutTagging,
+    objectPutPart,
+    objectPutCopyPart,
+    objectPutRetention,
+    objectRestore,
+    serviceGet,
+    websiteGet: website,
+    websiteHead: website,
+};
+
+module.exports = api;

lib/api/apiUtils/authorization/bucketOwner.js

@@ -0,0 +1,29 @@
+const { errors } = require('arsenal');
+const vault = require('../../../auth/vault');
+
+function checkExpectedBucketOwner(headers, bucket, log, cb) {
+    const expectedOwner = headers['x-amz-expected-bucket-owner'];
+    if (expectedOwner === undefined) {
+        return cb();
+    }
+
+    const bucketOwner = bucket.getOwner();
+    return vault.getAccountIds([bucketOwner], log, (error, res) => {
+        if (error) {
+            log.error('error fetch accountId from vault', {
+                method: 'checkExpectedBucketOwner',
+                error,
+            });
+        }
+
+        if (error || res[bucketOwner] !== expectedOwner) {
+            return cb(errors.AccessDenied);
+        }
+
+        return cb();
+    });
+}
+
+module.exports = {
+    checkExpectedBucketOwner,
+};

lib/api/apiUtils/authorization/permissionChecks.js

@@ -0,0 +1,641 @@
+const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
+const { errors } = require('arsenal');
+const { parseCIDR, isValid } = require('ipaddr.js');
+const constants = require('../../../../constants');
+const { config } = require('../../../Config');
+
+const {
+    allAuthedUsersId,
+    bucketOwnerActions,
+    logId,
+    publicId,
+    arrayOfAllowed,
+    assumedRoleArnResourceType,
+    backbeatLifecycleSessionName,
+    actionsToConsiderAsObjectPut,
+} = constants;
+
+// whitelist buckets to allow public read on objects
+const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
+    ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
+
+function getServiceAccountProperties(canonicalID) {
+    const canonicalIDArray = canonicalID.split('/');
+    const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
+    return constants.serviceAccountProperties[serviceName];
+}
+
+function isServiceAccount(canonicalID) {
+    return getServiceAccountProperties(canonicalID) !== undefined;
+}
+
+function isRequesterASessionUser(authInfo) {
+    const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
+    return regexpAssumedRoleArn.test(authInfo.getArn());
+}
+
+function isRequesterNonAccountUser(authInfo) {
+    return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
+}
+
+/**
+ * Checks the access control for a given bucket based on the request type and user's canonical ID.
+ *
+ * @param {Bucket} bucket - The bucket to check access control for.
+ * @param {string} requestType - The list of s3 actions to check within the API call.
+ * @param {string} canonicalID - The canonical ID of the user making the request.
+ * @param {string} mainApiCall - The main API call (first item of the requestType).
+ *
+ * @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
+ */
+
+function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
+    // Same logic applies on the Versioned APIs, so let's simplify it.
+    let requestTypeParsed = requestType.endsWith('Version') ?
+        requestType.slice(0, 'Version'.length * -1) : requestType;
+    requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
+        'objectPut' : requestTypeParsed;
+    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
+        'objectPut' : mainApiCall;
+    if (bucket.getOwner() === canonicalID) {
+        return true;
+    }
+    if (parsedMainApiCall === 'objectGet') {
+        if (requestTypeParsed === 'objectGetTagging') {
+            return true;
+        }
+    }
+    if (parsedMainApiCall === 'objectPut') {
+        if (arrayOfAllowed.includes(requestTypeParsed)) {
+            return true;
+        }
+    }
+
+    const bucketAcl = bucket.getAcl();
+    if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') {
+        if (bucketAcl.Canned === 'public-read'
+            || bucketAcl.Canned === 'public-read-write'
+            || (bucketAcl.Canned === 'authenticated-read'
+                && canonicalID !== publicId)) {
+            return true;
+        } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || bucketAcl.READ.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (bucketAcl.READ.indexOf(publicId) > -1
+            || (bucketAcl.READ.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
+    }
+    if (requestTypeParsed === 'bucketGetACL') {
+        if ((bucketAcl.Canned === 'log-delivery-write'
+            && canonicalID === logId)
+            || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (bucketAcl.READ_ACP.indexOf(publicId) > -1
+            || (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
+    }
+
+    if (requestTypeParsed === 'bucketPutACL') {
+        if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1
+            || (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
+    }
+
+    if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') {
+        if (bucketAcl.Canned === 'public-read-write'
+            || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || bucketAcl.WRITE.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (bucketAcl.WRITE.indexOf(publicId) > -1
+            || (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
+    }
+    // Note that an account can have the ability to do objectPutACL,
+    // objectGetACL, objectHead or objectGet even if the account has no rights
+    // to the bucket holding the object.  So, if the request type is
+    // objectPutACL, objectGetACL, objectHead or objectGet, the bucket
+    // authorization check should just return true so can move on to check
+    // rights at the object level.
+    return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
+    || requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
+}
+
+function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
+    isUserUnauthenticated, mainApiCall) {
+    const bucketOwner = bucket.getOwner();
+    const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
+        'objectPut' : requestType;
+    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
+        'objectPut' : mainApiCall;
+    // acls don't distinguish between users and accounts, so both should be allowed
+    if (bucketOwnerActions.includes(requestTypeParsed)
+        && (bucketOwner === canonicalID)) {
+        return true;
+    }
+    if (objectMD['owner-id'] === canonicalID) {
+        return true;
+    }
+
+    // Backward compatibility
+    if (parsedMainApiCall === 'objectGet') {
+        if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
+            && requestTypeParsed === 'objectGetTagging') {
+            return true;
+        }
+    }
+
+    if (!objectMD.acl) {
+        return false;
+    }
+
+    if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') {
+        if (objectMD.acl.Canned === 'public-read'
+            || objectMD.acl.Canned === 'public-read-write'
+            || (objectMD.acl.Canned === 'authenticated-read'
+                && canonicalID !== publicId)) {
+            return true;
+        } else if (objectMD.acl.Canned === 'bucket-owner-read'
+            && bucketOwner === canonicalID) {
+            return true;
+        } else if ((objectMD.acl.Canned === 'bucket-owner-full-control'
+            && bucketOwner === canonicalID)
+            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || objectMD.acl.READ.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (objectMD.acl.READ.indexOf(publicId) > -1
+            || (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
+    }
+
+    // User is already authorized on the bucket for FULL_CONTROL or WRITE or
+    // bucket has canned ACL public-read-write
+    if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') {
+        return true;
+    }
+
+    if (requestTypeParsed === 'objectPutACL') {
+        if ((objectMD.acl.Canned === 'bucket-owner-full-control'
+            && bucketOwner === canonicalID)
+            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1
+            || (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
+    }
+
+    if (requestTypeParsed === 'objectGetACL') {
+        if ((objectMD.acl.Canned === 'bucket-owner-full-control'
+            && bucketOwner === canonicalID)
+            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1
+            || (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
+    }
+
+    // allow public reads on buckets that are whitelisted for anonymous reads
+    // TODO: remove this after bucket policies are implemented
+    const bucketAcl = bucket.getAcl();
+    const allowPublicReads = publicReadBuckets.includes(bucket.getName())
+        && bucketAcl.Canned === 'public-read'
+        && (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
+    if (allowPublicReads) {
+        return true;
+    }
+    return false;
+}
+
+function _checkBucketPolicyActions(requestType, actions, log) {
+    const mappedAction = actionMaps.actionMapBP[requestType];
+    // Deny any action that isn't in list of controlled actions
+    if (!mappedAction) {
+        return false;
+    }
+    return evaluators.isActionApplicable(mappedAction, actions, log);
+}
+
+function _checkBucketPolicyResources(request, resource, log) {
+    if (!request || (Array.isArray(resource) && resource.length === 0)) {
+        return true;
+    }
+    // build request context from the request!
+    const requestContext = new RequestContext(request.headers, request.query,
+        request.bucketName, request.objectKey, null,
+        request.connection.encrypted, request.resourceType, 's3');
+    return evaluators.isResourceApplicable(requestContext, resource, log);
+}
+
+function _checkBucketPolicyConditions(request, conditions, log) {
+    const ip = request ? requestUtils.getClientIp(request, config) : undefined;
+    if (!conditions) {
+        return true;
+    }
+    // build request context from the request!
+    const requestContext = new RequestContext(request.headers, request.query,
+        request.bucketName, request.objectKey, ip,
+        request.connection.encrypted, request.resourceType, 's3', null, null,
+        null, null, null, null, null, null, null, null, null,
+        request.objectLockRetentionDays);
+    return evaluators.meetConditions(requestContext, conditions, log);
+}
+
+function _getAccountId(arn) {
+    // account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
+    return arn.substr(13, 12);
+}
+
+function _isAccountId(principal) {
+    return (principal.length === 12 && /^\d+$/.test(principal));
+}
+
+function _checkPrincipal(requester, principal) {
+    if (principal === '*') {
+        return true;
+    }
+    if (principal === requester) {
+        return true;
+    }
+    if (_isAccountId(principal)) {
+        return _getAccountId(requester) === principal;
+    }
+    if (principal.endsWith('root')) {
+        return _getAccountId(requester) === _getAccountId(principal);
+    }
+    return false;
+}
+
+function _checkPrincipals(canonicalID, arn, principal) {
+    if (principal === '*') {
+        return true;
+    }
+    if (principal.CanonicalUser) {
+        if (Array.isArray(principal.CanonicalUser)) {
+            return principal.CanonicalUser.some(p => _checkPrincipal(canonicalID, p));
+        }
+        return _checkPrincipal(canonicalID, principal.CanonicalUser);
+    }
+    if (principal.AWS) {
+        if (Array.isArray(principal.AWS)) {
+            return principal.AWS.some(p => _checkPrincipal(arn, p));
+        }
+        return _checkPrincipal(arn, principal.AWS);
+    }
+    return false;
+}
+
+function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) {
+    let permission = 'defaultDeny';
+    // if requester is user within bucket owner account, actions should be
+    // allowed unless explicitly denied (assumes allowed by IAM policy)
+    if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) {
+        permission = 'allow';
+    }
+    let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
+    while (copiedStatement.length > 0) {
+        const s = copiedStatement[0];
+        const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
+        const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
+        const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
+        const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
+
+        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') {
+            // explicit deny trumps any allows, so return immediately
+            return 'explicitDeny';
+        }
+        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') {
+            permission = 'allow';
+        }
+        copiedStatement = copiedStatement.splice(1);
+    }
+    return permission;
+}
+
+function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log,
+    request, aclPermission, results, actionImplicitDenies) {
+    const bucketPolicy = bucket.getBucketPolicy();
+    let processedResult = results[requestType];
+    if (!bucketPolicy) {
+        processedResult = actionImplicitDenies[requestType] === false && aclPermission;
+    } else {
+        const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
+            bucketOwner, log, request, actionImplicitDenies);
+
+        if (bucketPolicyPermission === 'explicitDeny') {
+            processedResult = false;
+        } else if (bucketPolicyPermission === 'allow') {
+            processedResult = true;
+        } else {
+            processedResult = actionImplicitDenies[requestType] === false && aclPermission;
+        }
+    }
+    return processedResult;
+}
+
+function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
+    actionImplicitDeniesInput = {}, isWebsite = false) {
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
+    const mainApiCall = requestTypes[0];
+    const results = {};
+    return requestTypes.every(_requestType => {
+        // By default, all missing actions are defined as allowed from IAM, to be
+        // backward compatible
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
+        // Check to see if user is authorized to perform a
+        // particular action on bucket based on ACLs.
+        // TODO: Add IAM checks
+        let requesterIsNotUser = true;
+        let arn = null;
+        if (authInfo) {
+            requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
+            arn = authInfo.getArn();
+        }
+        // if the bucket owner is an account, users should not have default access
+        if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
+            results[_requestType] = actionImplicitDenies[_requestType] === false;
+            return results[_requestType];
+        }
+        const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
+        // In case of error bucket access is checked with bucketGet
+        // For website, bucket policy only uses objectGet and ignores bucketGet
+        // https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
+        // bucketGet should be used to check acl but switched to objectGet for bucket policy
+        if (isWebsite && _requestType === 'bucketGet') {
+            // eslint-disable-next-line no-param-reassign
+            _requestType = 'objectGet';
+            actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
+        }
+        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
+            request, aclPermission, results, actionImplicitDenies);
+    });
+}
+
+function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {},
+    log, request) {
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
+    const results = {};
+    return requestTypes.every(_requestType => {
+        // By default, all missing actions are defined as allowed from IAM, to be
+        // backward compatible
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
+        let arn = null;
+        if (authInfo) {
+            arn = authInfo.getArn();
+        }
+        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
+        request, true, results, actionImplicitDenies);
+    });
+}
+
+function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
+    actionImplicitDeniesInput = {}, isWebsite = false) {
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
+    const results = {};
+    const mainApiCall = requestTypes[0];
+    return requestTypes.every(_requestType => {
+        // By default, all missing actions are defined as allowed from IAM, to be
+        // backward compatible
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
+        const parsedMethodName = _requestType.endsWith('Version')
+            ? _requestType.slice(0, -7) : _requestType;
+        const bucketOwner = bucket.getOwner();
+        if (!objectMD) {
+            // check bucket has read access
+            // 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
+            let permission = 'bucketGet';
+            if (actionsToConsiderAsObjectPut.includes(_requestType)) {
+                permission = 'objectPut';
+            }
+            results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
+                actionImplicitDenies, isWebsite);
+            // User is already authorized on the bucket for FULL_CONTROL or WRITE or
+            // bucket has canned ACL public-read-write
+            if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
+                && results[_requestType] === false) {
+                results[_requestType] = actionImplicitDenies[_requestType] === false;
+            }
+            return results[_requestType];
+        }
+        let requesterIsNotUser = true;
+        let arn = null;
+        let isUserUnauthenticated = false;
+        if (authInfo) {
+            requesterIsNotUser =  !isRequesterNonAccountUser(authInfo);
+            arn = authInfo.getArn();
+            isUserUnauthenticated = arn === undefined;
+        }
+        if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
+            results[_requestType] = actionImplicitDenies[_requestType] === false;
+            return results[_requestType];
+        }
+        // account is authorized if:
+        // - requesttype is included in bucketOwnerActions and
+        // - account is the bucket owner
+        // - requester is account, not user
+        if (bucketOwnerActions.includes(parsedMethodName)
+        && (bucketOwner === canonicalID)
+        && requesterIsNotUser) {
+            results[_requestType] = actionImplicitDenies[_requestType] === false;
+            return results[_requestType];
+        }
+        const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName,
+            canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall);
+        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner,
+            log, request, aclPermission, results, actionImplicitDenies);
+    });
+}
+
+function _checkResource(resource, bucketArn) {
+    if (resource === bucketArn) {
+        return true;
+    }
+    if (resource.includes('/')) {
+        const rSubs = resource.split('/');
+        return rSubs[0] === bucketArn;
+    }
+    return false;
+}
+
+// the resources specified in the bucket policy should contain the bucket name
+function validatePolicyResource(bucketName, policy) {
+    const bucketArn = `arn:aws:s3:::${bucketName}`;
+
+    return policy.Statement.every(s => {
+        if (Array.isArray(s.Resource)) {
+            return s.Resource.every(r => _checkResource(r, bucketArn));
+        }
+        if (typeof s.Resource === 'string') {
+            return _checkResource(s.Resource, bucketArn);
+        }
+        return false;
+    });
+}
+
+function checkIp(value) {
+    const errString = 'Invalid IP address in Conditions';
+
+    const values = Array.isArray(value) ? value : [value];
+
+    for (let i = 0; i < values.length; i++) {
+        // these preliminary checks are validating the provided
+        // ip address against ipaddr.js, the library we use when
+        // evaluating IP condition keys. It ensures compatibility,
+        // but additional checks are required to enforce the right
+        // notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
+        // we would accept different ip formats, which is not
+        // standard in an AWS use case.
+        try {
+            try {
+                parseCIDR(values[i]);
+            } catch (err) {
+                isValid(values[i]);
+            }
+        } catch (err) {
+            return errString;
+        }
+
+        // Apply the existing IP validation logic to each element
+        const validateIpRegex = ip => {
+            if (constants.ipv4Regex.test(ip)) {
+                return ip.split('.').every(part => parseInt(part, 10) <= 255);
+            }
+            if (constants.ipv6Regex.test(ip)) {
+                return ip.split(':').every(part => part.length <= 4);
+            }
+            return false;
+        };
+
+        if (validateIpRegex(values[i]) !== true) {
+            return errString;
+        }
+    }
+
+    // If the function hasn't returned by now, all elements are valid
+    return null;
+}
+
+// This function checks all bucket policy conditions if the values provided
+// are valid for the condition type. If not it returns a relevant Malformed policy error string
+function validatePolicyConditions(policy) {
+    const validConditions = [
+        { conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
+        { conditionKey: 's3:object-lock-remaining-retention-days' },
+    ];
+    // keys where value type does not seem to be checked by AWS:
+    // - s3:object-lock-remaining-retention-days
+
+    if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
+        return null;
+    }
+
+    // there can be multiple statements in the policy, each with a Condition enclosure
+    for (let i = 0; i < policy.Statement.length; i++) {
+        const s = policy.Statement[i];
+        if (s.Condition) {
+            const conditionOperators = Object.keys(s.Condition);
+            // there can be multiple condition operations in the Condition enclosure
+            // eslint-disable-next-line no-restricted-syntax
+            for (const conditionOperator of conditionOperators) {
+                const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
+                const conditionValue = s.Condition[conditionOperator][conditionKey];
+                const validCondition = validConditions.find(validCondition =>
+                    validCondition.conditionKey === conditionKey
+                );
+                // AWS returns does not return an error if the condition starts with 'aws:'
+                // so we reproduce this behaviour
+                if (!validCondition && !conditionKey.startsWith('aws:')) {
+                    return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
+                }
+                if (validCondition && validCondition.conditionValueTypeChecker) {
+                    const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
+                    if (conditionValueTypeError) {
+                        return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
+                    }
+                }
+            }
+        }
+    }
+    return null;
+}
+
+
+/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
+ * @param {string} arn - Amazon resource name - example:
+ * arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
+ * @return {boolean} true if Lifecycle assumed role session arn, false if not.
+ */
+function isLifecycleSession(arn) {
+    if (!arn) {
+        return false;
+    }
+
+    const arnSplits = arn.split(':');
+    const service = arnSplits[2];
+
+    const resourceNames = arnSplits[arnSplits.length - 1].split('/');
+
+    const resourceType = resourceNames[0];
+    const sessionName = resourceNames[resourceNames.length - 1];
+
+    return (service === 'sts'
+        && resourceType === assumedRoleArnResourceType
+        && sessionName === backbeatLifecycleSessionName);
+}
+
+module.exports = {
+    isBucketAuthorized,
+    isObjAuthorized,
+    getServiceAccountProperties,
+    isServiceAccount,
+    isRequesterASessionUser,
+    isRequesterNonAccountUser,
+    checkBucketAcls,
+    checkObjectAcls,
+    validatePolicyResource,
+    validatePolicyConditions,
+    isLifecycleSession,
+    evaluateBucketPolicyWithIAM,
+};

lib/api/apiUtils/authorization/prepareRequestContexts.js

@@ -0,0 +1,238 @@
+const { policies } = require('arsenal');
+const { config } = require('../../../Config');
+
+const { RequestContext, requestUtils } = policies;
+let apiMethodAfterVersionCheck;
+const apiMethodWithVersion = {
+    objectGetACL: true,
+    objectPutACL: true,
+    objectGet: true,
+    objectDelete: true,
+    objectPutTagging: true,
+    objectGetTagging: true,
+    objectDeleteTagging: true,
+    objectGetLegalHold: true,
+    objectPutLegalHold: true,
+    objectPutRetention: true,
+};
+
+function isHeaderAcl(headers) {
+    return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] ||
+    headers['x-amz-grant-write-acp'] || headers['x-amz-grant-full-control'] ||
+    headers['x-amz-acl'];
+}
+
+/**
+ * Prepares the requestContexts array to send to Vault for authorization
+ * @param {string} apiMethod - api being called
+ * @param {object} request - request object
+ * @param {string} sourceBucket - name of sourceBucket if copy request
+ * @param {string} sourceObject - name of sourceObject if copy request
+ * @param {string} sourceVersionId - value of sourceVersionId if copy request
+ * @return {RequestContext []} array of requestContexts
+ */
+function prepareRequestContexts(apiMethod, request, sourceBucket,
+    sourceObject, sourceVersionId) {
+    // if multiObjectDelete request, we want to authenticate
+    // before parsing the post body and creating multiple requestContexts
+    // so send null as requestContexts to Vault to avoid authorization
+    // checks at this point
+    //
+    // If bucketPut request, we want to do the authorization check in the API
+    // itself (once we parse the locationConstraint from the xml body) so send
+    // null as the requestContext to Vault so it will only do an authentication
+    // check.
+
+    const ip = requestUtils.getClientIp(request, config);
+
+    function generateRequestContext(apiMethod) {
+        return new RequestContext(request.headers,
+            request.query, request.bucketName, request.objectKey,
+            ip, request.connection.encrypted,
+            apiMethod, 's3');
+    }
+
+    if (apiMethod === 'bucketPut') {
+        return null;
+    }
+
+    if (apiMethodWithVersion[apiMethod] && request.query &&
+        request.query.versionId) {
+        apiMethodAfterVersionCheck = `${apiMethod}Version`;
+    } else {
+        apiMethodAfterVersionCheck = apiMethod;
+    }
+
+    const requestContexts = [];
+
+    if (apiMethod === 'multiObjectDelete') {
+        // MultiObjectDelete does not require any authorization when evaluating
+        // the API. Instead, we authorize each object passed.
+        // But in order to get any relevant information from the authorization service
+        // for example, the account quota, we must send a request context object
+        // with no `specificResource`. We expect the result to be an implicit deny.
+        // In the API, we then ignore these authorization results, and we can use
+        // any information returned, e.g., the quota.
+        const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
+          requestContexts.push(requestContextMultiObjectDelete);
+    } else if (apiMethodAfterVersionCheck === 'objectCopy'
+        || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
+        const objectGetAction = sourceVersionId ? 'objectGetVersion' :
+          'objectGet';
+        const reqQuery = Object.assign({}, request.query,
+            { versionId: sourceVersionId });
+        const getRequestContext = new RequestContext(request.headers,
+            reqQuery, sourceBucket, sourceObject,
+            ip, request.connection.encrypted,
+            objectGetAction, 's3');
+        const putRequestContext = generateRequestContext('objectPut');
+        requestContexts.push(getRequestContext, putRequestContext);
+        if (apiMethodAfterVersionCheck === 'objectCopy') {
+            // if tagging directive is COPY, "s3:PutObjectTagging" don't need
+            // to be included in the list of permitted actions in IAM policy
+            if (request.headers['x-amz-tagging'] &&
+                request.headers['x-amz-tagging-directive'] === 'REPLACE') {
+                const putTaggingRequestContext =
+                    generateRequestContext('objectPutTagging');
+                requestContexts.push(putTaggingRequestContext);
+            }
+            if (isHeaderAcl(request.headers)) {
+                const putAclRequestContext =
+                  generateRequestContext('objectPutACL');
+                requestContexts.push(putAclRequestContext);
+            }
+        }
+    } else if (apiMethodAfterVersionCheck === 'objectGet'
+               || apiMethodAfterVersionCheck === 'objectGetVersion') {
+        const objectGetTaggingAction = (request.query &&
+          request.query.versionId) ? 'objectGetTaggingVersion' :
+          'objectGetTagging';
+        if (request.headers['x-amz-version-id']) {
+            const objectGetVersionAction = 'objectGetVersion';
+            const getVersionResourceVersion =
+                generateRequestContext(objectGetVersionAction);
+            requestContexts.push(getVersionResourceVersion);
+        }
+        const getRequestContext =
+          generateRequestContext(apiMethodAfterVersionCheck);
+        const getTaggingRequestContext =
+          generateRequestContext(objectGetTaggingAction);
+        requestContexts.push(getRequestContext, getTaggingRequestContext);
+    } else if (apiMethodAfterVersionCheck === 'objectGetTagging') {
+        const objectGetTaggingAction = 'objectGetTagging';
+        const getTaggingResourceVersion =
+            generateRequestContext(objectGetTaggingAction);
+        requestContexts.push(getTaggingResourceVersion);
+        if (request.headers['x-amz-version-id']) {
+            const objectGetTaggingVersionAction = 'objectGetTaggingVersion';
+            const getTaggingVersionResourceVersion =
+                generateRequestContext(objectGetTaggingVersionAction);
+            requestContexts.push(getTaggingVersionResourceVersion);
+        }
+    } else if (apiMethodAfterVersionCheck === 'objectHead') {
+        const objectHeadAction = 'objectHead';
+        const headObjectAction =
+            generateRequestContext(objectHeadAction);
+        requestContexts.push(headObjectAction);
+        if (request.headers['x-amz-version-id']) {
+            const objectHeadVersionAction = 'objectGetVersion';
+            const headObjectVersion =
+                generateRequestContext(objectHeadVersionAction);
+            requestContexts.push(headObjectVersion);
+        }
+    } else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
+        const putObjectTaggingRequestContext =
+            generateRequestContext('objectPutTagging');
+        requestContexts.push(putObjectTaggingRequestContext);
+        if (request.headers['x-amz-version-id']) {
+            const putObjectVersionRequestContext =
+                generateRequestContext('objectPutTaggingVersion');
+            requestContexts.push(putObjectVersionRequestContext);
+        }
+    } else if (apiMethodAfterVersionCheck === 'objectPutCopyPart') {
+        const putObjectRequestContext =
+            generateRequestContext('objectPut');
+        requestContexts.push(putObjectRequestContext);
+        const getObjectRequestContext =
+            generateRequestContext('objectGet');
+        requestContexts.push(getObjectRequestContext);
+    } else if (apiMethodAfterVersionCheck === 'objectPut') {
+        // if put object with version
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
+            const putRequestContext =
+              generateRequestContext(apiMethodAfterVersionCheck);
+            requestContexts.push(putRequestContext);
+            // if put object (versioning) with tag set
+            if (request.headers['x-amz-tagging']) {
+                const putTaggingRequestContext =
+                  generateRequestContext('objectPutTagging');
+                requestContexts.push(putTaggingRequestContext);
+            }
+            if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
+                const putLegalHoldStatusAction =
+                    generateRequestContext('objectPutLegalHold');
+                requestContexts.push(putLegalHoldStatusAction);
+            }
+            // if put object (versioning) with ACL
+            if (isHeaderAcl(request.headers)) {
+                const putAclRequestContext =
+                  generateRequestContext('objectPutACL');
+                requestContexts.push(putAclRequestContext);
+            }
+            if (request.headers['x-amz-object-lock-mode']) {
+                const putObjectLockRequestContext =
+                  generateRequestContext('objectPutRetention');
+                requestContexts.push(putObjectLockRequestContext);
+            }
+            if (request.headers['x-amz-version-id']) {
+                const putObjectVersionRequestContext =
+                    generateRequestContext('objectPutTaggingVersion');
+                requestContexts.push(putObjectVersionRequestContext);
+            }
+        }
+    } else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
+      apiMethodAfterVersionCheck === 'objectPutPart' ||
+      apiMethodAfterVersionCheck === 'completeMultipartUpload'
+      ) {
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
+            const putRequestContext =
+              generateRequestContext(apiMethodAfterVersionCheck);
+            requestContexts.push(putRequestContext);
+        }
+
+        // if put object (versioning) with ACL
+        if (isHeaderAcl(request.headers)) {
+            const putAclRequestContext =
+              generateRequestContext('objectPutACL');
+            requestContexts.push(putAclRequestContext);
+        }
+
+        if (request.headers['x-amz-object-lock-mode']) {
+            const putObjectLockRequestContext =
+              generateRequestContext('objectPutRetention');
+            requestContexts.push(putObjectLockRequestContext);
+        }
+        if (request.headers['x-amz-version-id']) {
+            const putObjectVersionRequestContext =
+                generateRequestContext('objectPutTaggingVersion');
+            requestContexts.push(putObjectVersionRequestContext);
+        }
+    } else {
+        const requestContext =
+          generateRequestContext(apiMethodAfterVersionCheck);
+        requestContexts.push(requestContext);
+    }
+    return requestContexts;
+}
+
+module.exports = prepareRequestContexts;

lib/api/apiUtils/authorization/tagConditionKeys.js

@@ -0,0 +1,99 @@
+const async = require('async');
+
+const { auth, s3middleware } = require('arsenal');
+const metadata = require('../../../metadata/wrapper');
+const { decodeVersionId } = require('../object/versioning');
+
+const { parseTagXml } = s3middleware.tagging;
+
+function makeTagQuery(tags) {
+    return Object.entries(tags)
+        .map(i => i.join('='))
+        .join('&');
+}
+
+function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) {
+    async.waterfall([
+        next => {
+            if (request.headers['x-amz-tagging']) {
+                return next(null, request.headers['x-amz-tagging']);
+            }
+            if (request.post && apiMethod === 'objectPutTagging') {
+                return parseTagXml(request.post, log, (err, tags) => {
+                    if (err) {
+                        log.trace('error parsing request tags');
+                        return next(err);
+                    }
+                    return next(null, makeTagQuery(tags));
+                });
+            }
+            return next(null, null);
+        },
+        (requestTagsQuery, next) => {
+            const objectKey = request.objectKey;
+            const bucketName = request.bucketName;
+            const decodedVidResult = decodeVersionId(request.query);
+            if (decodedVidResult instanceof Error) {
+                log.trace('invalid versionId query', {
+                    versionId: request.query.versionId,
+                    error: decodedVidResult,
+                });
+                return next(decodedVidResult);
+            }
+            const reqVersionId = decodedVidResult;
+            return metadata.getObjectMD(
+                bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => {
+                    if (err) {
+                        // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
+                        if (err.NoSuchKey) {
+                            return next(null, requestTagsQuery, null);
+                        }
+                        log.trace('error getting request object tags');
+                        return next(err);
+                    }
+                    const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags);
+                    return next(null, requestTagsQuery, existingTagsQuery);
+                });
+        },
+    ], (err, requestTagsQuery, existingTagsQuery) => {
+        if (err) {
+            log.trace('error processing tag condition key evaluation');
+            return cb(err);
+        }
+        // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
+        // eslint-disable-next-line no-restricted-syntax
+        for (const rc of requestContexts) {
+            rc.setNeedTagEval(true);
+            if (requestTagsQuery) {
+                rc.setRequestObjTags(requestTagsQuery);
+            }
+            if (existingTagsQuery) {
+                rc.setExistingObjTag(existingTagsQuery);
+            }
+        }
+        return cb();
+    });
+}
+
+function tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log, cb) {
+    if (!authorizationResults) {
+        return cb();
+    }
+    if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
+        return cb(null, authorizationResults);
+    }
+
+    return updateRequestContextsWithTags(request, requestContexts, apiMethod, log, err => {
+        if (err) {
+            return cb(err);
+        }
+        return auth.server.doAuth(request, log,
+            (err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts);
+    });
+}
+
+module.exports = {
+    tagConditionKeyAuth,
+    updateRequestContextsWithTags,
+    makeTagQuery,
+};

lib/api/apiUtils/bucket/bucketCors.js

@@ -0,0 +1,356 @@
+const { parseString } = require('xml2js');
+const { errors, s3middleware } = require('arsenal');
+
+const escapeForXml = s3middleware.escapeForXml;
+
+/*
+    Format of xml request:
+
+    <CORSConfiguration>
+        <CORSRule>
+            <AllowedOrigin>http://www.example.com</AllowedOrigin>
+            <AllowedMethod>PUT</AllowedMethod>
+            <AllowedMethod>POST</AllowedMethod>
+            <AllowedMethod>DELETE</AllowedMethod>
+            <AllowedHeader>*</AllowedHeader>
+            <MaxAgeSeconds>3000</MaxAgeSec>
+            <ExposeHeader>x-amz-server-side-encryption</ExposeHeader>
+        </CORSRule>
+        <CORSRule>
+            <AllowedOrigin>*</AllowedOrigin>
+            <AllowedMethod>GET</AllowedMethod>
+            <AllowedHeader>*</AllowedHeader>
+            <MaxAgeSeconds>3000</MaxAgeSeconds>
+        </CORSRule>
+    </CORSConfiguration>
+*/
+
+const customizedErrs = {
+    numberRules: 'The number of CORS rules should not exceed allowed limit ' +
+    'of 100 rules.',
+    originAndMethodExist: 'Each CORSRule must identify at least one origin ' +
+    'and one method.',
+};
+
+// Helper validation methods
+const _validator = {
+    /** _validator.validateNumberWildcards - check if string has multiple
+    *   wildcards
+    @param {string} string - string to check for multiple wildcards
+    @return {boolean} - whether more than one wildcard in string
+    */
+    validateNumberWildcards(string) {
+        const firstIndex = string.indexOf('*');
+        if (firstIndex !== -1) {
+            return (string.indexOf('*', firstIndex + 1) === -1);
+        }
+        return true;
+    },
+    /** _validator.validateID - check value of optional ID
+    * @param {string[]} id - array containing id string
+    * @return {(Error|true|undefined)} - Arsenal error on failure, true on
+    *   success, undefined if ID does not exist
+    */
+    validateID(id) {
+        if (!id) {
+            return undefined; // to indicate ID does not exist
+        }
+        if (!Array.isArray(id) || id.length !== 1
+        || typeof id[0] !== 'string') {
+            return errors.MalformedXML;
+        }
+        if (id[0] === '') {
+            return undefined;
+        }
+        return true;
+    },
+    /** _validator.validateMaxAgeSeconds - check value of optional MaxAgeSeconds
+    * @param {string[]} seconds - array containing number string
+    * @return {(Error|parsedValue|undefined)} - Arsenal error on failure, parsed
+    *   value if valid, undefined if MaxAgeSeconds does not exist
+    */
+    validateMaxAgeSeconds(seconds) {
+        if (!seconds) {
+            return undefined;
+        }
+        if (!Array.isArray(seconds) || seconds.length !== 1) {
+            return errors.MalformedXML;
+        }
+        if (seconds[0] === '') {
+            return undefined;
+        }
+        const parsedValue = parseInt(seconds[0], 10);
+        const errMsg = `MaxAgeSeconds "${seconds[0]}" is not a valid value.`;
+        if (Number.isNaN(parsedValue) || parsedValue < 0) {
+            return errors.MalformedXML.customizeDescription(errMsg);
+        }
+        return parsedValue;
+    },
+    /** _validator.validateNumberRules - return if number of rules exceeds 100
+    * @param {number} length - array containing number string
+    * @return {(Error|true)} - Arsenal error on failure, true on success
+    */
+    validateNumberRules(length) {
+        if (length > 100) {
+            return errors.InvalidRequest
+            .customizeDescription(customizedErrs.numberRules);
+        }
+        return true;
+    },
+    /** _validator.validateOriginAndMethodExist
+    * @param {string[]} allowedMethods - array of AllowedMethod's
+    * @param {string[]} allowedOrigins - array of AllowedOrigin's
+    * @return {(Error|true)} - Arsenal error on failure, true on success
+    */
+    validateOriginAndMethodExist(allowedMethods, allowedOrigins) {
+        if (allowedOrigins && allowedMethods &&
+        Array.isArray(allowedOrigins) &&
+        Array.isArray(allowedMethods) &&
+        allowedOrigins.length > 0 &&
+        allowedMethods.length > 0) {
+            return true;
+        }
+        return errors.MalformedXML
+            .customizeDescription(customizedErrs.originAndMethodExist);
+    },
+    /** _validator.validateMethods - check values of AllowedMethod's
+    * @param {string[]} methods - array of AllowedMethod's
+    * @return {(Error|true)} - Arsenal error on failure, true on success
+    */
+    validateMethods(methods) {
+        let invalidMethod;
+        function isValidMethod(method) {
+            const acceptedValues = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE'];
+            if (acceptedValues.indexOf(method) !== -1) {
+                return true;
+            }
+            invalidMethod = method;
+            return false;
+        }
+        if (!methods.every(isValidMethod)) {
+            const errMsg = 'Found unsupported HTTP method in CORS config. ' +
+            `Unsupported method is "${invalidMethod}"`;
+            return errors.InvalidRequest.customizeDescription(errMsg);
+        }
+        return true;
+    },
+    /** _validator.validateAllowedOriginsOrHeaders - check values
+    * @param {string[]} elementArr - array of elements to check
+    * @param {string} typeElement - type of element being checked
+    * @return {(Error|true)} - Arsenal error on failure, true on success
+    */
+    validateAllowedOriginsOrHeaders(elementArr, typeElement) {
+        for (let i = 0; i < elementArr.length; i++) {
+            const element = elementArr[i];
+            if (typeof element !== 'string' || element === '') {
+                return errors.MalformedXML;
+            }
+            if (!this.validateNumberWildcards(element)) {
+                const errMsg = `${typeElement} "${element}" can not have ` +
+                'more than one wildcard.';
+                return errors.InvalidRequest.customizeDescription(errMsg);
+            }
+        }
+        return true;
+    },
+    /** _validator.validateAllowedHeaders - check values of AllowedHeader's
+    * @param {string[]} headers - array of AllowedHeader's
+    * @return {(Error|true|undefined)} - Arsenal error on failure, true if
+    *   valid, undefined if optional AllowedHeader's do not exist
+    */
+    validateAllowedHeaders(headers) {
+        if (!headers) {
+            return undefined; // to indicate AllowedHeaders do not exist
+        }
+        if (!Array.isArray(headers) || headers.length === 0) {
+            return errors.MalformedXML;
+        }
+        const result =
+            this.validateAllowedOriginsOrHeaders(headers, 'AllowedHeader');
+        if (result instanceof Error) {
+            return result;
+        }
+        return true;
+    },
+    /** _validator.validateExposeHeaders - check values of ExposeHeader's
+    * @param {string[]} headers - array of ExposeHeader's
+    * @return {(Error|true|undefined)} - Arsenal error on failure, true if
+    *   valid, undefined if optional ExposeHeader's do not exist
+    */
+    validateExposeHeaders(headers) {
+        if (!headers) {
+            return undefined; // indicate ExposeHeaders do not exist
+        }
+        if (!Array.isArray(headers) || headers.length === 0) {
+            return errors.MalformedXML;
+        }
+        for (let i = 0; i < headers.length; i++) {
+            const header = headers[i];
+            if (typeof header !== 'string') {
+                return errors.MalformedXML;
+            }
+            if (header.indexOf('*') !== -1) {
+                const errMsg = `ExposeHeader ${header} contains a wildcard. ` +
+                'Wildcards are currently not supported for ExposeHeader.';
+                return errors.InvalidRequest.customizeDescription(errMsg);
+            }
+            if (!/^[A-Za-z0-9-]*$/.test(header)) {
+                const errMsg = `ExposeHeader ${header} contains invalid ` +
+                'character.';
+                return errors.InvalidRequest.customizeDescription(errMsg);
+            }
+        }
+        return true;
+    },
+};
+
+/** _validateCorsXml - Validate XML, returning an error if any part is not valid
+* @param {object[]} rules - CORSRule collection parsed from xml to be validated
+* @param {string[]} [rules[].ID] - optional id to identify rule
+* @param {string[]} rules[].AllowedMethod - methods allowed for CORS
+* @param {string[]} rules[].AllowedOrigin - origins allowed for CORS
+* @param {string[]} [rules[].AllowedHeader] - headers allowed in an OPTIONS
+* request via the Access-Control-Request-Headers header
+* @param {string[]} [rules[].MaxAgeSeconds] - seconds browsers should cache
+* OPTIONS response
+* @param {string[]} [rules[].ExposeHeader] - headers exposed to applications
+* @return {(Error|object)} - return cors object on success; error on failure
+*/
+function _validateCorsXml(rules) {
+    const cors = [];
+    let result;
+
+    if (rules.length > 100) {
+        return errors.InvalidRequest
+            .customizeDescription(customizedErrs.numberRules);
+    }
+    for (let i = 0; i < rules.length; i++) {
+        const rule = rules[i];
+        const corsRule = {};
+
+        result = _validator.validateOriginAndMethodExist(rule.AllowedMethod,
+            rule.AllowedOrigin);
+        if (result instanceof Error) {
+            return result;
+        }
+
+        result = _validator.validateMethods(rule.AllowedMethod);
+        if (result instanceof Error) {
+            return result;
+        }
+        corsRule.allowedMethods = rule.AllowedMethod;
+
+        result = _validator.validateAllowedOriginsOrHeaders(rule.AllowedOrigin,
+            'AllowedOrigin');
+        if (result instanceof Error) {
+            return result;
+        }
+        corsRule.allowedOrigins = rule.AllowedOrigin;
+
+        result = _validator.validateID(rule.ID);
+        if (result instanceof Error) {
+            return result;
+        } else if (result) {
+            corsRule.id = rule.ID[0];
+        }
+
+        result = _validator.validateAllowedHeaders(rule.AllowedHeader);
+        if (result instanceof Error) {
+            return result;
+        } else if (result) {
+            corsRule.allowedHeaders = rule.AllowedHeader;
+        }
+
+        result = _validator.validateMaxAgeSeconds(rule.MaxAgeSeconds);
+        if (result instanceof Error) {
+            return result;
+        } else if (result) {
+            corsRule.maxAgeSeconds = result;
+        }
+
+        result = _validator.validateExposeHeaders(rule.ExposeHeader);
+        if (result instanceof Error) {
+            return result;
+        } else if (result) {
+            corsRule.exposeHeaders = rule.ExposeHeader;
+        }
+
+        cors.push(corsRule);
+    }
+    return cors;
+}
+
+/** parseCorsXml - Parse and validate xml body, returning cors object on success
+* @param {string} xml - xml body to parse and validate
+* @param {object} log - Werelogs logger
+* @param {function} cb - callback to server
+* @return {undefined} - calls callback with cors object on success, error on
+*   failure
+*/
+function parseCorsXml(xml, log, cb) {
+    parseString(xml, (err, result) => {
+        if (err) {
+            log.trace('xml parsing failed', {
+                error: err,
+                method: 'parseCorsXml',
+            });
+            log.debug('invalid xml', { xml });
+            return cb(errors.MalformedXML);
+        }
+
+        if (!result || !result.CORSConfiguration ||
+            !result.CORSConfiguration.CORSRule ||
+            !Array.isArray(result.CORSConfiguration.CORSRule)) {
+            const errMsg = 'Invalid cors configuration xml';
+            return cb(errors.MalformedXML.customizeDescription(errMsg));
+        }
+
+        const validationRes =
+            _validateCorsXml(result.CORSConfiguration.CORSRule);
+        if (validationRes instanceof Error) {
+            log.debug('xml validation failed', {
+                error: validationRes,
+                method: '_validateCorsXml',
+                xml,
+            });
+            return cb(validationRes);
+        }
+        // if no error, validation returns cors object
+        return cb(null, validationRes);
+    });
+}
+
+function convertToXml(arrayRules) {
+    const xml = [];
+    xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
+    '<CORSConfiguration>');
+    arrayRules.forEach(rule => {
+        xml.push('<CORSRule>');
+        ['allowedMethods', 'allowedOrigins', 'allowedHeaders', 'exposeHeaders']
+        .forEach(key => {
+            if (rule[key]) {
+                const element = key.charAt(0).toUpperCase() +
+                    key.slice(1, -1);
+                rule[key].forEach(value => {
+                    xml.push(`<${element}>${escapeForXml(value)}` +
+                    `</${element}>`);
+                });
+            }
+        });
+        if (rule.id) {
+            xml.push(`<ID>${escapeForXml(rule.id)}</ID>`);
+        }
+        if (rule.maxAgeSeconds) {
+            xml.push(`<MaxAgeSeconds>${rule.maxAgeSeconds}</MaxAgeSeconds>`);
+        }
+        xml.push('</CORSRule>');
+    });
+    xml.push('</CORSConfiguration>');
+    return xml.join('');
+}
+
+module.exports = {
+    _validator,
+    parseCorsXml,
+    convertToXml,
+};

lib/api/apiUtils/bucket/bucketCreation.js

@@ -0,0 +1,288 @@
+const assert = require('assert');
+const async = require('async');
+const { errors } = require('arsenal');
+
+const acl = require('../../../metadata/acl');
+const BucketInfo = require('arsenal').models.BucketInfo;
+const constants = require('../../../../constants');
+const createKeyForUserBucket = require('./createKeyForUserBucket');
+const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
+const metadata = require('../../../metadata/wrapper');
+const kms = require('../../../kms/wrapper');
+const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
+const { isServiceAccount } = require('../authorization/permissionChecks');
+
+const usersBucket = constants.usersBucket;
+const oldUsersBucket = constants.oldUsersBucket;
+const zenkoSeparator = constants.zenkoSeparator;
+const userBucketOwner = 'admin';
+
+
+function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
+    // BACKWARD: Simplify once do not have to deal with old
+    // usersbucket name and old splitter
+
+    // Get new format usersBucket to see if it exists
+    return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => {
+        if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) {
+            return cb(err);
+        }
+        const splitter = usersBucketAttrs ?
+            constants.splitter : constants.oldSplitter;
+        let key = createKeyForUserBucket(canonicalID, splitter, bucketName);
+        const omVal = {
+            creationDate: new Date().toJSON(),
+            ingestion: bucketMD.getIngestion(),
+        };
+        // If the new format usersbucket does not exist, try to put the
+        // key in the old usersBucket using the old splitter.
+        // Otherwise put the key in the new format usersBucket
+        const usersBucketBeingCalled = usersBucketAttrs ?
+            usersBucket : oldUsersBucket;
+        return metadata.putObjectMD(usersBucketBeingCalled, key,
+            omVal, {}, log, err => {
+                if (err?.is?.NoSuchBucket) {
+                    // There must be no usersBucket so createBucket
+                    // one using the new format
+                    log.trace('users bucket does not exist, ' +
+                        'creating users bucket');
+                    key = `${canonicalID}${constants.splitter}` +
+                        `${bucketName}`;
+                    const creationDate = new Date().toJSON();
+                    const freshBucket = new BucketInfo(usersBucket,
+                        userBucketOwner, userBucketOwner, creationDate,
+                        BucketInfo.currentModelVersion());
+                    return metadata.createBucket(usersBucket,
+                        freshBucket, log, err => {
+                            // Note: In the event that two
+                            // users' requests try to create the
+                            // usersBucket at the same time,
+                            // this will prevent one of the users
+                            // from getting a BucketAlreadyExists
+                            // error with respect
+                            // to the usersBucket.
+                            // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
+                            if (err && !err.BucketAlreadyExists) {
+                                log.error('error from metadata', {
+                                    error: err,
+                                });
+                                return cb(err);
+                            }
+                            log.trace('Users bucket created');
+                            // Finally put the key in the new format
+                            // usersBucket
+                            return metadata.putObjectMD(usersBucket,
+                                key, omVal, {}, log, cb);
+                        });
+                }
+                return cb(err);
+            });
+    });
+}
+
+function removeTransientOrDeletedLabel(bucket, log, callback) {
+    log.trace('removing transient or deleted label from bucket attributes');
+    const bucketName = bucket.getName();
+    bucket.removeTransientFlag();
+    bucket.removeDeletedFlag();
+    return metadata.updateBucket(bucketName, bucket, log, callback);
+}
+
+function freshStartCreateBucket(bucket, canonicalID, log, callback) {
+    const bucketName = bucket.getName();
+    metadata.createBucket(bucketName, bucket, log, err => {
+        if (err) {
+            log.debug('error from metadata', { error: err });
+            return callback(err);
+        }
+        log.trace('created bucket in metadata');
+        return addToUsersBucket(canonicalID, bucketName, bucket, log, err => {
+            if (err) {
+                return callback(err);
+            }
+            return removeTransientOrDeletedLabel(bucket, log, callback);
+        });
+    });
+}
+
+/**
+ * Finishes creating a bucket in transient state
+ * by putting an object in users bucket representing the created bucket
+ * and removing transient attribute of the created bucket
+ * @param {object} bucketMD - either the bucket metadata sent in the new request
+ * or the existing metadata if no new metadata sent
+ * (for example in an objectPut)
+ * @param {string} canonicalID - bucket owner's canonicalID
+ * @param {object} log - Werelogs logger
+ * @param {function} callback - callback with error or null as arguments
+ * @return {undefined}
+ */
+function cleanUpBucket(bucketMD, canonicalID, log, callback) {
+    const bucketName = bucketMD.getName();
+    return addToUsersBucket(canonicalID, bucketName, bucketMD, log, err => {
+        if (err) {
+            return callback(err);
+        }
+        return removeTransientOrDeletedLabel(bucketMD, log, callback);
+    });
+}
+
+/**
+ * Manage the server side encryption on bucket creation, as a side effect
+ * a bucket key is created in the kms
+ * @param {string} bucketName - name of bucket
+ * @param {object} headers - request headers
+ * @param {function} log - Werelogs logger
+ * @param {function} cb - called on completion
+ * @returns {undefined}
+ * @callback called with (err, sseInfo: object)
+ */
+function bucketLevelServerSideEncryption(bucketName, headers, log, cb) {
+    kms.bucketLevelEncryption(
+        bucketName, headers, log, (err, sseInfo) => {
+            if (err) {
+                log.debug('error getting bucket encryption info', {
+                    error: err,
+                });
+                return cb(err);
+            }
+            return cb(null, sseInfo);
+        });
+}
+
+/**
+ * Creates bucket
+ * @param {AuthInfo} authInfo - Instance of AuthInfo class with
+ *                              requester's info
+ * @param {string} bucketName - name of bucket
+ * @param {object} headers - request headers
+ * @param {string} locationConstraint - locationConstraint provided in
+ *                                      request body xml (if provided)
+ * @param {function} log - Werelogs logger
+ * @param {function} cb - callback to bucketPut
+ * @return {undefined}
+ */
+function createBucket(authInfo, bucketName, headers,
+    locationConstraint, log, cb) {
+    log.trace('Creating bucket');
+    assert.strictEqual(typeof bucketName, 'string');
+    const canonicalID = authInfo.getCanonicalID();
+    const ownerDisplayName =
+        authInfo.getAccountDisplayName();
+    const creationDate = new Date().toJSON();
+    const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
+    const headerObjectLock = headers['x-amz-bucket-object-lock-enabled'];
+    const objectLockEnabled
+        = headerObjectLock && headerObjectLock.toLowerCase() === 'true';
+    const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName,
+        creationDate, BucketInfo.currentModelVersion(), null, null, null, null,
+        null, null, null, null, null, null, null, null, null, isNFSEnabled,
+        null, null, objectLockEnabled);
+    let locationConstraintVal = null;
+
+    if (locationConstraint) {
+        const [locationConstraintStr, ingestion] =
+            locationConstraint.split(zenkoSeparator);
+        if (locationConstraintStr) {
+            locationConstraintVal = locationConstraintStr;
+            bucket.setLocationConstraint(locationConstraintStr);
+        }
+        if (ingestion === 'ingest') {
+            bucket.enableIngestion();
+            //automatically enable versioning for ingestion buckets
+            bucket.setVersioningConfiguration({ Status: 'Enabled' });
+        }
+    }
+    if (objectLockEnabled) {
+        // default versioning configuration AWS sets
+        // when a bucket is created with object lock
+        const versioningConfiguration = {
+            Status: 'Enabled',
+            MfaDelete: 'Disabled',
+        };
+        bucket.setVersioningConfiguration(versioningConfiguration);
+    }
+    const parseAclParams = {
+        headers,
+        resourceType: 'bucket',
+        acl: bucket.acl,
+        log,
+    };
+    async.parallel({
+        prepareNewBucketMD: function prepareNewBucketMD(callback) {
+            acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => {
+                if (err) {
+                    log.debug('error parsing acl from headers', {
+                        error: err,
+                    });
+                    return callback(err);
+                }
+                bucket.setFullAcl(parsedACL);
+                return callback(null, bucket);
+            });
+        },
+        getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
+            metadata.getBucket(bucketName, log, (err, data) => {
+                // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
+                if (err && err.NoSuchBucket) {
+                    return callback(null, 'NoBucketYet');
+                }
+                if (err) {
+                    return callback(err);
+                }
+                return callback(null, data);
+            });
+        },
+    },
+    // Function to run upon finishing both parallel requests
+    (err, results) => {
+        if (err) {
+            return cb(err);
+        }
+        const existingBucketMD = results.getAnyExistingBucketInfo;
+        if (existingBucketMD instanceof BucketInfo &&
+            existingBucketMD.getOwner() !== canonicalID &&
+            !isServiceAccount(canonicalID)) {
+            // return existingBucketMD to collect cors headers
+            return cb(errors.BucketAlreadyExists, existingBucketMD);
+        }
+        const newBucketMD = results.prepareNewBucketMD;
+        if (existingBucketMD === 'NoBucketYet') {
+            const sseConfig = parseBucketEncryptionHeaders(headers);
+            return bucketLevelServerSideEncryption(
+                bucketName, sseConfig, log,
+                (err, sseInfo) => {
+                    if (err) {
+                        return cb(err);
+                    }
+                    newBucketMD.setServerSideEncryption(sseInfo);
+                    log.trace(
+                        'new bucket without flags; adding transient label');
+                    newBucketMD.addTransientFlag();
+                    return freshStartCreateBucket(newBucketMD, canonicalID,
+                                                  log, cb);
+                });
+        }
+        if (existingBucketMD.hasTransientFlag() ||
+            existingBucketMD.hasDeletedFlag()) {
+            log.trace('bucket has transient flag or deleted flag. cleaning up');
+            return cleanUpBucket(newBucketMD, canonicalID, log, cb);
+        }
+        // If bucket already exists in non-transient and non-deleted
+        // state and owned by requester, then return BucketAlreadyOwnedByYou
+        // error unless old AWS behavior (us-east-1)
+        // Existing locationConstraint must have legacyAwsBehavior === true
+        // New locationConstraint should have legacyAwsBehavior === true
+        if (isLegacyAWSBehavior(locationConstraintVal) &&
+            isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) {
+            log.trace('returning 200 instead of 409 to mirror us-east-1');
+            return cb(null, existingBucketMD);
+        }
+        return cb(errors.BucketAlreadyOwnedByYou, existingBucketMD);
+    });
+}
+
+module.exports = {
+    cleanUpBucket,
+    createBucket,
+};

lib/api/apiUtils/bucket/bucketDeletion.js

@@ -0,0 +1,151 @@
+const assert = require('assert');
+const async = require('async');
+const { errors } = require('arsenal');
+
+const abortMultipartUpload = require('../object/abortMultipartUpload');
+const { pushMetric } = require('../../../utapi/utilities');
+
+const { splitter, oldSplitter, mpuBucketPrefix } =
+    require('../../../../constants');
+const metadata = require('../../../metadata/wrapper');
+const kms = require('../../../kms/wrapper');
+const deleteUserBucketEntry = require('./deleteUserBucketEntry');
+
+function _deleteMPUbucket(destinationBucketName, log, cb) {
+    const mpuBucketName =
+        `${mpuBucketPrefix}${destinationBucketName}`;
+    return metadata.deleteBucket(mpuBucketName, log, err => {
+        // If the mpu bucket does not exist, just move on
+        // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
+        if (err && err.NoSuchBucket) {
+            return cb();
+        }
+        return cb(err);
+    });
+}
+
+function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) {
+    async.mapLimit(mpus, 1, (mpu, next) => {
+        const splitterChar = mpu.key.includes(oldSplitter) ?
+            oldSplitter : splitter;
+        // `overview${splitter}${objectKey}${splitter}${uploadId}
+        const [, objectKey, uploadId] = mpu.key.split(splitterChar);
+        abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
+            (err, destBucket, partSizeSum) => {
+                pushMetric('abortMultipartUpload', log, {
+                    authInfo,
+                    canonicalID: bucketMD.getOwner(),
+                    bucket: bucketName,
+                    keys: [objectKey],
+                    byteLength: partSizeSum,
+                });
+                next(err);
+            }, request);
+    }, cb);
+}
+/**
+ * deleteBucket - Delete bucket from namespace
+ * @param {object} authInfo - authentication info
+ * @param {object} bucketMD - bucket attributes/metadata
+ * @param {string} bucketName - bucket in which objectMetadata is stored
+ * @param {string} canonicalID - account canonicalID of requester
+ * @param {object} request - request object given by router
+ *                           including normalized headers
+ * @param {object} log - Werelogs logger
+ * @param {function} cb - callback from async.waterfall in bucketDelete
+ * @return {undefined}
+ */
+function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, cb) {
+    log.trace('deleting bucket from metadata');
+    assert.strictEqual(typeof bucketName, 'string');
+    assert.strictEqual(typeof canonicalID, 'string');
+
+    return async.waterfall([
+        function checkForObjectsStep(next) {
+            const params = { maxKeys: 1, listingType: 'DelimiterVersions' };
+            // We list all the versions as we want to return BucketNotEmpty
+            // error if there are any versions or delete markers in the bucket.
+            // Works for non-versioned buckets as well since listing versions
+            // includes null (non-versioned) objects in the result.
+            return metadata.listObject(bucketName, params, log,
+                (err, list) => {
+                    if (err) {
+                        log.error('error from metadata', { error: err });
+                        return next(err);
+                    }
+                    const length = (list.Versions ? list.Versions.length : 0) +
+                        (list.DeleteMarkers ? list.DeleteMarkers.length : 0);
+                    log.debug('listing result', { length });
+                    if (length) {
+                        log.debug('bucket delete failed',
+                            { error: errors.BucketNotEmpty });
+                        return next(errors.BucketNotEmpty);
+                    }
+                    return next();
+                });
+        },
+
+        function deleteMPUbucketStep(next) {
+            const MPUBucketName = `${mpuBucketPrefix}${bucketName}`;
+            // check to see if there are any mpu overview objects (so ignore
+            // any orphaned part objects)
+            return metadata.listObject(MPUBucketName, { prefix: 'overview' },
+                log, (err, objectsListRes) => {
+                    // If no shadow bucket ever created, no ongoing MPU's, so
+                    // continue with deletion
+                    if (err?.is.NoSuchBucket) {
+                        return next();
+                    }
+                    if (err) {
+                        log.error('error from metadata', { error: err });
+                        return next(err);
+                    }
+                    if (objectsListRes.Contents.length) {
+                        return _deleteOngoingMPUs(authInfo, bucketName,
+                            bucketMD, objectsListRes.Contents, request, log, err => {
+                                if (err) {
+                                    return next(err);
+                                }
+                                log.trace('deleting shadow MPU bucket');
+                                return _deleteMPUbucket(bucketName, log, next);
+                            });
+                    }
+                    log.trace('deleting shadow MPU bucket');
+                    return _deleteMPUbucket(bucketName, log, next);
+                });
+        },
+        function addDeleteFlagStep(next) {
+            log.trace('adding deleted attribute to bucket attributes');
+            // Remove transient flag if any so never have both transient
+            // and deleted flags.
+            bucketMD.removeTransientFlag();
+            bucketMD.addDeletedFlag();
+            return metadata.updateBucket(bucketName, bucketMD, log, next);
+        },
+        function deleteUserBucketEntryStep(next) {
+            log.trace('deleting bucket name from user bucket');
+            return deleteUserBucketEntry(bucketName, canonicalID, log, next);
+        },
+    ],
+    // eslint-disable-next-line prefer-arrow-callback
+    function actualDeletionStep(err) {
+        if (err) {
+            return cb(err);
+        }
+        return metadata.deleteBucket(bucketName, log, err => {
+            log.trace('deleting bucket from metadata');
+            if (err) {
+                return cb(err);
+            }
+            const serverSideEncryption = bucketMD.getServerSideEncryption();
+            if (serverSideEncryption &&
+                serverSideEncryption.algorithm === 'AES256') {
+                const masterKeyId = serverSideEncryption.masterKeyId;
+                return kms.destroyBucketKey(masterKeyId, log, cb);
+            }
+            return cb();
+        });
+    });
+}
+
+module.exports = deleteBucket;

lib/api/apiUtils/bucket/bucketEncryption.js

@@ -0,0 +1,255 @@
+const { errors } = require('arsenal');
+const metadata = require('../../../metadata/wrapper');
+const kms = require('../../../kms/wrapper');
+const { parseString } = require('xml2js');
+
+/**
+ * ServerSideEncryptionInfo - user configuration for server side encryption
+ * @typedef {Object} ServerSideEncryptionInfo
+ * @property {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
+ * @property {string} masterKeyId - Key id for the kms key used to encrypt data keys.
+ * @property {string} configuredMasterKeyId - User configured master key id.
+ * @property {boolean} mandatory - Whether a default encryption policy has been enabled.
+*/
+
+/**
+ * @callback ServerSideEncryptionInfo~callback
+ * @param {Object} error - Instance of Arsenal error
+ * @param {ServerSideEncryptionInfo} - SSE configuration
+ */
+
+/**
+ * parseEncryptionXml - Parses and validates a ServerSideEncryptionConfiguration xml document
+ * @param {object} xml - ServerSideEncryptionConfiguration doc
+ * @param {object} log - logger
+ * @param {ServerSideEncryptionInfo~callback} cb - callback
+ * @returns {undefined}
+ */
+function parseEncryptionXml(xml, log, cb) {
+    return parseString(xml, (err, parsed) => {
+        if (err) {
+            log.trace('xml parsing failed', {
+                error: err,
+                method: 'parseEncryptionXml',
+            });
+            log.debug('invalid xml', { xml });
+            return cb(errors.MalformedXML);
+        }
+
+        if (!parsed
+            || !parsed.ServerSideEncryptionConfiguration
+            || !parsed.ServerSideEncryptionConfiguration.Rule) {
+            log.trace('error in sse config, invalid ServerSideEncryptionConfiguration section', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const { Rule } = parsed.ServerSideEncryptionConfiguration;
+
+        if (!Array.isArray(Rule)
+            || Rule.length > 1
+            || !Rule[0]
+            || !Rule[0].ApplyServerSideEncryptionByDefault
+            || !Rule[0].ApplyServerSideEncryptionByDefault[0]) {
+            log.trace('error in sse config, invalid ApplyServerSideEncryptionByDefault section', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const [encConfig] = Rule[0].ApplyServerSideEncryptionByDefault;
+
+        if (!encConfig.SSEAlgorithm || !encConfig.SSEAlgorithm[0]) {
+            log.trace('error in sse config, no SSEAlgorithm provided', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const [algorithm] = encConfig.SSEAlgorithm;
+
+        if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
+            log.trace('error in sse config, unknown SSEAlgorithm', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const result = { algorithm, mandatory: true };
+
+        if (encConfig.KMSMasterKeyID) {
+            if (algorithm === 'AES256') {
+                log.trace('error in sse config, can not specify KMSMasterKeyID when using AES256', {
+                    method: 'parseEncryptionXml',
+                });
+                return cb(errors.InvalidArgument.customizeDescription(
+                    'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'));
+            }
+
+            if (!encConfig.KMSMasterKeyID[0] || typeof encConfig.KMSMasterKeyID[0] !== 'string') {
+                log.trace('error in sse config, invalid KMSMasterKeyID', {
+                    method: 'parseEncryptionXml',
+                });
+                return cb(errors.MalformedXML);
+            }
+
+            result.configuredMasterKeyId = encConfig.KMSMasterKeyID[0];
+        }
+        return cb(null, result);
+    });
+}
+
+/**
+ * hydrateEncryptionConfig - Constructs a ServerSideEncryptionInfo object from arguments
+ * ensuring no invalid or undefined keys are added
+ *
+ * @param {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
+ * @param {string} configuredMasterKeyId - User configured master key id.
+ * @param {boolean} [mandatory] - Whether a default encryption policy has been enabled.
+ * @returns {ServerSideEncryptionInfo} - SSE configuration
+ */
+function hydrateEncryptionConfig(algorithm, configuredMasterKeyId, mandatory = null) {
+    if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
+        return {
+            algorithm: null,
+        };
+    }
+
+    const sseConfig = { algorithm, mandatory };
+
+    if (algorithm === 'aws:kms' && configuredMasterKeyId) {
+        sseConfig.configuredMasterKeyId = configuredMasterKeyId;
+    }
+
+    if (mandatory !== null) {
+        sseConfig.mandatory = mandatory;
+    }
+
+    return sseConfig;
+}
+
+/**
+ * parseBucketEncryptionHeaders - retrieves bucket level sse configuration from request headers
+ * @param {object} headers - Request headers
+ * @returns {ServerSideEncryptionInfo} - SSE configuration
+ */
+function parseBucketEncryptionHeaders(headers) {
+    const sseAlgorithm = headers['x-amz-scal-server-side-encryption'];
+    const configuredMasterKeyId = headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] || null;
+    return hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId, true);
+}
+
+/**
+ * parseObjectEncryptionHeaders - retrieves bucket level sse configuration from request headers
+ * @param {object} headers - Request headers
+ * @returns {ServerSideEncryptionInfo} - SSE configuration
+ */
+function parseObjectEncryptionHeaders(headers) {
+    const sseAlgorithm = headers['x-amz-server-side-encryption'];
+    const configuredMasterKeyId = headers['x-amz-server-side-encryption-aws-kms-key-id'] || null;
+
+    if (sseAlgorithm && sseAlgorithm !== 'AES256' && sseAlgorithm !== 'aws:kms') {
+        return {
+            error: errors.InvalidArgument.customizeDescription('The encryption method specified is not supported'),
+        };
+    }
+
+    if (sseAlgorithm !== 'aws:kms' && configuredMasterKeyId) {
+        return {
+            error: errors.InvalidArgument.customizeDescription(
+                'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'),
+        };
+    }
+    return { objectSSE: hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId) };
+}
+
+/**
+ * createDefaultBucketEncryptionMetadata - Creates master key and sets up default server side encryption configuration
+ * @param {BucketInfo} bucket - bucket metadata
+ * @param {object} log - werelogs logger
+ * @param {ServerSideEncryptionInfo~callback} cb - callback
+ * @returns {undefined}
+ */
+function createDefaultBucketEncryptionMetadata(bucket, log, cb) {
+    return kms.bucketLevelEncryption(
+        bucket.getName(),
+        { algorithm: 'AES256', mandatory: false },
+        log,
+        (error, sseConfig) => {
+            if (error) {
+                return cb(error);
+            }
+            bucket.setServerSideEncryption(sseConfig);
+            return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig));
+        });
+}
+
+/**
+ *
+ * @param {object} headers - request headers
+ * @param {BucketInfo} bucket - BucketInfo model
+ * @param {*} log - werelogs logger
+ * @param {ServerSideEncryptionInfo~callback} cb - callback
+ * @returns {undefined}
+ */
+function getObjectSSEConfiguration(headers, bucket, log, cb) {
+    const bucketSSE = bucket.getServerSideEncryption();
+    const { error, objectSSE } = parseObjectEncryptionHeaders(headers);
+    if (error) {
+        return cb(error);
+    }
+
+    // If a per object sse algo has been passed through
+    // x-amz-server-side-encryption
+    if (objectSSE.algorithm) {
+        // If aws:kms and a custom key id
+        // pass it through without updating the bucket md
+        if (objectSSE.algorithm === 'aws:kms' && objectSSE.configuredMasterKeyId) {
+            return cb(null, objectSSE);
+        }
+
+        // If the client has not specified a key id,
+        // and we have a default config, then we reuse
+        // it and pass it through
+        if (!objectSSE.configuredMasterKeyId && bucketSSE) {
+            // The default configs algo is overridden with the one passed in the
+            // request headers. Our implementations of AES256 and aws:kms are the
+            // same underneath so this is only cosmetic change.
+            const sseConfig = Object.assign({}, bucketSSE, { algorithm: objectSSE.algorithm });
+            return cb(null, sseConfig);
+        }
+
+        // If the client has not specified a key id, and we
+        // don't have a default config, generate it
+        if (!objectSSE.configuredMasterKeyId && !bucketSSE) {
+            return createDefaultBucketEncryptionMetadata(bucket, log, (error, sseConfig) => {
+                if (error) {
+                    return cb(error);
+                }
+                // Override the algorithm, for the same reasons as above.
+                Object.assign(sseConfig, { algorithm: objectSSE.algorithm });
+                return cb(null, sseConfig);
+            });
+        }
+    }
+
+    // If the bucket has a default encryption config, and it is mandatory
+    // (created with putBucketEncryption or legacy headers)
+    // pass it through
+    if (bucketSSE && bucketSSE.mandatory) {
+        return cb(null, bucketSSE);
+    }
+
+    // No encryption config
+    return cb(null, null);
+}
+
+module.exports = {
+    createDefaultBucketEncryptionMetadata,
+    getObjectSSEConfiguration,
+    hydrateEncryptionConfig,
+    parseEncryptionXml,
+    parseBucketEncryptionHeaders,
+    parseObjectEncryptionHeaders,
+};

lib/api/apiUtils/bucket/bucketShield.js

@@ -0,0 +1,43 @@
+const invisiblyDelete = require('./invisiblyDelete');
+const constants = require('../../../../constants');
+
+/**
+ * Checks whether to proceed with a request based on the bucket flags
+ * and sends a request to invisibly delete the bucket if applicable
+ * @param {object} bucket - bucket metadata
+ * @param {string} requestType - type of api request
+ * @return {boolean} true if the bucket should be shielded, false otherwise
+ */
+function bucketShield(bucket, requestType) {
+    const invisiblyDeleteRequests = constants.bucketOwnerActions.concat(
+        [
+            'bucketGet',
+            'bucketHead',
+            'bucketGetACL',
+            'objectGet',
+            'objectGetACL',
+            'objectHead',
+            'objectPutACL',
+            'objectDelete',
+        ]);
+    if (invisiblyDeleteRequests.indexOf(requestType) > -1 &&
+        bucket.hasDeletedFlag()) {
+        invisiblyDelete(bucket.getName(), bucket.getOwner());
+        return true;
+    }
+     // If request is initiateMultipartUpload (requestType objectPut),
+     // objectPut, bucketPutACL or bucketDelete, proceed with request.
+     // Otherwise return an error to the client
+    if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
+        (requestType !== 'objectPut' &&
+        requestType !== 'initiateMultipartUpload' &&
+        requestType !== 'objectPutPart' &&
+        requestType !== 'completeMultipartUpload' &&
+        requestType !== 'bucketPutACL' &&
+        requestType !== 'bucketDelete')) {
+        return true;
+    }
+    return false;
+}
+
+module.exports = bucketShield;