more controllers

LOG: Add endpoint log
FIX: Remove excess logging info
2017-06-15 10:23:44 -07:00 · 2017-06-15 10:23:44 -07:00 · 2017-06-15 10:23:44 -07:00 · 2017-06-15 10:23:44 -07:00 · 2017-06-06 17:55:58 -07:00 · 2017-06-06 17:25:42 -07:00
694 changed files with 16864 additions and 88560 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,9 +0,0 @@
 node_modules
 localData/*
 localMetadata/*
 # Keep the .git/HEAD file in order to properly report version
 .git/objects
 .github
 .tox
 coverage
 .DS_Store
--- a/.eslintrc
+++ b/.eslintrc
@ -1,54 +1 @@
-{
+{ "extends": "scality" }
    "extends": "scality",
    "plugins": [
        "mocha"
    ],
    "rules": {
        "import/extensions": "off",
        "lines-around-directive": "off",
        "no-underscore-dangle": "off",
        "indent": "off",
        "object-curly-newline": "off",
        "operator-linebreak": "off",
        "function-paren-newline": "off",
        "import/newline-after-import": "off",
        "prefer-destructuring": "off",
        "implicit-arrow-linebreak": "off",
        "no-bitwise": "off",
        "dot-location": "off",
        "comma-dangle": "off",
        "no-undef-init": "off",
        "global-require": "off",
        "import/no-dynamic-require": "off",
        "class-methods-use-this": "off",
        "no-plusplus": "off",
        "no-else-return": "off",
        "object-property-newline": "off",
        "import/order": "off",
        "no-continue": "off",
        "no-tabs": "off",
        "lines-between-class-members": "off",
        "prefer-spread": "off",
        "no-lonely-if": "off",
        "no-useless-escape": "off",
        "no-restricted-globals": "off",
        "no-buffer-constructor": "off",
        "import/no-extraneous-dependencies": "off",
        "space-unary-ops": "off",
        "no-useless-return": "off",
        "no-unexpected-multiline": "off",
        "no-mixed-operators": "off",
        "newline-per-chained-call": "off",
        "operator-assignment": "off",
        "spaced-comment": "off",
        "comma-style": "off",
        "no-restricted-properties": "off",
        "new-parens": "off",
        "no-multi-spaces": "off",
        "quote-props": "off",
        "mocha/no-exclusive-tests": "error",
    },
    "parserOptions": {
        "ecmaVersion": 2020
    }
 }
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@ -1,32 +1,19 @@
-# General support information
+# Issue template
-GitHub Issues are **reserved** for actionable bug reports (including
+If you are reporting a new issue, make sure that we do not have any
-documentation inaccuracies), and feature requests.
+duplicates already open. You can ensure this by searching the issue list for
-**All questions** (regarding configuration, use cases, performance, community,
+this repository. If there is a duplicate, please close your issue and add a
-events, setup and usage recommendations, among other things) should be asked on
+comment to the existing issue instead.
 the **[Zenko Forum](http://forum.zenko.io/)**.
-> Questions opened as GitHub issues will systematically be closed, and moved to
+## General support information
 > the [Zenko Forum](http://forum.zenko.io/).
--------------------------------------------------------------------------------
+GitHub Issues are reserved for actionable bug reports and feature requests.
-
+General questions should be sent to the
-## Avoiding duplicates
+[S3 scality server Forum](http://forum.scality.com/).
 When reporting a new issue/requesting a feature, make sure that we do not have
 any duplicates already open:
 - search the issue list for this repository (use the search bar, select
  "Issues" on the left pane after searching);
 - if there is a duplicate, please do not open your issue, and add a comment
  to the existing issue instead.
 --------------------------------------------------------------------------------
 ## Bug report information
-(delete this section (everything between the lines) if you're not reporting a bug
+(delete this section if not applicable)
 but requesting a feature)
 ### Description
@ -42,22 +29,13 @@ Describe the results you received
 ### Expected result
-Describe the results you expected
+Describe the results you expecteds
-### Additional information
+### Additional information: (Node.js version, Docker version, etc)
 - Node.js version,
 - Docker version,
 - yarn version,
 - distribution/OS,
 - optional: anything else you deem helpful to us.
 --------------------------------------------------------------------------------
 ## Feature Request
-(delete this section (everything between the lines) if you're not requesting
+(delete this section if not applicable)
 a feature but reporting a bug)
 ### Proposal
@ -74,14 +52,3 @@ What you would like to happen
 ### Use case
 Please provide use cases for changing the current behavior
 ### Additional information
 - Is this request for your company? Y/N
  - If Y: Company name:
  - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N
 - Are you willing to contribute this feature yourself?
 - Position/Title:
 - How did you hear about us?
 --------------------------------------------------------------------------------
--- a/.github/actions/setup-ci/action.yaml
+++ b/.github/actions/setup-ci/action.yaml
@ -1,43 +0,0 @@
 ---
 name: "Setup CI environment"
 description: "Setup Cloudserver CI environment"
 runs:
  using: composite
  steps:
    - name: Setup etc/hosts
      shell: bash
      run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
    - name: Setup Credentials
      shell: bash
      run: bash .github/scripts/credentials.bash
    - name: Setup job artifacts directory
      shell: bash
      run: |-
        set -exu;
        mkdir -p /tmp/artifacts/${JOB_NAME}/;
    - uses: actions/setup-node@v4
      with:
        node-version: '16'
        cache: 'yarn'
    - name: install dependencies
      shell: bash
      run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
    - uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip
    - uses: actions/setup-python@v4
      with:
        python-version: 3.9
    - name: Setup python2 test environment
      shell: bash
      run: |
        sudo apt-get install -y libdigest-hmac-perl
        pip install 's3cmd==2.3.0'
    - name: fix sproxyd.conf permissions
      shell: bash
      run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
    - name: ensure fuse kernel module is loaded (for sproxyd)
      shell: bash
      run: sudo modprobe fuse
--- a/.github/ceph/Dockerfile
+++ b/.github/ceph/Dockerfile
@ -1,25 +0,0 @@
 FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
 ENV CEPH_DAEMON demo
 ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
 ENV CEPH_DEMO_UID zenko
 ENV CEPH_DEMO_ACCESS_KEY accessKey1
 ENV CEPH_DEMO_SECRET_KEY verySecretKey1
 ENV CEPH_DEMO_BUCKET zenkobucket
 ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
 ENV MON_IP 0.0.0.0
 ENV NETWORK_AUTO_DETECT 4
 ENV RGW_CIVETWEB_PORT 8001
 RUN rm /etc/yum.repos.d/tcmu-runner.repo
 ADD ./entrypoint-wrapper.sh /
 RUN chmod +x /entrypoint-wrapper.sh && \
    yum install -y python-pip && \
    yum clean all && \
    pip install awscli && \
    rm -rf /root/.cache/pip
 ENTRYPOINT [ "/entrypoint-wrapper.sh" ]
--- a/.github/ceph/entrypoint-wrapper.sh
+++ b/.github/ceph/entrypoint-wrapper.sh
@ -1,37 +0,0 @@
 #!/bin/sh
 touch /artifacts/ceph.log
 mkfifo /tmp/entrypoint_output
 # We run this in the background so that we can tail the RGW log after init,
 # because entrypoint.sh never returns
 # The next line will be needed when ceph builds 3.2.2 so I'll leave it here
 # bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
 bash /entrypoint.sh > /tmp/entrypoint_output &
 entrypoint_pid="$!"
 while read -r line; do
    echo $line
    # When we find this line server has started
    if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
        break
    fi
 done < /tmp/entrypoint_output
 # Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
 s3cmd mb s3://cephbucket s3://cephbucket2
 mkdir /root/.aws
 cat > /root/.aws/credentials <<EOF
 [default]
 aws_access_key_id = accessKey1
 aws_secret_access_key = verySecretKey1
 EOF
 # Enable versioning on them
 for bucket in cephbucket cephbucket2; do
    echo "Enabling versiong for $bucket"
    aws --endpoint http://127.0.0.1:8001 s3api  put-bucket-versioning --bucket $bucket --versioning Status=Enabled
 done
 tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
 wait $entrypoint_pid
--- a/.github/ceph/wait_for_ceph.sh
+++ b/.github/ceph/wait_for_ceph.sh
@ -1,11 +0,0 @@
 #!/bin/sh
 # This script is needed because RADOS Gateway
 # will open the port before beginning to serve traffic
 # causing wait_for_local_port.bash to exit immediately
 echo 'Waiting for ceph'
 while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
    sleep 1
    echo -n "."
 done
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@ -1,10 +0,0 @@
 ---
 version: 2
 updates:
  - package-ecosystem: npm
    directory: "/"
    schedule:
      interval: daily
      time: "13:00"
    open-pull-requests-limit: 10
    target-branch: "development/7.4"
--- a/.github/docker/creds.env
+++ b/.github/docker/creds.env
@ -1,36 +0,0 @@
 azurebackend_AZURE_STORAGE_ACCESS_KEY
 azurebackend_AZURE_STORAGE_ACCOUNT_NAME
 azurebackend_AZURE_STORAGE_ENDPOINT
 azurebackend2_AZURE_STORAGE_ACCESS_KEY
 azurebackend2_AZURE_STORAGE_ACCOUNT_NAME
 azurebackend2_AZURE_STORAGE_ENDPOINT
 azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY
 azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME
 azurebackendmismatch_AZURE_STORAGE_ENDPOINT
 azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY
 azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME
 azurenonexistcontainer_AZURE_STORAGE_ENDPOINT
 azuretest_AZURE_BLOB_ENDPOINT
 b2backend_B2_ACCOUNT_ID
 b2backend_B2_STORAGE_ACCESS_KEY
 GOOGLE_SERVICE_EMAIL
 GOOGLE_SERVICE_KEY
 AWS_S3_BACKEND_ACCESS_KEY
 AWS_S3_BACKEND_SECRET_KEY
 AWS_S3_BACKEND_ACCESS_KEY_2
 AWS_S3_BACKEND_SECRET_KEY_2
 AWS_GCP_BACKEND_ACCESS_KEY
 AWS_GCP_BACKEND_SECRET_KEY
 AWS_GCP_BACKEND_ACCESS_KEY_2
 AWS_GCP_BACKEND_SECRET_KEY_2
 b2backend_B2_STORAGE_ENDPOINT
 gcpbackend2_GCP_SERVICE_EMAIL
 gcpbackend2_GCP_SERVICE_KEY
 gcpbackend2_GCP_SERVICE_KEYFILE
 gcpbackend_GCP_SERVICE_EMAIL
 gcpbackend_GCP_SERVICE_KEY
 gcpbackendmismatch_GCP_SERVICE_EMAIL
 gcpbackendmismatch_GCP_SERVICE_KEY
 gcpbackend_GCP_SERVICE_KEYFILE
 gcpbackendmismatch_GCP_SERVICE_KEYFILE
 gcpbackendnoproxy_GCP_SERVICE_KEYFILE
--- a/.github/docker/docker-compose.yaml
+++ b/.github/docker/docker-compose.yaml
@ -1,92 +0,0 @@
 services:
  cloudserver:
    image: ${CLOUDSERVER_IMAGE}
    command: sh -c "yarn start > /artifacts/s3.log"
    network_mode: "host"
    volumes:
      - /tmp/ssl:/ssl
      - /tmp/ssl-kmip:/ssl-kmip
      - ${HOME}/.aws/credentials:/root/.aws/credentials
      - /tmp/artifacts/${JOB_NAME}:/artifacts
    environment:
      - CI=true
      - ENABLE_LOCAL_CACHE=true
      - REDIS_HOST=0.0.0.0
      - REDIS_PORT=6379
      - REPORT_TOKEN=report-token-1
      - REMOTE_MANAGEMENT_DISABLE=1
      - HEALTHCHECKS_ALLOWFROM=0.0.0.0/0
      - DATA_HOST=0.0.0.0
      - METADATA_HOST=0.0.0.0
      - S3BACKEND
      - S3DATA
      - S3METADATA
      - MPU_TESTING
      - S3VAULT
      - S3_LOCATION_FILE
      - ENABLE_UTAPI_V2
      - BUCKET_DENY_FILTER
      - S3KMS
      - S3KMIP_PORT
      - S3KMIP_HOSTS
      - S3KMIP-COMPOUND_CREATE
      - S3KMIP_BUCKET_ATTRIBUTE_NAME
      - S3KMIP_PIPELINE_DEPTH
      - S3KMIP_KEY
      - S3KMIP_CERT
      - S3KMIP_CA
      - MONGODB_HOSTS=0.0.0.0:27018
      - MONGODB_RS=rs0
      - DEFAULT_BUCKET_KEY_FORMAT
      - METADATA_MAX_CACHED_BUCKETS
      - ENABLE_NULL_VERSION_COMPAT_MODE
      - SCUBA_HOST
      - SCUBA_PORT
      - SCUBA_HEALTHCHECK_FREQUENCY
      - S3QUOTA
      - QUOTA_ENABLE_INFLIGHTS
    env_file:
      - creds.env
    depends_on:
      - redis
    extra_hosts:
      - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
      - "pykmip.local:127.0.0.1"
  redis:
    image: redis:alpine
    network_mode: "host"
  squid:
    network_mode: "host"
    profiles: ['ci-proxy']
    image: scality/ci-squid
    command: >-
      sh -c 'mkdir -p /ssl &&
            openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
              -subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
              -keyout /ssl/myca.pem  -out /ssl/myca.pem &&
            cp /ssl/myca.pem /ssl/CA.pem &&
            squid -f /etc/squid/squid.conf -N -z &&
            squid -f /etc/squid/squid.conf -NYCd 1'
    volumes:
      - /tmp/ssl:/ssl
  pykmip:
    network_mode: "host"
    profiles: ['pykmip']
    image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
    volumes:
      - /tmp/artifacts/${JOB_NAME}:/artifacts
  mongo:
    network_mode: "host"
    profiles: ['mongo', 'ceph']
    image: ${MONGODB_IMAGE}
  ceph: 
    network_mode: "host"
    profiles: ['ceph']
    image: ghcr.io/scality/cloudserver/ci-ceph 
  sproxyd:
    network_mode: "host"
    profiles: ['sproxyd']
    image: sproxyd-standalone
    build: ./sproxyd
    user: 0:0
    privileged: yes
--- a/.github/docker/mongodb/Dockerfile
+++ b/.github/docker/mongodb/Dockerfile
@ -1,28 +0,0 @@
 FROM mongo:5.0.21
 ENV USER=scality \
    HOME_DIR=/home/scality \
    CONF_DIR=/conf \
    DATA_DIR=/data
 # Set up directories and permissions
 RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
    mkdir /logs; \
    adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
 # Set up environment variables and directories for scality user
 RUN mkdir ${CONF_DIR} && \
    chown -R ${USER} ${CONF_DIR} && \
    chown -R ${USER} ${DATA_DIR}
 # copy the mongo config file
 COPY /conf/mongod.conf /conf/mongod.conf
 COPY /conf/mongo-run.sh /conf/mongo-run.sh
 COPY /conf/initReplicaSet /conf/initReplicaSet.js
 EXPOSE 27017/tcp
 EXPOSE 27018
 # Set up CMD
 ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
 CMD ["bash", "/conf/mongo-run.sh"]
--- a/.github/docker/mongodb/conf/initReplicaSet
+++ b/.github/docker/mongodb/conf/initReplicaSet
@ -1,4 +0,0 @@
 rs.initiate({
    _id: "rs0",
    members: [{ _id: 0, host: "127.0.0.1:27018" }]
 });
--- a/.github/docker/mongodb/conf/mongo-run.sh
+++ b/.github/docker/mongodb/conf/mongo-run.sh
@ -1,10 +0,0 @@
 #!/bin/bash
 set -exo pipefail
 init_RS() {
  sleep 5
  mongo --port 27018 /conf/initReplicaSet.js
 }
 init_RS &
 mongod --bind_ip_all --config=/conf/mongod.conf
--- a/.github/docker/mongodb/conf/mongod.conf
+++ b/.github/docker/mongodb/conf/mongod.conf
@ -1,15 +0,0 @@
 storage:
   journal:
      enabled: true
   engine: wiredTiger
   dbPath: "/data/db"
 processManagement:
   fork: false
 net:
   port: 27018
   bindIp: 0.0.0.0
 replication:
   replSetName: "rs0"
   enableMajorityReadConcern: true
 security:
    authorization: disabled
--- a/.github/docker/sproxyd/Dockerfile
+++ b/.github/docker/sproxyd/Dockerfile
@ -1,3 +0,0 @@
 FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
 ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
 RUN chown root:root /conf/sproxyd0.conf
--- a/.github/docker/sproxyd/conf/fastcgi_params
+++ b/.github/docker/sproxyd/conf/fastcgi_params
@ -1,26 +0,0 @@
 fastcgi_param  QUERY_STRING       $query_string;
 fastcgi_param  REQUEST_METHOD     $request_method;
 fastcgi_param  CONTENT_TYPE       $content_type;
 fastcgi_param  CONTENT_LENGTH     $content_length;
 #fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;
 fastcgi_param  SCRIPT_NAME        /var/www;
 fastcgi_param  PATH_INFO          $document_uri;
 fastcgi_param  REQUEST_URI        $request_uri;
 fastcgi_param  DOCUMENT_URI       $document_uri;
 fastcgi_param  DOCUMENT_ROOT      $document_root;
 fastcgi_param  SERVER_PROTOCOL    $server_protocol;
 fastcgi_param  HTTPS              $https if_not_empty;
 fastcgi_param  GATEWAY_INTERFACE  CGI/1.1;
 fastcgi_param  SERVER_SOFTWARE    nginx/$nginx_version;
 fastcgi_param  REMOTE_ADDR        $remote_addr;
 fastcgi_param  REMOTE_PORT        $remote_port;
 fastcgi_param  SERVER_ADDR        $server_addr;
 fastcgi_param  SERVER_PORT        $server_port;
 fastcgi_param  SERVER_NAME        $server_name;
 # PHP only, required if PHP was built with --enable-force-cgi-redirect
 fastcgi_param  REDIRECT_STATUS    200;
--- a/.github/docker/sproxyd/conf/nginx.conf
+++ b/.github/docker/sproxyd/conf/nginx.conf
@ -1,88 +0,0 @@
 worker_processes 1;
 error_log  /logs/error.log;
 user root root;
 events {
    worker_connections  1000;
    reuse_port on;
    multi_accept on;
 }
 worker_rlimit_nofile    20000;
 http {
    root /var/www/;
    upstream sproxyds {
        least_conn;
        keepalive 40;
        server 127.0.0.1:20000;
    }
    server {
        client_max_body_size 0;
        client_body_timeout 150;
        client_header_timeout 150;
        postpone_output 0;
        client_body_postpone_size 0;
        keepalive_requests 1100;
        keepalive_timeout 300s;
        server_tokens off;
        default_type application/octet-stream;
        gzip off;
        tcp_nodelay on;
        tcp_nopush  on;
        sendfile on;
        listen 81;
        server_name localhost;
        rewrite ^/arc/(.*)$ /dc1/$1 permanent;
        location ~* ^/proxy/(.*)$ {
            rewrite ^/proxy/(.*)$ /$1 last;
        }
        allow 127.0.0.1;
        deny all;
        set $usermd '-';
        set $sentusermd '-';
    	set $elapsed_ms '-';
    	set $now '-';
    	log_by_lua '
    		if not(ngx.var.http_x_scal_usermd == nil) and  string.len(ngx.var.http_x_scal_usermd) > 2 then
    			ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
    		end
                    if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
                            ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
                    end
    		local elapsed_ms = tonumber(ngx.var.request_time)
    		if not (  elapsed_ms == nil) then
    			elapsed_ms = elapsed_ms * 1000
    			ngx.var.elapsed_ms = tostring(elapsed_ms)
    		end
    		local time = tonumber(ngx.var.msec) * 1000
    		ngx.var.now = time
    	';
        log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
                       '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
                       '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
 		       '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
 		       '"contentType":"$content_type","s3Address":"$remote_addr",'
 		       '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
                       '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
 		       '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
 		       '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
                       '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
                       '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
        access_log /dev/stdout irm;
    	error_log  /dev/stdout error;
        location / {
            proxy_request_buffering off;
            fastcgi_request_buffering off;
            fastcgi_no_cache 1;
            fastcgi_cache_bypass 1;
            fastcgi_buffering off;
            fastcgi_ignore_client_abort on;
            fastcgi_keep_conn on;
            include fastcgi_params;
            fastcgi_pass sproxyds;
            fastcgi_next_upstream error timeout;
            fastcgi_send_timeout 285s;
            fastcgi_read_timeout 285s;
        }
    }
 }
--- a/.github/docker/sproxyd/conf/sproxyd0.conf
+++ b/.github/docker/sproxyd/conf/sproxyd0.conf
@ -1,12 +0,0 @@
 {
    "general": {
        "ring": "DATA",
        "port": 20000,
        "syslog_facility": "local0"
    },
    "ring_driver:0": {
        "alias": "dc1",
        "type": "local",
        "queue_path": "/tmp/ring-objs"
    },
 }
--- a/.github/docker/sproxyd/conf/supervisord.conf
+++ b/.github/docker/sproxyd/conf/supervisord.conf
@ -1,43 +0,0 @@
 [supervisord]
 nodaemon = true
 loglevel = info
 logfile  = %(ENV_LOG_DIR)s/supervisord.log
 pidfile  = %(ENV_SUP_RUN_DIR)s/supervisord.pid
 logfile_maxbytes = 20MB
 logfile_backups = 2
 [unix_http_server]
 file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
 [rpcinterface:supervisor]
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 [supervisorctl]
 serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
 [program:nginx]
 directory=%(ENV_SUP_RUN_DIR)s
 command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
 stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
 stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
 stdout_logfile_maxbytes=100MB
 stdout_logfile_backups=7
 stderr_logfile_maxbytes=100MB
 stderr_logfile_backups=7
 autorestart=true
 autostart=true
 user=root
 [program:sproxyd]
 directory=%(ENV_SUP_RUN_DIR)s
 process_name=%(program_name)s-%(process_num)s
 numprocs=1
 numprocs_start=0
 command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
 stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
 stdout_logfile_maxbytes=100MB
 stdout_logfile_backups=7
 redirect_stderr=true
 autorestart=true
 autostart=true
 user=root
--- a/.github/pykmip/Dockerfile
+++ b/.github/pykmip/Dockerfile
@ -1,29 +0,0 @@
 FROM python:3.10-alpine
 RUN apk add --no-cache \
        libressl && \
    apk add --no-cache --virtual .build-deps \
        python3-dev \
        libffi-dev \
        libressl-dev \
        sqlite-dev \
        build-base \
        curl
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 RUN pip3 install -U pip && \
    pip3 install pykmip requests && \
    apk del .build-deps && \
    mkdir /pykmip
 ADD ./bin /usr/local/bin
 ADD ./certs /ssl
 ADD policy.json /etc/pykmip/policies/policy.json
 ADD server.conf /etc/pykmip/server.conf
 ADD docker-entrypoint.sh /
 RUN chmod +x /docker-entrypoint.sh
 ENTRYPOINT ["/docker-entrypoint.sh"]
--- a/.github/pykmip/bin/create_certificates.py
+++ b/.github/pykmip/bin/create_certificates.py
@ -1,156 +0,0 @@
 #!/usr/bin/env python
 from cryptography import x509
 from cryptography.hazmat import backends
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
 import datetime
 import argparse
 import sys
 def get_args():
    parser = argparse.ArgumentParser(
        prog=sys.argv[0],
        description='Tool to generate a x509 CA root, server and client certs')
    parser.add_argument('-c', '--common-name', action='store',
                        default='localhost',
                        help='Set the common name for the server-side cert')
    return parser.parse_args()
 def create_rsa_private_key(key_size=2048, public_exponent=65537):
    private_key = rsa.generate_private_key(
        public_exponent=public_exponent,
        key_size=key_size,
        backend=backends.default_backend()
    )
    return private_key
 def create_self_signed_certificate(subject_name,
                                   private_key,
                                   days_valid=36500):
    subject = x509.Name([
        x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),
        x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name)
    ])
    certificate = x509.CertificateBuilder().subject_name(
        subject
    ).issuer_name(
        subject
    ).public_key(
        private_key.public_key()
    ).serial_number(
        x509.random_serial_number()
    ).not_valid_before(
        datetime.datetime.utcnow()
    ).not_valid_after(
        datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
    ).add_extension(
        x509.BasicConstraints(True, None),
        critical=True
    ).sign(private_key, hashes.SHA256(), backends.default_backend())
    return certificate
 def create_certificate(subject_name,
                       private_key,
                       signing_certificate,
                       signing_key,
                       days_valid=36500,
                       client_auth=False):
    subject = x509.Name([
        x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),
        x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name)
    ])
    builder = x509.CertificateBuilder().subject_name(
        subject
    ).issuer_name(
        signing_certificate.subject
    ).public_key(
        private_key.public_key()
    ).serial_number(
        x509.random_serial_number()
    ).not_valid_before(
        datetime.datetime.utcnow()
    ).not_valid_after(
        datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
    )
    if client_auth:
        builder = builder.add_extension(
            x509.ExtendedKeyUsage([x509.ExtendedKeyUsageOID.CLIENT_AUTH]),
            critical=True
        )
    certificate = builder.sign(
        signing_key,
        hashes.SHA256(),
        backends.default_backend()
    )
    return certificate
 def main(common_name):
    root_key = create_rsa_private_key()
    root_certificate = create_self_signed_certificate(
        u"Root CA",
        root_key
    )
    server_key = create_rsa_private_key()
    server_certificate = create_certificate(
        common_name,
        server_key,
        root_certificate,
        root_key
    )
    john_doe_client_key = create_rsa_private_key()
    john_doe_client_certificate = create_certificate(
        u"John Doe",
        john_doe_client_key,
        root_certificate,
        root_key,
        client_auth=True
    )
    with open("certs/kmip-ca.pem", "wb") as f:
        f.write(
            root_certificate.public_bytes(
                serialization.Encoding.PEM
            )
        )
    with open("certs/kmip-key.pem", "wb") as f:
        f.write(server_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ))
    with open("certs/kmip-cert.pem", "wb") as f:
        f.write(
            server_certificate.public_bytes(
                serialization.Encoding.PEM
            )
        )
    with open("certs/kmip-client-key.pem", "wb") as f:
        f.write(john_doe_client_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ))
    with open("certs/kmip-client-cert.pem", "wb") as f:
        f.write(
            john_doe_client_certificate.public_bytes(
                serialization.Encoding.PEM
            )
        )
 if __name__ == '__main__':
    args = get_args()
    main(args.common_name)
--- a/.github/pykmip/bin/run_server.py
+++ b/.github/pykmip/bin/run_server.py
@ -1,26 +0,0 @@
 #!/usr/bin/env python
 # Copyright (c) 2016 The Johns Hopkins University/Applied Physics Laboratory
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
 import logging  # noqa: E402
 logging.basicConfig(level=logging.DEBUG)
 from kmip.services.server import server  # noqa: E402
 if __name__ == '__main__':
    print('Starting PyKMIP server on 0.0.0.0:5696')
    server.main()
--- a/.github/pykmip/certs/kmip-ca.pem
+++ b/.github/pykmip/certs/kmip-ca.pem
@ -1,18 +0,0 @@
 -----BEGIN CERTIFICATE-----
 MIIC6zCCAdOgAwIBAgIUPIpMY95b4HjKAk+FyydZApAEFskwDQYJKoZIhvcNAQEL
 BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
 MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJDEQMA4GA1UECgwHU2NhbGl0eTEQ
 MA4GA1UEAwwHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
 AKqLFEsWtfRTxnoZrQe63tq+rQnVgninHMahRmXkzyjK/uNhoKnIh8bXdTC/eCZ6
 FBROqBYNL0TJb0HDv1FzcZS1UCUldRqTlvr6wZb0pfrp40fvztsqQgAh1t/Blg5i
 Zv5+ESSlNs5rWbFTxtq+FbMW/ERYTrVfnMkBiLg4Gq0HwID9a5jvJatzrrno2s1m
 OfZCT3HaE3tMZ6vvYuoamvLNdvdH+9KeTmBCursfNejt0rSGjIqfi6DvFJSayydQ
 is5DMSTbCLGdKQmA85VfEQmlQ8v0232WDSd6gVfp2tthDEDHnCbgWkEd1vsTyS85
 ubdt5v4CWGOWV+mu3bf8xM0CAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkq
 hkiG9w0BAQsFAAOCAQEARTjc2zV/ol1/LsSzZy6l1R0uFBmR2KumH+Se1Yq2vKpY
 Dv6xmrvmjOUr5RBO77nRhIgdcQA+LyAg8ii2Dfzc8r1RTD+j1bYOxESXctBOBcXM
 Chy6FEBydR6m7S8qQyL+caJWO1WZWp2tapcm6sUG1oRVznWtK1/SHKIzOBwsmJ07
 79KsCJ6wf9tzD05EDTI2QhAObE9/thy+zc8l8cmv9A6p3jKkx9rwXUttSUqTn0CW
 w45bgKg6+DDcrhZ+MATbzuTfhuA4NFUTzK7KeX9sMuOV03Zs8SA3VhAOXmu063M3
 0f9X7P/0RmGTTp7GGCqEINcZdbLh3k7CpFb2Ox998Q==
 -----END CERTIFICATE-----
--- a/.github/pykmip/certs/kmip-cert.pem
+++ b/.github/pykmip/certs/kmip-cert.pem
@ -1,18 +0,0 @@
 -----BEGIN CERTIFICATE-----
 MIIC2zCCAcOgAwIBAgIUIlE8UAkqQ+6mbJDtrt9kkmi8aJYwDQYJKoZIhvcNAQEL
 BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
 MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowKTEQMA4GA1UECgwHU2NhbGl0eTEV
 MBMGA1UEAwwMcHlrbWlwLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
 CgKCAQEAtxr7pq/lnzVeZz4z52Yc3DeaPqjNfRSyW5cPUlT7ABXFb7+tja7K2C7u
 DYVK+Q+2yJCQwYJY47aKJB++ewam9t2V8Xy0Z8S+0I2ImCwuyeihaD/f6uJZRzms
 ycdECH22BA6tCPlQLnlboRiZzI6rcIvXAbUMvLvFm3nyYIs9qidExRnfyMjISknM
 V+83LT5QW4IcHgKYqzdz2ZmOnk+f4wmMmitcivTdIZCL8Z0cxr7BJlOh5JZ/V5uj
 WUXeNa+ttW0RKKBlg9T+wj0JvwoJBPZTmsMAy3tI9tjLg3DwGYKsflbFeU2tebXI
 gncGFZ/dFxj331GGtq3kz1PzAUYf2wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB1
 8HgJ0fu6/pCrDxAm90eESFjmaTFyTN8q00zhq4Cb3zAT9KMWzAygkZ9n4ZFgELPo
 7kBE2H6RcDdoBmjVYd8HnBloDdYzYbncKgt5YBvxRaMSF4/l65BM8wjatyXErqnH
 QLLTRe5AuF0/F0KtPeDQ2JFVu8dZ35W3fyKGPRsEdVOSCTHROmqpGhZCpscyUP4W
 Hb0dBTESQ9mQHw14OCaaahARd0X5WdcA/E+m0fpGqj1rQCXS+PrRcSLe1E1hqPlK
 q/hXSXD5nybwipktELvJCbB7l4HmJr2pIpldeR5+ef68Cs8hqs6DRlsJX9sK2ng+
 TFe5v6SCarqZ9kFvr6Yp
 -----END CERTIFICATE-----
--- a/.github/pykmip/certs/kmip-client-cert.pem
+++ b/.github/pykmip/certs/kmip-client-cert.pem
@ -1,18 +0,0 @@
 -----BEGIN CERTIFICATE-----
 MIIC8zCCAdugAwIBAgIUBs6nVXQXhrFbClub3aSLg72/DiYwDQYJKoZIhvcNAQEL
 BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
 MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJTEQMA4GA1UECgwHU2NhbGl0eTER
 MA8GA1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
 AQC6neSYoBoWh/i2mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4L
 jLLRsPGt9KkJlUhHGWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jt
 UDnQUwshj+hRFe0iKAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsH
 PVju9yZBYzYhwAMyYFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFip
 yR2Fh3WGSwWh45HgMT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+p
 SMLm0T4iNxedQWBtDM7ts4EjAgMBAAGjGjAYMBYGA1UdJQEB/wQMMAoGCCsGAQUF
 BwMCMA0GCSqGSIb3DQEBCwUAA4IBAQCMi9HEhZc5jHJMj18Wq00fZy4O9XtjCe0J
 nntW9tzi3rTQcQWKA7i9uVdDoCg+gMFVxWMvV7luFEUc/VYV1v8hFfbIFygzFsZY
 xwv4GQaIwbsgzD+oziia53w0FSuNL0uE0MeKvrt3yzHxCxylHyl+TQd/UdAtAo+k
 RL1sI0mBZx5qo6d1J7ZMCxzAGaT7KjnJvziFr/UbfSNnwDsxsUwGaI1ZeAxJN8DI
 zTrg3f3lrrmHcauEgKnuQwIqaMZR6veG6RkjtcYSlJYID1irkE6njs7+wivOAkzt
 fBt/0PD76FmAI0VArgU/zDB8dGyYzrq39W749LuEfm1TPmlnUtDr
 -----END CERTIFICATE-----
--- a/.github/pykmip/certs/kmip-client-key.pem
+++ b/.github/pykmip/certs/kmip-client-key.pem
@ -1,28 +0,0 @@
 -----BEGIN PRIVATE KEY-----
 MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6neSYoBoWh/i2
 mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4LjLLRsPGt9KkJlUhH
 GWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jtUDnQUwshj+hRFe0i
 KAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsHPVju9yZBYzYhwAMy
 YFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFipyR2Fh3WGSwWh45Hg
 MT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+pSMLm0T4iNxedQWBt
 DM7ts4EjAgMBAAECggEANNXdUeUKXdSzcycPV/ea/c+0XFcy8e9B46lfQTpTqQOx
 xD8GbWD1L/gdk6baJgT43+ukEWdSsJbmdtLXti29Ta8OF2VtIDhIbCVtvs3dq3zt
 vrvugsiVDr8nkP306qOrKrNIVIFE+igmEmSaXsu/h/33ladxeeV9/s2DC7NOOjWN
 Mu4KYr5BBbu3qAavdzbrcz7Sch+GzsYqK/pBounCTQu3o9E4TSUcmcsasWmtHN3u
 e6G2UjObdzEW7J0wWvvtJ0wHQUVRueHfqwqKf0dymcZ3xOlx3ZPhKPz5n4F1UGUt
 RQaNazqs5SzZpUgDuPw4k8h/aCHK21Yexw/l4+O9KQKBgQD1WZSRK54zFoExBQgt
 OZSBNZW3Ibti5lSiF0M0g+66yNZSWfPuABEH0tu5CXopdPDXo4kW8NLGEqQStWTX
 RGK0DE9buEL3eebOfjIdS2IZ3t3dX3lMypplVCj4HzAgITlweSH1LLTyAtaaOpwa
 jksqfcn5Zw+XGkyc6GBBVaZetQKBgQDCt6Xf/g26+zjvHscjdzsfBhnYvTOrr6+F
 xqFFxOEOocGr+mL7UTAs+a9m/6lOWhlagk+m+TIZNL8o3IN7KFTYxPYPxTiewgVE
 rIm3JBmPxRiPn01P3HrtjaqfzsXF30j3ele7ix5OxieZq4vsW7ZXP3GZE34a08Ov
 12sE1DlvdwKBgQDzpYQOLhyqazzcqzyVfMrnDYmiFVN7QXTmiudobWRUBUIhAcdl
 oJdJB7K/rJOuO704x+RJ7dnCbZyWH6EGzZifaGIemXuXO21jvpqR0NyZCGOXhUp2
 YfS1j8AntwEZxyS9du2sBjui4gKvomiHTquChOxgSmKHEcznPTTpbN8MyQKBgF5F
 LVCZniolkLXsL7tS8VOez4qoZ0i6wP7CYLf3joJX+/z4N023S9yqcaorItvlMRsp
 tciAIyoi6F2vDRTmPNXJ3dtav4PVKVnLMs1w89MwOCjoljSQ6Q7zpGTEZenbpWbz
 W2BYBS9cLjXu4MpoyInLFINo9YeleLs8TvrCiKAXAoGBANsduqLnlUW/f5zDb5Fe
 SB51+KhBjsVIeYmU+8xtur9Z7IxZXK28wpoEsm7LmX7Va5dERjI+tItBiJ5+Unu1
 Xs2ljDg35ARKHs0dWBJGpbnZg4dbT6xpIL4YMPXm1Zu++PgRpxPIMn646xqd8GlH
 bavm6Km/fXNG58xus+EeLpV5
 -----END PRIVATE KEY-----
--- a/.github/pykmip/certs/kmip-key.pem
+++ b/.github/pykmip/certs/kmip-key.pem
@ -1,28 +0,0 @@
 -----BEGIN PRIVATE KEY-----
 MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC3Gvumr+WfNV5n
 PjPnZhzcN5o+qM19FLJblw9SVPsAFcVvv62NrsrYLu4NhUr5D7bIkJDBgljjtook
 H757Bqb23ZXxfLRnxL7QjYiYLC7J6KFoP9/q4llHOazJx0QIfbYEDq0I+VAueVuh
 GJnMjqtwi9cBtQy8u8WbefJgiz2qJ0TFGd/IyMhKScxX7zctPlBbghweApirN3PZ
 mY6eT5/jCYyaK1yK9N0hkIvxnRzGvsEmU6Hkln9Xm6NZRd41r621bREooGWD1P7C
 PQm/CgkE9lOawwDLe0j22MuDcPAZgqx+VsV5Ta15tciCdwYVn90XGPffUYa2reTP
 U/MBRh/bAgMBAAECggEABCvcMcbuDztzBB0Zp5re63Fk1SqZS9Et4wJE+hYvhaf5
 UHtoY8LoohYnnC0+MQBXpKgOdCoZBk8BRKNofnr/UL5pjQ/POFH2GuAujXDsO/NN
 wgc6fapcaE/7DLm6ZgsfG2aOMJclaXmgScI6trtFUpIM+t/6A06vyMP1bpeddwPW
 Fqu7NvpDiEcTRUGd+z1JooYgUhGgC7peYUx5+9zqFrwoDBKxnUOnz3BkDsXBy3qm
 65Vu0BSjuJzf6vVMpNGUHY6JXjopVNWku+JAX0wD+iikOd5sziNVdIj1fnZ+IHIf
 7G5h5owHpvSGzJFQ18/g5VHtJdCm+4WQSnbSJRsCAQKBgQDu4IH8yspyeH44fhoS
 PAp/OtILqSP+Da0zAp2LbhrOgyzyuSTdEAYyptqjqHS6QkB1Bu1H44FS0BYUxRXc
 iu2e9AndiLVCGngsE7TpA/ZVLN1B0LEZEHjM6p4d6zZM6iveKVnPAOkTWTBAgzCt
 b31nj4jL8PdlPKQil1AMrOlRAQKBgQDEOwshzIdr2Iy6B/n4CuBViEtwnbAd5f/c
 atA9bcfF8kCahokJsI4eCCLgBwDZpYKD+v0AwOBlacF6t6TX+vdlJsi5EP7uxZ22
 ILsuWqVm/0H77PACuckc5/qLZoGGC81l0DhnpoeMEb6r/TKOo5xAK1gxdlwNNrq+
 nP1zdZnU2wKBgBAS92xFUR4m0YeHpMV5WNN658t1FEDyNqdqE6PgQtmGpi2nG73s
 aB5cb/X3TfOCpce6MZlWy8sAyZuYL4Jprte1YDySCHBsS43bvZ64b4kHvdPB8UjY
 fOh9GSq2Oy8tysnmSm7NhuGQbNjKeyoQiIXBeNkQW/VqATl6qR5RPFoBAoGACNqV
 JQBCd/Y8W0Ry3eM3vgQ5SyqCQMcY5UwYez0Rz3efvJknY72InAhH8o2+VxOlsOjJ
 M5iAR3MfHLdeg7Q6J2E5m0gOCJ34ALi3WV8TqXMI+iH1rlnNnjVFU7bbTz4HFXnw
 oZSc9w/x53a0KkVtjmOmRg0OGDaI9ILG2MfMmhMCgYB8ZqJtX8qZ2TqKU3XdLZ4z
 T2N7xMFuKohWP420r5jKm3Xw85IC+y1SUTB9XGcL79r2eJzmzmdKQ3A3sf3oyUH3
 RdYWxtKcZ5PAE8hVRtn1ETZqUgxASGOUn/6w0npkYSOXPU5bc0W6RSLkjES0i+c3
 fv3OMNI8qpmQhEjpHHQS1g==
 -----END PRIVATE KEY-----
--- a/.github/pykmip/docker-entrypoint.sh
+++ b/.github/pykmip/docker-entrypoint.sh
@ -1,3 +0,0 @@
 #!/bin/sh
 python3 /usr/local/bin/run_server.py 2>&1 | tee -a /artifacts/pykmip.log
--- a/.github/pykmip/policy.json
+++ b/.github/pykmip/policy.json
@ -1,168 +0,0 @@
 {
    "example": {
        "preset": {
            "CERTIFICATE": {
                "LOCATE": "ALLOW_ALL",
                "CHECK": "ALLOW_ALL",
                "GET": "ALLOW_ALL",
                "GET_ATTRIBUTES": "ALLOW_ALL",
                "GET_ATTRIBUTE_LIST": "ALLOW_ALL",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_ALL",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            },
            "SYMMETRIC_KEY": {
                "REKEY": "ALLOW_OWNER",
                "REKEY_KEY_PAIR": "ALLOW_OWNER",
                "DERIVE_KEY": "ALLOW_OWNER",
                "LOCATE": "ALLOW_OWNER",
                "CHECK": "ALLOW_OWNER",
                "GET": "ALLOW_OWNER",
                "GET_ATTRIBUTES": "ALLOW_OWNER",
                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_OWNER",
                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            },
            "PUBLIC_KEY": {
                "LOCATE": "ALLOW_ALL",
                "CHECK": "ALLOW_ALL",
                "GET": "ALLOW_ALL",
                "GET_ATTRIBUTES": "ALLOW_ALL",
                "GET_ATTRIBUTE_LIST": "ALLOW_ALL",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_ALL",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            },
            "PRIVATE_KEY": {
                "REKEY": "ALLOW_OWNER",
                "REKEY_KEY_PAIR": "ALLOW_OWNER",
                "DERIVE_KEY": "ALLOW_OWNER",
                "LOCATE": "ALLOW_OWNER",
                "CHECK": "ALLOW_OWNER",
                "GET": "ALLOW_OWNER",
                "GET_ATTRIBUTES": "ALLOW_OWNER",
                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_OWNER",
                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            },
            "SPLIT_KEY": {
                "REKEY": "ALLOW_OWNER",
                "REKEY_KEY_PAIR": "ALLOW_OWNER",
                "DERIVE_KEY": "ALLOW_OWNER",
                "LOCATE": "ALLOW_OWNER",
                "CHECK": "ALLOW_OWNER",
                "GET": "ALLOW_OWNER",
                "GET_ATTRIBUTES": "ALLOW_OWNER",
                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_OWNER",
                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            },
            "TEMPLATE": {
                "LOCATE": "ALLOW_OWNER",
                "GET": "ALLOW_OWNER",
                "GET_ATTRIBUTES": "ALLOW_OWNER",
                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER"
            },
            "SECRET_DATA": {
                "REKEY": "ALLOW_OWNER",
                "REKEY_KEY_PAIR": "ALLOW_OWNER",
                "DERIVE_KEY": "ALLOW_OWNER",
                "LOCATE": "ALLOW_OWNER",
                "CHECK": "ALLOW_OWNER",
                "GET": "ALLOW_OWNER",
                "GET_ATTRIBUTES": "ALLOW_OWNER",
                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_OWNER",
                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            },
            "OPAQUE_DATA": {
                "REKEY": "ALLOW_OWNER",
                "REKEY_KEY_PAIR": "ALLOW_OWNER",
                "DERIVE_KEY": "ALLOW_OWNER",
                "LOCATE": "ALLOW_OWNER",
                "CHECK": "ALLOW_OWNER",
                "GET": "ALLOW_OWNER",
                "GET_ATTRIBUTES": "ALLOW_OWNER",
                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_OWNER",
                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            },
            "PGP_KEY": {
                "REKEY": "ALLOW_OWNER",
                "REKEY_KEY_PAIR": "ALLOW_OWNER",
                "DERIVE_KEY": "ALLOW_OWNER",
                "LOCATE": "ALLOW_OWNER",
                "CHECK": "ALLOW_OWNER",
                "GET": "ALLOW_OWNER",
                "GET_ATTRIBUTES": "ALLOW_OWNER",
                "GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
                "ADD_ATTRIBUTE": "ALLOW_OWNER",
                "MODIFY_ATTRIBUTE": "ALLOW_OWNER",
                "DELETE_ATTRIBUTE": "ALLOW_OWNER",
                "OBTAIN_LEASE": "ALLOW_OWNER",
                "GET_USAGE_ALLOCATION": "ALLOW_OWNER",
                "ACTIVATE": "ALLOW_OWNER",
                "REVOKE": "ALLOW_OWNER",
                "DESTROY": "ALLOW_OWNER",
                "ARCHIVE": "ALLOW_OWNER",
                "RECOVER": "ALLOW_OWNER"
            }
        }
    }
 }
--- a/.github/pykmip/server.conf
+++ b/.github/pykmip/server.conf
@ -1,15 +0,0 @@
 [server]
 hostname=0.0.0.0
 port=5696
 certificate_path=/ssl/kmip-cert.pem
 key_path=/ssl/kmip-key.pem
 ca_path=/ssl/kmip-ca.pem
 auth_suite=TLS1.2
 policy_path=/etc/pykmip/policies
 enable_tls_client_auth=True
 database_path=/pykmip/pykmip.db
 tls_cipher_suites=
    TLS_RSA_WITH_AES_128_CBC_SHA256
    TLS_RSA_WITH_AES_256_CBC_SHA256
    TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384
 logging_level=DEBUG
--- a/.github/scripts/credentials.bash
+++ b/.github/scripts/credentials.bash
@ -1,20 +0,0 @@
 #!/bin/bash -x
 set -x #echo on
 set -e #exit at the first error
 mkdir -p $HOME/.aws
 cat >>$HOME/.aws/credentials <<EOF
 [default]
 aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
 aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
 [default_2]
 aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
 aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
 [google]
 aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY
 aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY
 [google_2]
 aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY_2
 aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY_2
 EOF
--- a/.github/workflows/alerts.yaml
+++ b/.github/workflows/alerts.yaml
@ -1,35 +0,0 @@
 name: Test alerts
 on:
  push:
    branches-ignore:
      - 'development/**'
      - 'q/*/**'
 jobs:
  run-alert-tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        tests:
          - name: 1 minute interval tests
            file: monitoring/alerts.test.yaml
          - name: 10 seconds interval tests
            file: monitoring/alerts.10s.test.yaml
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Render and test ${{ matrix.tests.name }}
        uses: scality/action-prom-render-test@1.0.3
        with:
          alert_file_path: monitoring/alerts.yaml
          test_file_path: ${{ matrix.tests.file }}
          alert_inputs: |
            namespace=zenko
            service=artesca-data-connector-s3api-metrics
            reportJob=artesca-data-ops-report-handler
            replicas=3
          github_token: ${{ secrets.GITHUB_TOKEN }}
--- a/.github/workflows/codeql.yaml
+++ b/.github/workflows/codeql.yaml
@ -1,25 +0,0 @@
 ---
 name: codeQL
 on:
  push:
    branches: [w/**, q/*]
  pull_request:
    branches: [development/*, stabilization/*, hotfix/*]
  workflow_dispatch:
 jobs:
  analyze:
    name: Static analysis with CodeQL
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: javascript, python, ruby
      - name: Build and analyze
        uses: github/codeql-action/analyze@v3
--- a/.github/workflows/dependency-review.yaml
+++ b/.github/workflows/dependency-review.yaml
@ -1,16 +0,0 @@
 ---
 name: dependency review
 on:
  pull_request:
    branches: [development/*, stabilization/*, hotfix/*]
 jobs:
  dependency-review:
    runs-on: ubuntu-latest
    steps:
      - name: 'Checkout Repository'
        uses: actions/checkout@v4
      - name: 'Dependency Review'
        uses: actions/dependency-review-action@v4
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@ -1,80 +0,0 @@
 ---
 name: release
 run-name: release ${{ inputs.tag }}
 on:
  workflow_dispatch:
    inputs:
      tag:
        description: 'Tag to be released'
        required: true
 env:
  PROJECT_NAME: ${{ github.event.repository.name }}
 jobs:
  build-federation-image:
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GitHub Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ github.token }}
      - name: Build and push image for federation
        uses: docker/build-push-action@v5
        with:
          push: true
          context: .
          file: images/svc-base/Dockerfile
          tags: |
            ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
          cache-from: type=gha,scope=federation
          cache-to: type=gha,mode=max,scope=federation
  release:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Docker Buildk
        uses: docker/setup-buildx-action@v3
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ github.token }}
      - name: Push dashboards into the production namespace
        run: |
          oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
            dashboard.json:application/grafana-dashboard+json \
            alerts.yaml:application/prometheus-alerts+yaml
        working-directory: monitoring
      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
      - name: Create Release
        uses: softprops/action-gh-release@v2
        env:
          GITHUB_TOKEN: ${{ github.token }}
        with:
          name: Release ${{ github.event.inputs.tag }}
          tag_name: ${{ github.event.inputs.tag }}
          generate_release_notes: true
          target_commitish: ${{ github.sha }}
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@ -1,533 +0,0 @@
 ---
 name: tests
 on:
  workflow_dispatch:
  push:
    branches-ignore:
      - 'development/**'
      - 'q/*/**'
 env:
  # Secrets
  azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
  azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
  azurebackend_AZURE_STORAGE_ENDPOINT: >-
    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
  azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
    ${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
  azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
  azurebackend2_AZURE_STORAGE_ENDPOINT: >-
    ${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
  azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
  azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
  azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
  azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
  azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
  azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
  azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
  b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
  b2backend_B2_STORAGE_ACCESS_KEY: >-
    ${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
  GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
  GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
  AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
  AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
  AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
  AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
  AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
  AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
  AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
  AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
  b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
  gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
  gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
  gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
  gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
  gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
  gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
    ${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
  gcpbackendmismatch_GCP_SERVICE_KEY: >-
    ${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
  gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
  gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
  gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
  gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
  # Configs
  ENABLE_LOCAL_CACHE: "true"
  REPORT_TOKEN: "report-token-1"
  REMOTE_MANAGEMENT_DISABLE: "1"
  # https://github.com/git-lfs/git-lfs/issues/5749
  GIT_CLONE_PROTECTION_ACTIVE: 'false'
 jobs:
  linting-coverage:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - uses: actions/setup-node@v4
      with:
        node-version: '16'
        cache: yarn
    - name: install dependencies
      run: yarn install --frozen-lockfile --network-concurrency 1
    - uses: actions/setup-python@v5
      with:
        python-version: '3.9'
    - uses: actions/cache@v4
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip
    - name: Install python deps
      run: pip install flake8
    - name: Lint Javascript
      run: yarn run --silent lint -- --max-warnings 0
    - name: Lint Markdown
      run: yarn run --silent lint_md
    - name: Lint python
      run: flake8 $(git ls-files "*.py")
    - name: Lint Yaml
      run: yamllint -c yamllint.yml $(git ls-files "*.yml")
    - name: Unit Coverage
      run: |
        set -ex
        mkdir -p $CIRCLE_TEST_REPORTS/unit
        yarn test
        yarn run test_legacy_location
      env:
        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
        CIRCLE_TEST_REPORTS: /tmp
        CIRCLE_ARTIFACTS: /tmp
        CI_REPORTS: /tmp
    - name: Unit Coverage logs
      run: find /tmp/unit -exec cat {} \;
    - name: preparing junit files for upload
      run: |
        mkdir -p artifacts/junit
        find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
      if: always()
    - name: Upload files to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: artifacts
      if: always()
  build:
    runs-on: ubuntu-20.04
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GitHub Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ github.token }}
      - name: Build and push cloudserver image
        uses: docker/build-push-action@v5
        with:
          push: true
          context: .
          provenance: false
          tags: |
            ghcr.io/${{ github.repository }}:${{ github.sha }}
          labels: |
            git.repository=${{ github.repository }}
            git.commit-sha=${{ github.sha }}
          cache-from: type=gha,scope=cloudserver
          cache-to: type=gha,mode=max,scope=cloudserver
      - name: Build and push pykmip image
        uses: docker/build-push-action@v5
        with:
          push: true
          context: .github/pykmip
          tags: |
            ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
          labels: |
            git.repository=${{ github.repository }}
            git.commit-sha=${{ github.sha }}
          cache-from: type=gha,scope=pykmip
          cache-to: type=gha,mode=max,scope=pykmip
      - name: Build and push MongoDB
        uses: docker/build-push-action@v5
        with:
          push: true
          context: .github/docker/mongodb
          tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
          cache-from: type=gha,scope=mongodb
          cache-to: type=gha,mode=max,scope=mongodb
  multiple-backend:
    runs-on: ubuntu-latest
    needs: build
    env:
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      S3BACKEND: mem
      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
      S3DATA: multiple
      JOB_NAME: ${{ github.job }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Login to Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ github.token }}
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - name: Setup CI services
      run: docker compose --profile sproxyd up -d
      working-directory: .github/docker
    - name: Run multiple backend test
      run: |-
        set -o pipefail;
        bash wait_for_local_port.bash 8000 40
        bash wait_for_local_port.bash 81 40
        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
      env:
        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
  mongo-v0-ft-tests:
    runs-on: ubuntu-latest
    needs: build
    env:
      S3BACKEND: mem
      MPU_TESTING: "yes" 
      S3METADATA: mongodb
      S3KMS: file
      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
      DEFAULT_BUCKET_KEY_FORMAT: v0
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      JOB_NAME: ${{ github.job }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - name: Setup CI services
      run: docker compose --profile mongo up -d
      working-directory: .github/docker
    - name: Run functional tests
      run: |-
        set -o pipefail;
        bash wait_for_local_port.bash 8000 40
        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
      env:
        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
  mongo-v1-ft-tests:
    runs-on: ubuntu-latest
    needs: build
    env:
      S3BACKEND: mem
      MPU_TESTING: "yes" 
      S3METADATA: mongodb
      S3KMS: file
      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
      DEFAULT_BUCKET_KEY_FORMAT: v1
      METADATA_MAX_CACHED_BUCKETS: 1
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      JOB_NAME: ${{ github.job }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - name: Setup CI services
      run: docker compose --profile mongo up -d
      working-directory: .github/docker
    - name: Run functional tests
      run: |-
        set -o pipefail;        
        bash wait_for_local_port.bash 8000 40
        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
        yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
      env:
        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
  file-ft-tests:
    strategy:
      matrix:
        include:
          - job-name: file-ft-tests
    name: ${{ matrix.job-name }}
    runs-on: ubuntu-latest
    needs: build
    env:
      S3BACKEND: file
      S3VAULT: mem
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      MPU_TESTING: "yes"
      JOB_NAME: ${{ matrix.job-name }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - name: Setup matrix job artifacts directory
      shell: bash
      run: |
        set -exu
        mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
    - name: Setup CI services
      run: docker compose up -d
      working-directory: .github/docker
    - name: Run file ft tests
      run: |-
        set -o pipefail;
        bash wait_for_local_port.bash 8000 40
        yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
  utapi-v2-tests:
    runs-on: ubuntu-latest
    needs: build
    env:
      ENABLE_UTAPI_V2: t
      S3BACKEND: mem
      BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      JOB_NAME: ${{ github.job }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - name: Setup CI services
      run: docker compose up -d
      working-directory: .github/docker
    - name: Run file utapi v2 tests
      run: |-
        set -ex -o pipefail;
        bash wait_for_local_port.bash 8000 40
        yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
  quota-tests:
    runs-on: ubuntu-latest
    needs: build
    strategy:
      matrix:
        inflights:
          - name: "With Inflights"
            value: "true"
          - name: "Without Inflights"
            value: "false"
    env:
      S3METADATA: mongodb
      S3BACKEND: mem
      S3QUOTA: scuba
      QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
      SCUBA_HOST: localhost
      SCUBA_PORT: 8100
      SCUBA_HEALTHCHECK_FREQUENCY: 100
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      JOB_NAME: ${{ github.job }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - name: Setup CI services
      run: docker compose --profile mongo up -d
      working-directory: .github/docker
    - name: Run quota tests
      run: |-
        set -ex -o pipefail;
        bash wait_for_local_port.bash 8000 40
        yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
  kmip-ft-tests:
    runs-on: ubuntu-latest
    needs: build
    env:
      S3BACKEND: file
      S3VAULT: mem
      MPU_TESTING: "yes" 
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      JOB_NAME: ${{ github.job }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - name: Copy KMIP certs
      run: cp -r ./certs /tmp/ssl-kmip
      working-directory: .github/pykmip
    - name: Setup CI services
      run: docker compose --profile pykmip up -d
      working-directory: .github/docker
    - name: Run file KMIP tests
      run: |-
        set -ex -o pipefail;
        bash wait_for_local_port.bash 8000 40
        bash wait_for_local_port.bash 5696 40
        yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
  ceph-backend-test:
    runs-on: ubuntu-latest
    needs: build
    env:
      S3BACKEND: mem
      S3DATA: multiple
      S3KMS: file
      CI_CEPH: 'true'
      MPU_TESTING: "yes" 
      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
      JOB_NAME: ${{ github.job }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Login to GitHub Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ github.token }}
    - name: Setup CI environment
      uses: ./.github/actions/setup-ci
    - uses: ruby/setup-ruby@v1
      with:
        ruby-version: '2.5.9'
    - name: Install Ruby dependencies
      run: |
        gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
    - name: Install Java dependencies
      run: |
        sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
    - name: Setup CI services
      run: docker compose --profile ceph up -d
      working-directory: .github/docker
      env: 
        S3METADATA: mongodb
    - name: Run Ceph multiple backend tests
      run: |-
        set -ex -o pipefail;
        bash .github/ceph/wait_for_ceph.sh
        bash wait_for_local_port.bash 27018 40
        bash wait_for_local_port.bash 8000 40
        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
      env:
        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
        S3METADATA: mem
    - name: Run Java tests
      run: |-
        set -ex -o pipefail;
        mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
      working-directory: tests/functional/jaws
    - name: Run Ruby tests
      run: |-
        set -ex -o pipefail;
        rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
      working-directory: tests/functional/fog
    - name: Run Javascript AWS SDK tests
      run: |-
        set -ex -o pipefail;
        yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
        yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
      env:
        S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
        S3BACKEND: file
        S3VAULT: mem
        S3METADATA: mongodb
    - name: Upload logs to artifacts
      uses: scality/action-artifacts@v4
      with:
        method: upload
        url: https://artifacts.scality.net
        user: ${{ secrets.ARTIFACTS_USER }}
        password: ${{ secrets.ARTIFACTS_PASSWORD }}
        source: /tmp/artifacts
      if: always()
--- a/.gitignore
+++ b/.gitignore
@ -22,14 +22,6 @@ coverage
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
 # Sphinx build dir
 _build
 # Dependency directory
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
 yarn.lock
 .tox
 # Junit directory
 junit
--- a/DESIGN.md
+++ b/DESIGN.md
@ -56,8 +56,6 @@ Right now, the following operations are implemented:
 - Put Bucket Website
 - Get Bucket Website
 - Delete Bucket Website
 - Put Bucket Versioning
 - Get Bucket Versioning
 - v2 Authentication
 - v4 Authentication (Transferring Payload in a Single Chunk)
 - v4 Authentication (Transferring Payload in Multiple Chunks)
--- a/DOCKER.md
+++ b/DOCKER.md
@ -0,0 +1,189 @@
 # Using S3 for continuous integration testing or in production with Docker
 * [For continuous integration with Docker](#for-continuous-integration-with-docker)
  * [Environment Variables](#environment-variables)
 * [In production with Docker](#in-production-with-docker)
  * [Using Docker Volume in production](#using-docker-volume-in-production)
  * [Adding modifying or deleting accounts or users credentials](#adding-modifying-or-deleting-accounts-or-users-credentials)
  * [Specifying your own host name](#specifying-your-own-host-name)
  * [Running as an unprivileged user](#running-as-an-unprivileged-user)
 ## For continuous integration with Docker
 When you start the Docker Scality S3 server image, you can adjust the
 configuration of the Scality S3 server instance by passing one or more
 environment variables on the docker run command line.
 ### Environment Variables
 #### S3DATA=multiple
 This runs Scality S3 server with multiple data backends.
 [More info](https://github.com/scality/S3#run-it-with-multiple-data-backends)
 ```shell
 docker run -d --name s3server -p 8000:8000 -e S3DATA=multiple scality/s3server
 ```
 #### HOST_NAME
 This variable specifies a host name.
 If you have a domain such as new.host.com, by specifying that here,
 you and your users can direct s3 server requests to new.host.com.
 ```shell
 docker run -d --name s3server -p 8000:8000 -e HOST_NAME=new.host.com scality/s3server
 ```
 Note: In your `/etc/hosts` file on Linux, OS X, or Unix with root permissions,
 make sure to associate 127.0.0.1 with `new.host.com`
 #### SCALITY_ACCESS_KEY_ID and SCALITY_SECRET_ACCESS_KEY
 These variables specify authentication credentials for an account
 named "CustomAccount".
 You can set credentials for many accounts by editing `conf/authdata.json`
 (see below for further info),
 but if you just want to specify one set of your own,
 you can use these environment variables.
 ```shell
 docker run -d --name s3server -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey
 -e SCALITY_SECRET_ACCESS_KEY=newSecretKey scality/s3server
 ```
 Note: Anything in the `authdata.json` file will be ignored.
 Note: The old `ACCESS_KEY` and `SECRET_KEY` environment variables are now
 deprecated
 #### LOG_LEVEL
 This variable allows you to change the log level: info, debug or trace.
 The default is info. Debug will give you more detailed logs and trace
 will give you the most detailed.
 ```shell
 docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
 ```
 #### SSL
 This variable specifies the Common Name `<DOMAIN_NAME>` used to create the
 Certificate Signing Request using OpenSSL. This allows you to run S3 with SSL:
 **Note**:
 In your `/etc/hosts` file on Linux, OS X, or Unix with root permissions,
 make sure to associate 127.0.0.1 with `<SUBDOMAIN>.<DOMAIN_NAME>`
 **Warning**:
 These certs, being self-signed (and the CA being generated inside the container)
 will be untrusted by any clients, and could disappear on a container upgrade.
 That's ok as long as it's for quick testing.
 Also, best security practice for non-testing would be to use an extra container
 to do SSL/TLS termination such as haproxy/nginx/stunnel to limit what an exploit
 on either component could expose, as well as certificates in a mounted volume
 ```shell
 docker run -d --name s3server -p 8000:8000 -e SSL=<DOMAIN_NAME> -e HOST_NAME=<SUBDOMAIN>.<DOMAIN_NAME>
 scality/s3server
 ```
 More information about how to use S3 server with SSL
 [here](https://s3.scality.com/v1.0/page/scality-with-ssl)
 ## In production with Docker
 ### Using Docker Volume in production
 S3 server runs with a file backend by default.
 So, by default, the data is stored inside your S3 server Docker container.
 However, if you want your data and metadata to persist, you **MUST** use Docker
 volumes to host your data and metadata outside your s3 server Docker container.
 Otherwise, the data and metadata will be destroyed when you erase the container.
 ```shell
 docker run -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
 -p 8000:8000 -d scality/s3server
 ```
 This command mounts the host directory, `./data`, into the container at
 /usr/src/app/localData and the host directory, `./metadata`, into the container
 at /usr/src/app/localMetaData. It can also be any host mount point,
 like `/mnt/data` and `/mnt/metadata`.
 ### Adding modifying or deleting accounts or users credentials
 1. Create locally a customized `authdata.json`.
 2. Use [Docker Volume](https://docs.docker.com/engine/tutorials/dockervolumes/)
 to override the default `authdata.json` through a docker file mapping.
 For example:
 ```shell
 docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
 scality/s3server
 ```
 ### Specifying your own host name
 To specify a host name (e.g. s3.domain.name),
 you can provide your own
 [config.json](https://github.com/scality/S3/blob/master/config.json)
 using [Docker Volume](https://docs.docker.com/engine/tutorials/dockervolumes/).
 First add a new key-value pair in the restEndpoints section of your config.json.
 The key in the key-value pair should be the host name you would like to add
 and the value is the default location_constraint for this endpoint.
 For example, `s3.example.com` is mapped to `us-east-1` which is one of
 the `location_constraints` listed in your locationConfig.json file
 [here](https://github.com/scality/S3/blob/master/locationConfig.json).
 More information about location configuration
 [here](https://github.com/scality/S3/blob/master/README.md#location-configuration)
 ```json
 "restEndpoints": {
    "localhost": "file",
    "127.0.0.1": "file",
    ...
    "s3.example.com": "us-east-1"
 },
 ```
 Then, run your Scality S3 Server using
 [Docker Volume](https://docs.docker.com/engine/tutorials/dockervolumes/):
 ```shell
 docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
 ```
 Your local `config.json` file will override the default one through a docker
 file mapping.
 ### Running as an unprivileged user
 S3 Server runs as root by default.
 You can change that by modifing the dockerfile and specifying a user before the entrypoint.
 The user needs to exist within the container,
 and own the folder **/usr/src/app** for Scality S3 Server to run properly.
 For instance, you can modify these lines in the dockerfile:
 ```shell
 ...
 && groupadd -r -g 1001 scality \
 && useradd -u 1001 -g 1001 -d /usr/src/app -r scality \
 && chown -R scality:scality /usr/src/app
 ...
 USER scality
 ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
 ```
--- a/66
+++ b/66
@ -1,60 +1,22 @@
-ARG NODE_VERSION=16.20-bullseye-slim
+FROM node:6-slim
-
+MAINTAINER Giorgio Regni <gr@scality.com>
 FROM node:${NODE_VERSION} as builder
 WORKDIR /usr/src/app
 COPY . /usr/src/app
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
+    && apt-get install -y python git build-essential --no-install-recommends \
-        build-essential \
+    && npm install --production \
-        ca-certificates \
+    && apt-get autoremove --purge -y python git build-essential \
-        curl \
+    && rm -rf /var/lib/apt/lists/* \
-        git \
+    && npm cache clear \
-        gnupg2 \
+    && rm -rf ~/.node-gyp \
-        jq \
+    && rm -rf /tmp/npm-*
        python3 \
        ssh \
        wget \
        libffi-dev \
        zlib1g-dev \
    && apt-get clean \
    && mkdir -p /root/ssh \
    && ssh-keyscan -H github.com > /root/ssh/known_hosts
 ENV PYTHON=python3
 COPY package.json yarn.lock /usr/src/app/
 RUN npm install typescript -g
 RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
 ################################################################################
 FROM node:${NODE_VERSION}
 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
        jq \
    && rm -rf /var/lib/apt/lists/*
 ENV NO_PROXY localhost,127.0.0.1
 ENV no_proxy localhost,127.0.0.1
 EXPOSE 8000
 EXPOSE 8002
 RUN apt-get update && \
    apt-get install -y --no-install-recommends \
        jq \
        tini \
    && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/app
 # Keep the .git directory in order to properly report version
 COPY . /usr/src/app
 COPY --from=builder /usr/src/app/node_modules ./node_modules/
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
-ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
+ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
 CMD [ "npm", "start" ]
-CMD [ "yarn", "start" ]
+EXPOSE 8000
--- a/10
+++ b/10
@ -6,17 +6,17 @@ WORKDIR /usr/src/app
 COPY . /usr/src/app
 RUN apt-get update \
-    && apt-get install -y jq python git build-essential --no-install-recommends \
+    && apt-get install -y python git build-essential --no-install-recommends \
-    && yarn install --production \
+    && npm install --production \
    && apt-get autoremove --purge -y python git build-essential \
    && rm -rf /var/lib/apt/lists/* \
-    && yarn cache clean \
+    && npm cache clear \
    && rm -rf ~/.node-gyp \
-    && rm -rf /tmp/yarn-*
+    && rm -rf /tmp/npm-*
 ENV S3BACKEND mem
 ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
-CMD [ "yarn", "start" ]
+CMD [ "npm", "start" ]
 EXPOSE 8000
--- a/Healthchecks.md
+++ b/Healthchecks.md
@ -1,7 +1,6 @@
 # S3 Healthcheck
-Scality S3 exposes a healthcheck route `/live` on the port used
+Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
 for the metrics (defaults to port 8002) which returns a
 response with HTTP code
 - 200 OK
--- a/README.md
+++ b/README.md
@ -1,81 +1,511 @@
-# Zenko CloudServer with Vitastor Backend
+# Scality S3 Server
-![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
+![S3 Server logo](res/Scality-S3-Server-Logo-Large.png)
-## Overview
+[![CircleCI][badgepub]](https://circleci.com/gh/scality/S3)
 [![Scality CI][badgepriv]](http://ci.ironmann.io/gh/scality/S3)
-CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
+## Learn more at [s3.scality.com](http://s3.scality.com)
 object storage server that is part of [Zenko](https://www.zenko.io),
 Scality’s Open Source Multi-Cloud Data Controller.
-CloudServer provides a single AWS S3 API interface to access multiple
+## Contributing
 backend data storage both on-premise or public in the cloud.
-This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
+In order to contribute, please follow the
-backend support.
+[Contributing Guidelines](
 https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
-## Quick Start with Vitastor
+## Installation
-Vitastor Backend is in experimental status, however you can already try to
+### Dependencies
 run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
 it works too 😊.
-Installation instructions:
+Building and running the Scality S3 Server requires node.js 6.9.5 and npm v3
 . Up-to-date versions can be found at
 [Nodesource](https://github.com/nodesource/distributions).
-### Install Vitastor
+### Clone source code
-Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
+```shell
-
+git clone https://github.com/scality/S3.git
 ### Install Zenko with Vitastor Backend
 - Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
 - Install dependencies: `npm install --omit dev` or just `npm install`
 - Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
 - Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
  You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
 - Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
 ### Install and Configure MongoDB
 Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
 ### Setup Zenko
 - Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
 - Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
 - In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
 - Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
 - Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
  and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
  access keys, but it's not published, so let's use a file for now.
 - Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
  You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
  in this file.
 Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
 instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
 ### Start Zenko
 Start the S3 server with: `node index.js`
 If you use default settings, Zenko CloudServer starts on port 8000.
 The default access key is `accessKey1` with a secret key of `verySecretKey1`.
 Now you can access your S3 with `s3cmd` or `geesefs`:
 ```
 s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
 ```
-```
+### Install js dependencies
-AWS_ACCESS_KEY_ID=accessKey1 \
+
-    AWS_SECRET_ACCESS_KEY=verySecretKey1 \
+Go to the ./S3 folder,
-    geesefs --endpoint http://localhost:8000 testbucket mountdir
+
 ```shell
 npm install
 ```
-# Author & License
+## Run it with a file backend
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
+```shell
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
+npm start
-  (a "network copyleft" license based on AGPL/SSPL, but worded in a better way)
+```
 This starts an S3 server on port 8000. Two additional ports 9990 and
 9991 are also open locally for internal transfer of metadata and data,
 respectively.
 The default access key is accessKey1 with
 a secret key of verySecretKey1.
 By default the metadata files will be saved in the
 localMetadata directory and the data files will be saved
 in the localData directory within the ./S3 directory on your
 machine.  These directories have been pre-created within the
 repository.  If you would like to save the data or metadata in
 different locations of your choice, you must specify them with absolute paths.
 So, when starting the server:
 ```shell
 mkdir -m 700 $(pwd)/myFavoriteDataPath
 mkdir -m 700 $(pwd)/myFavoriteMetadataPath
 export S3DATAPATH="$(pwd)/myFavoriteDataPath"
 export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
 npm start
 ```
 ## Run it with multiple data backends
 ```shell
 export S3DATA='multiple'
 npm start
 ```
 This starts an S3 server on port 8000.
 The default access key is accessKey1 with
 a secret key of verySecretKey1.
 With multiple backends, you have the ability to
 choose where each object will be saved by setting
 the following header with a locationConstraint on
 a PUT request:
 ```shell
 'x-amz-meta-scal-location-constraint':'myLocationConstraint'
 ```
 If no header is sent with a PUT object request, the
 location constraint of the bucket will determine
 where the data is saved. If the bucket has no location
 constraint, the endpoint of the PUT request will be
 used to determine location.
 See the Configuration section below to learn how to set
 location constraints.
 ## Run it with an in-memory backend
 ```shell
 npm run mem_backend
 ```
 This starts an S3 server on port 8000.
 The default access key is accessKey1 with
 a secret key of verySecretKey1.
 ## Setting your own access key and secret key pairs
 You can set credentials for many accounts by editing `conf/authdata.json` but if
 you want to specify one set of your own credentials, you can use
 `SCALITY_ACCESS_KEY_ID` and `SCALITY_SECRET_ACCESS_KEY` environment variables.
 ### SCALITY_ACCESS_KEY_ID and SCALITY_SECRET_ACCESS_KEY
 These variables specify authentication credentials for an account
 named "CustomAccount".
 Note: Anything in the `authdata.json` file will be ignored.
 ```shell
 SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey npm start
 ```
 ## Run it for continuous integration testing or in production with Docker
 [DOCKER.md](DOCKER.md)
 ## Testing
 You can run the unit tests with the following command:
 ```shell
 npm test
 ```
 You can run the multiple backend unit tests with:
 ```shell
 npm run multiple_backend_test
 ```
 You can run the linter with:
 ```shell
 npm run lint
 ```
 Running functional tests locally:
 The test suite requires additional tools, **s3cmd** and **Redis** installed
 in the environment the tests are running in.
 * Install [s3cmd](http://s3tools.org/download)
 * Install [redis](https://redis.io/download) and start Redis.
 * Add localCache section to your `config.json`:
 ```
 "localCache": {
    "host": REDIS_HOST,
    "port": REDIS_PORT
 }
 ```
 where `REDIS_HOST` is your Redis instance IP address (`"127.0.0.1"` if your
 Redis is running locally)
 and `REDIS_PORT` is your Redis instance port (`6379` by default)
 * Add the following to the etc/hosts file on your machine:
 ```shell
 127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
 ```
 * Start the S3 server in memory and run the functional tests:
 ```shell
 npm run mem_backend
 npm run ft_test
 ```
 ## Configuration
 There are three configuration files for your Scality S3 Server:
 1. `conf/authdata.json`, described above for authentication
 2. `locationConfig.json`, to set up configuration options for
    where data will be saved
 3. `config.json`, for general configuration options
 ### Location Configuration
 You must specify at least one locationConstraint in your
 locationConfig.json (or leave as pre-configured).
 For instance, the following locationConstraint will save data
 sent to `myLocationConstraint` to the file backend:
 ```json
 "myLocationConstraint": {
    "type": "file",
    "legacyAwsBehavior": false,
    "details": {}
 },
 ```
 Each locationConstraint must include the `type`, `legacyAwsBehavior`,
 and `details` keys. `type` indicates which backend will be used
 for that region. Currently, mem, file, and scality are the
 supported backends. `legacyAwsBehavior` indicates whether the
 region will have the same behavior as the AWS S3 'us-east-1'
 region. If the locationConstraint type is scality, `details` should
 contain connector information for sproxyd. If the
 locationConstraint type is mem or file, `details` should be empty.
 Once you have your locationConstraints in your locationConfig.json,
 you can specify a default locationConstraint for each of your
 endpoints.
 For instance, the following sets the `localhost` endpoint to the
 `myLocationConstraint` data backend defined above:
 ```json
 "restEndpoints": {
     "localhost": "myLocationConstraint"
 },
 ```
 If you would like to use an endpoint other than localhost for your
 Scality S3 Server, that endpoint MUST be listed in your `restEndpoints`.
 Otherwise if your server is running with a:
 - __file backend__: your default location constraint will be `file`
 - __memory backend__: your default location constraint will be `mem`
 ## Endpoints
 Note that our S3server supports both:
 - path-style: http://myhostname.com/mybucket
 - hosted-style: http://mybucket.myhostname.com
 However, hosted-style requests will not hit the server if you are
 using an ip address for your host.
 So, make sure you are using path-style requests in that case.
 For instance, if you are using the AWS SDK for JavaScript,
 you would instantiate your client like this:
 ```js
 const s3 = new aws.S3({
   endpoint: 'http://127.0.0.1:8000',
   s3ForcePathStyle: true,
 });
 ```
 [badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
 [badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
 ## Getting started: List of applications that have been tested with S3 Server
 ### GUI
 #### [Cyberduck](https://cyberduck.io/?l=en)
 - https://www.youtube.com/watch?v=-n2MCt4ukUg
 - https://www.youtube.com/watch?v=IyXHcu4uqgU
 #### [Cloud Explorer](https://www.linux-toys.com/?p=945)
 - https://www.youtube.com/watch?v=2hhtBtmBSxE
 #### [CloudBerry Lab](http://www.cloudberrylab.com)
 - https://youtu.be/IjIx8g_o0gY
 ### Command Line Tools
 #### [s3curl](https://github.com/rtdp/s3curl)
 https://github.com/scality/S3/blob/master/tests/functional/s3curl/s3curl.pl
 #### [aws-cli](http://docs.aws.amazon.com/cli/latest/reference/)
 `~/.aws/credentials` on Linux, OS X, or Unix or
 `C:\Users\USERNAME\.aws\credentials` on Windows
 ```shell
 [default]
 aws_access_key_id = accessKey1
 aws_secret_access_key = verySecretKey1
 ```
 `~/.aws/config` on Linux, OS X, or Unix or
 `C:\Users\USERNAME\.aws\config` on Windows
 ```shell
 [default]
 region = us-east-1
 ```
 Note: `us-east-1` is the default region, but you can specify any region.
 See all buckets:
 ```shell
 aws s3 ls --endpoint-url=http://localhost:8000
 ```
 Create bucket:
 ```shell
 aws --endpoint-url=http://localhost:8000 s3 mb s3://mybucket
 ```
 #### [s3cmd](http://s3tools.org/s3cmd)
 If using s3cmd as a client to S3 be aware that v4 signature format
 is buggy in s3cmd versions < 1.6.1.
 `~/.s3cfg` on Linux, OS X, or Unix or
 `C:\Users\USERNAME\.s3cfg` on Windows
 ```shell
 [default]
 access_key = accessKey1
 secret_key = verySecretKey1
 host_base = localhost:8000
 host_bucket = %(bucket).localhost:8000
 signature_v2 = False
 use_https = False
 ```
 See all buckets:
 ```shell
 s3cmd ls
 ```
 #### [rclone](http://rclone.org/s3/)
 `~/.rclone.conf` on Linux, OS X, or Unix or
 `C:\Users\USERNAME\.rclone.conf` on Windows
 ```shell
 [remote]
 type = s3
 env_auth = false
 access_key_id = accessKey1
 secret_access_key = verySecretKey1
 region = other-v2-signature
 endpoint = http://localhost:8000
 location_constraint =
 acl = private
 server_side_encryption =
 storage_class =
 ```
 See all buckets:
 ```shell
 rclone lsd remote:
 ```
 ### JavaScript
 #### [AWS JavaScript SDK](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html)
 ```javascript
 const AWS = require('aws-sdk');
 const s3 = new AWS.S3({
    accessKeyId: 'accessKey1',
    secretAccessKey: 'verySecretKey1',
    endpoint: 'localhost:8000',
    sslEnabled: false,
    s3ForcePathStyle: true,
 });
 ```
 ### JAVA
 #### [AWS JAVA SDK](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Client.html)
 ```java
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.S3ClientOptions;
 import com.amazonaws.services.s3.model.Bucket;
 public class S3 {
    public static void main(String[] args) {
        AWSCredentials credentials = new BasicAWSCredentials("accessKey1",
        "verySecretKey1");
        // Create a client connection based on credentials
        AmazonS3 s3client = new AmazonS3Client(credentials);
        s3client.setEndpoint("http://localhost:8000");
        // Using path-style requests
        // (deprecated) s3client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
        s3client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        // Create bucket
        String bucketName = "javabucket";
        s3client.createBucket(bucketName);
        // List off all buckets
        for (Bucket bucket : s3client.listBuckets()) {
            System.out.println(" - " + bucket.getName());
        }
    }
 }
 ```
 ## Ruby
 ### [AWS SDK for Ruby - Version 2](http://docs.aws.amazon.com/sdkforruby/api/)
 ```ruby
 require 'aws-sdk'
 s3 = Aws::S3::Client.new(
  :access_key_id => 'accessKey1',
  :secret_access_key => 'verySecretKey1',
  :endpoint => 'http://localhost:8000',
  :force_path_style => true
 )
 resp = s3.list_buckets
 ```
 #### [fog](http://fog.io/storage/)
 ```ruby
 require "fog"
 connection = Fog::Storage.new(
 {
    :provider => "AWS",
    :aws_access_key_id => 'accessKey1',
    :aws_secret_access_key => 'verySecretKey1',
    :endpoint => 'http://localhost:8000',
    :path_style => true,
    :scheme => 'http',
 })
 ```
 ### Python
 #### [boto2](http://boto.cloudhackers.com/en/latest/ref/s3.html)
 ```python
 import boto
 from boto.s3.connection import S3Connection, OrdinaryCallingFormat
 connection = S3Connection(
    aws_access_key_id='accessKey1',
    aws_secret_access_key='verySecretKey1',
    is_secure=False,
    port=8000,
    calling_format=OrdinaryCallingFormat(),
    host='localhost'
 )
 connection.create_bucket('mybucket')
 ```
 #### [boto3](http://boto3.readthedocs.io/en/latest/index.html)
 ``` python
 import boto3
 client = boto3.client(
    's3',
    aws_access_key_id='accessKey1',
    aws_secret_access_key='verySecretKey1',
    endpoint_url='http://localhost:8000'
 )
 lists = client.list_buckets()
 ```
 ### PHP
 Should use v3 over v2 because v2 would create virtual-hosted style URLs
 while v3 generates path-style URLs.
 #### [AWS PHP SDK v3](https://docs.aws.amazon.com/aws-sdk-php/v3/guide)
 ```php
 use Aws\S3\S3Client;
 $client = S3Client::factory([
    'region'  => 'us-east-1',
    'version'   => 'latest',
    'endpoint' => 'http://localhost:8000',
    'credentials' => [
         'key'    => 'accessKey1',
         'secret' => 'verySecretKey1'
    ]
 ]);
 $client->createBucket(array(
    'Bucket' => 'bucketphp',
 ));
 ```
--- a/_config.yml
+++ b/_config.yml
@ -1,2 +1,2 @@
 ---
-theme: jekyll-theme-modernist
+theme: jekyll-theme-minimal
--- a/authdata.json.example
+++ b/authdata.json.example
@ -1,56 +0,0 @@
 {
    "accounts": [{
        "name": "Bart",
        "email": "sampleaccount1@sampling.com",
        "arn": "arn:aws:iam::123456789012:root",
        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
        "shortid": "123456789012",
        "keys": [{
            "access": "accessKey1",
            "secret": "verySecretKey1"
        }]
    }, {
        "name": "Lisa",
        "email": "sampleaccount2@sampling.com",
        "arn": "arn:aws:iam::123456789013:root",
        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
        "shortid": "123456789013",
        "keys": [{
            "access": "accessKey2",
            "secret": "verySecretKey2"
        }]
    },
    {
        "name": "Clueso",
        "email": "inspector@clueso.info",
        "arn": "arn:aws:iam::123456789014:root",
        "canonicalID": "http://acs.zenko.io/accounts/service/clueso",
        "shortid": "123456789014",
        "keys": [{
            "access": "cluesoKey1",
            "secret": "cluesoSecretKey1"
        }]
    },
    {
        "name": "Replication",
        "email": "inspector@replication.info",
        "arn": "arn:aws:iam::123456789015:root",
        "canonicalID": "http://acs.zenko.io/accounts/service/replication",
        "shortid": "123456789015",
        "keys": [{
            "access": "replicationKey1",
            "secret": "replicationSecretKey1"
        }]
    },
    {
        "name": "Lifecycle",
        "email": "inspector@lifecycle.info",
        "arn": "arn:aws:iam::123456789016:root",
        "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle",
        "shortid": "123456789016",
        "keys": [{
            "access": "lifecycleKey1",
            "secret": "lifecycleSecretKey1"
        }]
    }]
 }
--- a/bin/create_bucket_with_NFS_enabled.js
+++ b/bin/create_bucket_with_NFS_enabled.js
@ -1,4 +0,0 @@
 #!/usr/bin/env node
 'use strict'; // eslint-disable-line strict
 require('../lib/nfs/utilities.js').createBucketWithNFSEnabled();
--- a/bin/search_bucket.js
+++ b/bin/search_bucket.js
@ -1,108 +0,0 @@
 #!/bin/sh
 // 2>/dev/null ; exec "$(which nodejs 2>/dev/null || which node)" "$0" "$@"
 'use strict'; // eslint-disable-line strict
 const { auth } = require('arsenal');
 const commander = require('commander');
 const http = require('http');
 const https = require('https');
 const logger = require('../lib/utilities/logger');
 function _performSearch(host,
                        port,
                        bucketName,
                        query,
                        listVersions,
                        accessKey,
                        secretKey,
                        sessionToken,
                        verbose, ssl) {
    const escapedSearch = encodeURIComponent(query);
    const options = {
        host,
        port,
        method: 'GET',
        path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`,
        headers: {
            'Content-Length': 0,
        },
        rejectUnauthorized: false,
        versions: '',
    };
    if (sessionToken) {
        options.headers['x-amz-security-token'] = sessionToken;
    }
    const transport = ssl ? https : http;
    const request = transport.request(options, response => {
        if (verbose) {
            logger.info('response status code', {
                statusCode: response.statusCode,
            });
            logger.info('response headers', { headers: response.headers });
        }
        const body = [];
        response.setEncoding('utf8');
        response.on('data', chunk => body.push(chunk));
        response.on('end', () => {
            if (response.statusCode >= 200 && response.statusCode < 300) {
                logger.info('Success');
                process.stdout.write(body.join(''));
                process.exit(0);
            } else {
                logger.error('request failed with HTTP Status ', {
                    statusCode: response.statusCode,
                    body: body.join(''),
                });
                process.exit(1);
            }
        });
    });
    // generateV4Headers exepects request object with path that does not
    // include query
    request.path = `/${bucketName}`;
    const requestData = listVersions ? { search: query, versions: '' } : { search: query };
    auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3');
    request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`;
    if (verbose) {
        logger.info('request headers', { headers: request._headers });
    }
    request.end();
 }
 /**
 * This function is used as a binary to send a request to S3 to perform a
 * search on the objects in a bucket
 *
 * @return {undefined}
 */
 function searchBucket() {
    // TODO: Include other bucket listing possible query params?
    commander
        .version('0.0.1')
        .option('-a, --access-key <accessKey>', 'Access key id')
        .option('-k, --secret-key <secretKey>', 'Secret access key')
        .option('-t, --session-token <sessionToken>', 'Session token')
        .option('-b, --bucket <bucket>', 'Name of the bucket')
        .option('-q, --query <query>', 'Search query')
        .option('-h, --host <host>', 'Host of the server')
        .option('-p, --port <port>', 'Port of the server')
        .option('-s', '--ssl', 'Enable ssl')
        .option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
            'otherwise only list the latest version')
        .option('-v, --verbose')
        .parse(process.argv);
    const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
        commander;
    if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
        logger.error('missing parameter');
        commander.outputHelp();
        process.exit(1);
    }
    _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose,
        ssl);
 }
 searchBucket();
--- a/circle.yml
+++ b/circle.yml
@ -0,0 +1,134 @@
 ---
 general:
  branches:
    ignore:
      - /^ultron\/.*/   # Ignore ultron/* branches
  artifacts:
    - coverage/
 machine:
  node:
    version: 6.9.5
  ruby:
    version: "2.4.1"
  services:
    - redis
  environment:
    CXX: g++-4.9
    ENABLE_LOCAL_CACHE: true
  hosts:
    bucketwebsitetester.s3-website-us-east-1.amazonaws.com: 127.0.0.1
 dependencies:
  post:
    - sudo pip install flake8 yamllint
    - sudo pip install s3cmd==1.6.1
    # fog and ruby testing dependencies
    - gem install fog-aws -v 1.3.0
    - gem install mime-types -v 3.1
    - gem install rspec -v 3.5
    - gem install json
    - gem install digest
    # java sdk dependencies
    - sudo apt-get install -y -q default-jdk
 test:
  override:
    - npm run --silent lint -- --max-warnings 0
    - npm run --silent lint_md
    - flake8 $(git ls-files '*.py')
    - yamllint $(git ls-files '*.yml')
    - mkdir -p $CIRCLE_TEST_REPORTS/unit
    - npm run unit_coverage
    # Run S3 with mem Backend ; run ft_tests
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_java.txt
        & bash wait_for_local_port.bash 8000 40
        && cd ./tests/functional/jaws && mvn test
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_fog.txt
        & bash wait_for_local_port.bash 8000 40
        && cd tests/functional/fog && rspec tests.rb
    - S3BACKEND=mem MPU_TESTING=yes npm start
            > $CIRCLE_ARTIFACTS/server_mem_awssdk.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_awssdk
    - S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start
            > $CIRCLE_ARTIFACTS/server_mem_multiple_awsskd.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_awssdk
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_s3cmd.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_s3cmd
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_s3curl.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_s3curl
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_rawnode.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_node
    # Run S3 with mem Backend + KMS Encryption ; run ft_tests
    - S3BACKEND=mem MPU_TESTING=yes npm start
            > $CIRCLE_ARTIFACTS/server_mem_kms_awssdk.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_awssdk
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_kms_s3cmd.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_s3cmd
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_kms_s3curl.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_s3curl
    - S3BACKEND=mem npm start
            > $CIRCLE_ARTIFACTS/server_mem_kms_rawnode.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_node
    # Run S3 with file Backend ; run ft_tests
    - S3BACKEND=file S3VAULT=mem MPU_TESTING=yes npm start
            > $CIRCLE_ARTIFACTS/server_file_awssdk.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_awssdk
    - S3BACKEND=file S3VAULT=mem npm start
            > $CIRCLE_ARTIFACTS/server_file_s3cmd.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_s3cmd
    - S3BACKEND=file S3VAULT=mem npm start
            > $CIRCLE_ARTIFACTS/server_file_s3curl.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_s3curl
    - S3BACKEND=file S3VAULT=mem npm start
            > $CIRCLE_ARTIFACTS/server_file_rawnode.txt
        & bash wait_for_local_port.bash 8000 40
        && npm run ft_node
    # Run S3 with file Backend + KMS Encryption ; run ft_tests
    - S3BACKEND=file S3VAULT=mem MPU_TESTING=yes npm start
            > $CIRCLE_ARTIFACTS/server_file_kms_awssdk.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_awssdk
    - S3BACKEND=file S3VAULT=mem npm start
            > $CIRCLE_ARTIFACTS/server_file_kms_s3cmd.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_s3cmd
    - S3BACKEND=file S3VAULT=mem npm start
            > $CIRCLE_ARTIFACTS/server_file_kms_s3curl.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_s3curl
    - S3BACKEND=file S3VAULT=mem npm start
            > $CIRCLE_ARTIFACTS/server_file_kms_rawnode.txt
        & bash wait_for_local_port.bash 8000 40
        && ENABLE_KMS_ENCRYPTION=true npm run ft_node
    # Run S3 with multiple data backends ; run multiple_backend_test
    - S3BACKEND=mem S3DATA=multiple npm run start_dmd
            > $CIRCLE_ARTIFACTS/server_multiple_data_backends.txt
        & bash wait_for_local_port.bash 9991 40
        && npm run multiple_backend_test
--- a/conf/authdata.json
+++ b/conf/authdata.json
@ -0,0 +1,42 @@
 {
    "accounts": [{
        "name": "Bart",
        "email": "sampleaccount1@sampling.com",
        "arn": "aws::iam:123456789012:root",
        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
        "shortid": "123456789012",
        "keys": [{
            "access": "accessKey1",
            "secret": "verySecretKey1"
        }],
        "users": [{
            "name": "Bart Jr",
            "email": "user1.sampleaccount2@sampling.com",
            "arn": "aws::iam:123456789013:bart",
            "keys": [{
                "access": "USERBARTFUNACCESSKEY",
                "secret": "verySecretKey1"
            }]
        }]
    }, {
        "name": "Lisa",
        "email": "sampleaccount2@sampling.com",
        "arn": "aws::iam:accessKey2:user/Lisa",
        "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
        "shortid": "123456789012",
        "keys": [{
            "access": "accessKey2",
            "secret": "verySecretKey2"
        }]
    }, {
        "name": "Docker",
        "email": "sampleaccount3@sampling.com",
        "arn": "aws::iam:accessKeyDocker:user/Docker",
        "canonicalID": "sd359df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47eh3hd",
        "shortid": "123456789012",
        "keys": [{
            "access": "accessKeyDocker",
            "secret": "verySecretKeyDocker"
        }]
    }]
 }
--- a/config.json
+++ b/config.json
@ -0,0 +1,57 @@
 {
    "port": 8000,
    "listenOn": [],
    "replicationGroupId": "RG001",
    "restEndpoints": {
        "localhost": "file",
        "127.0.0.1": "file",
        "s3.docker.test": "us-east-1",
        "127.0.0.2": "us-east-1",
        "s3.amazonaws.com": "us-east-1"
    },
    "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
                        "s3-website.us-east-2.amazonaws.com",
                        "s3-website-us-west-1.amazonaws.com",
                        "s3-website-us-west-2.amazonaws.com",
                        "s3-website.ap-south-1.amazonaws.com",
                        "s3-website.ap-northeast-2.amazonaws.com",
                        "s3-website-ap-southeast-1.amazonaws.com",
                        "s3-website-ap-southeast-2.amazonaws.com",
                        "s3-website-ap-northeast-1.amazonaws.com",
                        "s3-website.eu-central-1.amazonaws.com",
                        "s3-website-eu-west-1.amazonaws.com",
                        "s3-website-sa-east-1.amazonaws.com",
                        "s3-website.localhost",
                        "s3-website.scality.test"],
    "bucketd": {
        "bootstrap": ["localhost"]
    },
    "vaultd": {
        "host": "localhost",
        "port": 8500
    },
    "clusters": 10,
    "log": {
        "logLevel": "info",
        "dumpLevel": "error"
    },
    "healthChecks": {
        "allowFrom": ["127.0.0.1/8", "::1"]
    },
    "metadataClient": {
        "host": "localhost",
        "port": 9990
    },
    "dataClient": {
        "host": "localhost",
        "port": 9991
    },
    "metadataDaemon": {
        "bindAddress": "localhost",
        "port": 9990
    },
    "dataDaemon": {
        "bindAddress": "localhost",
        "port": 9991
    }
 }
--- a/config.json.example
+++ b/config.json.example
@ -1,143 +0,0 @@
 {
    "port": 8000,
    "listenOn": [],
    "metricsPort": 8002,
    "metricsListenOn": [],
    "replicationGroupId": "RG001",
    "workers": 4,
    "restEndpoints": {
        "localhost": "us-east-1",
        "127.0.0.1": "us-east-1",
        "cloudserver-front": "us-east-1",
        "s3.docker.test": "us-east-1",
        "127.0.0.2": "us-east-1",
        "s3.amazonaws.com": "us-east-1",
        "zenko-cloudserver-replicator": "us-east-1",
        "lb": "us-east-1"
    },
    "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
                        "s3-website.us-east-2.amazonaws.com",
                        "s3-website-us-west-1.amazonaws.com",
                        "s3-website-us-west-2.amazonaws.com",
                        "s3-website.ap-south-1.amazonaws.com",
                        "s3-website.ap-northeast-2.amazonaws.com",
                        "s3-website-ap-southeast-1.amazonaws.com",
                        "s3-website-ap-southeast-2.amazonaws.com",
                        "s3-website-ap-northeast-1.amazonaws.com",
                        "s3-website.eu-central-1.amazonaws.com",
                        "s3-website-eu-west-1.amazonaws.com",
                        "s3-website-sa-east-1.amazonaws.com",
                        "s3-website.localhost",
                        "s3-website.scality.test",
                        "zenkoazuretest.blob.core.windows.net"],
    "replicationEndpoints": [{
        "site": "zenko",
        "servers": ["127.0.0.1:8000"],
        "default": true
    }, {
        "site": "us-east-2",
        "type": "aws_s3"
    }],
    "backbeat": {
        "host": "localhost",
        "port": 8900
    },
    "workflowEngineOperator": {
        "host": "localhost",
        "port": 3001
    },
    "cdmi": {
        "host": "localhost",
        "port": 81,
        "path": "/dewpoint",
        "readonly": true
    },
    "bucketd": {
        "bootstrap": ["localhost:9000"]
    },
    "vaultd": {
        "host": "localhost",
        "port": 8500
    },
    "clusters": 1,
    "log": {
        "logLevel": "info",
        "dumpLevel": "error"
    },
    "healthChecks": {
        "allowFrom": ["127.0.0.1/8", "::1"]
    },
    "metadataClient": {
        "host": "localhost",
        "port": 9990
    },
    "dataClient": {
        "host": "localhost",
        "port": 9991
    },
    "pfsClient": {
        "host": "localhost",
        "port": 9992
    },
    "metadataDaemon": {
        "bindAddress": "localhost",
        "port": 9990
    },
    "dataDaemon": {
        "bindAddress": "localhost",
        "port": 9991
    },
    "pfsDaemon": {
        "bindAddress": "localhost",
        "port": 9992
    },
    "recordLog": {
        "enabled": true,
        "recordLogName": "s3-recordlog"
    },
    "mongodb": {
        "replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
        "writeConcern": "majority",
        "replicaSet": "rs0",
        "readPreference": "primary",
        "database": "metadata"
    },
    "authdata": "authdata.json",
    "backends": {
        "auth": "file",
        "data": "file",
        "metadata": "mongodb",
        "kms": "file",
        "quota": "none"
    },
    "externalBackends": {
        "aws_s3": {
            "httpAgent": {
                "keepAlive": false,
                "keepAliveMsecs": 1000,
                "maxFreeSockets": 256,
                "maxSockets": null
            }
        },
        "gcp": {
            "httpAgent": {
                "keepAlive": true,
                "keepAliveMsecs": 1000,
                "maxFreeSockets": 256,
                "maxSockets": null
            }
        }
    },
    "requests": {
        "viaProxy": false,
        "trustedProxyCIDRs": [],
        "extractClientIPFromHeader": ""
    },
    "bucketNotificationDestinations": [
        {
            "resource": "target1",
            "type": "dummy",
            "host": "localhost:6000"
        }
    ]
 }
--- a/config.json.vitastor
+++ b/config.json.vitastor
@ -1,71 +0,0 @@
 {
    "port": 8000,
    "listenOn": [],
    "metricsPort": 8002,
    "metricsListenOn": [],
    "replicationGroupId": "RG001",
    "restEndpoints": {
        "localhost": "STANDARD",
        "127.0.0.1": "STANDARD",
        "yourhostname.ru": "STANDARD"
    },
    "websiteEndpoints": [
        "static.yourhostname.ru"
    ],
    "replicationEndpoints": [ {
        "site": "zenko",
        "servers": ["127.0.0.1:8000"],
        "default": true
    } ],
    "log": {
        "logLevel": "info",
        "dumpLevel": "error"
    },
    "healthChecks": {
        "allowFrom": ["127.0.0.1/8", "::1"]
    },
    "backends": {
        "metadata": "mongodb"
    },
    "mongodb": {
        "replicaSetHosts": "127.0.0.1:27017",
        "writeConcern": "majority",
        "replicaSet": "rs0",
        "readPreference": "primary",
        "database": "s3",
        "authCredentials": {
            "username": "s3",
            "password": ""
        }
    },
    "externalBackends": {
        "aws_s3": {
            "httpAgent": {
                "keepAlive": false,
                "keepAliveMsecs": 1000,
                "maxFreeSockets": 256,
                "maxSockets": null
            }
        },
        "gcp": {
            "httpAgent": {
                "keepAlive": true,
                "keepAliveMsecs": 1000,
                "maxFreeSockets": 256,
                "maxSockets": null
            }
        }
    },
    "requests": {
        "viaProxy": false,
        "trustedProxyCIDRs": [],
        "extractClientIPFromHeader": ""
    },
    "bucketNotificationDestinations": [
        {
            "resource": "target1",
            "type": "dummy",
            "host": "localhost:6000"
        }
    ]
 }
--- a/constants.js
+++ b/constants.js
@ -38,9 +38,6 @@ const constants = {
    // by the name of the final destination bucket for the object
    // once the multipart upload is complete.
    mpuBucketPrefix: 'mpuShadowBucket',
    blacklistedPrefixes: { bucket: [], object: [] },
    // GCP Object Tagging Prefix
    gcpTaggingPrefix: 'aws-tag-',
    // PublicId is used as the canonicalID for a request that contains
    // no authentication information.  Requestor can access
    // only public resources
@ -66,183 +63,37 @@ const constants = {
    // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
    minimumAllowedPartSize: 5242880,
    // AWS sets a maximum total parts limit
    // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
    maximumAllowedPartCount: 10000,
    gcpMaximumAllowedPartCount: 1024,
    // Max size on put part or copy part is 5GB. For functional
-    // testing use 110 MB as max
+    // testing use 100 MB as max
-    maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :
+    maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 104857600 :
        5368709120,
    // Max size allowed in a single put object request is 5GB
    // https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
    maximumAllowedUploadSize: 5368709120,
    // AWS states max size for user-defined metadata (x-amz-meta- headers) is
    // 2 KB: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
    // In testing, AWS seems to allow up to 88 more bytes, so we do the same.
    maximumMetaHeadersSize: 2136,
    // Maximum HTTP headers size allowed
    maxHttpHeadersSize: 14122,
    // hex digest of sha256 hash of empty string:
    emptyStringHash: crypto.createHash('sha256')
        .update('', 'binary').digest('hex'),
    // Queries supported by AWS that we do not currently support.
-    // Non-bucket queries
+    unsupportedQueries: {
-    unsupportedQueries: [
+        'accelerate': true,
-        'accelerate',
+        'analytics': true,
-        'analytics',
+        'inventory': true,
-        'inventory',
+        'lifecycle': true,
-        'logging',
+        'list-type': true,
-        'metrics',
+        'logging': true,
-        'policyStatus',
+        'metrics': true,
-        'publicAccessBlock',
+        'notification': true,
-        'requestPayment',
+        'policy': true,
-        'torrent',
+        'requestPayment': true,
-    ],
+        'restore': true,
-
+        'torrent': true,
-    // Headers supported by AWS that we do not currently support.
+    },
    unsupportedHeaders: [
        'x-amz-server-side-encryption-customer-algorithm',
        'x-amz-server-side-encryption-context',
        'x-amz-server-side-encryption-customer-key',
        'x-amz-server-side-encryption-customer-key-md5',
    ],
    // user metadata header to set object locationConstraint
-    objectLocationConstraintHeader: 'x-amz-storage-class',
+    objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
    lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
    legacyLocations: ['sproxyd', 'legacy'],
    // declare here all existing service accounts and their properties
    // (if any, otherwise an empty object)
    serviceAccountProperties: {
        replication: {},
        lifecycle: {},
        gc: {},
        'md-ingestion': {
            canReplicate: true,
        },
    },
    /* eslint-disable camelcase */
    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
    // some of the available data backends  (if called directly rather
    // than through the multiple backend gateway) need a key provided
    // as a string as first parameter of the get/delete methods.
    clientsRequireStringKey: { sproxyd: true, cdmi: true },
    // healthcheck default call from nginx is every 2 seconds
    // for external backends, don't call unless at least 1 minute
    // (60,000 milliseconds) since last call
    externalBackendHealthCheckInterval: 60000,
    versioningNotImplBackends: { azure: true, gcp: true },
    mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
    skipBatchDeleteBackends: { azure: true, gcp: true },
    s3HandledBackends: { azure: true, gcp: true },
    hasCopyPartBackends: { aws_s3: true, gcp: true },
    /* eslint-enable camelcase */
    mpuMDStoredOnS3Backend: { azure: true },
    azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
    base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
        '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
    productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
    // location constraint delimiter
    zenkoSeparator: ':',
    // user metadata applied on zenko objects
    zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
    bucketOwnerActions: [
        'bucketDeleteCors',
        'bucketDeleteLifecycle',
        'bucketDeletePolicy',
        'bucketDeleteReplication',
        'bucketDeleteWebsite',
        'bucketGetCors',
        'bucketGetLifecycle',
        'bucketGetLocation',
        'bucketGetPolicy',
        'bucketGetReplication',
        'bucketGetVersioning',
        'bucketGetWebsite',
        'bucketPutCors',
        'bucketPutLifecycle',
        'bucketPutPolicy',
        'bucketPutReplication',
        'bucketPutVersioning',
        'bucketPutWebsite',
        'objectDeleteTagging',
        'objectGetTagging',
        'objectPutTagging',
        'objectPutLegalHold',
        'objectPutRetention',
    ],
    // response header to be sent when there are invalid
    // user metadata in the object's metadata
    invalidObjectUserMetadataHeader: 'x-amz-missing-meta',
    // Bucket specific queries supported by AWS that we do not currently support
    // these queries may or may not be supported at object level
    unsupportedBucketQueries: [
    ],
    suppressedUtapiEventFields: [
        'object',
        'location',
        'versionId',
    ],
    allowedUtapiEventFilterFields: [
        'operationId',
        'location',
        'account',
        'user',
        'bucket',
    ],
    arrayOfAllowed: [
        'objectPutTagging',
        'objectPutLegalHold',
        'objectPutRetention',
    ],
    allowedUtapiEventFilterStates: ['allow', 'deny'],
    allowedRestoreObjectRequestTierValues: ['Standard'],
    lifecycleListing: {
        CURRENT_TYPE: 'current',
        NON_CURRENT_TYPE: 'noncurrent',
        ORPHAN_DM_TYPE: 'orphan',
    },
    multiObjectDeleteConcurrency: 50,
    maxScannedLifecycleListingEntries: 10000,
    overheadField: [
        'content-length',
        'owner-id',
        'versionId',
        'isNull',
        'isDeleteMarker',
    ],
    unsupportedSignatureChecksums: new Set([
        'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
    ]),
    supportedSignatureChecksums: new Set([
        'UNSIGNED-PAYLOAD',
        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
    ]),
    ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
    ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
    // The AWS assumed Role resource type
    assumedRoleArnResourceType: 'assumed-role',
    // Session name of the backbeat lifecycle assumed role session.
    backbeatLifecycleSessionName: 'backbeat-lifecycle',
    actionsToConsiderAsObjectPut: [
        'initiateMultipartUpload',
        'objectPutPart',
        'completeMultipartUpload',
    ],
    // if requester is not bucket owner, bucket policy actions should be denied with
    // MethodNotAllowed error
    onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
 };
 module.exports = constants;
--- a/controllers/BucketController.js
+++ b/controllers/BucketController.js
@ -0,0 +1,25 @@
 class BucketController {
    static putBucket(req, res, cb) {
    }
    static putBucketVersioning(req, res, cb) {
    }
    static putObject(req, res, cb) {
    }
    static putObjectCors(req, res, cb) {
    }
    static putObjectVersioning(req, res, cb) {
    }
 }
 module.exports = BucketController;
--- a/controllers/HealthcheckController.js
+++ b/controllers/HealthcheckController.js
--- a/controllers/ObjectController.js
+++ b/controllers/ObjectController.js
@ -0,0 +1,14 @@
 class ObjectController {
    static putObject(req, res, cb) {
    }
    static getObject(req, res, cb) {
    }
 }
 module.exports = ObjectController;
--- a/dataserver.js
+++ b/dataserver.js
@ -4,30 +4,16 @@ const arsenal = require('arsenal');
 const { config } = require('./lib/Config.js');
 const logger = require('./lib/utilities/logger');
 process.on('uncaughtException', err => {
    logger.fatal('caught error', {
        error: err.message,
        stack: err.stack,
        workerId: this.worker ? this.worker.id : undefined,
        workerPid: this.worker ? this.worker.process.pid : undefined,
    });
    process.exit(1);
 });
 if (config.backends.data === 'file' ||
    (config.backends.data === 'multiple' &&
     config.backends.metadata !== 'scality')) {
-    const dataServer = new arsenal.network.rest.RESTServer({
+    const dataServer = new arsenal.network.rest.RESTServer(
-        bindAddress: config.dataDaemon.bindAddress,
+        { bindAddress: config.dataDaemon.bindAddress,
-        port: config.dataDaemon.port,
+          port: config.dataDaemon.port,
-        dataStore: new arsenal.storage.data.file.DataFileStore({
+          dataStore: new arsenal.storage.data.file.DataFileStore(
-            dataPath: config.dataDaemon.dataPath,
+              { dataPath: config.dataDaemon.dataPath,
-            log: config.log,
+                log: config.log }),
-            noSync: config.dataDaemon.noSync,
+          log: config.log });
            noCache: config.dataDaemon.noCache,
        }),
        log: config.log,
    });
    dataServer.setup(err => {
        if (err) {
            logger.error('Error initializing REST data server',
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@ -3,218 +3,48 @@
 # set -e stops the execution of a script if a command or pipeline has an error
 set -e
-# modifying config.json
+if [[ "$HOST_NAME" ]]; then
-JQ_FILTERS_CONFIG="."
+    sed -i "s/s3.docker.test/$HOST_NAME/" ./config.json
-
+    echo "Host name has been modified to $HOST_NAME"
-# ENDPOINT var can accept comma separated values
+    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with $HOST_NAME"
 # for multiple endpoint locations
 if [[ "$ENDPOINT" ]]; then
    IFS="," read -ra HOST_NAMES <<< "$ENDPOINT"
    for host in "${HOST_NAMES[@]}"; do
        JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$host\"]=\"us-east-1\""
    done
    echo "Host name has been modified to ${HOST_NAMES[@]}"
    echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with ${HOST_NAMES[@]}"
 fi
 if [[ "$LOG_LEVEL" ]]; then
    if [[ "$LOG_LEVEL" == "info" || "$LOG_LEVEL" == "debug" || "$LOG_LEVEL" == "trace" ]]; then
-        JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .log.logLevel=\"$LOG_LEVEL\""
+        sed -i "s/\"logLevel\": \"info\"/\"logLevel\": \"$LOG_LEVEL\"/" ./config.json
        echo "Log level has been modified to $LOG_LEVEL"
    else
        echo "The log level you provided is incorrect (info/debug/trace)"
    fi
 fi
-if [[ "$SSL" && "$HOST_NAMES" ]]; then
+if [[ "$SSL" ]]; then
-# This condition makes sure that the certificates are not generated twice. (for docker restart)
+    if [[ -z "$HOST_NAME" ]]; then
-if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
+        echo "WARNING! No HOST_NAME has been provided"
-# Compute config for utapi tests
+    fi
-cat >>req.cfg <<EOF
+    # This condition makes sure that the certificates are not generated twice. (for docker restart)
-[req]
+    if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
-distinguished_name = req_distinguished_name
+        ## Generate SSL key and certificates
-prompt = no
+        # Generate a private key for your CSR
-req_extensions = s3_req
+        openssl genrsa -out ca.key 2048
-
+        # Generate a self signed certificate for your local Certificate Authority
-[req_distinguished_name]
+        openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=$SSL"
-CN = ${HOST_NAMES[0]}
+        # Generate a key for S3 Server
-
+        openssl genrsa -out server.key 2048
-[s3_req]
+        # Generate a Certificate Signing Request for S3 Server
-subjectAltName = @alt_names
+        openssl req -new -key server.key -out server.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.$SSL"
-extendedKeyUsage = serverAuth, clientAuth
+        # Generate a local-CA-signed certificate for S3 Server
-
+        openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 -sha256
-[alt_names]
+    fi
-DNS.1 = *.${HOST_NAMES[0]}
+    ## Update S3Server config.json
-DNS.2 = ${HOST_NAMES[0]}
+    # This condition makes sure that certFilePaths section is not added twice. (for docker restart)
-
+    if ! grep -q "certFilePaths" ./config.json; then
-EOF
+        sed -i "0,/,/s//,\n    \"certFilePaths\": { \"key\": \".\/server.key\", \"cert\": \".\/server.crt\", \"ca\": \".\/ca.crt\" },/" ./config.json
-
+    fi
 ## Generate SSL key and certificates
 # Generate a private key for your CSR
 openssl genrsa -out ca.key 2048
 # Generate a self signed certificate for your local Certificate Authority
 openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=S3 CA Server"
 # Generate a key for S3 Server
 openssl genrsa -out server.key 2048
 # Generate a Certificate Signing Request for S3 Server
 openssl req -new -key server.key -out server.csr -config req.cfg
 # Generate a local-CA-signed certificate for S3 Server
 openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 -sha256 -extfile req.cfg -extensions s3_req
 fi
 ## Update S3Server config.json
 # This condition makes sure that certFilePaths section is not added twice. (for docker restart)
 if ! grep -q "certFilePaths" ./config.json; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .certFilePaths= { \"key\": \".\/server.key\", \"cert\": \".\/server.crt\", \"ca\": \".\/ca.crt\" }"
 fi
 fi
-if [[ "$LISTEN_ADDR" ]]; then
+if [[ "$S3DATA" == "multiple" ]]; then
-    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
+    export S3DATA="$S3DATA"
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
 fi
 if [[ "$REPLICATION_GROUP_ID" ]] ; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
 fi
 if [[ "$DATA_HOST" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
 fi
 if [[ "$METADATA_HOST" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
 fi
 if [[ "$PFSD_HOST" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
 fi
 if [[ "$MONGODB_HOSTS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
 fi
 if [[ "$MONGODB_RS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSet=\"$MONGODB_RS\""
 fi
 if [[ "$MONGODB_DATABASE" ]]; then
   JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.database=\"$MONGODB_DATABASE\""
 fi
 if [ -z "$REDIS_HA_NAME" ]; then
    REDIS_HA_NAME='mymaster'
 fi
 if [[ "$REDIS_SENTINELS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.name=\"$REDIS_HA_NAME\""
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.sentinels=\"$REDIS_SENTINELS\""
 elif [[ "$REDIS_HOST" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.host=\"$REDIS_HOST\""
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=6379"
 fi
 if [[ "$REDIS_PORT" ]] && [[ ! "$REDIS_SENTINELS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=$REDIS_PORT"
 fi
 if [[ "$REDIS_SENTINELS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.name=\"$REDIS_HA_NAME\""
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.sentinels=\"$REDIS_SENTINELS\""
 elif [[ "$REDIS_HA_HOST" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.host=\"$REDIS_HA_HOST\""
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=6379"
 fi
 if [[ "$REDIS_HA_PORT" ]] && [[ ! "$REDIS_SENTINELS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=$REDIS_HA_PORT"
 fi
 if [[ "$RECORDLOG_ENABLED" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true"
 fi
 if [[ "$STORAGE_LIMIT_ENABLED" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.metrics[.utapi.metrics | length]=\"location\""
 fi
 if [[ "$CRR_METRICS_HOST" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\""
 fi
 if [[ "$CRR_METRICS_PORT" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
 fi
 if [[ "$WE_OPERATOR_HOST" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
 fi
 if [[ "$WE_OPERATOR_PORT" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
 fi
 if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
 fi
 # external backends http(s) agent config
 # AWS
 if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
 fi
 if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
 fi
 if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
 fi
 if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
 fi
 #GCP
 if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
 fi
 if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
 fi
 if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
 fi
 if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
 fi
 if [[ -n "$BUCKET_DENY_FILTER" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
 fi
 if [[ "$TESTING_MODE" ]]; then
    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
 fi
 if [[ $JQ_FILTERS_CONFIG != "." ]]; then
    jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
    mv config.json.tmp config.json
 fi
 if test -v INITIAL_INSTANCE_ID && test -v S3METADATAPATH && ! test -f ${S3METADATAPATH}/uuid ; then
    echo -n ${INITIAL_INSTANCE_ID} > ${S3METADATAPATH}/uuid
 fi
 # s3 secret credentials for Zenko
 if [ -r /run/secrets/s3-credentials ] ; then
    . /run/secrets/s3-credentials
 fi
 exec "$@"
--- a/docs/ARCHITECTURE.rst
+++ b/docs/ARCHITECTURE.rst
@ -1,993 +0,0 @@
 .. role:: raw-latex(raw)
   :format: latex
 ..
 Architecture
 ++++++++++++
 Versioning
 ==========
 This document describes Zenko CloudServer's support for the AWS S3 Bucket
 Versioning feature.
 AWS S3 Bucket Versioning
 ------------------------
 See AWS documentation for a description of the Bucket Versioning
 feature:
 -  `Bucket
   Versioning <http://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html>`__
 -  `Object
   Versioning <http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html>`__
 This document assumes familiarity with the details of Bucket Versioning,
 including null versions and delete markers, described in the above
 links.
 Implementation of Bucket Versioning in Zenko CloudServer
 --------------------------------------------------------
 Overview of Metadata and API Component Roles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Each version of an object is stored as a separate key in metadata. The
 S3 API interacts with the metadata backend to store, retrieve, and
 delete version metadata.
 The implementation of versioning within the metadata backend is naive.
 The metadata backend does not evaluate any information about bucket or
 version state (whether versioning is enabled or suspended, and whether a
 version is a null version or delete marker). The S3 front-end API
 manages the logic regarding versioning information, and sends
 instructions to metadata to handle the basic CRUD operations for version
 metadata.
 The role of the S3 API can be broken down into the following:
 -  put and delete version data
 -  store extra information about a version, such as whether it is a
   delete marker or null version, in the object's metadata
 -  send instructions to metadata backend to store, retrieve, update and
   delete version metadata based on bucket versioning state and version
   metadata
 -  encode version ID information to return in responses to requests, and
   decode version IDs sent in requests
 The implementation of Bucket Versioning in S3 is described in this
 document in two main parts. The first section, `"Implementation of
 Bucket Versioning in
 Metadata" <#implementation-of-bucket-versioning-in-metadata>`__,
 describes the way versions are stored in metadata, and the metadata
 options for manipulating version metadata.
 The second section, `"Implementation of Bucket Versioning in
 API" <#implementation-of-bucket-versioning-in-api>`__, describes the way
 the metadata options are used in the API within S3 actions to create new
 versions, update their metadata, and delete them. The management of null
 versions and creation of delete markers is also described in this
 section.
 Implementation of Bucket Versioning in Metadata
 -----------------------------------------------
 As mentioned above, each version of an object is stored as a separate
 key in metadata. We use version identifiers as the suffix for the keys
 of the object versions, and a special version (the `"Master
 Version" <#master-version>`__) to represent the latest version.
 An example of what the metadata keys might look like for an object
 ``foo/bar`` with three versions (with `.` representing a null character):
 +------------------------------------------------------+
 | key                                                  |
 +======================================================+
 | foo/bar                                              |
 +------------------------------------------------------+
 | foo/bar.098506163554375999999PARIS 0.a430a1f85c6ec   |
 +------------------------------------------------------+
 | foo/bar.098506163554373999999PARIS 0.41b510cd0fdf8   |
 +------------------------------------------------------+
 | foo/bar.098506163554373999998PARIS 0.f9b82c166f695   |
 +------------------------------------------------------+
 The most recent version created is represented above in the key
 ``foo/bar`` and is the master version. This special version is described
 further in the section `"Master Version" <#master-version>`__.
 Version ID and Metadata Key Format
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 The version ID is generated by the metadata backend, and encoded in a
 hexadecimal string format by S3 before sending a response to a request.
 S3 also decodes the hexadecimal string received from a request before
 sending to metadata to retrieve a particular version.
 The format of a ``version_id`` is: ``ts`` ``rep_group_id`` ``seq_id``
 where:
 -  ``ts``: is the combination of epoch and an increasing number
 -  ``rep_group_id``: is the name of deployment(s) considered one unit
   used for replication
 -  ``seq_id``: is a unique value based on metadata information.
 The format of a key in metadata for a version is:
 ``object_name separator version_id`` where:
 -  ``object_name``: is the key of the object in metadata
 -  ``separator``: we use the ``null`` character (``0x00`` or ``\0``) as
   the separator between the ``object_name`` and the ``version_id`` of a
   key
 -  ``version_id``: is the version identifier; this encodes the ordering
   information in the format described above as metadata orders keys
   alphabetically
 An example of a key in metadata:
 ``foo\01234567890000777PARIS 1234.123456`` indicating that this specific
 version of ``foo`` was the ``000777``\ th entry created during the epoch
 ``1234567890`` in the replication group ``PARIS`` with ``1234.123456``
 as ``seq_id``.
 Master Version
 ~~~~~~~~~~~~~~
 We store a copy of the latest version of an object's metadata using
 ``object_name`` as the key; this version is called the master version.
 The master version of each object facilitates the standard GET
 operation, which would otherwise need to scan among the list of versions
 of an object for its latest version.
 The following table shows the layout of all versions of ``foo`` in the
 first example stored in the metadata (with dot ``.`` representing the
 null separator):
 +----------+---------+
 | key      | value   |
 +==========+=========+
 | foo      | B       |
 +----------+---------+
 | foo.v2   | B       |
 +----------+---------+
 | foo.v1   | A       |
 +----------+---------+
 Metadata Versioning Options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Zenko CloudServer sends instructions to the metadata engine about whether to
 create a new version or overwrite, retrieve, or delete a specific
 version by sending values for special options in PUT, GET, or DELETE
 calls to metadata. The metadata engine can also list versions in the
 database, which is used by Zenko CloudServer to list object versions.
 These only describe the basic CRUD operations that the metadata engine
 can handle. How these options are used by the S3 API to generate and
 update versions is described more comprehensively in `"Implementation of
 Bucket Versioning in
 API" <#implementation-of-bucket-versioning-in-api>`__.
 Note: all operations (PUT and DELETE) that generate a new version of an
 object will return the ``version_id`` of the new version to the API.
 PUT
 ^^^
 -  no options: original PUT operation, will update the master version
 -  ``versioning: true`` create a new version of the object, then update
   the master version with this version.
 -  ``versionId: <versionId>`` create or update a specific version (for updating
   version's ACL or tags, or remote updates in geo-replication)
   *  if the version identified by ``versionId`` happens to be the latest
      version, the master version will be updated as well
   *  if the master version is not as recent as the version identified by
      ``versionId``, as may happen with cross-region replication, the master
      will be updated as well
   *  note that with ``versionId`` set to an empty string ``''``, it will
      overwrite the master version only (same as no options, but the master
      version will have a ``versionId`` property set in its metadata like
      any other version). The ``versionId`` will never be exposed to an
      external user, but setting this internal-only ``versionID`` enables
      Zenko CloudServer to find this version later if it is no longer the master.
      This option of ``versionId`` set to ``''`` is used for creating null
      versions once versioning has been suspended, which is discussed in
      `"Null Version Management" <#null-version-management>`__.
 In general, only one option is used at a time. When ``versionId`` and
 ``versioning`` are both set, only the ``versionId`` option will have an effect.
 DELETE
 ^^^^^^
 -  no options: original DELETE operation, will delete the master version
 -  ``versionId: <versionId>`` delete a specific version
 A deletion targeting the latest version of an object has to:
 -  delete the specified version identified by ``versionId``
 -  replace the master version with a version that is a placeholder for
   deletion
    -  this version contains a special keyword, 'isPHD', to indicate the
         master version was deleted and needs to be updated
 -  initiate a repair operation to update the value of the master
   version:
    -  involves listing the versions of the object and get the latest
       version to replace the placeholder delete version
    -  if no more versions exist, metadata deletes the master version,
       removing the key from metadata
 Note: all of this happens in metadata before responding to the front-end api,
 and only when the metadata engine is instructed by Zenko CloudServer to delete
 a specific version or the master version.
 See section `"Delete Markers" <#delete-markers>`__ for a description of what
 happens when a Delete Object request is sent to the S3 API.
 GET
 ^^^
 -  no options: original GET operation, will get the master version
 -  ``versionId: <versionId>`` retrieve a specific version
 The implementation of a GET operation does not change compared to the
 standard version. A standard GET without versioning information would
 get the master version of a key. A version-specific GET would retrieve
 the specific version identified by the key for that version.
 LIST
 ^^^^
 For a standard LIST on a bucket, metadata iterates through the keys by
 using the separator (``\0``, represented by ``.`` in examples) as an
 extra delimiter. For a listing of all versions of a bucket, there is no
 change compared to the original listing function. Instead, the API
 component returns all the keys in a List Objects call and filters for
 just the keys of the master versions in a List Object Versions call.
 For example, a standard LIST operation against the keys in a table below
 would return from metadata the list of
 ``[ foo/bar, bar, qux/quz, quz ]``.
 +--------------+
 | key          |
 +==============+
 | foo/bar      |
 +--------------+
 | foo/bar.v2   |
 +--------------+
 | foo/bar.v1   |
 +--------------+
 | bar          |
 +--------------+
 | qux/quz      |
 +--------------+
 | qux/quz.v2   |
 +--------------+
 | qux/quz.v1   |
 +--------------+
 | quz          |
 +--------------+
 | quz.v2       |
 +--------------+
 | quz.v1       |
 +--------------+
 Implementation of Bucket Versioning in API
 ------------------------------------------
 Object Metadata Versioning Attributes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 To access all the information needed to properly handle all cases that
 may exist in versioned operations, the API stores certain
 versioning-related information in the metadata attributes of each
 version's object metadata.
 These are the versioning-related metadata properties:
 -  ``isNull``: whether the version being stored is a null version.
 -  ``nullVersionId``: the unencoded version ID of the latest null
   version that existed before storing a non-null version.
 -  ``isDeleteMarker``: whether the version being stored is a delete
   marker.
 The metadata engine also sets one additional metadata property when
 creating the version.
 -  ``versionId``: the unencoded version ID of the version being stored.
 Null versions and delete markers are described in further detail in
 their own subsections.
 Creation of New Versions
 ~~~~~~~~~~~~~~~~~~~~~~~~
 When versioning is enabled in a bucket, APIs which normally result in
 the creation of objects, such as Put Object, Complete Multipart Upload
 and Copy Object, will generate new versions of objects.
 Zenko CloudServer creates a new version and updates the master version using the
 ``versioning: true`` option in PUT calls to the metadata engine. As an
 example, when two consecutive Put Object requests are sent to the Zenko
 CloudServer for a versioning-enabled bucket with the same key names, there
 are two corresponding metadata PUT calls with the ``versioning`` option
 set to true.
 The PUT calls to metadata and resulting keys are shown below:
 (1) PUT foo (first put), versioning: ``true``
 +----------+---------+
 | key      | value   |
 +==========+=========+
 | foo      | A       |
 +----------+---------+
 | foo.v1   | A       |
 +----------+---------+
 (2) PUT foo (second put), versioning: ``true``
 +----------+---------+
 | key      | value   |
 +==========+=========+
 | foo      | B       |
 +----------+---------+
 | foo.v2   | B       |
 +----------+---------+
 | foo.v1   | A       |
 +----------+---------+
 Null Version Management
 ^^^^^^^^^^^^^^^^^^^^^^^
 In a bucket without versioning, or when versioning is suspended, putting
 an object with the same name twice should result in the previous object
 being overwritten. This is managed with null versions.
 Only one null version should exist at any given time, and it is
 identified in Zenko CloudServer requests and responses with the version
 id "null".
 Case 1: Putting Null Versions
 '''''''''''''''''''''''''''''
 With respect to metadata, since the null version is overwritten by
 subsequent null versions, the null version is initially stored in the
 master key alone, as opposed to being stored in the master key and a new
 version. Zenko CloudServer checks if versioning is suspended or has never been
 configured, and sets the ``versionId`` option to ``''`` in PUT calls to
 the metadata engine when creating a new null version.
 If the master version is a null version, Zenko CloudServer also sends a DELETE
 call to metadata prior to the PUT, in order to clean up any pre-existing null
 versions which may, in certain edge cases, have been stored as a separate
 version. [1]_
 The tables below summarize the calls to metadata and the resulting keys if
 we put an object 'foo' twice, when versioning has not been enabled or is
 suspended.
 (1) PUT foo (first put), versionId: ``''``
 +--------------+---------+
 | key          | value   |
 +==============+=========+
 | foo (null)   | A       |
 +--------------+---------+
 (2A) DELETE foo (clean-up delete before second put),
 versionId: ``<version id of master version>``
 +--------------+---------+
 | key          | value   |
 +==============+=========+
 |              |         |
 +--------------+---------+
 (2B) PUT foo (second put), versionId: ``''``
 +--------------+---------+
 | key          | value   |
 +==============+=========+
 | foo (null)   | B       |
 +--------------+---------+
 The S3 API also sets the ``isNull`` attribute to ``true`` in the version
 metadata before storing the metadata for these null versions.
 .. [1]  Some examples of these cases are: (1) when there is a null version
        that is the second-to-latest version, and the latest version has been
        deleted, causing metadata to repair the master value with the value of
        the null version and (2) when putting object tag or ACL on a null
        version that is the master version, as explained in `"Behavior of
        Object-Targeting APIs" <#behavior-of-object-targeting-apis>`__.
 Case 2: Preserving Existing Null Versions in Versioning-Enabled Bucket
 ''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
 Null versions are preserved when new non-null versions are created after
 versioning has been enabled or re-enabled.
 If the master version is the null version, the S3 API preserves the
 current null version by storing it as a new key ``(3A)`` in a separate
 PUT call to metadata, prior to overwriting the master version ``(3B)``.
 This implies the null version may not necessarily be the latest or
 master version.
 To determine whether the master version is a null version, the S3 API
 checks if the master version's ``isNull`` property is set to ``true``,
 or if the ``versionId`` attribute of the master version is undefined
 (indicating it is a null version that was put before bucket versioning
 was configured).
 Continuing the example from Case 1, if we enabled versioning and put
 another object, the calls to metadata and resulting keys would resemble
 the following:
 (3A) PUT foo, versionId: ``<versionId of master version>`` if defined or
 ``<non-versioned object id>``
 +-----------------+---------+
 | key             | value   |
 +=================+=========+
 | foo             | B       |
 +-----------------+---------+
 | foo.v1 (null)   | B       |
 +-----------------+---------+
 (3B) PUT foo, versioning: ``true``
 +-----------------+---------+
 | key             | value   |
 +=================+=========+
 | foo             | C       |
 +-----------------+---------+
 | foo.v2          | C       |
 +-----------------+---------+
 | foo.v1 (null)   | B       |
 +-----------------+---------+
 To prevent issues with concurrent requests, Zenko CloudServer ensures the null
 version is stored with the same version ID by using ``versionId`` option.
 Zenko CloudServer sets the ``versionId`` option to the master version's
 ``versionId`` metadata attribute value during the PUT. This creates a new
 version with the same version ID of the existing null master version.
 The null version's ``versionId`` attribute may be undefined because it
 was generated before the bucket versioning was configured. In that case,
 a version ID is generated using the max epoch and sequence values
 possible so that the null version will be properly ordered as the last
 entry in a metadata listing. This value ("non-versioned object id") is
 used in the PUT call with the ``versionId`` option.
 Case 3: Overwriting a Null Version That is Not Latest Version
 '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
 Normally when versioning is suspended, Zenko CloudServer uses the
 ``versionId: ''`` option in a PUT to metadata to create a null version.
 This also overwrites an existing null version if it is the master version.
 However, if there is a null version that is not the latest version,
 Zenko CloudServer cannot rely on the ``versionId: ''`` option will not
 overwrite the existing null version. Instead, before creating a new null
 version, the Zenko CloudServer API must send a separate DELETE call to metadata
 specifying the version id of the current null version for delete.
 To do this, when storing a null version (3A above) before storing a new
 non-null version, Zenko CloudServer records the version's ID in the
 ``nullVersionId`` attribute of the non-null version. For steps 3A and 3B above,
 these are the values stored in the ``nullVersionId`` of each version's metadata:
 (3A) PUT foo, versioning: ``true``
 +-----------------+---------+-----------------------+
 | key             | value   | value.nullVersionId   |
 +=================+=========+=======================+
 | foo             | B       | undefined             |
 +-----------------+---------+-----------------------+
 | foo.v1 (null)   | B       | undefined             |
 +-----------------+---------+-----------------------+
 (3B) PUT foo, versioning: ``true``
 +-----------------+---------+-----------------------+
 | key             | value   | value.nullVersionId   |
 +=================+=========+=======================+
 | foo             | C       | v1                    |
 +-----------------+---------+-----------------------+
 | foo.v2          | C       | v1                    |
 +-----------------+---------+-----------------------+
 | foo.v1 (null)   | B       | undefined             |
 +-----------------+---------+-----------------------+
 If defined, the ``nullVersionId`` of the master version is used with the
 ``versionId`` option in a DELETE call to metadata if a Put Object
 request is received when versioning is suspended in a bucket.
 (4A) DELETE foo, versionId: ``<nullVersionId of master version>`` (v1)
 +----------+---------+
 | key      | value   |
 +==========+=========+
 | foo      | C       |
 +----------+---------+
 | foo.v2   | C       |
 +----------+---------+
 Then the master version is overwritten with the new null version:
 (4B) PUT foo, versionId: ``''``
 +--------------+---------+
 | key          | value   |
 +==============+=========+
 | foo (null)   | D       |
 +--------------+---------+
 | foo.v2       | C       |
 +--------------+---------+
 The ``nullVersionId`` attribute is also used to retrieve the correct
 version when the version ID "null" is specified in certain object-level
 APIs, described further in the section `"Null Version
 Mapping" <#null-version-mapping>`__.
 Specifying Versions in APIs for Putting Versions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Since Zenko CloudServer does not allow an overwrite of existing version data,
 Put Object, Complete Multipart Upload and Copy Object return
 ``400 InvalidArgument`` if a specific version ID is specified in the
 request query, e.g. for a ``PUT /foo?versionId=v1`` request.
 PUT Example
 ~~~~~~~~~~~
 When Zenko CloudServer receives a request to PUT an object:
 -  It checks first if versioning has been configured
 -  If it has not been configured, Zenko CloudServer proceeds to puts the new
   data, puts the metadata by overwriting the master version, and proceeds to
   delete any pre-existing data
 If versioning has been configured, Zenko CloudServer checks the following:
 Versioning Enabled
 ^^^^^^^^^^^^^^^^^^
 If versioning is enabled and there is existing object metadata:
 -  If the master version is a null version (``isNull: true``) or has no
   version ID (put before versioning was configured):
   -  store the null version metadata as a new version
   -  create a new version and overwrite the master version
      -  set ``nullVersionId``: version ID of the null version that was
         stored
 If versioning is enabled and the master version is not null; or there is
 no existing object metadata:
 -  create a new version and store it, and overwrite the master version
 Versioning Suspended
 ^^^^^^^^^^^^^^^^^^^^
 If versioning is suspended and there is existing object metadata:
 -  If the master version has no version ID:
   -  overwrite the master version with the new metadata (PUT ``versionId: ''``)
   -  delete previous object data
 - If the master version is a null version:
   -  delete the null version using the `versionId` metadata attribute of the
      master version (PUT ``versionId: <versionId of master object MD>``)
   -  put a new null version (PUT ``versionId: ''``)
 -  If master is not a null version and ``nullVersionId`` is defined in
   the object’s metadata:
   -  delete the current null version metadata and data
   -  overwrite the master version with the new metadata
 If there is no existing object metadata, create the new null version as
 the master version.
 In each of the above cases, set ``isNull`` metadata attribute to true
 when creating the new null version.
 Behavior of Object-Targeting APIs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 API methods which can target existing objects or versions, such as Get
 Object, Head Object, Get Object ACL, Put Object ACL, Copy Object and
 Copy Part, will perform the action on the latest version of an object if
 no version ID is specified in the request query or relevant request
 header (``x-amz-copy-source-version-id`` for Copy Object and Copy Part
 APIs).
 Two exceptions are the Delete Object and Multi-Object Delete APIs, which
 will instead attempt to create delete markers, described in the
 following section, if no version ID is specified.
 No versioning options are necessary to retrieve the latest version from
 metadata, since the master version is stored in a key with the name of
 the object. However, when updating the latest version, such as with the
 Put Object ACL API, Zenko CloudServer sets the ``versionId`` option in the
 PUT call to metadata to the value stored in the object metadata's ``versionId``
 attribute. This is done in order to update the metadata both in the
 master version and the version itself, if it is not a null version. [2]_
 When a version id is specified in the request query for these APIs, e.g.
 ``GET /foo?versionId=v1``, Zenko CloudServer will attempt to decode the version
 ID and perform the action on the appropriate version. To do so, the API sets
 the value of the ``versionId`` option to the decoded version ID in the
 metadata call.
 Delete Markers
 ^^^^^^^^^^^^^^
 If versioning has not been configured for a bucket, the Delete Object
 and Multi-Object Delete APIs behave as their standard APIs.
 If versioning has been configured, Zenko CloudServer deletes object or version
 data only if a specific version ID is provided in the request query, e.g.
 ``DELETE /foo?versionId=v1``.
 If no version ID is provided, S3 creates a delete marker by creating a
 0-byte version with the metadata attribute ``isDeleteMarker: true``. The
 S3 API will return a ``404 NoSuchKey`` error in response to requests
 getting or heading an object whose latest version is a delete maker.
 To restore a previous version as the latest version of an object, the
 delete marker must be deleted, by the same process as deleting any other
 version.
 The response varies when targeting an object whose latest version is a
 delete marker for other object-level APIs that can target existing
 objects and versions, without specifying the version ID.
 -  Get Object, Head Object, Get Object ACL, Object Copy and Copy Part
   return ``404 NoSuchKey``.
 -  Put Object ACL and Put Object Tagging return
   ``405 MethodNotAllowed``.
 These APIs respond to requests specifying the version ID of a delete
 marker with the error ``405 MethodNotAllowed``, in general. Copy Part
 and Copy Object respond with ``400 Invalid Request``.
 See section `"Delete Example" <#delete-example>`__ for a summary.
 Null Version Mapping
 ^^^^^^^^^^^^^^^^^^^^
 When the null version is specified in a request with the version ID
 "null", the S3 API must use the ``nullVersionId`` stored in the latest
 version to retrieve the current null version, if the null version is not
 the latest version.
 Thus, getting the null version is a two step process:
 1. Get the latest version of the object from metadata. If the latest
   version's ``isNull`` property is ``true``, then use the latest
   version's metadata. Otherwise,
 2. Get the null version of the object from metadata, using the internal
   version ID of the current null version stored in the latest version's
   ``nullVersionId`` metadata attribute.
 DELETE Example
 ~~~~~~~~~~~~~~
 The following steps are used in the delete logic for delete marker
 creation:
 -  If versioning has not been configured: attempt to delete the object
 -  If request is version-specific delete request: attempt to delete the
   version
 -  otherwise, if not a version-specific delete request and versioning
   has been configured:
   -  create a new 0-byte content-length version
   -  in version's metadata, set a 'isDeleteMarker' property to true
 -  Return the version ID of any version deleted or any delete marker
   created
 -  Set response header ``x-amz-delete-marker`` to true if a delete
   marker was deleted or created
 The Multi-Object Delete API follows the same logic for each of the
 objects or versions listed in an xml request. Note that a delete request
 can result in the creation of a deletion marker even if the object
 requested to delete does not exist in the first place.
 Object-level APIs which can target existing objects and versions perform
 the following checks regarding delete markers:
 -  If not a version-specific request and versioning has been configured,
   check the metadata of the latest version
 -  If the 'isDeleteMarker' property is set to true, return
   ``404 NoSuchKey`` or ``405 MethodNotAllowed``
 -  If it is a version-specific request, check the object metadata of the
   requested version
 -  If the ``isDeleteMarker`` property is set to true, return
   ``405 MethodNotAllowed`` or ``400 InvalidRequest``
 .. [2]  If it is a null version, this call will overwrite the null version
        if it is stored in its own key (``foo\0<versionId>``). If the null
        version is stored only in the master version, this call will both
        overwrite the master version *and* create a new key
        (``foo\0<versionId>``), resulting in the edge case referred to by the
        previous footnote [1]_.
 Data-metadata daemon Architecture and Operational guide
 =======================================================
 This document presents the architecture of the data-metadata daemon
 (dmd) used for the community edition of Zenko CloudServer. It also provides a
 guide on how to operate it.
 The dmd is responsible for storing and retrieving Zenko CloudServer data and
 metadata, and is accessed by Zenko CloudServer connectors through socket.io
 (metadata) and REST (data) APIs.
 It has been designed such that more than one Zenko CloudServer connector can
 access the same buckets by communicating with the dmd. It also means that
 the dmd can be hosted on a separate container or machine.
 Operation
 ---------
 Startup
 ~~~~~~~
 The simplest deployment is still to launch with yarn start, this will
 start one instance of the Zenko CloudServer connector and will listen on the
 locally bound dmd ports 9990 and 9991 (by default, see below).
 The dmd can be started independently from the Zenko CloudServer by running this
 command in the Zenko CloudServer directory:
 ::
   yarn run start_dmd
 This will open two ports:
 - one is based on socket.io and is used for metadata transfers (9990 by
  default)
 - the other is a REST interface used for data transfers (9991 by
  default)
 Then, one or more instances of Zenko CloudServer without the dmd can be started
 elsewhere with:
 .. code:: sh
   yarn run start_s3server
 Configuration
 ~~~~~~~~~~~~~
 Most configuration happens in ``config.json`` for Zenko CloudServer, local
 storage paths can be changed where the dmd is started using environment
 variables, like before: ``S3DATAPATH`` and ``S3METADATAPATH``.
 In ``config.json``, the following sections are used to configure access
 to the dmd through separate configuration of the data and metadata
 access:
 ::
   "metadataClient": {
       "host": "localhost",
       "port": 9990
   },
   "dataClient": {
       "host": "localhost",
       "port": 9991
   },
 To run a remote dmd, you have to do the following:
 - change both ``"host"`` attributes to the IP or host name where the
  dmd is run.
 - Modify the ``"bindAddress"`` attributes in ``"metadataDaemon"`` and
  ``"dataDaemon"`` sections where the dmd is run to accept remote
  connections (e.g. ``"::"``)
 Architecture
 ------------
 This section gives a bit more insight on how it works internally.
 .. figure:: ./images/data_metadata_daemon_arch.png
  :alt: Architecture diagram
  ./images/data\_metadata\_daemon\_arch.png
 Metadata on socket.io
 ~~~~~~~~~~~~~~~~~~~~~
 This communication is based on an RPC system based on socket.io events
 sent by Zenko CloudServerconnectors, received by the DMD and acknowledged back
 to the Zenko CloudServer connector.
 The actual payload sent through socket.io is a JSON-serialized form of
 the RPC call name and parameters, along with some additional information
 like the request UIDs, and the sub-level information, sent as object
 attributes in the JSON request.
 With introduction of versioning support, the updates are now gathered in
 the dmd for some number of milliseconds max, before being batched as a
 single write to the database. This is done server-side, so the API is
 meant to send individual updates.
 Four RPC commands are available to clients: ``put``, ``get``, ``del``
 and ``createReadStream``. They more or less map the parameters accepted
 by the corresponding calls in the LevelUp implementation of LevelDB.
 They differ in the following:
 - The ``sync`` option is ignored (under the hood, puts are gathered
  into batches which have their ``sync`` property enforced when they
  are committed to the storage)
 - Some additional versioning-specific options are supported
 - ``createReadStream`` becomes asynchronous, takes an additional
  callback argument and returns the stream in the second callback
  parameter
 Debugging the socket.io exchanges can be achieved by running the daemon
 with ``DEBUG='socket.io*'`` environment variable set.
 One parameter controls the timeout value after which RPC commands sent
 end with a timeout error, it can be changed either:
 - via the ``DEFAULT_CALL_TIMEOUT_MS`` option in
  ``lib/network/rpc/rpc.js``
 - or in the constructor call of the ``MetadataFileClient`` object (in
  ``lib/metadata/bucketfile/backend.js`` as ``callTimeoutMs``.
 Default value is 30000.
 A specific implementation deals with streams, currently used for listing
 a bucket. Streams emit ``"stream-data"`` events that pack one or more
 items in the listing, and a special ``“stream-end”`` event when done.
 Flow control is achieved by allowing a certain number of “in flight”
 packets that have not received an ack yet (5 by default). Two options
 can tune the behavior (for better throughput or getting it more robust
 on weak networks), they have to be set in ``mdserver.js`` file directly,
 as there is no support in ``config.json`` for now for those options:
 - ``streamMaxPendingAck``: max number of pending ack events not yet
  received (default is 5)
 - ``streamAckTimeoutMs``: timeout for receiving an ack after an output
  stream packet is sent to the client (default is 5000)
 Data exchange through the REST data port
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Data is read and written with REST semantic.
 The web server recognizes a base path in the URL of ``/DataFile`` to be
 a request to the data storage service.
 PUT
 ^^^
 A PUT on ``/DataFile`` URL and contents passed in the request body will
 write a new object to the storage.
 On success, a ``201 Created`` response is returned and the new URL to
 the object is returned via the ``Location`` header (e.g.
 ``Location: /DataFile/50165db76eecea293abfd31103746dadb73a2074``). The
 raw key can then be extracted simply by removing the leading
 ``/DataFile`` service information from the returned URL.
 GET
 ^^^
 A GET is simply issued with REST semantic, e.g.:
 ::
   GET /DataFile/50165db76eecea293abfd31103746dadb73a2074 HTTP/1.1
 A GET request can ask for a specific range. Range support is complete
 except for multiple byte ranges.
 DELETE
 ^^^^^^
 DELETE is similar to GET, except that a ``204 No Content`` response is
 returned on success.
 Listing
 =======
 Listing Types
 -------------
 We use three different types of metadata listing for various operations.
 Here are the scenarios we use each for:
 - 'Delimiter' - when no versions are possible in the bucket since it is
  an internally-used only bucket which is not exposed to a user.
  Namely,
 1. to list objects in the "user's bucket" to respond to a GET SERVICE
   request and
 2. to do internal listings on an MPU shadow bucket to complete multipart
   upload operations.
 - 'DelimiterVersion' - to list all versions in a bucket
 - 'DelimiterMaster' - to list just the master versions of objects in a
  bucket
 Algorithms
 ----------
 The algorithms for each listing type can be found in the open-source
 `scality/Arsenal <https://github.com/scality/Arsenal>`__ repository, in
 `lib/algos/list <https://github.com/scality/Arsenal/tree/master/lib/algos/list>`__.
 Encryption
 ===========
 With CloudServer, there are two possible methods of at-rest encryption.
 (1) We offer bucket level encryption where Scality CloudServer itself handles at-rest
 encryption for any object that is in an 'encrypted' bucket, regardless of what
 the location-constraint for the data is and
 (2) If the location-constraint specified for the data is of type AWS,
 you can choose to use AWS server side encryption.
 Note: bucket level encryption is not available on the standard AWS
 S3 protocol, so normal AWS S3 clients will not provide the option to send a
 header when creating a bucket. We have created a simple tool to enable you
 to easily create an encrypted bucket.
 Example:
 --------
 Creating encrypted bucket using our encrypted bucket tool in the bin directory
 .. code:: shell
    ./create_encrypted_bucket.js -a accessKey1 -k verySecretKey1 -b bucketname -h localhost -p 8000
 AWS backend
 ------------
 With real AWS S3 as a location-constraint, you have to configure the
 location-constraint as follows
 .. code:: json
    "awsbackend": {
        "type": "aws_s3",
        "legacyAwsBehavior": true,
        "details": {
            "serverSideEncryption": true,
            ...
        }
    },
 Then, every time an object is put to that data location, we pass the following
 header to AWS: ``x-amz-server-side-encryption: AES256``
 Note: due to these options, it is possible to configure encryption by both
 CloudServer and AWS S3 (if you put an object to a CloudServer bucket which has
 the encryption flag AND the location-constraint for the data is AWS S3 with
 serverSideEncryption set to true).
--- a/docs/BUCKET_POLICIES.md
+++ b/docs/BUCKET_POLICIES.md
@ -1,146 +0,0 @@
 # Bucket Policy Documentation
 ## Description
 Bucket policy is a method of controlling access to a user's account at the
 resource level.
 There are three associated APIs:
 - PUT Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTpolicy.html)
 - GET Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETpolicy.html)
 - DELETE Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEpolicy.html)
 More information on bucket policies in general can be found at
 https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html.
 ## Requirements
 To prevent loss of access to a bucket, the root owner of a bucket will always
 be able to perform any of the three bucket policy-related operations, even
 if permission is explicitly denied.
 All other users must have permission to perform the desired operation.
 ## Design
 On a PUTBucketPolicy request, the user provides a policy in JSON format.
 The policy is evaluated against our policy schema in Arsenal and, once
 validated, is stored as part of the bucket's metadata.
 On a GETBucketPolicy request, the policy is retrieved from the bucket's
 metadata.
 On a DELETEBucketPolicy request, the policy is deleted from the bucket's
 metadata.
 All other APIs are updated to check if a bucket policy is attached to the bucket
 the request is made on. If there is a policy, user authorization to perform
 the requested action is checked.
 ### Differences Between Bucket and IAM Policies
 IAM policies are attached to an IAM identity and define what actions that
 identity is allowed to or denied from doing on what resource.
 Bucket policies attach only to buckets and define what actions are allowed or
 denied for which principles on that bucket. Permissions specified in a bucket
 policy apply to all objects in that bucket unless otherwise specified.
 Besides their attachment origins, the main structural difference between
 IAM policy and bucket policy is the requirement of a "Principal" element in
 bucket policies. This field is redundant in IAM policies.
 ### Policy Validation
 For general guidelines for bucket policy structure, see examples here:
 https://docs.aws.amazon.com/AmazonS3/latest/dev//example-bucket-policies.html.
 Each bucket policy statement object requires at least four keys:
 "Effect", "Principle", "Resource", and "Action".
 "Effect" defines the effect of the policy and can have a string value of either
 "Allow" or "Deny".
 "Resource" defines to which bucket or list of buckets a policy is attached.
 An object within the bucket is also a valid resource. The element value can be
 either a single bucket or object ARN string or an array of ARNs.
 "Action" lists which action(s) the policy controls. Its value can also be either
 a string or array of S3 APIs. Each action is the API name prepended by "s3:".
 "Principle" specifies which user(s) are granted or denied access to the bucket
 resource. Its value can be a string or an object containing an array of users.
 Valid users can be identified with an account ARN, account id, or user ARN.
 There are also two optional bucket policy statement keys: Sid and Condition.
 "Sid" stands for "statement id". If this key is not included, one will be
 generated for the statement.
 "Condition" lists the condition under which a statement will take affect.
 The possibilities are as follows:
 - ArnEquals
 - ArnEqualsIfExists
 - ArnLike
 - ArnLikeIfExists
 - ArnNotEquals
 - ArnNotEqualsIfExists
 - ArnNotLike
 - ArnNotLikeIfExists
 - BinaryEquals
 - BinaryEqualsIfExists
 - BinaryNotEquals
 - BinaryNotEqualsIfExists
 - Bool
 - BoolIfExists
 - DateEquals
 - DateEqualsIfExists
 - DateGreaterThan
 - DateGreaterThanEquals
 - DateGreaterThanEqualsIfExists
 - DateGreaterThanIfExists
 - DateLessThan
 - DateLessThanEquals
 - DateLessThanEqualsIfExists
 - DateLessThanIfExists
 - DateNotEquals
 - DateNotEqualsIfExists
 - IpAddress
 - IpAddressIfExists
 - NotIpAddress
 - NotIpAddressIfExists
 - Null
 - NumericEquals
 - NumericEqualsIfExists
 - NumericGreaterThan
 - NumericGreaterThanEquals
 - NumericGreaterThanEqualsIfExists
 - NumericGreaterThanIfExists
 - NumericLessThan
 - NumericLessThanEquals
 - NumericLessThanEqualsIfExists
 - NumericLessThanIfExists
 - NumericNotEquals
 - NumericNotEqualsIfExists
 - StringEquals
 - StringEqualsIfExists
 - StringEqualsIgnoreCase
 - StringEqualsIgnoreCaseIfExists
 - StringLike
 - StringLikeIfExists
 - StringNotEquals
 - StringNotEqualsIfExists
 - StringNotEqualsIgnoreCase
 - StringNotEqualsIgnoreCaseIfExists
 - StringNotLike
 - StringNotLikeIfExists
 The value of the Condition key will be an object containing the desired
 condition name as that key. The value of inner object can be a string, boolean,
 number, or object, depending on the condition.
 ## Authorization with Multiple Access Control Mechanisms
 In the case where multiple access control mechanisms (such as IAM policies,
 bucket policies, and ACLs) refer to the same resource, the principle of
 least-privilege is applied. Unless an action is explicitly allowed, access will
 by default be denied. An explicit DENY in any policy will trump another
 policy's ALLOW for an action. The request will only be allowed if at least one
 policy specifies an ALLOW, and there is no overriding DENY.
 The following diagram illustrates this logic:
 ![Access_Control_Authorization_Chart](./images/access_control_authorization.png)
--- a/docs/CLIENTS.rst
+++ b/docs/CLIENTS.rst
@ -1,345 +0,0 @@
 Clients
 =========
 List of applications that have been tested with Zenko CloudServer.
 GUI
 ~~~
 `Cyberduck <https://cyberduck.io/?l=en>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 -  https://www.youtube.com/watch?v=-n2MCt4ukUg
 -  https://www.youtube.com/watch?v=IyXHcu4uqgU
 `Cloud Explorer <https://www.linux-toys.com/?p=945>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 -  https://www.youtube.com/watch?v=2hhtBtmBSxE
 `CloudBerry Lab <http://www.cloudberrylab.com>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 -  https://youtu.be/IjIx8g\_o0gY
 Command Line Tools
 ~~~~~~~~~~~~~~~~~~
 `s3curl <https://github.com/rtdp/s3curl>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 https://github.com/scality/S3/blob/master/tests/functional/s3curl/s3curl.pl
 `aws-cli <http://docs.aws.amazon.com/cli/latest/reference/>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 ``~/.aws/credentials`` on Linux, OS X, or Unix or
 ``C:\Users\USERNAME\.aws\credentials`` on Windows
 .. code:: shell
    [default]
    aws_access_key_id = accessKey1
    aws_secret_access_key = verySecretKey1
 ``~/.aws/config`` on Linux, OS X, or Unix or
 ``C:\Users\USERNAME\.aws\config`` on Windows
 .. code:: shell
    [default]
    region = us-east-1
 Note: ``us-east-1`` is the default region, but you can specify any
 region.
 See all buckets:
 .. code:: shell
    aws s3 ls --endpoint-url=http://localhost:8000
 Create bucket:
 .. code:: shell
    aws --endpoint-url=http://localhost:8000 s3 mb s3://mybucket
 `s3cmd <http://s3tools.org/s3cmd>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 If using s3cmd as a client to S3 be aware that v4 signature format is
 buggy in s3cmd versions < 1.6.1.
 ``~/.s3cfg`` on Linux, OS X, or Unix or ``C:\Users\USERNAME\.s3cfg`` on
 Windows
 .. code:: shell
    [default]
    access_key = accessKey1
    secret_key = verySecretKey1
    host_base = localhost:8000
    host_bucket = %(bucket).localhost:8000
    signature_v2 = False
    use_https = False
 See all buckets:
 .. code:: shell
    s3cmd ls
 `rclone <http://rclone.org/s3/>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 ``~/.rclone.conf`` on Linux, OS X, or Unix or
 ``C:\Users\USERNAME\.rclone.conf`` on Windows
 .. code:: shell
    [remote]
    type = s3
    env_auth = false
    access_key_id = accessKey1
    secret_access_key = verySecretKey1
    region = other-v2-signature
    endpoint = http://localhost:8000
    location_constraint =
    acl = private
    server_side_encryption =
    storage_class =
 See all buckets:
 .. code:: shell
    rclone lsd remote:
 JavaScript
 ~~~~~~~~~~
 `AWS JavaScript SDK <http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code:: javascript
    const AWS = require('aws-sdk');
    const s3 = new AWS.S3({
        accessKeyId: 'accessKey1',
        secretAccessKey: 'verySecretKey1',
        endpoint: 'localhost:8000',
        sslEnabled: false,
        s3ForcePathStyle: true,
    });
 JAVA
 ~~~~
 `AWS JAVA SDK <http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Client.html>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code:: java
    import com.amazonaws.auth.AWSCredentials;
    import com.amazonaws.auth.BasicAWSCredentials;
    import com.amazonaws.services.s3.AmazonS3;
    import com.amazonaws.services.s3.AmazonS3Client;
    import com.amazonaws.services.s3.S3ClientOptions;
    import com.amazonaws.services.s3.model.Bucket;
    public class S3 {
        public static void main(String[] args) {
            AWSCredentials credentials = new BasicAWSCredentials("accessKey1",
            "verySecretKey1");
            // Create a client connection based on credentials
            AmazonS3 s3client = new AmazonS3Client(credentials);
            s3client.setEndpoint("http://localhost:8000");
            // Using path-style requests
            // (deprecated) s3client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
            s3client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
            // Create bucket
            String bucketName = "javabucket";
            s3client.createBucket(bucketName);
            // List off all buckets
            for (Bucket bucket : s3client.listBuckets()) {
                System.out.println(" - " + bucket.getName());
            }
        }
    }
 Ruby
 ~~~~
 `AWS SDK for Ruby - Version 2 <http://docs.aws.amazon.com/sdkforruby/api/>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code:: ruby
    require 'aws-sdk'
    s3 = Aws::S3::Client.new(
      :access_key_id => 'accessKey1',
      :secret_access_key => 'verySecretKey1',
      :endpoint => 'http://localhost:8000',
      :force_path_style => true
    )
    resp = s3.list_buckets
 `fog <http://fog.io/storage/>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code:: ruby
    require "fog"
    connection = Fog::Storage.new(
    {
        :provider => "AWS",
        :aws_access_key_id => 'accessKey1',
        :aws_secret_access_key => 'verySecretKey1',
        :endpoint => 'http://localhost:8000',
        :path_style => true,
        :scheme => 'http',
    })
 Python
 ~~~~~~
 `boto2 <http://boto.cloudhackers.com/en/latest/ref/s3.html>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code:: python
    import boto
    from boto.s3.connection import S3Connection, OrdinaryCallingFormat
    connection = S3Connection(
        aws_access_key_id='accessKey1',
        aws_secret_access_key='verySecretKey1',
        is_secure=False,
        port=8000,
        calling_format=OrdinaryCallingFormat(),
        host='localhost'
    )
    connection.create_bucket('mybucket')
 `boto3 <http://boto3.readthedocs.io/en/latest/index.html>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Client integration
 .. code:: python
    import boto3
    client = boto3.client(
        's3',
        aws_access_key_id='accessKey1',
        aws_secret_access_key='verySecretKey1',
        endpoint_url='http://localhost:8000'
    )
    lists = client.list_buckets()
 Full integration (with object mapping)
 .. code:: python
    import os
    from botocore.utils import fix_s3_host
    import boto3
    os.environ['AWS_ACCESS_KEY_ID'] = "accessKey1"
    os.environ['AWS_SECRET_ACCESS_KEY'] = "verySecretKey1"
    s3 = boto3.resource(service_name='s3', endpoint_url='http://localhost:8000')
    s3.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
    for bucket in s3.buckets.all():
        print(bucket.name)
 PHP
 ~~~
 Should force path-style requests even though v3 advertises it does by default.
 `AWS PHP SDK v3 <https://docs.aws.amazon.com/aws-sdk-php/v3/guide>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code:: php
    use Aws\S3\S3Client;
    $client = S3Client::factory([
        'region'  => 'us-east-1',
        'version'   => 'latest',
        'endpoint' => 'http://localhost:8000',
        'use_path_style_endpoint' => true,
        'credentials' => [
             'key'    => 'accessKey1',
             'secret' => 'verySecretKey1'
        ]
    ]);
    $client->createBucket(array(
        'Bucket' => 'bucketphp',
    ));
 Go
 ~~
 `AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code:: go
   package main
   import (
      "context"
      "fmt"
      "log"
      "os"
      "time"
      "github.com/aws/aws-sdk-go/aws"
      "github.com/aws/aws-sdk-go/aws/endpoints"
      "github.com/aws/aws-sdk-go/aws/session"
      "github.com/aws/aws-sdk-go/service/s3"
   )
   func main() {
      os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
      os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
      endpoint := "http://localhost:8000"
      timeout := time.Duration(10) * time.Second
      sess := session.Must(session.NewSession())
      // Create a context with a timeout that will abort the upload if it takes
      // more than the passed in timeout.
      ctx, cancel := context.WithTimeout(context.Background(), timeout)
      defer cancel()
      svc := s3.New(sess, &aws.Config{
         Region:   aws.String(endpoints.UsEast1RegionID),
         Endpoint: &endpoint,
      })
      out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
      if err != nil {
         log.Fatal(err)
      } else {
         fmt.Println(out)
      }
   }
--- a/docs/CONTRIBUTING.rst
+++ b/docs/CONTRIBUTING.rst
@ -1,24 +0,0 @@
 Contributing
 ============
 Need help?
 ----------
 We're always glad to help out. Simply open a
 `GitHub issue <https://github.com/scality/S3/issues>`__ and we'll give you
 insight. If what you want is not available, and if you're willing to help us
 out, we'll be happy to welcome you in the team, whether for a small fix or for
 a larger feature development. Thanks for your interest!
 Got an idea? Get started!
 -------------------------
 In order to contribute, please follow the `Contributing
 Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__.
 If anything is unclear to you, reach out to us on
 `forum <https://forum.zenko.io/>`__ or via a GitHub issue.
 Don't write code? There are other ways to help!
 -----------------------------------------------
 We're always eager to learn about our users' stories. If you can't contribute
 code, but would love to help us, please shoot us an email at zenko@scality.com,
 and tell us what our software enables you to do! Thanks for your time!
--- a/docs/DOCKER.rst
+++ b/docs/DOCKER.rst
@ -1,371 +0,0 @@
 Docker
 ======
 .. _environment-variables:
 Environment Variables
 ---------------------
 S3DATA
 ~~~~~~
 S3DATA=multiple
 ^^^^^^^^^^^^^^^
 This variable enables running CloudServer with multiple data backends, defined
 as regions.
 For multiple data backends, a custom locationConfig.json file is required.
 This file enables you to set custom regions. You must provide associated 
 rest_endpoints for each custom region in config.json.
 `Learn more about multiple-backend configurations <GETTING_STARTED.html#location-configuration>`__
 If you are using Scality RING endpoints, refer to your customer documentation.
 Running CloudServer with an AWS S3-Hosted Backend
 """""""""""""""""""""""""""""""""""""""""""""""""
 To run CloudServer with an S3 AWS backend, add a new section to the 
 ``locationConfig.json`` file with the ``aws_s3`` location type:
 .. code:: json
    (...)
    "awsbackend": {
        "type": "aws_s3",
        "details": {
            "awsEndpoint": "s3.amazonaws.com",
            "bucketName": "yourawss3bucket",
            "bucketMatch": true,
            "credentialsProfile": "aws_hosted_profile"
        }
    }
    (...)
 Edit your AWS credentials file to enable your preferred command-line tool.
 This file must mention credentials for all backends in use. You can use 
 several profiles if multiple profiles are configured.
 .. code:: json
    [default]
    aws_access_key_id=accessKey1
    aws_secret_access_key=verySecretKey1
    [aws_hosted_profile]
    aws_access_key_id={{YOUR_ACCESS_KEY}}
    aws_secret_access_key={{YOUR_SECRET_KEY}}
 As with locationConfig.json, the AWS credentials file must be mounted at 
 run time: ``-v ~/.aws/credentials:/root/.aws/credentials`` on Unix-like 
 systems (Linux, OS X, etc.), or 
 ``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows
 .. note:: One account cannot copy to another account with a source and
   destination on real AWS unless the account associated with the 
   accessKey/secretKey pairs used for the destination bucket has source 
   bucket access privileges. To enable this, update ACLs directly on AWS.
 S3BACKEND
 ~~~~~~~~~
 S3BACKEND=file
 ^^^^^^^^^^^^^^
 For stored file data to persist, you must mount Docker volumes
 for both data and metadata. See :ref:`In Production with a Docker-Hosted CloudServer <in-production-w-a-Docker-hosted-cloudserver>`
 S3BACKEND=mem
 ^^^^^^^^^^^^^
 This is ideal for testing: no data remains after the container is shut down.
 ENDPOINT
 ~~~~~~~~
 This variable specifies the endpoint. To direct CloudServer requests to 
 new.host.com, for example, specify the endpoint with:
 .. code-block:: shell
    $ docker run -d --name cloudserver -p 8000:8000 -e ENDPOINT=new.host.com zenko/cloudserver
 .. note:: On Unix-like systems (Linux, OS X, etc.) edit /etc/hosts
   to associate 127.0.0.1 with new.host.com.
 REMOTE_MANAGEMENT_DISABLE
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 CloudServer is a part of `Zenko <https://www.zenko.io/>`__. When you run CloudServer standalone it will still try to connect to Orbit by default (browser-based graphical user interface for Zenko).
 Setting this variable to true(1) will default to accessKey1 and verySecretKey1 for credentials and disable the automatic Orbit management:
 .. code-block:: shell
    $ docker run -d --name cloudserver -p 8000:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
 SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 These variables specify authentication credentials for an account named
 “CustomAccount”.
 Set account credentials for multiple accounts by editing conf/authdata.json
 (see below for further details). To specify one set for personal use, set these 
 environment variables:
 .. code-block:: shell
   $ docker run -d --name cloudserver -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey \
   -e SCALITY_SECRET_ACCESS_KEY=newSecretKey zenko/cloudserver
 .. note:: This takes precedence over the contents of the authdata.json 
 	  file. The authdata.json file is ignored. 
 .. note:: The ACCESS_KEY and SECRET_KEY environment variables are 
 	  deprecated.
 LOG\_LEVEL
 ~~~~~~~~~~
 This variable changes the log level. There are three levels: info, debug, 
 and trace. The default is info. Debug provides more detailed logs, and trace
 provides the most detailed logs.
 .. code-block:: shell
    $ docker run -d --name cloudserver -p 8000:8000 -e LOG_LEVEL=trace zenko/cloudserver
 SSL
 ~~~
 Set true, this variable runs CloudServer with SSL.
 If SSL is set true: 
 * The ENDPOINT environment variable must also be specified.
 * On Unix-like systems (Linux, OS X, etc.), 127.0.0.1 must be associated with
  <YOUR_ENDPOINT> in /etc/hosts.
   .. Warning:: Self-signed certs with a CA generated within the container are 
      suitable for testing purposes only. Clients cannot trust them, and they may
      disappear altogether on a container upgrade. The best security practice for 
      production environments is to use an extra container, such as 
      haproxy/nginx/stunnel, for SSL/TLS termination and to pull certificates
      from a mounted volume, limiting what an exploit on either component
      can expose. 
 .. code:: shell
     $ docker run -d --name cloudserver -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT> \
     zenko/cloudserver
  For more information about using ClousdServer with SSL, see `Using SSL <GETTING_STARTED.html#Using SSL>`__
 LISTEN\_ADDR
 ~~~~~~~~~~~~
 This variable causes CloudServer and its data and metadata components to 
 listen on the specified address. This allows starting the data or metadata 
 servers as standalone services, for example.
 .. code:: shell
    docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0
    scality/s3server yarn run start_dataserver
 DATA\_HOST and METADATA\_HOST
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 These variables configure the data and metadata servers to use,
 usually when they are running on another host and only starting the stateless
 Zenko CloudServer.
 .. code:: shell
    $ docker run -d --name cloudserver -e DATA_HOST=cloudserver-data \
    -e METADATA_HOST=cloudserver-metadata zenko/cloudserver yarn run start_s3server
 REDIS\_HOST
 ~~~~~~~~~~~
 Use this variable to connect to the redis cache server on another host than
 localhost.
 .. code:: shell
    $ docker run -d --name cloudserver -p 8000:8000 \
    -e REDIS_HOST=my-redis-server.example.com zenko/cloudserver
 REDIS\_PORT
 ~~~~~~~~~~~
 Use this variable to connect to the Redis cache server on a port other 
 than the default 6379.
 .. code:: shell
    $ docker run -d --name cloudserver -p 8000:8000 \
    -e REDIS_PORT=6379 zenko/cloudserver
 .. _tunables-and-setup-tips:
 Tunables and Setup Tips
 -----------------------
 Using Docker Volumes
 ~~~~~~~~~~~~~~~~~~~~
 CloudServer runs with a file backend by default, meaning that data is 
 stored inside the CloudServer’s Docker container.
 For data and metadata to persist, data and metadata must be hosted in Docker 
 volumes outside the CloudServer’s Docker container. Otherwise, the data
 and metadata are destroyed when the container is erased.
 .. code-block:: shell
    $ docker run -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
    -p 8000:8000 -d zenko/cloudserver
 This command mounts the ./data host directory to the container
 at /usr/src/app/localData and the ./metadata host directory to
 the container at /usr/src/app/localMetaData. 
 .. tip:: These host directories can be mounted to any accessible mount 
   point, such as /mnt/data and /mnt/metadata, for example.
 Adding, Modifying, or Deleting Accounts or Credentials
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 1. Create a customized authdata.json file locally based on /conf/authdata.json.
 2. Use `Docker volumes <https://docs.docker.com/storage/volumes/>`__
   to override the default ``authdata.json`` through a Docker file mapping.
 For example:
 .. code-block:: shell
    $ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d \
    zenko/cloudserver
 Specifying a Host Name
 ~~~~~~~~~~~~~~~~~~~~~~
 To specify a host name (for example, s3.domain.name), provide your own
 `config.json <https://github.com/scality/cloudserver/blob/master/config.json>`__
 file using `Docker volumes <https://docs.docker.com/storage/volumes/>`__.
 First, add a new key-value pair to the restEndpoints section of your
 config.json. Make the key the host name you want, and the value the default 
 location\_constraint for this endpoint.
 For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one
 of the ``location_constraints`` listed in your locationConfig.json file
 `here <https://github.com/scality/S3/blob/master/locationConfig.json>`__.
 For more information about location configuration, see:
 `GETTING STARTED <GETTING_STARTED.html#location-configuration>`__
 .. code:: json
    "restEndpoints": {
        "localhost": "file",
        "127.0.0.1": "file",
        ...
        "cloudserver.example.com": "us-east-1"
    },
 Next, run CloudServer using a `Docker volume 
 <https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
 .. code-block:: shell
    $ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d zenko/cloudserver
 The local ``config.json`` file overrides the default one through a Docker 
 file mapping.
 Running as an Unprivileged User
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 CloudServer runs as root by default.
 To change this, modify the dockerfile and specify a user before the 
 entry point.
 The user must exist within the container, and must own the 
 /usr/src/app directory for CloudServer to run.
 For example, the following dockerfile lines can be modified:
 .. code-block:: shell
    ...
    && groupadd -r -g 1001 scality \
    && useradd -u 1001 -g 1001 -d /usr/src/app -r scality \
    && chown -R scality:scality /usr/src/app
    ...
    USER scality
    ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
 .. _continuous-integration-with-docker-hosted-cloudserver:
 Continuous Integration with a Docker-Hosted CloudServer
 -------------------------------------------------------
 When you start the Docker CloudServer image, you can adjust the
 configuration of the CloudServer instance by passing one or more
 environment variables on the ``docker run`` command line.
 To run CloudServer for CI with custom locations (one in-memory, 
 one hosted on AWS), and custom credentials mounted:
 .. code-block:: shell
   $ docker run --name CloudServer -p 8000:8000 \
   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
   -v ~/.aws/credentials:/root/.aws/credentials \
   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
 To run CloudServer for CI with custom locations, (one in-memory, one
 hosted on AWS, and one file), and custom credentials `set as environment 
 variables <GETTING_STARTED.html#scality-access-key-id-and-scality-secret-access-key>`__):
 .. code-block:: shell
   $ docker run --name CloudServer -p 8000:8000 \
   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
   -v ~/.aws/credentials:/root/.aws/credentials \
   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
   -e SCALITY_ACCESS_KEY_ID=accessKey1 \
   -e SCALITY_SECRET_ACCESS_KEY=verySecretKey1 \
   -e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
 .. _in-production-w-a-Docker-hosted-cloudserver:
 In Production with a Docker-Hosted CloudServer
 ----------------------------------------------
 Because data must persist in production settings, CloudServer offers
 multiple-backend capabilities. This requires a custom endpoint 
 and custom credentials for local storage.
 Customize these with:
 .. code-block:: shell
   $ docker run -d --name CloudServer \
   -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
   -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
   -v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple \
   -e ENDPOINT=custom.endpoint.com \
   -p 8000:8000 -d zenko/cloudserver \
--- a/docs/GETTING_STARTED.rst
+++ b/docs/GETTING_STARTED.rst
@ -1,436 +0,0 @@
 Getting Started
 ===============
 .. figure:: ../res/scality-cloudserver-logo.png
   :alt: Zenko CloudServer logo
 Dependencies
 ------------
 Building and running the Scality Zenko CloudServer requires node.js 10.x and
 yarn v1.17.x. Up-to-date versions can be found at
 `Nodesource <https://github.com/nodesource/distributions>`__.
 Installation
 ------------
 1. Clone the source code
   .. code-block:: shell
      $ git clone https://github.com/scality/cloudserver.git
 2. Go to the cloudserver directory and use yarn to install the js dependencies.
   .. code-block:: shell
      $ cd cloudserver
      $ yarn install
 Running CloudServer with a File Backend
 ---------------------------------------
 .. code-block:: shell
   $ yarn start
 This starts a Zenko CloudServer on port 8000. Two additional ports, 9990 
 and 9991, are also open locally for internal transfer of metadata and 
 data, respectively.
 The default access key is accessKey1. The secret key is verySecretKey1.
 By default, metadata files are saved in the localMetadata directory and 
 data files are saved in the localData directory in the local ./cloudserver 
 directory. These directories are pre-created within the repository. To 
 save data or metadata in different locations, you must specify them using 
 absolute paths. Thus, when starting the server:
 .. code-block:: shell
   $ mkdir -m 700 $(pwd)/myFavoriteDataPath
   $ mkdir -m 700 $(pwd)/myFavoriteMetadataPath
   $ export S3DATAPATH="$(pwd)/myFavoriteDataPath"
   $ export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
   $ yarn start
 Running CloudServer with Multiple Data Backends
 -----------------------------------------------
 .. code-block:: shell
   $ export S3DATA='multiple'
   $ yarn start
 This starts a Zenko CloudServer on port 8000. 
 The default access key is accessKey1. The secret key is verySecretKey1.
 With multiple backends, you can choose where each object is saved by setting
 the following header with a location constraint in a PUT request:
 .. code-block:: shell
    'x-amz-meta-scal-location-constraint':'myLocationConstraint'
 If no header is sent with a PUT object request, the bucket’s location
 constraint determines where the data is saved. If the bucket has no
 location constraint, the endpoint of the PUT request determines location.
 See the Configuration_ section to set location constraints.
 Run CloudServer with an In-Memory Backend
 -----------------------------------------
 .. code-block:: shell
   $ yarn run mem_backend
 This starts a Zenko CloudServer on port 8000. 
 The default access key is accessKey1. The secret key is verySecretKey1.
 Run CloudServer with Vault User Management
 ------------------------------------------
 .. code:: shell
   export S3VAULT=vault
   yarn start
 Note: Vault is proprietary and must be accessed separately.
 This starts a Zenko CloudServer using Vault for user management.
 Run CloudServer for Continuous Integration Testing or in Production with Docker
 -------------------------------------------------------------------------------
 Run Cloudserver with `DOCKER <DOCKER.html>`__
 Testing
 ~~~~~~~
 Run unit tests with the command:
 .. code-block:: shell
   $ yarn test
 Run multiple-backend unit tests with:
 .. code-block:: shell
   $ CI=true S3DATA=multiple yarn start
   $ yarn run multiple_backend_test
 Run the linter with:
 .. code-block:: shell
   $ yarn run lint
 Running Functional Tests Locally
 --------------------------------
 To pass AWS and Azure backend tests locally, modify 
 tests/locationConfig/locationConfigTests.json so that ``awsbackend`` 
 specifies the bucketname of a bucket you have access to based on your
 credentials, and modify ``azurebackend`` with details for your Azure account.
 The test suite requires additional tools, **s3cmd** and **Redis**
 installed in the environment the tests are running in.
 1. Install `s3cmd <http://s3tools.org/download>`__
 2. Install `redis <https://redis.io/download>`__ and start Redis.
 3. Add localCache section to ``config.json``:
   .. code:: json
      "localCache": {
        "host": REDIS_HOST,
        "port": REDIS_PORT
 	}
   where ``REDIS_HOST`` is the Redis instance IP address (``"127.0.0.1"``
   if Redis is running locally) and ``REDIS_PORT`` is the Redis instance
   port (``6379`` by default)
 4. Add the following to the local etc/hosts file:
   .. code-block:: shell
      127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
 5. Start Zenko CloudServer in memory and run the functional tests:
   .. code-block:: shell
      $ CI=true yarn run mem_backend
      $ CI=true yarn run ft_test
 .. _Configuration:
 Configuration
 -------------
 There are three configuration files for Zenko CloudServer:
 * ``conf/authdata.json``, for authentication.
 * ``locationConfig.json``, to configure where data is saved.
 * ``config.json``, for general configuration options.
 .. _location-configuration:
 Location Configuration
 ~~~~~~~~~~~~~~~~~~~~~~
 You must specify at least one locationConstraint in locationConfig.json
 (or leave it as pre-configured).
 You must also specify 'us-east-1' as a locationConstraint. If you put a 
 bucket to an unknown endpoint and do not specify a locationConstraint in
 the PUT bucket call, us-east-1 is used.
 For instance, the following locationConstraint saves data sent to
 ``myLocationConstraint`` to the file backend:
 .. code:: json
   "myLocationConstraint": {
       "type": "file",
       "legacyAwsBehavior": false,
       "details": {}
   },
 Each locationConstraint must include the ``type``, ``legacyAwsBehavior``,
 and ``details`` keys. ``type`` indicates which backend is used for that
 region. Supported backends are mem, file, and scality.``legacyAwsBehavior``
 indicates whether the region behaves the same as the AWS S3 'us-east-1' 
 region. If the locationConstraint type is ``scality``, ``details`` must 
 contain connector information for sproxyd. If the locationConstraint type
 is ``mem`` or ``file``, ``details`` must be empty.
 Once locationConstraints is set in locationConfig.json, specify a default
 locationConstraint for each endpoint.
 For instance, the following sets the ``localhost`` endpoint to the
 ``myLocationConstraint`` data backend defined above:
 .. code:: json
    "restEndpoints": {
         "localhost": "myLocationConstraint"
    },
 To use an endpoint other than localhost for Zenko CloudServer, the endpoint
 must be listed in ``restEndpoints``. Otherwise, if the server is running
 with a:
 *  **file backend**: The default location constraint is ``file``
 *  **memory backend**: The default location constraint is ``mem``
 Endpoints
 ~~~~~~~~~
 The Zenko CloudServer supports endpoints that are rendered in either:
 * path style: http://myhostname.com/mybucket or
 * hosted style: http://mybucket.myhostname.com
 However, if an IP address is specified for the host, hosted-style requests
 cannot reach the server. Use path-style requests in that case. For example,
 if you are using the AWS SDK for JavaScript, instantiate your client like this:
 .. code:: js
    const s3 = new aws.S3({
       endpoint: 'http://127.0.0.1:8000',
       s3ForcePathStyle: true,
    });
 Setting Your Own Access and Secret Key Pairs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Credentials can be set for many accounts by editing ``conf/authdata.json``, 
 but use the ``SCALITY_ACCESS_KEY_ID`` and ``SCALITY_SECRET_ACCESS_KEY`` 
 environment variables to specify your own credentials.
 _`scality-access-key-id-and-scality-secret-access-key`
 SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 These variables specify authentication credentials for an account named
 “CustomAccount”.
 .. note:: Anything in the ``authdata.json`` file is ignored.
 .. code-block:: shell
   $ SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey yarn start
 .. _Using_SSL:
 Using SSL
 ~~~~~~~~~
 To use https with your local CloudServer, you must set up
 SSL certificates. 
 1. Deploy CloudServer using `our DockerHub page
   <https://hub.docker.com/r/zenko/cloudserver/>`__ (run it with a file
   backend).
   .. Note:: If Docker is not installed locally, follow the
      `instructions to install it for your distribution 
      <https://docs.docker.com/engine/installation/>`__
 2. Update the CloudServer container’s config 
   Add your certificates to your container. To do this, 
   #. exec inside the CloudServer container. 
   #. Run ``$> docker ps`` to find the container’s ID (the corresponding 
      image name is ``scality/cloudserver``. 
   #. Copy the corresponding container ID (``894aee038c5e`` in the present
      example), and run:
      .. code-block:: shell
         $> docker exec -it 894aee038c5e bash
      This puts you inside your container, using an interactive terminal.
 3. Generate the SSL key and certificates. The paths where the different
   files are stored are defined after the ``-out`` option in each of the 
   following commands.
    #. Generate a private key for your certificate signing request (CSR):
       .. code-block:: shell
 	  $> openssl genrsa -out ca.key 2048
    #. Generate a self-signed certificate for your local certificate 
       authority (CA):
       .. code:: shell
 	  $> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999  -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
    #. Generate a key for the CloudServer:
       .. code:: shell
          $> openssl genrsa -out test.key 2048
    #. Generate a CSR for CloudServer:
       .. code:: shell
          $> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
    #. Generate a certificate for CloudServer signed by the local CA:
       .. code:: shell
          $> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
 4. Update Zenko CloudServer ``config.json``. Add a ``certFilePaths`` 
   section to ``./config.json`` with appropriate paths:
   .. code:: json
        "certFilePaths": {
            "key": "./test.key",
            "cert": "./test.crt",
            "ca": "./ca.crt"
        }
 5. Run your container with the new config. 
   #. Exit the container by running ``$> exit``. 
   #. Restart the container with ``$> docker restart cloudserver``.
 6. Update the host configuration by adding s3.scality.test 
   to /etc/hosts:
   .. code:: bash
      127.0.0.1      localhost s3.scality.test
 7. Copy the local certificate authority (ca.crt in step 4) from your 
   container. Choose the path to save this file to (in the present 
   example, ``/root/ca.crt``), and run:
   .. code:: shell
      $> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
   .. note:: Your container ID will be different, and your path to 
 	     ca.crt may be different.
 Test the Config
 ^^^^^^^^^^^^^^^
 If aws-sdk is not installed, run ``$> yarn install aws-sdk``. 
 Paste the following script into a file named "test.js":
 .. code:: js
    const AWS = require('aws-sdk');
    const fs = require('fs');
    const https = require('https');
    const httpOptions = {
        agent: new https.Agent({
            // path on your host of the self-signed certificate
            ca: fs.readFileSync('./ca.crt', 'ascii'),
        }),
    };
    const s3 = new AWS.S3({
        httpOptions,
        accessKeyId: 'accessKey1',
        secretAccessKey: 'verySecretKey1',
        // The endpoint must be s3.scality.test, else SSL will not work
        endpoint: 'https://s3.scality.test:8000',
        sslEnabled: true,
        // With this setup, you must use path-style bucket access
        s3ForcePathStyle: true,
    });
    const bucket = 'cocoriko';
    s3.createBucket({ Bucket: bucket }, err => {
        if (err) {
            return console.log('err createBucket', err);
        }
        return s3.deleteBucket({ Bucket: bucket }, err => {
            if (err) {
                return console.log('err deleteBucket', err);
            }
            return console.log('SSL is cool!');
        });
    });
 Now run this script with:
 .. code::
   $> nodejs test.js
 On success, the script outputs ``SSL is cool!``.
 .. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg
   :target: https://circleci.com/gh/scality/S3
 .. |Scality CI| image:: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
   :target: http://ci.ironmann.io/gh/scality/S3
--- a/docs/GET_BUCKET_V2.md
+++ b/docs/GET_BUCKET_V2.md
@ -1,69 +0,0 @@
 # Get Bucket Version 2 Documentation
 ## Description
 This feature implements version 2 of the GET Bucket (List Objects)
 operation, following AWS specifications
 (see https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html).
 ## Requirements
 The user must have READ access to the bucket.
 ## Design
 ### Request
 The `delimiter`, `encoding-type`, `max-keys`, and `prefix` request parameters
 from GET Bucket v1 remain unchanged.
 In order to specify v2, the parameter `list-type` must be included and
 set to `2`.
 The `marker` v1 parameter's functionality has been split in two and replaced by
 `start-after` and `continuation-token` in v2. The `start-after` parameter is
 a specific object key after which the API will return key names. It is only
 valid in the first GET request. If both the `start-after` and
 `continuation-token` parameters are included in a request, the API will
 ignore the `start-after` parameter in favor of the `continuation-token`.
 If the GET Bucket v2 response is truncated, a `NextContinuationToken` will
 also be included. To list the next set of objects, the `NextContinuationToken`
 can be used as the `continuation-token` in the next request. The continuation
 token is an obfuscated string of 57 characters that CloudServer understands and
 interprets.
 By default, the v2 response does not include object owner information. To
 include owner information like the default v1 response, use the `fetch-owner`
 request parameter set to `true`.
 ### Response
 The GET Bucket v1 and v2 responses are largely the same, with only a few changes.
 The `NextMarker` v1 parameter has been replaced by the
 `NextContinuationToken`. The `NextContinuationToken` is included with any
 truncated response, even if no delimiter is sent in the request. Its value is an
 obfuscated string that can be passed at the `continuation-token` in the next
 request, which will be interpreted by CloudServer.
 The `KeyCounter` parameter is returned in every response. Its value is the
 number of keys included in the response. It is always less than or equal to
 the `MaxKeys` value.
 If the `start-after` or `continuation-token` parameter is used in the
 request, it is also included in the response.
 By default, the v2 response does not include object owner information, unlike
 the v1 response. See the `Request` section for including it.
 ### Continuation Token
 An example continuation token:
 ```
 NextContinuationToken: '1bunC4s+crlZNAAbKUGBLyajJUQKp22TOdUR6/01snxD2cZtjJD0ugA=='
 ```
 In order to generate a comparable token, CloudServer uses base64 encoding to
 obfuscate the key name of the next object to be listed.
 Encoded continuation tokens are similarly decoded in order for listing to
 continue from the correct object.
 ## Performing Get Bucket V2 Operation
 When performing the GET Bucket V2 operation, if the request is built manually,
 the parameter `list-type` must be included and set to `2`.
 Using the AWS cli client, the command becomes `list-objects-v2`.
--- a/docs/INTEGRATIONS.rst
+++ b/docs/INTEGRATIONS.rst
@ -1,565 +0,0 @@
 Integrations
 ++++++++++++
 High Availability
 =================
 `Docker Swarm <https://docs.docker.com/engine/swarm/>`__ is a clustering tool
 developed by Docker for use with its containers. It can be used to start
 services, which we define to ensure CloudServer's continuous availability to
 end users. A swarm defines a manager and *n* workers among *n* + 1 servers.
 This tutorial shows how to perform a basic setup with three servers, which
 provides strong service resiliency, while remaining easy to use and
 maintain. We will use NFS through Docker to share data and
 metadata between the different servers.
 Sections are labeled **On Server**, **On Clients**, or
 **On All Machines**, referring respectively to NFS server, NFS clients, or
 NFS server and clients. In the present example, the server’s IP address is
 **10.200.15.113** and the client IP addresses are **10.200.15.96** and
 **10.200.15.97**
 1. Install Docker (on All Machines)
   Docker 17.03.0-ce is used for this tutorial. Docker 1.12.6 and later will
   likely work, but is not tested.
   * On Ubuntu 14.04
     Install Docker CE for Ubuntu as `documented at Docker
     <https://docs.docker.com/install/linux/docker-ce/ubuntu/>`__.
     Install the aufs dependency as recommended by Docker. The required
     commands are:
     .. code:: sh
      $> sudo apt-get update
      $> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
      $> sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
      $> curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
      $> sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
      $> sudo apt-get update
      $> sudo apt-get install docker-ce
    * On CentOS 7
      Install Docker CE as `documented at Docker
      <https://docs.docker.com/install/linux/docker-ce/centos/>`__.
      The required commands are:
      .. code:: sh
        $> sudo yum install -y yum-utils
        $> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
        $> sudo yum makecache fast
        $> sudo yum install docker-ce
        $> sudo systemctl start docker
 2. Install NFS on Client(s)
   NFS clients mount Docker volumes over the NFS server’s shared folders.
   If the NFS commons are installed, manual mounts are no longer needed.
   * On Ubuntu 14.04
     Install the NFS commons with apt-get:
     .. code:: sh
       $> sudo apt-get install nfs-common
   * On CentOS 7
     Install the NFS utils; then start required services:
     .. code:: sh
      $> yum install nfs-utils
      $> sudo systemctl enable rpcbind
      $> sudo systemctl enable nfs-server
      $> sudo systemctl enable nfs-lock
      $> sudo systemctl enable nfs-idmap
      $> sudo systemctl start rpcbind
      $> sudo systemctl start nfs-server
      $> sudo systemctl start nfs-lock
      $> sudo systemctl start nfs-idmap
 3. Install NFS (on Server)
   The NFS server hosts the data and metadata. The package(s) to install on it
   differs from the package installed on the clients.
   * On Ubuntu 14.04
     Install the NFS server-specific package and the NFS commons:
     .. code:: sh
      $> sudo apt-get install nfs-kernel-server nfs-common
   * On CentOS 7
     Install the NFS utils and start the required services:
     .. code:: sh
       $> yum install nfs-utils
       $> sudo systemctl enable rpcbind
       $> sudo systemctl enable nfs-server
       $> sudo systemctl enable nfs-lock
       $> sudo systemctl enable nfs-idmap
       $> sudo systemctl start rpcbind
       $> sudo systemctl start nfs-server
       $> sudo systemctl start nfs-lock
       $> sudo systemctl start nfs-idmap
   For both distributions:
   #. Choose where shared data and metadata from the local
      `CloudServer <http://www.zenko.io/cloudserver/>`__ shall be stored (The
      present example uses /var/nfs/data and /var/nfs/metadata). Set permissions
      for these folders for
      sharing over NFS:
      .. code:: sh
        $> mkdir -p /var/nfs/data /var/nfs/metadata
        $> chmod -R 777 /var/nfs/
   #. The /etc/exports file configures network permissions and r-w-x permissions
      for NFS access. Edit /etc/exports, adding the following lines:
      .. code:: sh
        /var/nfs/data        10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
        /var/nfs/metadata    10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
      Ubuntu applies the no\_subtree\_check option by default, so both
      folders are declared with the same permissions, even though they’re in
      the same tree.
   #. Export this new NFS table:
      .. code:: sh
        $> sudo exportfs -a
   #. Edit the ``MountFlags`` option in the Docker config in
      /lib/systemd/system/docker.service to enable NFS mount from Docker volumes
      on other machines:
      .. code:: sh
        MountFlags=shared
   #. Restart the NFS server and Docker daemons to apply these changes.
      * On Ubuntu 14.04
        .. code:: sh
          $> sudo service nfs-kernel-server restart
          $> sudo service docker restart
      * On CentOS 7
        .. code:: sh
         $> sudo systemctl restart nfs-server
         $> sudo systemctl daemon-reload
         $> sudo systemctl restart docker
 4. Set Up a Docker Swarm
  * On all machines and distributions:
    Set up the Docker volumes to be mounted to the NFS server for CloudServer’s
    data and metadata storage. The following commands must be replicated on all
    machines:
    .. code:: sh
     $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
     $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
    There is no need to ``docker exec`` these volumes to mount them: the
    Docker Swarm manager does this when the Docker service is started.
  * On a server:
    To start a Docker service on a Docker Swarm cluster, initialize the cluster
    (that is, define a manager), prompt workers/nodes to join in, and then start
    the service.
    Initialize the swarm cluster, and review its response:
    .. code:: sh
      $> docker swarm init --advertise-addr 10.200.15.113
      Swarm initialized: current node (db2aqfu3bzfzzs9b1kfeaglmq) is now a manager.
      To add a worker to this swarm, run the following command:
      docker swarm join \
      --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka \
      10.200.15.113:2377
      To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
  * On clients:
    Copy and paste the command provided by your Docker Swarm init. A successful
    request/response will resemble:
    .. code:: sh
      $> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
      This node joined a swarm as a worker.
 Set Up Docker Swarm on Clients on a Server
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Start the service on the Swarm cluster.
 .. code:: sh
  $> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/cloudserver
 On a successful installation, ``docker service ls`` returns the following
 output:
 .. code:: sh
    $> docker service ls
    ID            NAME  MODE        REPLICAS  IMAGE
    ocmggza412ft  s3    replicated  1/1       scality/cloudserver:latest
 If the service does not start, consider disabling apparmor/SELinux.
 Testing the High-Availability CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 On all machines (client/server) and distributions (Ubuntu and CentOS),
 determine where CloudServer is running using ``docker ps``. CloudServer can
 operate on any node of the Swarm cluster, manager or worker. When you find
 it, you can kill it with ``docker stop <container id>``. It will respawn
 on a different node. Now, if one server falls, or if Docker stops
 unexpectedly, the end user will still be able to access your the local CloudServer.
 Troubleshooting
 ~~~~~~~~~~~~~~~
 To troubleshoot the service, run:
 .. code:: sh
    $> docker service ps s3docker service ps s3
    ID                         NAME      IMAGE             NODE                               DESIRED STATE  CURRENT STATE       ERROR
    0ar81cw4lvv8chafm8pw48wbc  s3.1      scality/cloudserver  localhost.localdomain.localdomain  Running        Running 7 days ago
    cvmf3j3bz8w6r4h0lf3pxo6eu   \_ s3.1  scality/cloudserver  localhost.localdomain.localdomain  Shutdown       Failed 7 days ago   "task: non-zero exit (137)"
 If the error is truncated, view the error in detail by inspecting the
 Docker task ID:
 .. code:: sh
    $> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu
 Off you go!
 ~~~~~~~~~~~
 Let us know how you use this and if you'd like any specific developments 
 around it. Even better: come and contribute to our `Github repository 
 <https://github.com/scality/s3/>`__! We look forward to meeting you!
 S3FS
 ====
 You can export buckets as a filesystem with s3fs on CloudServer.
 `s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source
 tool, available both on Debian and RedHat distributions, that enables
 you to mount an S3 bucket on a filesystem-like backend. This tutorial uses
 an Ubuntu 14.04 host to deploy and use s3fs over CloudServer.
 Deploying Zenko CloudServer with SSL
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 First, deploy CloudServer with a file backend using `our DockerHub page
 <https://hub.docker.com/r/zenko/cloudserver>`__.
 .. note::
  If Docker is not installed on your machine, follow
  `these instructions <https://docs.docker.com/engine/installation/>`__
  to install it for your distribution.
 You must also set up SSL with CloudServer to use s3fs. See `Using SSL
 <./GETTING_STARTED#Using_SSL>`__ for instructions.
 s3fs Setup
 ~~~~~~~~~~
 Installing s3fs
 ---------------
 Follow the instructions in the s3fs `README
 <https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation-from-pre-built-packages>`__,
 Check that s3fs is properly installed. A version check should return
 a response resembling:
 .. code:: sh
    $> s3fs --version
    Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL
    Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
    License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>
    This is free software: you are free to change and redistribute it.
    There is NO WARRANTY, to the extent permitted by law.
 Configuring s3fs
 ----------------
 s3fs expects you to provide it with a password file. Our file is
 ``/etc/passwd-s3fs``. The structure for this file is
 ``ACCESSKEYID:SECRETKEYID``, so, for CloudServer, you can run:
 .. code:: sh
    $> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs
    $> chmod 600 /etc/passwd-s3fs
 Using CloudServer with s3fs
 ---------------------------
 1. Use /mnt/tests3fs as a mount point.
   .. code:: sh
    $> mkdir /mnt/tests3fs
 2. Create a bucket on your local CloudServer. In the present example it is
   named “tests3fs”.
   .. code:: sh
    $> s3cmd mb s3://tests3fs
 3. Mount the bucket to your mount point with s3fs:
   .. code:: sh
    $> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style
   The structure of this command is:
   ``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``. Of these mandatory
   options:
   * ``passwd_file`` specifies the path to the password file.
   * ``url`` specifies the host name used by your SSL provider.
   * ``use_path_request_style`` forces the path style (by default,
       s3fs uses DNS-style subdomains).
 Once the bucket is mounted, files added to the mount point or
 objects added to the bucket will appear in both locations.
 Example
 -------
   Create two files, and then a directory with a file in our mount point:
   .. code:: sh
      $> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
      $> mkdir /mnt/tests3fs/dir1
      $> touch /mnt/tests3fs/dir1/file3
   Now, use s3cmd to show what is in CloudServer:
   .. code:: sh
      $> s3cmd ls -r s3://tests3fs
      2017-02-28 17:28         0   s3://tests3fs/dir1/
      2017-02-28 17:29         0   s3://tests3fs/dir1/file3
      2017-02-28 17:28         0   s3://tests3fs/file1
      2017-02-28 17:28         0   s3://tests3fs/file2
   Now you can enjoy a filesystem view on your local CloudServer.
 Duplicity
 =========
 How to back up your files with CloudServer.
 Installing Duplicity and its Dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 To install `Duplicity <http://duplicity.nongnu.org/>`__,
 go to `this site <https://code.launchpad.net/duplicity/0.7-series>`__.
 Download the latest tarball. Decompress it and follow the instructions
 in the README.
 .. code:: sh
   $> tar zxvf duplicity-0.7.11.tar.gz
   $> cd duplicity-0.7.11
   $> python setup.py install
 You may receive error messages indicating the need to install some or all
 of the following dependencies:
 .. code:: sh
    $> apt-get install librsync-dev gnupg
    $> apt-get install python-dev python-pip python-lockfile
    $> pip install -U boto
 Testing the Installation
 ------------------------
 1. Check that CloudServer is running. Run ``$> docker ps``. You should
   see one container named ``scality/cloudserver``. If you do not, run
   ``$> docker start cloudserver`` and check again.
 2. Duplicity uses a module called “Boto” to send requests to S3. Boto
   requires a configuration file located in ``/etc/boto.cfg`` to store
   your credentials and preferences. A minimal configuration
   you can fine tune `following these instructions
   <http://boto.cloudhackers.com/en/latest/getting_started.html>`__ is
   shown here:
 ::
    [Credentials]
    aws_access_key_id = accessKey1
    aws_secret_access_key = verySecretKey1
    [Boto]
    # If using SSL, set to True
    is_secure = False
    # If using SSL, unmute and provide absolute path to local CA certificate
    # ca_certificates_file = /absolute/path/to/ca.crt
 .. note:: To set up SSL with CloudServer, check out our `Using SSL
 	   <./GETTING_STARTED#Using_SSL>`__ in GETTING STARTED.
 3. At this point all requirements to run CloudServer as a backend to Duplicity
   have been met. A local folder/file should back up to the local S3.
   Try it with the decompressed Duplicity folder:
 .. code:: sh
    $> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/"
 .. note:: Duplicity will prompt for a symmetric encryption passphrase.
 	  Save it carefully, as you will need it to recover your data.
 	  Alternatively, you can add the ``--no-encryption`` flag
 	  and the data will be stored plain.
   If this command is successful, you will receive an output resembling:
   .. code:: sh
      --------------[ Backup Statistics ]--------------
      StartTime 1486486547.13 (Tue Feb  7 16:55:47 2017)
      EndTime 1486486547.40 (Tue Feb  7 16:55:47 2017)
      ElapsedTime 0.27 (0.27 seconds)
      SourceFiles 388
      SourceFileSize 6634529 (6.33 MB)
      NewFiles 388
      NewFileSize 6634529 (6.33 MB)
      DeletedFiles 0
      ChangedFiles 0
      ChangedFileSize 0 (0 bytes)
      ChangedDeltaSize 0 (0 bytes)
      DeltaEntries 388
      RawDeltaSize 6392865 (6.10 MB)
      TotalDestinationSizeChange 2003677 (1.91 MB)
      Errors 0
      -------------------------------------------------
 Congratulations! You can now back up to your local S3 through Duplicity.
 Automating Backups
 ------------------
 The easiest way to back up files periodically is to write a bash script
 and add it to your crontab. A suggested script follows.
 .. code:: sh
    #!/bin/bash
    # Export your passphrase so you don't have to type anything
    export PASSPHRASE="mypassphrase"
    # To use a GPG key, put it here and uncomment the line below
    #GPG_KEY=
    # Define your backup bucket, with localhost specified
    DEST="s3://127.0.0.1:8000/testbucketcloudserver/"
    # Define the absolute path to the folder to back up
    SOURCE=/root/testfolder
    # Set to "full" for full backups, and "incremental" for incremental backups
    # Warning: you must perform one full backup befor you can perform
    # incremental ones on top of it
    FULL=incremental
    # How long to keep backups. If you don't want to delete old backups, keep
    # this value empty; otherwise, the syntax is "1Y" for one year, "1M" for
    # one month, "1D" for one day.
    OLDER_THAN="1Y"
    # is_running checks whether Duplicity is currently completing a task
    is_running=$(ps -ef | grep duplicity  | grep python | wc -l)
    # If Duplicity is already completing a task, this will not run
    if [ $is_running -eq 0 ]; then
        echo "Backup for ${SOURCE} started"
        # To delete backups older than a certain time, do it here
        if [ "$OLDER_THAN" != "" ]; then
            echo "Removing backups older than ${OLDER_THAN}"
            duplicity remove-older-than ${OLDER_THAN} ${DEST}
        fi
        # This is where the actual backup takes place
        echo "Backing up ${SOURCE}..."
        duplicity ${FULL} \
            ${SOURCE} ${DEST}
            # If you're using GPG, paste this in the command above
            # --encrypt-key=${GPG_KEY} --sign-key=${GPG_KEY} \
            # If you want to exclude a subfolder/file, put it below and
            # paste this
            # in the command above
            # --exclude=/${SOURCE}/path_to_exclude \
        echo "Backup for ${SOURCE} complete"
        echo "------------------------------------"
    fi
    # Forget the passphrase...
    unset PASSPHRASE
 Put this file in ``/usr/local/sbin/backup.sh``. Run ``crontab -e`` and
 paste your configuration into the file that opens. If you're unfamiliar
 with Cron, here is a good `HowTo
 <https://help.ubuntu.com/community/CronHowto>`__. If the folder being
 backed up is a folder to be modified permanently during the work day,
 we can set incremental backups every 5 minutes from 8 AM to 9 PM Monday
 through Friday by pasting the following line into crontab:
 .. code:: sh
    */5 8-20 * * 1-5 /usr/local/sbin/backup.sh
 Adding or removing files from the folder being backed up will result in
 incremental backups in the bucket.
--- a/docs/MD_SEARCH.rst
+++ b/docs/MD_SEARCH.rst
@ -1,263 +0,0 @@
 Metadata Search Documentation
 =============================
 Description
 -----------
 This feature enables metadata search to be performed on the metadata of objects
 stored in Zenko.
 Requirements
 ------------
 * MongoDB
 Design
 ------
 The Metadata Search feature expands on the existing :code:`GET Bucket` S3 API by
 enabling users to conduct metadata searches by adding the custom Zenko query
 string parameter, :code:`search`. The :code:`search` parameter is structured as a pseudo
 SQL WHERE clause, and supports basic SQL operators. For example:
 :code:`"A=1 AND B=2 OR C=3"` (complex queries can be built using nesting
 operators, :code:`(` and :code:`)`).
 The search process is as follows:
 * Zenko receives a :code:`GET` request.
  .. code::
    # regular getBucket request
    GET /bucketname HTTP/1.1
    Host: 127.0.0.1:8000
    Date: Wed, 18 Oct 2018 17:50:00 GMT
    Authorization: authorization string
    # getBucket versions request
    GET /bucketname?versions HTTP/1.1
    Host: 127.0.0.1:8000
    Date: Wed, 18 Oct 2018 17:50:00 GMT
    Authorization: authorization string
    # search getBucket request
    GET /bucketname?search=key%3Dsearch-item HTTP/1.1
    Host: 127.0.0.1:8000
    Date: Wed, 18 Oct 2018 17:50:00 GMT
    Authorization: authorization string
 * If the request does *not* contain the :code:`search` query parameter, Zenko performs
  a normal bucket listing and returns an XML result containing the list of
  objects.
 * If the request *does* contain the :code:`search` query parameter, Zenko parses and
  validates the search string.
  - If the search string is invalid, Zenko returns an :code:`InvalidArgument` error.
    .. code::
      <?xml version=\"1.0\" encoding=\"UTF-8\"?>
      <Error>
        <Code>InvalidArgument</Code>
        <Message>Invalid sql where clause sent as search query</Message>
        <Resource></Resource>
        <RequestId>d1d6afc64345a8e1198e</RequestId>
      </Error>
  - If the search string is valid, Zenko parses it and generates an abstract
    syntax tree (AST). The AST is then passed to the MongoDB backend to be
    used as the query filter for retrieving objects from a bucket that
    satisfies the requested search conditions. Zenko parses the filtered
    results and returns them as the response.
 Metadata search results have the same structure as a :code:`GET Bucket` response:
 .. code:: xml
  <?xml version="1.0" encoding="UTF-8"?>
  <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
      <Name>bucketname</Name>
      <Prefix/>
      <Marker/>
      <MaxKeys>1000</MaxKeys>
      <IsTruncated>false</IsTruncated>
      <Contents>
          <Key>objectKey</Key>
          <LastModified>2018-04-19T18:31:49.426Z</LastModified>
          <ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
          <Size>0</Size>
          <Owner>
              <ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
              <DisplayName>Bart</DisplayName>
          </Owner>
          <StorageClass>STANDARD</StorageClass>
      </Contents>
      <Contents>
          ...
      </Contents>
  </ListBucketResult>
 Performing Metadata Searches with Zenko
 ---------------------------------------
 You can perform metadata searches by:
 + Using the :code:`search_bucket` tool in the
  `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository.
 + Creating a signed HTTP request to Zenko in your preferred programming
  language.
 Using the S3 Tool
 +++++++++++++++++
 After cloning the `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository
 and installing the necessary dependencies, run the following command in the S3
 project’s root directory to access the search tool:
 .. code::
  node bin/search_bucket
 This generates the following output:
 .. code::
    Usage: search_bucket [options]
    Options:
      -V, --version                 output the version number
      -a, --access-key <accessKey>  Access key id
      -k, --secret-key <secretKey>  Secret access key
      -b, --bucket <bucket>         Name of the bucket
      -q, --query <query>           Search query
      -h, --host <host>             Host of the server
      -p, --port <port>             Port of the server
      -s                            --ssl
      -v, --verbose
      -h, --help                    output usage information
 In the following examples, Zenko Server is accessible on endpoint
 :code:`http://127.0.0.1:8000` and contains the bucket :code:`zenkobucket`.
 .. code::
    # search for objects with metadata "blue"
    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
        -q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
    # search for objects tagged with "type=color"
    node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
        -q "tags.type=color" -h 127.0.0.1 -p 8000
 Coding Examples
 +++++++++++++++
 Search requests can be also performed by making HTTP requests authenticated
 with one of the AWS Signature schemes: version 2 or version 4. \
 For more about authentication scheme, see:
 * https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
 * http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
 * http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
 You can also view examples for making requests with Auth V4 in various
 languages `here <../../../examples>`__.
 Specifying Metadata Fields
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 To search system metadata headers:
 .. code::
    {system-metadata-key}{supported SQL op}{search value}
    # example
    key = blueObject
    size > 0
    key LIKE "blue.*"
 To search custom user metadata:
 .. code::
    # metadata must be prefixed with "x-amz-meta-"
    x-amz-meta-{user-metadata-key}{supported SQL op}{search value}
    # example
    x-amz-meta-color = blue
    x-amz-meta-color != red
    x-amz-meta-color LIKE "b.*"
 To search tags:
 .. code::
    # tag searches must be prefixed with "tags."
    tags.{tag-key}{supported SQL op}{search value}
    # example
    tags.type = color
 Examples queries:
 .. code::
    # searching for objects with custom metadata "color"=blue" and are tagged
    # "type"="color"
    tags.type="color" AND  x-amz-meta-color="blue"
    # searching for objects with the object key containing the substring "blue"
    # or (custom metadata "color"=blue" and are tagged "type"="color")
    key LIKE '.*blue.*' OR (x-amz-meta-color="blue" AND tags.type="color")
 Differences from SQL
 ++++++++++++++++++++
 Zenko metadata search queries are similar to SQL-query :code:`WHERE` clauses, but
 differ in that:
 * They follow the :code:`PCRE` format
 * They do not require values with hyphens to be enclosed in
  backticks, :code:``(`)``
  .. code::
        # SQL query
        `x-amz-meta-search-item` = `ice-cream-cone`
        # MD Search query
        x-amz-meta-search-item = ice-cream-cone
 * Search queries do not support all SQL operators.
  .. code::
    # Supported SQL operators:
    =, <, >, <=, >=, !=, AND, OR, LIKE, <>
    # Unsupported SQL operators:
    NOT, BETWEEN, IN, IS, +, -, %, ^, /, *, !
 Using Regular Expressions in Metadata Search
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Regular expressions in Zenko metadata search differ from SQL in the following
 ways:
 + Wildcards are represented with :code:`.*` instead of :code:`%`.
 + Regex patterns must be wrapped in quotes. Failure to do this can lead to
  misinterpretation of patterns.
 + As with :code:`PCRE`, regular expressions can be entered in either the
  :code:`/pattern/` syntax or as the pattern itself if regex options are
  not required.
 Example regular expressions:
 .. code::
    # search for strings containing word substring "helloworld"
        ".*helloworld.*"
        "/.*helloworld.*/"
        "/.*helloworld.*/i"
--- a/docs/Makefile
+++ b/docs/Makefile
@ -1,21 +0,0 @@
 # Minimal makefile for Sphinx documentation
 #
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 SPHINXPROJ    = Zenko
 SOURCEDIR     = .
 BUILDDIR      = _build
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 .PHONY: help Makefile
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
--- a/docs/OBJECT_LOCK_TEST_PLAN.md
+++ b/docs/OBJECT_LOCK_TEST_PLAN.md
@ -1,161 +0,0 @@
 # Object Lock Feature Test Plan
 ## Feature Component Description
 Implementing Object Lock will introduce six new APIs:
 - putObjectLockConfiguration
 - getObjectLockConfiguration
 - putObjectRetention
 - getObjectRetention
 - putObjectLegalHold
 - getObjectLegalHold
 Along with these APIs, putBucket, putObject, deleteObject, and multiObjectDelete
 be affected. In Arsenal, both the BucketInfo and ObjectMD models will be
 updated. Bucket policy and IAM policy permissions will be updated to include
 the new API actions.
 ## Functional Tests
 ### putBucket tests
 - passing option to enable object lock updates bucket metadata and enables
    bucket versioning
 ### putBucketVersioning tests
 - suspending versioning on bucket with object lock enabled returns error
 ### putObject tests
 - putting retention configuration on object should be allowed
 - putting invalid retention configuration returns error
 ### getObject tests
 - getting object with retention information should include retention information
 ### copyObject tests
 - copying object with retention information should include retention information
 ### initiateMultipartUpload tests
 - mpu object initiated with retention information should include retention
    information
 ### putObjectLockConfiguration tests
 - putting configuration as non-bucket-owner user returns AccessDenied error
 - disabling object lock on bucket created with object lock returns error
 - enabling object lock on bucket created without object lock returns
    InvalidBucketState error
 - enabling object lock with token on bucket created without object lock succeeds
 - putting valid object lock configuration when bucket does not have object
    lock enabled returns error (InvalidRequest?)
 - putting valid object lock configuration updates bucket metadata
 - putting invalid object lock configuration returns error
    - ObjectLockEnabled !== "Enabled"
    - Rule object doesn't contain DefaultRetention key
    - Mode !== "GOVERNANCE" or "COMPLIANCE"
    - Days are not an integer
    - Years are not an integer
 ### getObjectLockConfiguration tests
 - getting configuration as non-bucket-owner user returns AccessDenied error
 - getting configuration when none is set returns
    ObjectLockConfigurationNotFoundError error
 - getting configuration returns correct object lock configuration for bucket
 ### putObjectRetention
 - putting retention as non-bucket-owner user returns AccessDenied error
 - putting retention on object in bucket without object lock enabled returns
    InvalidRequest error
 - putting valid retention period updates object metadata
 ### getObjectRetention
 - getting retention as non-bucket-owner user returns AccessDenied error
 - getting retention when none is set returns NoSuchObjectLockConfiguration
    error
 - getting retention returns correct object retention period
 ### putObjectLegalHold
 - putting legal hold as non-bucket-owner user returns AccessDenied error
 - putting legal hold on object in bucket without object lock enabled returns
        InvalidRequest error
 - putting valid legal hold updates object metadata
 ### getObjectLegalHold
 - getting legal hold as non-bucket-owner user returns AccessDenied error
 - getting legal hold when none is set returns NoSuchObjectLockConfiguration
    error
 - getting legal hold returns correct object legal hold
 ## End to End Tests
 ### Scenarios
 - Create bucket with object lock enabled. Put object. Put object lock
    configuration. Put another object.
    - Ensure object put before configuration does not have retention period set
    - Ensure object put after configuration does have retention period set
 - Create bucket without object lock. Put object. Enable object lock with token
    and put object lock configuration. Put another object.
    - Ensure object put before configuration does not have retention period set
    - Ensure object put after configuration does have retention period set
 - Create bucket with object lock enabled and put configuration with COMPLIANCE
    mode. Put object.
    - Ensure object cannot be deleted (returns AccessDenied error).
    - Ensure object cannot be overwritten.
 - Create bucket with object lock enabled and put configuration with GOVERNANCE
    mode. Put object.
    - Ensure user without permission cannot delete object
    - Ensure user without permission cannot overwrite object
    - Ensure user with permission can delete object
    - Ensure user with permission can overwrite object
    - Ensure user with permission can lengthen retention period
    - Ensure user with permission cannot shorten retention period
 - Create bucket with object lock enabled and put configuration. Edit bucket
    metadata so retention period is expired. Put object.
    - Ensure object can be deleted.
    - Ensure object can be overwritten.
 - Create bucket with object lock enabled and put configuration. Edit bucket
    metadata so retention period is expired. Put object. Put new retention
    period on object.
    - Ensure object cannot be deleted.
    - Ensure object cannot be overwritten.
 - Create bucket with object locked enabled and put configuration. Put object.
    Edit object metadata so retention period is past expiration.
    - Ensure object can be deleted.
    - Ensure object can be overwritten.
 - Create bucket with object lock enabled and put configuration. Edit bucket
    metadata so retention period is expired. Put object. Put legal hold
    on object.
    - Ensure object cannot be deleted.
    - Ensure object cannot be overwritten.
 - Create bucket with object lock enabled and put configuration. Put object.
    Check object retention. Change bucket object lock configuration.
    - Ensure object retention period has not changed with bucket configuration.
 - Create bucket with object lock enabled. Put object with legal hold.
    - Ensure object cannot be deleted.
    - Ensure object cannot be overwritten.
 - Create bucket with object lock enabled. Put object with legal hold. Remove
    legal hold.
    - Ensure object can be deleted.
    - Ensure object can be overwritten.
--- a/docs/RELEASE.md
+++ b/docs/RELEASE.md
@ -1,73 +0,0 @@
 # Cloudserver Release Plan
 ## Docker Image Generation
 Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
 CloudServer has a few images there:
 * Cloudserver container image: ghcr.io/scality/cloudserver
 * Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
 * Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
 With every CI build, the CI will push images, tagging the
 content with the developer branch's short SHA-1 commit hash.
 This allows those images to be used by developers, CI builds,
 build chain and so on.
 Tagged versions of cloudserver will be stored in the production namespace.
 ## How to Pull Docker Images
 ```sh
 docker pull ghcr.io/scality/cloudserver:<commit hash>
 docker pull ghcr.io/scality/cloudserver:<tag>
 ```
 ## Release Process
 To release a production image:
 * Create a PR to bump the package version
  Update Cloudserver's `package.json` by bumping it to the relevant next
  version in a new PR. Per example if the last released version was
  `8.4.7`, the next version would be `8.4.8`.
 ```js
 {
  "name": "cloudserver",
  "version": "8.4.8", <--- Here
  [...]
 }
 ```
 * Review & merge the PR
 * Create the release on GitHub
  * Go the Release tab (https://github.com/scality/cloudserver/releases);
  * Click on the `Draft new release button`;
  * In the `tag` field, type the name of the release (`8.4.8`), and confirm
    to create the tag on publish;
  * Click on `Generate release notes` button to fill the fields;
  * Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
  * Click to `Publish the release` to create the GitHub release and git tag
  Notes:
  * the Git tag will be created automatically.
  * this should be done as soon as the PR is merged, so that the tag
    is put on the "version bump" commit.
 * With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
  * Branch Name: The one used for the tag earlier. In this example `development/8.4`
  * Override Stage: 'release'
  * Extra properties:
    * name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
 * Release the release version on Jira
  * Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
  * Create a next version
    * Name: `[next version]`, in this example `8.4.9`
  * Click `...` and select `Release` on the recently released version (`8.4.8`)
  * Fill in the field to move incomplete version to the next one
--- a/docs/USING_PUBLIC_CLOUDS.rst
+++ b/docs/USING_PUBLIC_CLOUDS.rst
@ -1,398 +0,0 @@
 .. _use-public-cloud:
 Using Public Clouds as data backends
 ====================================
 Introduction
 ------------
 As stated in our `GETTING STARTED guide <GETTING_STARTED.html#location-configuration>`__,
 new data backends can be added by creating a region (also called location
 constraint) with the right endpoint and credentials.
 This section of the documentation shows you how to set up our currently
 supported public cloud backends:
 - `Amazon S3 <#aws-s3-as-a-data-backend>`__ ;
 - `Microsoft Azure <#microsoft-azure-as-a-data-backend>`__ .
 For each public cloud backend, you will have to edit your CloudServer
 :code:`locationConfig.json` and do a few setup steps on the applicable public
 cloud backend.
 AWS S3 as a data backend
 ------------------------
 From the AWS S3 Console (or any AWS S3 CLI tool)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Create a bucket where you will host your data for this new location constraint.
 This bucket must have versioning enabled:
 - This is an option you may choose to activate at step 2 of Bucket Creation in
  the Console;
 - With AWS CLI, use :code:`put-bucket-versioning` from the :code:`s3api`
  commands on your bucket of choice;
 - Using other tools, please refer to your tool's documentation.
 In this example, our bucket will be named ``zenkobucket`` and has versioning
 enabled.
 From the CloudServer repository
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 locationConfig.json
 ^^^^^^^^^^^^^^^^^^^
 Edit this file to add a new location constraint. This location constraint will
 contain the information for the AWS S3 bucket to which you will be writing your
 data whenever you create a CloudServer bucket in this location.
 There are a few configurable options here:
 - :code:`type` : set to :code:`aws_s3` to indicate this location constraint is
  writing data to AWS S3;
 - :code:`legacyAwsBehavior` : set to :code:`true` to indicate this region should
  behave like AWS S3 :code:`us-east-1` region, set to :code:`false` to indicate
  this region should behave like any other AWS S3 region;
 - :code:`bucketName` : set to an *existing bucket* in your AWS S3 Account; this
  is the bucket in which your data will be stored for this location constraint;
 - :code:`awsEndpoint` : set to your bucket's endpoint, usually :code:`s3.amazonaws.com`;
 - :code:`bucketMatch` : set to :code:`true` if you want your object name to be the
  same in your local bucket and your AWS S3 bucket; set to :code:`false` if you
  want your object name to be of the form :code:`{{localBucketName}}/{{objectname}}`
  in your AWS S3 hosted bucket;
 - :code:`credentialsProfile` and :code:`credentials` are two ways to provide
  your AWS S3 credentials for that bucket, *use only one of them* :
  - :code:`credentialsProfile` : set to the profile name allowing you to access
    your AWS S3 bucket from your :code:`~/.aws/credentials` file;
  - :code:`credentials` : set the two fields inside the object (:code:`accessKey`
    and :code:`secretKey`) to their respective values from your AWS credentials.
 .. code:: json
    (...)
    "aws-test": {
        "type": "aws_s3",
        "legacyAwsBehavior": true,
        "details": {
            "awsEndpoint": "s3.amazonaws.com",
            "bucketName": "zenkobucket",
            "bucketMatch": true,
            "credentialsProfile": "zenko"
        }
    },
    (...)
 .. code:: json
    (...)
    "aws-test": {
        "type": "aws_s3",
        "legacyAwsBehavior": true,
        "details": {
            "awsEndpoint": "s3.amazonaws.com",
            "bucketName": "zenkobucket",
            "bucketMatch": true,
            "credentials": {
                "accessKey": "WHDBFKILOSDDVF78NPMQ",
                "secretKey": "87hdfGCvDS+YYzefKLnjjZEYstOIuIjs/2X72eET"
            }
        }
    },
    (...)
 .. WARNING::
   If you set :code:`bucketMatch` to :code:`true`, we strongly advise that you
   only have one local bucket per AWS S3 location.
   Without :code:`bucketMatch` set to :code:`false`, your object names in your
   AWS S3 bucket will not be prefixed with your Cloud Server bucket name. This
   means that if you put an object :code:`foo` to your CloudServer bucket
   :code:`zenko1` and you then put a different :code:`foo` to your CloudServer
   bucket :code:`zenko2` and both :code:`zenko1` and :code:`zenko2` point to the
   same AWS bucket, the second :code:`foo` will overwrite the first :code:`foo`.
 ~/.aws/credentials
 ^^^^^^^^^^^^^^^^^^
 .. TIP::
   If you explicitly set your :code:`accessKey` and :code:`secretKey` in the 
   :code:`credentials` object of your :code:`aws_s3` location in your
   :code:`locationConfig.json` file, you may skip this section
 Make sure your :code:`~/.aws/credentials` file has a profile matching the one
 defined in your :code:`locationConfig.json`. Following our previous example, it
 would look like:
 .. code:: shell
    [zenko]
    aws_access_key_id=WHDBFKILOSDDVF78NPMQ
    aws_secret_access_key=87hdfGCvDS+YYzefKLnjjZEYstOIuIjs/2X72eET
 Start the server with the ability to write to AWS S3
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Inside the repository, once all the files have been edited, you should be able
 to start the server and start writing data to AWS S3 through CloudServer.
 .. code:: shell
   # Start the server locally
   $> S3DATA=multiple yarn start
 Run the server as a docker container with the ability to write to AWS S3
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. TIP::
   If you set the :code:`credentials` object in your
   :code:`locationConfig.json` file, you don't need to mount your
   :code:`.aws/credentials` file
 Mount all the files that have been edited to override defaults, and do a
 standard Docker run; then you can start writing data to AWS S3 through
 CloudServer.
 .. code:: shell
   # Start the server in a Docker container
   $> sudo docker run -d --name CloudServer \
   -v $(pwd)/data:/usr/src/app/localData \
   -v $(pwd)/metadata:/usr/src/app/localMetadata \
   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
   -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
   -v ~/.aws/credentials:/root/.aws/credentials \
   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 \
   -d scality/cloudserver
 Testing: put an object to AWS S3 using CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In order to start testing pushing to AWS S3, you will need to create a local
 bucket in the AWS S3 location constraint - this local bucket will only store the
 metadata locally, while both the data and any user metadata (:code:`x-amz-meta`
 headers sent with a PUT object, and tags) will be stored on AWS S3.
 This example is based on all our previous steps.
 .. code:: shell
   # Create a local bucket storing data in AWS S3
   $> s3cmd --host=127.0.0.1:8000 mb s3://zenkobucket --region=aws-test
   # Put an object to AWS S3, and store the metadata locally
   $> s3cmd --host=127.0.0.1:8000 put /etc/hosts s3://zenkobucket/testput
    upload: '/etc/hosts' -> 's3://zenkobucket/testput'  [1 of 1]
     330 of 330   100% in    0s   380.87 B/s  done
   # List locally to check you have the metadata
   $> s3cmd --host=127.0.0.1:8000 ls s3://zenkobucket
    2017-10-23 10:26       330   s3://zenkobucket/testput
 Then, from the AWS Console, if you go into your bucket, you should see your
 newly uploaded object:
 .. figure:: ../res/aws-console-successful-put.png
   :alt: AWS S3 Console upload example
 Troubleshooting
 ~~~~~~~~~~~~~~~
 Make sure your :code:`~/.s3cfg` file has credentials matching your local
 CloudServer credentials defined in :code:`conf/authdata.json`. By default, the
 access key is :code:`accessKey1` and the secret key is :code:`verySecretKey1`.
 For more informations, refer to our template `~/.s3cfg <./CLIENTS/#s3cmd>`__ .
 Pre-existing objects in your AWS S3 hosted bucket can unfortunately not be
 accessed by CloudServer at this time.
 Make sure versioning is enabled in your remote AWS S3 hosted bucket. To check,
 using the AWS Console, click on your bucket name, then on "Properties" at the
 top, and then you should see something like this:
 .. figure:: ../res/aws-console-versioning-enabled.png
   :alt: AWS Console showing versioning enabled
 Microsoft Azure as a data backend
 ---------------------------------
 From the MS Azure Console
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 From your Storage Account dashboard, create a container where you will host your
 data for this new location constraint.
 You will also need to get one of your Storage Account Access Keys, and to
 provide it to CloudServer.
 This can be found from your Storage Account dashboard, under "Settings, then
 "Access keys".
 In this example, our container will be named ``zenkontainer``, and will belong
 to the ``zenkomeetups`` Storage Account.
 From the CloudServer repository
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 locationConfig.json
 ^^^^^^^^^^^^^^^^^^^
 Edit this file to add a new location constraint. This location constraint will
 contain the information for the MS Azure container to which you will be writing
 your data whenever you create a CloudServer bucket in this location.
 There are a few configurable options here:
 - :code:`type` : set to :code:`azure` to indicate this location constraint is
  writing data to MS Azure;
 - :code:`legacyAwsBehavior` : set to :code:`true` to indicate this region should
  behave like AWS S3 :code:`us-east-1` region, set to :code:`false` to indicate
  this region should behave like any other AWS S3 region (in the case of MS Azure
  hosted data, this is mostly relevant for the format of errors);
 - :code:`azureStorageEndpoint` : set to your storage account's endpoint, usually
  :code:`https://{{storageAccountName}}.blob.core.windows.net`;
 - :code:`azureContainerName` : set to an *existing container* in your MS Azure
  storage account; this is the container in which your data will be stored for
  this location constraint;
 - :code:`bucketMatch` : set to :code:`true` if you want your object name to be
  the same in your local bucket and your MS Azure container; set to
  :code:`false` if you want your object name to be of the form
  :code:`{{localBucketName}}/{{objectname}}` in your MS Azure container ;
 - :code:`azureStorageAccountName` : the MS Azure Storage Account to which your
  container belongs;
 - :code:`azureStorageAccessKey` : one of the Access Keys associated to the above
  defined MS Azure Storage Account.
 .. code:: json
    (...)
    "azure-test": {
 	"type": "azure",
        "legacyAwsBehavior": false,
        "details": {
          "azureStorageEndpoint": "https://zenkomeetups.blob.core.windows.net/",
 	  "bucketMatch": true,
          "azureContainerName": "zenkontainer",
 	  "azureStorageAccountName": "zenkomeetups",
 	  "azureStorageAccessKey": "auhyDo8izbuU4aZGdhxnWh0ODKFP3IWjsN1UfFaoqFbnYzPj9bxeCVAzTIcgzdgqomDKx6QS+8ov8PYCON0Nxw=="
 	}
    },
    (...)
 .. WARNING::
   If you set :code:`bucketMatch` to :code:`true`, we strongly advise that you
   only have one local bucket per MS Azure location.
   Without :code:`bucketMatch` set to :code:`false`, your object names in your
   MS Azure container will not be prefixed with your Cloud Server bucket name.
   This means that if you put an object :code:`foo` to your CloudServer bucket
   :code:`zenko1` and you then put a different :code:`foo` to your CloudServer
   bucket :code:`zenko2` and both :code:`zenko1` and :code:`zenko2` point to the
   same MS Azure container, the second :code:`foo` will overwrite the first
   :code:`foo`.
 .. TIP::
   You may export environment variables to **override** some of your
   :code:`locationConfig.json` variable ; the syntax for them is
   :code:`{{region-name}}_{{ENV_VAR_NAME}}`; currently, the available variables
   are those shown below, with the values used in the current example:
   .. code:: shell
      $> export azure-test_AZURE_STORAGE_ACCOUNT_NAME="zenkomeetups"
      $> export azure-test_AZURE_STORAGE_ACCESS_KEY="auhyDo8izbuU4aZGdhxnWh0ODKFP3IWjsN1UfFaoqFbnYzPj9bxeCVAzTIcgzdgqomDKx6QS+8ov8PYCON0Nxw=="
      $> export azure-test_AZURE_STORAGE_ENDPOINT="https://zenkomeetups.blob.core.windows.net/"
 Start the server with the ability to write to MS Azure
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Inside the repository, once all the files have been edited, you should be able
 to start the server and start writing data to MS Azure through CloudServer.
 .. code:: shell
   # Start the server locally
   $> S3DATA=multiple yarn start
 Run the server as a docker container with the ability to write to MS Azure
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Mount all the files that have been edited to override defaults, and do a
 standard Docker run; then you can start writing data to MS Azure through
 CloudServer.
 .. code:: shell
   # Start the server in a Docker container
   $> sudo docker run -d --name CloudServer \
   -v $(pwd)/data:/usr/src/app/localData \
   -v $(pwd)/metadata:/usr/src/app/localMetadata \
   -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
   -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
   -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
   -d scality/cloudserver
 Testing: put an object to MS Azure using CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In order to start testing pushing to MS Azure, you will need to create a local
 bucket in the MS Azure region - this local bucket will only store the metadata
 locally, while both the data and any user metadata (:code:`x-amz-meta` headers
 sent with a PUT object, and tags) will be stored on MS Azure.
 This example is based on all our previous steps.
 .. code:: shell
   # Create a local bucket storing data in MS Azure
   $> s3cmd --host=127.0.0.1:8000 mb s3://zenkontainer --region=azure-test
   # Put an object to MS Azure, and store the metadata locally
   $> s3cmd --host=127.0.0.1:8000 put /etc/hosts s3://zenkontainer/testput
    upload: '/etc/hosts' -> 's3://zenkontainer/testput'  [1 of 1]
     330 of 330   100% in    0s   380.87 B/s  done
   # List locally to check you have the metadata
   $> s3cmd --host=127.0.0.1:8000 ls s3://zenkobucket
    2017-10-24 14:38       330   s3://zenkontainer/testput
 Then, from the MS Azure Console, if you go into your container, you should see
 your newly uploaded object:
 .. figure:: ../res/azure-console-successful-put.png
   :alt: MS Azure Console upload example
 Troubleshooting
 ~~~~~~~~~~~~~~~
 Make sure your :code:`~/.s3cfg` file has credentials matching your local
 CloudServer credentials defined in :code:`conf/authdata.json`. By default, the
 access key is :code:`accessKey1` and the secret key is :code:`verySecretKey1`.
 For more informations, refer to our template `~/.s3cfg <./CLIENTS/#s3cmd>`__ .
 Pre-existing objects in your MS Azure container can unfortunately not be
 accessed by CloudServer at this time.
 For any data backend
 --------------------
 From the CloudServer repository
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 config.json
 ^^^^^^^^^^^
 .. IMPORTANT::
   You only need to follow this section if you want to define a given location
   as the default for a specific endpoint
 Edit the :code:`restEndpoint` section of your :code:`config.json` file to add
 an endpoint definition matching the location you want to use as a default for an
 endpoint to this specific endpoint.
 In this example, we'll make :code:`custom-location` our default location for the
 endpoint :code:`zenkotos3.com`:
 .. code:: json
    (...)
    "restEndpoints": {
        "localhost": "us-east-1",
        "127.0.0.1": "us-east-1",
        "cloudserver-front": "us-east-1",
        "s3.docker.test": "us-east-1",
        "127.0.0.2": "us-east-1",
        "zenkotos3.com": "custom-location"
    },
    (...)
--- a/docs/conf.py
+++ b/docs/conf.py
@ -1,161 +0,0 @@
 # -*- coding: utf-8 -*-
 #
 # Zope docs documentation build configuration file, created by
 # sphinx-quickstart on Fri Feb 20 16:22:03 2009.
 #
 # This file is execfile()d with the current directory set to its containing
 # dir.
 #
 # The contents of this file are pickled, so don't put values in the namespace
 # that aren't pickleable (module imports are okay, they're removed
 # automatically).
 #
 # Note that not all possible configuration values are present in this
 # autogenerated file.
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 # import sys
 # import os
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 # sys.path.append(os.path.abspath('.'))
 # General configuration
 # ---------------------
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = []
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 # The suffix of source filenames.
 source_suffix = '.rst'
 # The encoding of source files.
 # source_encoding = 'utf-8'
 # The master toctree document.
 master_doc = 'index'
 # General information about the project.
 project = u'scality-zenko-cloudserver'
 copyright = u'Apache License Version 2.0, 2004 http://www.apache.org/licenses/'
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
 version = '7.0.0'
 # The full version, including alpha/beta/rc tags.
 release = '7.0.0'
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 # language = None
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
 # today = ''
 # Else, today_fmt is used as the format for a strftime call.
 # today_fmt = '%B %d, %Y'
 # List of documents that shouldn't be included in the build.
 # unused_docs = []
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
 exclude_trees = ['_build']
 # The reST default role (used for this markup: `text`) to use for
 # all documents.
 # default_role = None
 # If true, '()' will be appended to :func: etc. cross-reference text.
 # add_function_parentheses = True
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
 # add_module_names = True
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
 # show_authors = False
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 # Options for HTML output
 # -----------------------
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
 # given in html_static_path.
 html_style = 'css/default.css'
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 # html_title = None
 # A shorter title for the navigation bar.  Default is the same as html_title.
 # html_short_title = None
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
 html_logo = '../res/scality-cloudserver-logo.png'
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
 # html_favicon = None
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 # html_last_updated_fmt = '%b %d, %Y'
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
 # html_use_smartypants = True
 # Custom sidebar templates, maps document names to template names.
 # html_sidebars = {}
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
 # html_additional_pages = {}
 # If false, no module index is generated.
 # html_use_modindex = True
 # If false, no index is generated.
 # html_use_index = True
 # If true, the index is split into individual pages for each letter.
 # html_split_index = False
 # If true, the reST sources are included in the HTML build as _sources/<name>.
 # html_copy_source = True
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
 # html_use_opensearch = ''
 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
 # html_file_suffix = ''
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'ZenkoCloudServerdoc'
--- a/docs/data_metadata_daemon.md
+++ b/docs/data_metadata_daemon.md
@ -0,0 +1,166 @@
 # Data-metadata daemon Architecture and Operational guide
 This document presents the architecture of the data-metadata daemon
 (dmd) used for the community edition of S3 server. It also provides a
 guide on how to operate it.
 The dmd is responsible for storing and retrieving S3 data and
 metadata, and is accessed by S3 connectors through socket.io
 (metadata) and REST (data) APIs.
 It has been designed such that more than one S3 connector can access
 the same buckets by communicating with the dmd. It also means that the
 dmd can be hosted on a separate container or machine.
 ## Operation
 ### Startup
 The simplest deployment is still to launch with npm start, this will
 start one instance of the S3 connector and will listen on the locally
 bound dmd ports 9990 and 9991 (by default, see below).
 The dmd can be started independently from the S3 server by running
 this command in the S3 directory:
    npm run start_dmd
 This will open two ports:
 * one is based on socket.io and is used for metadata transfers (9990
  by default)
 * the other is a REST interface used for data transfers (9991 by
  default)
 Then, one or more instances of S3 server without the dmd can be
 started elsewhere with:
    npm run start_s3server
 ### Configuration
 Most configuration happens in `config.json` for S3 server, local
 storage paths can be changed where the dmd is started using
 environment variables, like before: `S3DATAPATH` and `S3METADATAPATH`.
 In `config.json`, the following sections are used to configure access
 to the dmd through separate configuration of the data and metadata
 access:
    "metadataClient": {
        "host": "localhost",
        "port": 9990
    },
    "dataClient": {
        "host": "localhost",
        "port": 9991
    },
 To run a remote dmd, you have to do the following:
 * change both `"host"` attributes to the IP or host name where the dmd
  is run.
 * Modify the `"bindAddress"` attributes in `"metadataDaemon"` and
  `"dataDaemon"` sections where the dmd is run to accept remote
  connections (e.g. `"::"`)
 ## Architecture
 This section gives a bit more insight on how it works internally.
 ![./images/data_metadata_daemon_arch.png](./images/data_metadata_daemon_arch.png
 "Architecture diagram")
 ### Metadata on socket.io
 This communication is based on an RPC system based on socket.io events
 sent by S3 connectors, received by the DMD and acknowledged back to
 the S3 connector.
 The actual payload sent through socket.io is a JSON-serialized form of
 the RPC call name and parameters, along with some additional
 information like the request UIDs, and the sub-level information, sent
 as object attributes in the JSON request.
 With introduction of versioning support, the updates are now gathered
 in the dmd for some number of milliseconds max, before being batched
 as a single write to the database. This is done server-side, so the
 API is meant to send individual updates.
 Four RPC commands are available to clients: `put`, `get`, `del` and
 `createReadStream`. They more or less map the parameters accepted by
 the corresponding calls in the LevelUp implementation of LevelDB. They
 differ in the following:
 * The `sync` option is ignored (under the hood, puts are gathered into
  batches which have their `sync` property enforced when they are
  committed to the storage)
 * Some additional versioning-specific options are supported
 * `createReadStream` becomes asynchronous, takes an additional
  callback argument and returns the stream in the second callback
  parameter
 Debugging the socket.io exchanges can be achieved by running the
 daemon with `DEBUG='socket.io*'` environment variable set.
 One parameter controls the timeout value after which RPC commands sent
 end with a timeout error, it can be changed either:
 * via the `DEFAULT_CALL_TIMEOUT_MS` option in `lib/network/rpc/rpc.js`
 * or in the constructor call of the `MetadataFileClient` object (in
  `lib/metadata/bucketfile/backend.js` as `callTimeoutMs`.
 Default value is 30000.
 A specific implementation deals with streams, currently used for
 listing a bucket. Streams emit `"stream-data"` events that pack one or
 more items in the listing, and a special `“stream-end”` event when
 done. Flow control is achieved by allowing a certain number of “in
 flight” packets that have not received an ack yet (5 by default). Two
 options can tune the behavior (for better throughput or getting it
 more robust on weak networks), they have to be set in `mdserver.js`
 file directly, as there is no support in `config.json` for now for
 those options:
 * `streamMaxPendingAck`: max number of pending ack events not yet
  received (default is 5)
 * `streamAckTimeoutMs`: timeout for receiving an ack after an output
  stream packet is sent to the client (default is 5000)
 ### Data exchange through the REST data port
 Data is read and written with REST semantic.
 The web server recognizes a base path in the URL of `/DataFile` to be
 a request to the data storage service.
 #### PUT
 A PUT on `/DataFile` URL and contents passed in the request body will
 write a new object to the storage.
 On success, a `201 Created` response is returned and the new URL to
 the object is returned via the `Location` header (e.g.
 `Location: /DataFile/50165db76eecea293abfd31103746dadb73a2074`).
 The raw key can then be extracted simply by removing the leading
 `/DataFile` service information from the returned URL.
 #### GET
 A GET is simply issued with REST semantic, e.g.:
    GET /DataFile/50165db76eecea293abfd31103746dadb73a2074 HTTP/1.1
 A GET request can ask for a specific range. Range support is complete
 except for multiple byte ranges.
 #### DELETE
 DELETE is similar to GET, except that a `204 No Content` response is
 returned on success.
--- a/docs/developers/getting-started.rst
+++ b/docs/developers/getting-started.rst
@ -1,79 +0,0 @@
 ============================================
 Add New Backend Storage To Zenko CloudServer
 ============================================
 This set of documents aims at bootstrapping developers with Zenko's CloudServer
 module, so they can then go on and contribute features.
 .. toctree::
    :maxdepth: 2
    non-s3-compatible-backend
    s3-compatible-backend
 We always encourage our community to offer new extensions to Zenko,
 and new backend support is paramount to meeting more community needs.
 If that is something you want to contribute (or just do on your own
 version of the cloudserver image), this is the guid to read. Please
 make sure you follow our `Contributing Guidelines`_/.
 If you need help with anything, please search our `forum`_ for more
 information.
 Add support for a new backend
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Currently the main public cloud protocols are supported and more can
 be added. There are two main types of backend: those compatible with
 Amazon's S3 protocol and those not compatible.
 ================= ========== ============ ===========
 Backend type      Supported  Active WIP   Not started
 ================= ========== ============ ===========
 Private disk/fs       x
 AWS S3                x
 Microsoft Azure       x
 Backblaze B2                    x
 Google Cloud          x
 Openstack Swift                               x
 ================= ========== ============ ===========
 .. important:: Should you want to request for a new backend to be
               supported, please do so by opening a `Github issue`_,
               and filling out the "Feature Request" section of our
               template.
 To add support for a new backend support to CloudServer official
 repository, please follow these steps:
 - familiarize yourself with our `Contributing Guidelines`_
 - open a `Github issue`_ and fill out Feature Request form, and
  specify you would like to contribute it yourself;
 - wait for our core team to get back to you with an answer on whether
  we are interested in taking that contribution in (and hence
  committing to maintaining it over time);
 - once approved, fork the repository and start your development;
 - use the `forum`_ with any question you may have during the
  development process;
 - when you think it's ready, let us know so that we create a feature
  branch against which we'll compare and review your code;
 - open a pull request with your changes against that dedicated feature
  branch;
 - once that pull request gets merged, you're done.
 .. tip::
    While we do take care of the final rebase (when we merge your feature
    branch on the latest default branch), we do ask that you keep up to date with our latest default branch
    until then.
 .. important::
    If we do not approve your feature request, you may of course still
    work on supporting a new backend: all our "no" means is that we do not
    have the resources, as part of our core development team, to maintain
    this feature for the moment.
 .. _GitHub issue: https://github.com/scality/S3/issues
 .. _Contributing Guidelines: https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md
 .. _forum: https://forum.zenko.io
--- a/docs/developers/new-backend.rst
+++ b/docs/developers/new-backend.rst
@ -1,53 +0,0 @@
 =================
 Add A New Backend
 =================
 Supporting all possible public cloud storage APIs is CloudServer's
 ultimate goal. As an open source project, contributions are welcome.
 The first step is to get familiar with building a custom Docker image
 for CloudServer.
 Build a Custom Docker Image
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Clone Zenko's CloudServer, install all dependencies and start the
 service:
 .. code-block:: shell
  $ git clone https://github.com/scality/cloudserver
  $ cd cloudserver
  $ yarn install
  $ yarn start
 .. tip::
    Some optional dependencies may fail, resulting in you seeing `yarn
    WARN` messages; these can safely be ignored.  Refer to the User
    documentation for all available options.
 Build the Docker image:
 .. code-block:: shell
     # docker build . -t
     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
 Push the newly created Docker image to your own hub:
 .. code-block:: shell
     # docker push
     # {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
 .. note::
    To perform this last operation, you need to be authenticated with DockerHub
 There are two main types of backend you could want Zenko to support:
 == link:S3_COMPATIBLE_BACKENDS.adoc[S3 compatible data backends]
 == link:NON_S3_COMPATIBLE_BACKENDS.adoc[Data backends using another protocol than the S3 protocol]
--- a/docs/developers/non-s3-compatible-backend.rst
+++ b/docs/developers/non-s3-compatible-backend.rst
@ -1,530 +0,0 @@
 ==========================================================
 Adding support for data backends not supporting the S3 API
 ==========================================================
 These backends abstract the complexity of multiple APIs to let users
 work on a single common namespace across multiple clouds.
 This documents aims at introducing you to the right files in
 CloudServer (the Zenko stack's subcomponent in charge of API
 translation, among other things) to add support to your own backend of
 choice.
 General configuration
 ~~~~~~~~~~~~~~~~~~~~~
 There are a number of constants and environment variables to define to support a
 new data backend; here is a list and where to find them:
 :file:`/constants.js`
 ---------------------
 * give your backend type a name, as part of the `externalBackends` object;
 * specify whether versioning is implemented, as part of the
   `versioningNotImplemented` object;
 :file:`/lib/Config.js`
 ----------------------
 * this is where you should put common utility functions, like the ones to parse
   the location object from `locationConfig.json`;
 * make sure you define environment variables (like `GCP_SERVICE_EMAIL` as we'll
   use those internally for the CI to test against the real remote backend;
 :file:`/lib/data/external/{{backendName}}Client.js`
 ---------------------------------------------------
 * this file is where you'll instantiate your backend client; this should be a
   class with a constructor taking the config object built in `/lib/Config.js` as
   parameter;
 * over time, you may need some utility functions which we've defined in the
   folder `/api/apiUtils`, and in the file `/lib/data/external/utils`;
 :file:`/lib/data/external/utils.js`
 -----------------------------------
 * make sure to add options for `sourceLocationConstraintType` to be equal to
   the name you gave your backend in :file:`/constants.js`;
 :file:`/lib/data/external/{{BackendName}}_lib/`
 -----------------------------------------------
 * this folder is where you'll put the functions needed for supporting your
   backend; keep your files as atomic as possible;
 :file:`/tests/locationConfig/locationConfigTests.json`
 ------------------------------------------------------
 * this file is where you'll create location profiles to be used by your
   functional tests;
 :file:`/lib/data/locationConstraintParser.js`
 ---------------------------------------------
 * this is where you'll instantiate your client if the operation the end user
   sent effectively writes to your backend; everything happens inside the
   function `parseLC()`; you should add a condition that executes if
   `locationObj.type` is the name of your backend (that you defined in
   `constants.js`), and instantiates a client of yours. See pseudocode below,
   assuming location type name is `ztore`:
 .. code-block:: js
   :linenos:
   :emphasize-lines: 12
    (...) //<1>
    const ZtoreClient = require('./external/ZtoreClient');
    const { config } = require('../Config'); //<1>
    function parseLC(){ //<1>
    (...) //<1>
        Object.keys(config.locationConstraints).forEach(location => { //<1>
            const locationObj = config.locationConstraints[location]; //<1>
            (...) //<1>
            if (locationObj.type === 'ztore' {
                const ztoreEndpoint = config.getZtoreEndpoint(location);
                const ztoreCredentials = config.getZtoreCredentials(location); //<2>
                clients[location] = new ZtoreClient({
                    ztoreEndpoint,
                    ztoreCredentials,
                    ztoreBucketname: locationObj.details.ztoreBucketName,
                    bucketMatch:  locationObj.details.BucketMatch,
                    dataStoreName: location,
                }); //<3>
                clients[location].clientType = 'ztore';
            });
            (...) //<1>
        });
    }
 1. Code that is already there
 2. You may need more utility functions depending on your backend specs
 3. You may have more fields required in your constructor object depending on
   your backend specs
 Operation of type PUT
 ~~~~~~~~~~~~~~~~~~~~~
 PUT routes are usually where people get started, as it's the easiest to check!
 Simply go on your remote backend console and you'll be able to see whether your
 object actually went up in the cloud...
 These are the files you'll need to edit:
 :file:`/lib/data/external/{{BackendName}}Client.js`
 ---------------------------------------------------
 - the function that is going to call your `put()` function is also called
  `put()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
 - define a function with signature like
  `put(stream, size, keyContext, reqUids, callback)`; this is worth exploring a
  bit more as these parameters are the same for all backends:
  //TODO: generate this from jsdoc
 - `stream`: the stream of data you want to put in the cloud; if you're
   unfamiliar with node.js streams, we suggest you start training, as we use
   them a lot !
 - `size`: the size of the object you're trying to put;
 - `keyContext`: an object with metadata about the operation; common entries are
   `namespace`, `buckerName`, `owner`, `cipherBundle`, and `tagging`; if these
   are not sufficient for your integration, contact us to get architecture
   validation before adding new entries;
 - `reqUids`: the request unique ID used for logging;
 - `callback`: your function's callback (should handle errors);
 :file:`/lib/data/external/{{backendName}}_lib/`
 -----------------------------------------------
 - this is where you should put all utility functions for your PUT operation, and
  then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
  your code clean;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/put/put{{BackendName}}js`
 -----------------------------------------------------------------------------------
 - every contribution should come with thorough functional tests, showing
  nominal context gives expected behaviour, and error cases are handled in a way
  that is standard with the backend (including error messages and code);
 - the ideal setup is if you simulate your backend locally, so as not to be
  subjected to network flakiness in the CI; however, we know there might not be
  mockups available for every client; if that is the case of your backend, you
  may test against the "real" endpoint of your data backend;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
 -------------------------------------------------------------------
 - where you'll define a constant for your backend location matching your
  :file:`/tests/locationConfig/locationConfigTests.json`
 - depending on your backend, the sample `keys[]` and associated made up objects
  may not work for you (if your backend's key format is different, for example);
  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
  function returning ajusted `keys[]` to your tests.
 Operation of type GET
 ~~~~~~~~~~~~~~~~~~~~~
 GET routes are easy to test after PUT routes are implemented, hence why we're
 covering them second.
 These are the files you'll need to edit:
 :file:`/lib/data/external/{{BackendName}}Client.js`
 ---------------------------------------------------
 - the function that is going to call your `get()` function is also called
  `get()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
 - define a function with signature like
  `get(objectGetInfo, range, reqUids, callback)`; this is worth exploring a
  bit more as these parameters are the same for all backends:
 //TODO: generate this from jsdoc
 - `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
   data store, and `client`, the data store name;
 - `range`: the range of bytes you will get, for "get-by-range" operations (we
   recommend you do simple GETs first, and then look at this);
 - `reqUids`: the request unique ID used for logging;
 - `callback`: your function's callback (should handle errors);
 :file:`/lib/data/external/{{backendName}}_lib/`
 -----------------------------------------------
 - this is where you should put all utility functions for your GET operation, and
  then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
  your code clean;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
 -----------------------------------------------------------------------------------
 - every contribution should come with thorough functional tests, showing
  nominal context gives expected behaviour, and error cases are handled in a way
  that is standard with the backend (including error messages and code);
 - the ideal setup is if you simulate your backend locally, so as not to be
  subjected to network flakiness in the CI; however, we know there might not be
  mockups available for every client; if that is the case of your backend, you
  may test against the "real" endpoint of your data backend;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
 -------------------------------------------------------------------
 .. note:: You should need this section if you have followed the tutorial in order
          (that is, if you have covered the PUT operation already)
 - where you'll define a constant for your backend location matching your
  :file:`/tests/locationConfig/locationConfigTests.json`
 - depending on your backend, the sample `keys[]` and associated made up objects
  may not work for you (if your backend's key format is different, for example);
  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
 Operation of type DELETE
 ~~~~~~~~~~~~~~~~~~~~~~~~
 DELETE routes are easy to test after PUT routes are implemented, and they are
 similar to GET routes in our implementation, hence why we're covering them
 third.
 These are the files you'll need to edit:
 :file:`/lib/data/external/{{BackendName}}Client.js`
 ---------------------------------------------------
 - the function that is going to call your `delete()` function is also called
  `delete()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
 - define a function with signature like
  `delete(objectGetInfo, reqUids, callback)`; this is worth exploring a
  bit more as these parameters are the same for all backends:
 //TODO: generate this from jsdoc
 * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
   data store, and `client`, the data store name;
 * `reqUids`: the request unique ID used for logging;
 * `callback`: your function's callback (should handle errors);
 :file:`/lib/data/external/{{backendName}}_lib/`
 -----------------------------------------------
 - this is where you should put all utility functions for your DELETE operation,
  and then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
  your code clean;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/delete/delete{{BackendName}}js`
 -----------------------------------------------------------------------------------------
 - every contribution should come with thorough functional tests, showing
  nominal context gives expected behaviour, and error cases are handled in a way
  that is standard with the backend (including error messages and code);
 - the ideal setup is if you simulate your backend locally, so as not to be
  subjected to network flakiness in the CI; however, we know there might not be
  mockups available for every client; if that is the case of your backend, you
  may test against the "real" endpoint of your data backend;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
 -------------------------------------------------------------------
 .. note:: You should need this section if you have followed the
          tutorial in order (that is, if you have covered the PUT operation
          already)
 - where you'll define a constant for your backend location matching your
  :file:`/tests/locationConfig/locationConfigTests.json`
 - depending on your backend, the sample `keys[]` and associated made up objects
  may not work for you (if your backend's key format is different, for example);
  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
 Operation of type HEAD
 ~~~~~~~~~~~~~~~~~~~~~~
 HEAD routes are very similar to DELETE routes in our implementation, hence why
 we're covering them fourth.
 These are the files you'll need to edit:
 :file:`/lib/data/external/{{BackendName}}Client.js`
 ---------------------------------------------------
 - the function that is going to call your `head()` function is also called
  `head()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
 - define a function with signature like
  `head(objectGetInfo, reqUids, callback)`; this is worth exploring a
  bit more as these parameters are the same for all backends:
 // TODO:: generate this from jsdoc
 * `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
   data store, and `client`, the data store name;
 * `reqUids`: the request unique ID used for logging;
 * `callback`: your function's callback (should handle errors);
 :file:`/lib/data/external/{{backendName}}_lib/`
 -----------------------------------------------
 - this is where you should put all utility functions for your HEAD operation,
  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
  your code clean;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
 -----------------------------------------------------------------------------------
 - every contribution should come with thorough functional tests, showing
  nominal context gives expected behaviour, and error cases are handled in a way
  that is standard with the backend (including error messages and code);
 - the ideal setup is if you simulate your backend locally, so as not to be
  subjected to network flakiness in the CI; however, we know there might not be
  mockups available for every client; if that is the case of your backend, you
  may test against the "real" endpoint of your data backend;
 :file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
 -------------------------------------------------------------------
 .. note:: You should need this section if you have followed the tutorial in order
          (that is, if you have covered the PUT operation already)
 - where you'll define a constant for your backend location matching your
  :file:`/tests/locationConfig/locationConfigTests.json`
 - depending on your backend, the sample `keys[]` and associated made up objects
  may not work for you (if your backend's key format is different, for example);
  if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
 Healthcheck
 ~~~~~~~~~~~
 Healtchecks are used to make sure failure to write to a remote cloud is due to
 a problem on that remote cloud, an not on Zenko's side.
 This is usually done by trying to create a bucket that already exists, and
 making sure you get the expected answer.
 These are the files you'll need to edit:
 :file:`/lib/data/external/{{BackendName}}Client.js`
 ---------------------------------------------------
 - the function that is going to call your `healthcheck()` function is called
  `checkExternalBackend()` and it's defined in
  :file:`/lib/data/multipleBackendGateway.js`; you will need to add your own;
 - your healtcheck function should get `location` as a parameter, which is an
  object comprising:`
 * `reqUids`: the request unique ID used for logging;
 * `callback`: your function's callback (should handle errors);
 :file:`/lib/data/external/{{backendName}}_lib/{{backendName}}_create_bucket.js`
 -------------------------------------------------------------------------------
 - this is where you should write the function performing the actual bucket
  creation;
 :file:`/lib/data/external/{{backendName}}_lib/utils.js`
 -------------------------------------------------------
 - add an object named per your backend's name to the `backendHealth` dictionary,
  with proper `response` and `time` entries;
 :file:`lib/data/multipleBackendGateway.js`
 ------------------------------------------
 - edit the `healthcheck` function to add your location's array, and call your
  healthcheck; see pseudocode below for a sample implementation, provided your
  backend name is `ztore`
 .. code-block:: js
   :linenos:
    (...) //<1>
        healthcheck: (flightCheckOnStartUp, log, callback) => { //<1>
            (...) //<1>
            const ztoreArray = []; //<2>
            async.each(Object.keys(clients), (location, cb) => { //<1>
                (...) //<1>
                } else if (client.clientType === 'ztore' {
                    ztoreArray.push(location); //<3>
                    return cb();
                }
            (...) //<1>
            multBackendResp[location] = { code: 200, message: 'OK' }; //<1>
            return cb();
        }, () => { //<1>
            async.parallel([
                (...) //<1>
                next => checkExternalBackend( //<4>
                    clients, ztoreArray, 'ztore', flightCheckOnStartUp,
                    externalBackendHealthCheckInterval, next),
            ] (...) //<1>
            });
            (...) //<1>
        });
    }
 1. Code that is already there
 2. The array that will store all locations of type 'ztore'
 3. Where you add locations of type 'ztore' to the array
 4. Where you actually call the healthcheck function on all 'ztore' locations
 Multipart upload (MPU)
 ~~~~~~~~~~~~~~~~~~~~~~
 This is the final part to supporting a new backend! MPU is far from
 the easiest subject, but you've come so far it shouldn't be a problem.
 These are the files you'll need to edit:
 :file:`/lib/data/external/{{BackendName}}Client.js`
 ---------------------------------------------------
 You'll be creating four functions with template signatures:
 - `createMPU(Key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
  cacheControl, contentDisposition, contentEncoding, log, callback)` will
  initiate the multi part upload process; now, here, all parameters are
  metadata headers except for:
 * `Key`, the key id for the final object (collection of all parts);
 * `bucketName`, the name of the bucket to which we will do an MPU;
 * `log`, the logger;
 - `uploadPart(request, streamingV4Params, stream, size, key, uploadId, partNumber, bucketName, log, callback)`
   will be called for each part; the parameters can be explicited as follow:
 * `request`, the request object for putting the part;
 * `streamingV4Params`, parameters for auth V4 parameters against S3;
 * `stream`, the node.js readable stream used to put the part;
 * `size`, the size of the part;
 * `key`, the key of the object;
 * `uploadId`, multipart upload id string;
 * `partNumber`, the number of the part in this MPU (ordered);
 * `bucketName`, the name of the bucket to which we will do an MPU;
 * `log`, the logger;
 - `completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback)` will
   end the MPU process once all parts are uploaded; parameters can be explicited
   as follows:
 * `jsonList`, user-sent list of parts to include in final mpu object;
 * `mdInfo`, object containing 3 keys: storedParts, mpuOverviewKey, and
   splitter;
 * `key`, the key of the object;
 * `uploadId`, multipart upload id string;
 * `bucketName`, name of bucket;
 * `log`, logger instance:
 - `abortMPU(key, uploadId, bucketName, log, callback)` will handle errors, and
  make sure that all parts that may have been uploaded will be deleted if the
  MPU ultimately fails; the parameters are:
 * `key`, the key of the object;
 * `uploadId`, multipart upload id string;
 * `bucketName`, name of bucket;
 * `log`, logger instance.
 :file:`/lib/api/objectPutPart.js`
 ---------------------------------
 - you'll need to add your backend type in appropriate sections (simply look for
  other backends already implemented).
 :file:`/lib/data/external/{{backendName}}_lib/`
 -----------------------------------------------
 - this is where you should put all utility functions for your MPU operations,
  and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
  your code clean;
 :file:`lib/data/multipleBackendGateway.js`
 ------------------------------------------
 - edit the `createMPU` function to add your location type, and call your
  `createMPU()`; see pseudocode below for a sample implementation, provided your
  backend name is `ztore`
 .. code-block:: javascript
   :linenos:
    (...) //<1>
        createMPU:(key, metaHeaders, bucketName, websiteRedirectHeader, //<1>
         location, contentType, cacheControl, contentDisposition,
         contentEncoding, log, cb) => {
            const client = clients[location]; //<1>
            if (client.clientType === 'aws_s3') { //<1>
                return client.createMPU(key, metaHeaders, bucketName,
                websiteRedirectHeader, contentType, cacheControl,
                contentDisposition, contentEncoding, log, cb);
            } else if (client.clientType === 'ztore') { //<2>
                return client.createMPU(key, metaHeaders, bucketName,
                  websiteRedirectHeader, contentType, cacheControl,
                  contentDisposition, contentEncoding, log, cb);
            }
            return cb();
        };
    (...) //<1>
 1. Code that is already there
 2. Where the `createMPU()` of your client is actually called
 Add functional tests
 ~~~~~~~~~~~~~~~~~~~~
 * :file:`tests/functional/aws-node-sdk/test/multipleBackend/initMPU/{{BackendName}}InitMPU.js`
 * :file:`tests/functional/aws-node-sdk/test/multipleBackend/listParts/{{BackendName}}ListPart.js`
 * :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/{{BackendName}}AbortMPU.js`
 * :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/{{BackendName}}CompleteMPU.js`
 * :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/{{BackendName}}UploadPart.js`
 Adding support in Orbit, Zenko's UI for simplified Multi Cloud Management
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 This can only be done by our core developers' team. Once your backend
 integration is merged, you may open a feature request on the
 `Zenko repository`_, and we will
 get back to you after we evaluate feasability and maintainability.
 .. _Zenko repository: https://www.github.com/scality/Zenko/issues/new
--- a/docs/developers/s3-compatible-backend.rst
+++ b/docs/developers/s3-compatible-backend.rst
@ -1,43 +0,0 @@
 ======================
 S3-Compatible Backends
 ======================
 Adding Support in CloudServer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 This is the easiest case for backend support integration: there is nothing to do
 but configuration!  Follow the steps described in our
 :ref:`use-public-cloud` and make sure you:
 - set ``details.awsEndpoint`` to your storage provider endpoint;
 - use ``details.credentials`` and *not* ``details.credentialsProfile`` to set your
  credentials for that S3-compatible backend.
 For example, if you’re using a Wasabi bucket as a backend, then your region
 definition for that backend will look something like:
 ::
    "wasabi-bucket-zenkobucket": {
    "type": "aws_s3",
    "legacyAwsBehavior": true,
    "details": {
    "awsEndpoint": "s3.wasabisys.com",
    "bucketName": "zenkobucket",
    "bucketMatch": true,
    "credentials": {
    "accessKey": "\\{YOUR_WASABI_ACCESS_KEY}",
    "secretKey": "\\{YOUR_WASABI_SECRET_KEY}"
    }
    }
    },
 Adding Support in Zenko Orbit
 #############################
 This can only be done by our core developpers' team. If that’s what you’re
 after, open a feature request on the `Zenko repository`_, and we will
 get back to you after we evaluate feasability and maintainability.
 .. _Zenko repository: https://www.github.com/scality/Zenko/issues/new
--- a/docs/images/access_control_authorization.png
+++ b/docs/images/access_control_authorization.png
--- a/docs/index.rst
+++ b/docs/index.rst
@ -1,18 +0,0 @@
 Scality Zenko CloudServer
 =========================
 .. _user-docs:
 .. toctree::
   :maxdepth: 2
   :caption: Documentation
   :glob:
   CONTRIBUTING
   GETTING_STARTED
   USING_PUBLIC_CLOUDS
   CLIENTS
   DOCKER
   INTEGRATIONS
   ARCHITECTURE
   developers/*
--- a/docs/listing.md
+++ b/docs/listing.md
@ -0,0 +1,20 @@
 # Listing
 ## Listing Types
 We use three different types of metadata listing for various operations.
 Here are the scenarios we use each for:
 - 'Delimiter' - when no versions are possible in the bucket since it is an
   internally-used only bucket which is not exposed to a user. Namely,
  1. to list objects in the "user's bucket" to respond to a GET SERVICE
  request and
  2. to do internal listings on an MPU shadow bucket to complete multipart
  upload operations.
 - 'DelimiterVersion' - to list all versions in a bucket
 - 'DelimiterMaster' - to list just the master versions of objects in a bucket
 ## Algorithms
 The algorithms for each listing type can be found in the open-source
 [scality/Arsenal](https://github.com/scality/Arsenal) repository, in [lib/algos/list](https://github.com/scality/Arsenal/tree/master/lib/algos/list).
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@ -1,5 +0,0 @@
 ---
 # http://www.mkdocs.org/user-guide/configuration/
 # https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes
 site_name: Scality Zenko CloudServer documentation
--- a/docs/requirements.in
+++ b/docs/requirements.in
@ -1,2 +0,0 @@
 Sphinx >= 1.7.5
 recommonmark >= 0.4.0
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@ -1,119 +0,0 @@
 #
 # This file is autogenerated by pip-compile
 # To update, run:
 #
 #    tox -e pip-compile
 #
 alabaster==0.7.12 \
    --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
    --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 \
    # via sphinx
 babel==2.6.0 \
    --hash=sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669 \
    --hash=sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23 \
    # via sphinx
 certifi==2018.10.15 \
    --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
    --hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a \
    # via requests
 chardet==3.0.4 \
    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
    # via requests
 commonmark==0.5.4 \
    --hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5 \
    # via recommonmark
 docutils==0.14 \
    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
    # via recommonmark, sphinx
 idna==2.7 \
    --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
    --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
    # via requests
 imagesize==1.1.0 \
    --hash=sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8 \
    --hash=sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5 \
    # via sphinx
 jinja2==2.10 \
    --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
    --hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
    # via sphinx
 markupsafe==1.1.0 \
    --hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \
    --hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \
    --hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \
    --hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \
    --hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \
    --hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \
    --hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \
    --hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \
    --hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \
    --hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \
    --hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \
    --hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \
    --hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \
    --hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \
    --hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \
    --hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \
    --hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \
    --hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \
    --hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \
    --hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \
    --hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \
    --hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \
    --hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \
    --hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \
    --hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \
    --hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \
    --hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \
    --hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 \
    # via jinja2
 packaging==18.0 \
    --hash=sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807 \
    --hash=sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9 \
    # via sphinx
 pygments==2.2.0 \
    --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
    --hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc \
    # via sphinx
 pyparsing==2.3.0 \
    --hash=sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b \
    --hash=sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592 \
    # via packaging
 pytz==2018.7 \
    --hash=sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca \
    --hash=sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6 \
    # via babel
 recommonmark==0.4.0 \
    --hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d \
    --hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7
 requests==2.20.1 \
    --hash=sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54 \
    --hash=sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263 \
    # via sphinx
 six==1.11.0 \
    --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
    --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \
    # via packaging, sphinx
 snowballstemmer==1.2.1 \
    --hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 \
    --hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \
    # via sphinx
 sphinx==1.8.2 \
    --hash=sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea \
    --hash=sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64
 sphinxcontrib-websupport==1.1.0 \
    --hash=sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd \
    --hash=sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9 \
    # via sphinx
 typing==3.6.6 \
    --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \
    --hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
    --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
    # via sphinx
 urllib3==1.24.1 \
    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
    # via requests
--- a/examples/go-md-search.go
+++ b/examples/go-md-search.go
@ -1,46 +0,0 @@
 package main
 import (
    "fmt"
    "time"
    "bytes"
    "net/http"
    "net/url"
    "io/ioutil"
    "github.com/aws/aws-sdk-go/aws/credentials"
    "github.com/aws/aws-sdk-go/aws/signer/v4"
 )
 func main() {
    // Input AWS access key, secret key
    aws_access_key_id := "accessKey1"
    aws_secret_access_key := "verySecretKey1"
    endpoint := "http://localhost:8000"
    bucket_name := "bucketname"
    searchQuery := url.QueryEscape("x-amz-meta-color=blue")
    buf := bytes.NewBuffer([]byte{})
    requestUrl := fmt.Sprintf("%s/%s?search=%s",
        endpoint, bucket_name, searchQuery)
    request, err := http.NewRequest("GET", requestUrl, buf)
    if err != nil {
        panic(err)
    }
    reader := bytes.NewReader(buf.Bytes())
    credentials := credentials.NewStaticCredentials(aws_access_key_id,
        aws_secret_access_key, "")
    signer := v4.NewSigner(credentials)
    signer.Sign(request, reader, "s3", "us-east-1", time.Now())
    client := &http.Client{}
    resp, err := client.Do(request)
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }
    fmt.Println(string(body))
 }
--- a/examples/node-md-search.js
+++ b/examples/node-md-search.js
@ -1,28 +0,0 @@
 const { S3 } = require('aws-sdk');
 const config = {
    sslEnabled: false,
    endpoint: 'http://127.0.0.1:8000',
    signatureCache: false,
    signatureVersion: 'v4',
    region: 'us-east-1',
    s3ForcePathStyle: true,
    accessKeyId: 'accessKey1',
    secretAccessKey: 'verySecretKey1',
 };
 const s3Client = new S3(config);
 const encodedSearch =
    encodeURIComponent('x-amz-meta-color="blue"');
 const req = s3Client.listObjects({ Bucket: 'bucketname' });
 // the build event
 req.on('build', () => {
    req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
 });
 req.on('success', res => {
    process.stdout.write(`Result ${res.data}`);
 });
 req.on('error', err => {
    process.stdout.write(`Error ${err}`);
 });
 req.send();
--- a/examples/python-md-search.py
+++ b/examples/python-md-search.py
@ -1,79 +0,0 @@
 import datetime
 import hashlib
 import hmac
 import urllib
 # pip install requests
 import requests
 access_key = 'accessKey1'
 secret_key = 'verySecretKey1'
 method = 'GET'
 service = 's3'
 host = 'localhost:8000'
 region = 'us-east-1'
 canonical_uri = '/bucketname'
 query = 'x-amz-meta-color=blue'
 canonical_querystring = 'search=%s' % (urllib.quote(query))
 algorithm = 'AWS4-HMAC-SHA256'
 t = datetime.datetime.utcnow()
 amz_date = t.strftime('%Y%m%dT%H%M%SZ')
 date_stamp = t.strftime('%Y%m%d')
 # Key derivation functions. See:
 # http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
 def sign(key, msg):
    return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
 def getSignatureKey(key, date_stamp, regionName, serviceName):
    kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
    kRegion = sign(kDate, regionName)
    kService = sign(kRegion, serviceName)
    kSigning = sign(kService, 'aws4_request')
    return kSigning
 payload_hash = hashlib.sha256('').hexdigest()
 canonical_headers = \
    'host:{0}\nx-amz-content-sha256:{1}\nx-amz-date:{2}\n' \
    .format(host, payload_hash, amz_date)
 signed_headers = 'host;x-amz-content-sha256;x-amz-date'
 canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
    .format(method, canonical_uri, canonical_querystring, canonical_headers,
            signed_headers, payload_hash)
 print(canonical_request)
 credential_scope = '{0}/{1}/{2}/aws4_request' \
    .format(date_stamp, region, service)
 string_to_sign = '{0}\n{1}\n{2}\n{3}' \
    .format(algorithm, amz_date, credential_scope,
            hashlib.sha256(canonical_request).hexdigest())
 signing_key = getSignatureKey(secret_key, date_stamp, region, service)
 signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
                     hashlib.sha256).hexdigest()
 authorization_header = \
    '{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}' \
    .format(algorithm, access_key, credential_scope, signed_headers, signature)
 # The 'host' header is added automatically by the Python 'requests' library.
 headers = {
    'X-Amz-Content-Sha256': payload_hash,
    'X-Amz-Date': amz_date,
    'Authorization': authorization_header
 }
 endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
 r = requests.get(endpoint, headers=headers)
 print(r.text)
--- a/images/svc-base/Dockerfile
+++ b/images/svc-base/Dockerfile
@ -1,28 +0,0 @@
 FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
 ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
 ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
 COPY . ${HOME_DIR}/s3
 RUN chown -R ${USER} ${HOME_DIR}
 RUN pip3 install redis===3.5.3 requests==2.27.1 && \
    apt-get install -y git-lfs
 USER ${USER}
 WORKDIR ${HOME_DIR}/s3
 RUN rm -f ~/.gitconfig && \
    git config --global --add safe.directory . && \
    git lfs install && \
    GIT_LFS_SKIP_SMUDGE=1 && \
    yarn global add typescript && \
    yarn install --frozen-lockfile --production --network-concurrency 1 && \
    yarn cache clean --all && \
    yarn global remove typescript
 # run symlinking separately to avoid yarn installation errors
 # we might have to check if the symlinking is really needed!
 RUN ln -sf /scality-kms node_modules
 EXPOSE 8000
 CMD bash -c "source ${CONF_DIR}/env && export && supervisord -c ${CONF_DIR}/supervisord.conf"
--- a/index.js
+++ b/index.js
@ -1,10 +1,3 @@
 'use strict'; // eslint-disable-line strict
 require('werelogs').stderrUtils.catchAndTimestampStderr(
    undefined,
    // Do not exit as workers have their own listener that will exit
    // But primary don't have another listener
    require('cluster').isPrimary ? 1 : null,
 );
 require('./lib/server.js')();
--- a/lib/Config.js
+++ b/lib/Config.js
--- a/lib/RedisClient.js
+++ b/lib/RedisClient.js
@ -0,0 +1,54 @@
 const Redis = require('ioredis');
 const logger = require('./utilities/logger');
 module.exports = class RedisClient {
    /**
    * @constructor
    * @param {string} host - Redis host
    * @param {number} port - Redis port
    */
    constructor(host, port) {
        this._client = new Redis({
            host,
            port,
        });
        this._client.on('error', err =>
            logger.trace('error from redis', {
                error: err,
                method: 'RedisClient.constructor',
                redisHost: host,
                redisPort: port,
            })
        );
        return this;
    }
    /**
    * increment value of a key by 1 and set a ttl
    * @param {string} key - key holding the value
    * @param {number} expiry - expiry in seconds
    * @param {callback} cb - callback
    * @return {undefined}
    */
    incrEx(key, expiry, cb) {
        return this._client
            .multi([['incr', key], ['expire', key, expiry]])
            .exec(cb);
    }
    /**
    * execute a batch of commands
    * @param {string[]} cmds - list of commands
    * @param {callback} cb - callback
    * @return {undefined}
    */
    batch(cmds, cb) {
        return this._client.pipeline(cmds).exec(cb);
    }
    clear(cb) {
        return this._client.flushDb(cb);
    }
 };
--- a/lib/StatsClient.js
+++ b/lib/StatsClient.js
@ -0,0 +1,156 @@
 const async = require('async');
 class StatsClient {
    /**
    * @constructor
    * @param {object} redisClient - RedisClient instance
    * @param {number} interval - sampling interval
    * @param {number} expiry - sampling duration
    */
    constructor(redisClient, interval, expiry) {
        this._redis = redisClient;
        this._interval = interval;
        this._expiry = expiry;
        return this;
    }
    /*
    * Utility function to use when callback is undefined
    */
    _noop() {}
    /**
    * normalize to the nearest interval
    * @param {object} d - Date instance
    * @return {number} timestamp - normalized to the nearest interval
    */
    _normalizeTimestamp(d) {
        const s = d.getSeconds();
        return d.setSeconds(s - s % this._interval, 0);
    }
    /**
    * set timestamp to the previous interval
    * @param {object} d - Date instance
    * @return {number} timestamp - set to the previous interval
    */
    _setPrevInterval(d) {
        return d.setSeconds(d.getSeconds() - this._interval);
    }
    /**
    * build redis key to get total number of requests received
    * @param {object} d - Date instance
    * @return {string} key
    */
    _buildRequestsKey(d) {
        return `s3:requests:${this._normalizeTimestamp(d)}`;
    }
    /**
    * build redis key to get total number of 500s occurred on the server
    * @param {object} d - Date instance
    * @return {string} key - key for redis
    */
    _build500sKey(d) {
        return `s3:500s:${this._normalizeTimestamp(d)}`;
    }
    /**
    * reduce the array of values to a single value
    * typical input looks like [[null, '1'], [null, '2'], [null, null]...]
    * @param {array} arr - Date instance
    * @return {string} key - key for redis
    */
    _getCount(arr) {
        return arr.reduce((prev, a) => {
            let num = parseInt(a[1], 10);
            num = Number.isNaN(num) ? 0 : num;
            return prev + num;
        }, 0);
    }
    /**
    * report/record a new request received on the server
    * @param {callback} cb - callback
    * @return {undefined}
    */
    reportNewRequest(cb) {
        if (!this._redis) {
            return undefined;
        }
        const callback = cb || this._noop;
        const key = this._buildRequestsKey(new Date());
        return this._redis.incrEx(key, this._expiry, callback);
    }
    /**
    * report/record a request that ended up being a 500 on the server
    * @param {callback} cb - callback
    * @return {undefined}
    */
    report500(cb) {
        if (!this._redis) {
            return undefined;
        }
        const callback = cb || this._noop;
        const key = this._build500sKey(new Date());
        return this._redis.incrEx(key, this._expiry, callback);
    }
    /**
    * get stats for the last x seconds, x being the sampling duration
    * @param {object} log - Werelogs request logger
    * @param {callback} cb - callback to call with the err/result
    * @return {undefined}
    */
    getStats(log, cb) {
        if (!this._redis) {
            return cb(null, {});
        }
        const d = new Date();
        const totalKeys = Math.floor(this._expiry / this._interval);
        const reqsKeys = [];
        const req500sKeys = [];
        for (let i = 0; i < totalKeys; i++) {
            reqsKeys.push(['get', this._buildRequestsKey(d)]);
            req500sKeys.push(['get', this._build500sKey(d)]);
            this._setPrevInterval(d);
        }
        return async.parallel([
            next => this._redis.batch(reqsKeys, next),
            next => this._redis.batch(req500sKeys, next),
        ], (err, results) => {
            /**
            * Batch result is of the format
            * [ [null, '1'], [null, '2'], [null, '3'] ] where each
            * item is the result of the each batch command
            * Foreach item in the result, index 0 signifies the error and
            * index 1 contains the result
            */
            const statsRes = {
                'requests': 0,
                '500s': 0,
                'sampleDuration': this._expiry,
            };
            if (err) {
                log.error('error getting stats', {
                    error: err,
                    method: 'StatsClient.getStats',
                });
                /**
                * Redis for stats is not a critial component, ignoring
                * any error here as returning an InternalError
                * would be confused with the health of the service
                */
                return cb(null, statsRes);
            }
            statsRes.requests = this._getCount(results[0]);
            statsRes['500s'] = this._getCount(results[1]);
            return cb(null, statsRes);
        });
    }
 }
 module.exports = StatsClient;
--- a/lib/api/api.js
+++ b/lib/api/api.js
@ -1,129 +1,58 @@
-const { auth, errors, policies } = require('arsenal');
+const { auth, errors } = require('arsenal');
 const async = require('async');
 const bucketDelete = require('./bucketDelete');
 const bucketDeleteCors = require('./bucketDeleteCors');
 const bucketDeleteEncryption = require('./bucketDeleteEncryption');
 const bucketDeleteWebsite = require('./bucketDeleteWebsite');
-const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
+const bucketGet = require('./bucketGet');
 const bucketDeletePolicy = require('./bucketDeletePolicy');
 const bucketDeleteQuota = require('./bucketDeleteQuota');
 const { bucketGet } = require('./bucketGet');
 const bucketGetACL = require('./bucketGetACL');
 const bucketGetCors = require('./bucketGetCors');
 const bucketGetVersioning = require('./bucketGetVersioning');
 const bucketGetWebsite = require('./bucketGetWebsite');
 const bucketGetLocation = require('./bucketGetLocation');
 const bucketGetLifecycle = require('./bucketGetLifecycle');
 const bucketGetNotification = require('./bucketGetNotification');
 const bucketGetObjectLock = require('./bucketGetObjectLock');
 const bucketGetPolicy = require('./bucketGetPolicy');
 const bucketGetQuota = require('./bucketGetQuota');
 const bucketGetEncryption = require('./bucketGetEncryption');
 const bucketHead = require('./bucketHead');
 const { bucketPut } = require('./bucketPut');
 const bucketPutACL = require('./bucketPutACL');
 const bucketPutCors = require('./bucketPutCors');
 const bucketPutVersioning = require('./bucketPutVersioning');
 const bucketPutTagging = require('./bucketPutTagging');
 const bucketDeleteTagging = require('./bucketDeleteTagging');
 const bucketGetTagging = require('./bucketGetTagging');
 const bucketPutWebsite = require('./bucketPutWebsite');
 const bucketPutReplication = require('./bucketPutReplication');
 const bucketPutLifecycle = require('./bucketPutLifecycle');
 const bucketPutNotification = require('./bucketPutNotification');
 const bucketPutEncryption = require('./bucketPutEncryption');
 const bucketPutPolicy = require('./bucketPutPolicy');
 const bucketPutObjectLock = require('./bucketPutObjectLock');
 const bucketUpdateQuota = require('./bucketUpdateQuota');
 const bucketGetReplication = require('./bucketGetReplication');
 const bucketDeleteReplication = require('./bucketDeleteReplication');
 const corsPreflight = require('./corsPreflight');
 const completeMultipartUpload = require('./completeMultipartUpload');
 const initiateMultipartUpload = require('./initiateMultipartUpload');
 const listMultipartUploads = require('./listMultipartUploads');
 const listParts = require('./listParts');
 const metadataSearch = require('./metadataSearch');
 const { multiObjectDelete } = require('./multiObjectDelete');
 const multipartDelete = require('./multipartDelete');
 const objectCopy = require('./objectCopy');
-const { objectDelete } = require('./objectDelete');
+const objectDelete = require('./objectDelete');
 const objectDeleteTagging = require('./objectDeleteTagging');
 const objectGet = require('./objectGet');
 const objectGetACL = require('./objectGetACL');
 const objectGetLegalHold = require('./objectGetLegalHold');
 const objectGetRetention = require('./objectGetRetention');
 const objectGetTagging = require('./objectGetTagging');
 const objectHead = require('./objectHead');
 const objectPut = require('./objectPut');
 const objectPutACL = require('./objectPutACL');
 const objectPutLegalHold = require('./objectPutLegalHold');
 const objectPutTagging = require('./objectPutTagging');
 const objectPutPart = require('./objectPutPart');
 const objectPutCopyPart = require('./objectPutCopyPart');
 const objectPutRetention = require('./objectPutRetention');
 const objectRestore = require('./objectRestore');
 const prepareRequestContexts
    = require('./apiUtils/authorization/prepareRequestContexts');
 const serviceGet = require('./serviceGet');
 const vault = require('../auth/vault');
-const website = require('./website');
+const websiteGet = require('./websiteGet');
 const websiteHead = require('./websiteHead');
 const writeContinue = require('../utilities/writeContinue');
 const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
 const parseCopySource = require('./apiUtils/object/parseCopySource');
 const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
 const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
 const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
 const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
 auth.setHandler(vault);
 /* eslint-disable no-param-reassign */
 const api = {
    callApiMethod(apiMethod, request, response, log, callback) {
        // Attach the apiMethod method to the request, so it can used by monitoring in the server
        // eslint-disable-next-line no-param-reassign
        request.apiMethod = apiMethod;
        // Array of end of API callbacks, used to perform some logic
        // at the end of an API.
        // eslint-disable-next-line no-param-reassign
        request.finalizerHooks = [];
        const actionLog = monitoringMap[apiMethod];
        if (!actionLog &&
            apiMethod !== 'websiteGet' &&
            apiMethod !== 'websiteHead' &&
            apiMethod !== 'corsPreflight') {
            log.error('callApiMethod(): No actionLog for this api method', {
                apiMethod,
            });
        }
        log.addDefaultFields({
            service: 's3',
            action: actionLog,
            bucketName: request.bucketName,
        });
        if (request.objectKey) {
            log.addDefaultFields({
                objectKey: request.objectKey,
            });
        }
        let returnTagCount = true;
        const validationRes = validateQueryAndHeaders(request, log);
        if (validationRes.error) {
            log.debug('request query / header validation failed', {
                error: validationRes.error,
                method: 'api.callApiMethod',
            });
            return process.nextTick(callback, validationRes.error);
        }
        // no need to check auth on website or cors preflight requests
        if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
        apiMethod === 'corsPreflight') {
            request.actionImplicitDenies = false;
            return this[apiMethod](request, log, callback);
        }
@ -133,245 +62,126 @@ const api = {
            log.debug('error parsing copy source', {
                error: parsingError,
            });
-            return process.nextTick(callback, parsingError);
+            return callback(parsingError);
        }
        const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers);
        if (httpHeadersSizeError) {
            log.debug('http header size limit exceeded', {
                error: httpHeadersSizeError,
            });
            return process.nextTick(callback, httpHeadersSizeError);
        }
        const requestContexts = prepareRequestContexts(apiMethod, request,
            sourceBucket, sourceObject, sourceVersionId);
-        // Extract all the _apiMethods and store them in an array
+        return auth.server.doAuth(request, log, (err, userInfo,
-        const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : [];
+            authorizationResults, streamingV4Params) => {
        // Attach the names to the current request
        // eslint-disable-next-line no-param-reassign
        request.apiMethods = apiMethods;
        function checkAuthResults(authResults) {
            let returnTagCount = true;
            const isImplicitDeny = {};
            let isOnlyImplicitDeny = true;
            if (apiMethod === 'objectGet') {
                // first item checks s3:GetObject(Version) action
                if (!authResults[0].isAllowed && !authResults[0].isImplicit) {
                    log.trace('get object authorization denial from Vault');
                    return errors.AccessDenied;
                }
                // TODO add support for returnTagCount in the bucket policy
                // checks
                isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
                // second item checks s3:GetObject(Version)Tagging action
                if (!authResults[1].isAllowed) {
                    log.trace('get tagging authorization denial ' +
                    'from Vault');
                    returnTagCount = false;
                }
            } else {
                for (let i = 0; i < authResults.length; i++) {
                    isImplicitDeny[authResults[i].action] = true;
                    if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
                        // Any explicit deny rejects the current API call
                        log.trace('authorization denial from Vault');
                        return errors.AccessDenied;
                    }
                    if (authResults[i].isAllowed) {
                        // If the action is allowed, the result is not implicit
                        // Deny.
                        isImplicitDeny[authResults[i].action] = false;
                        isOnlyImplicitDeny = false;
                    }
                }
            }
            // These two APIs cannot use ACLs or Bucket Policies, hence, any
            // implicit deny from vault must be treated as an explicit deny.
            if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
                return errors.AccessDenied;
            }
            return { returnTagCount, isImplicitDeny };
        }
        return async.waterfall([
            next => auth.server.doAuth(
                request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
                    if (err) {
                        // VaultClient returns standard errors, but the route requires
                        // Arsenal errors
                        const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
                        log.trace('authentication error', { error: err });
                        return next(arsenalError);
                    }
                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                }, 's3', requestContexts),
            (userInfo, authorizationResults, streamingV4Params, infos, next) => {
                const authNames = { accountName: userInfo.getAccountDisplayName() };
                if (userInfo.isRequesterAnIAMUser()) {
                    authNames.userName = userInfo.getIAMdisplayName();
                }
                if (isRequesterASessionUser(userInfo)) {
                    authNames.sessionName = userInfo.getShortid().split(':')[1];
                }
                log.addDefaultFields(authNames);
                if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                }
                // issue 100 Continue to the client
                writeContinue(request, response);
                const MAX_POST_LENGTH = request.method === 'POST' ?
                      1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
                const post = [];
                let postLength = 0;
                request.on('data', chunk => {
                    postLength += chunk.length;
                    // Sanity check on post length
                    if (postLength <= MAX_POST_LENGTH) {
                        post.push(chunk);
                    }
                });
                request.on('error', err => {
                    log.trace('error receiving request', {
                        error: err,
                    });
                    return next(errors.InternalError);
                });
                request.on('end', () => {
                    if (postLength > MAX_POST_LENGTH) {
                        log.error('body length is too long for request type',
                                  { postLength });
                        return next(errors.InvalidRequest);
                    }
                    // Convert array of post buffers into one string
                    request.post = Buffer.concat(post, postLength).toString();
                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                });
                return undefined;
            },
            // Tag condition keys require information from CloudServer for evaluation
            (userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
                authorizationResults,
                request,
                requestContexts,
                apiMethod,
                log,
                (err, authResultsWithTags) => {
                    if (err) {
                        log.trace('tag authentication error', { error: err });
                        return next(err);
                    }
                    return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
                },
            ),
        ], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
            if (err) {
                log.trace('authentication error', { error: err });
                return callback(err);
            }
            request.accountQuotas = infos?.accountQuota;
            if (authorizationResults) {
-                const checkedResults = checkAuthResults(authorizationResults);
+                if (apiMethod === 'objectGet') {
-                if (checkedResults instanceof Error) {
+                    // first item checks s3:GetObject(Version) action
-                    return callback(checkedResults);
+                    if (!authorizationResults[0].isAllowed) {
                        log.trace('get object authorization denial from Vault');
                        return callback(errors.AccessDenied);
                    }
                    // second item checks s3:GetObject(Version)Tagging action
                    if (!authorizationResults[1].isAllowed) {
                        log.trace('get tagging authorization denial ' +
                        'from Vault');
                        returnTagCount = false;
                    }
                } else {
                    for (let i = 0; i < authorizationResults.length; i++) {
                        if (!authorizationResults[i].isAllowed) {
                            log.trace('authorization denial from Vault');
                            return callback(errors.AccessDenied);
                        }
                    }
                }
                returnTagCount = checkedResults.returnTagCount;
                request.actionImplicitDenies = checkedResults.isImplicitDeny;
            } else {
                // create an object of keys apiMethods with all values to false:
                // for backward compatibility, all apiMethods are allowed by default
                // thus it is explicitly allowed, so implicit deny is false
                request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
                    acc[curr] = false;
                    return acc;
                }, {});
            }
-            const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
+            // issue 100 Continue to the client
-                    (hook, done) => hook(err, done),
+            writeContinue(request, response);
                    () => callback(err, ...results));
            if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
                request._response = response;
                return this[apiMethod](userInfo, request, streamingV4Params,
-                    log, methodCallback, authorizationResults);
+                    log, callback);
            }
-            if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
+            const MAX_POST_LENGTH = request.method.toUpperCase() === 'POST' ?
-                return this[apiMethod](userInfo, request, sourceBucket,
+                1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
-                    sourceObject, sourceVersionId, log, methodCallback);
+            const post = [];
-            }
+            let postLength = 0;
-            if (apiMethod === 'objectGet') {
+            request.on('data', chunk => {
-                return this[apiMethod](userInfo, request, returnTagCount, log, callback);
+                postLength += chunk.length;
-            }
+                // Sanity check on post length
-            return this[apiMethod](userInfo, request, log, methodCallback);
+                if (postLength <= MAX_POST_LENGTH) {
-        });
+                    post.push(chunk);
                }
                return undefined;
            });
            request.on('error', err => {
                log.trace('error receiving request', {
                    error: err,
                });
                return callback(errors.InternalError);
            });
            request.on('end', () => {
                if (postLength > MAX_POST_LENGTH) {
                    log.error('body length is too long for request type',
                        { postLength });
                    return callback(errors.InvalidRequest);
                }
                // Convert array of post buffers into one string
                request.post = Buffer.concat(post, postLength).toString();
                if (apiMethod === 'objectCopy' ||
                    apiMethod === 'objectPutCopyPart') {
                    return this[apiMethod](userInfo, request, sourceBucket,
                        sourceObject, sourceVersionId, log, callback);
                }
                if (apiMethod === 'objectGet') {
                    return this[apiMethod](userInfo, request,
                      returnTagCount, log, callback);
                }
                return this[apiMethod](userInfo, request, log, callback);
            });
            return undefined;
        }, 's3', requestContexts);
    },
    bucketDelete,
    bucketDeleteCors,
    bucketDeleteEncryption,
    bucketDeleteWebsite,
    bucketGet,
    bucketGetACL,
    bucketGetCors,
    bucketGetObjectLock,
    bucketGetVersioning,
    bucketGetWebsite,
    bucketGetLocation,
    bucketGetEncryption,
    bucketHead,
    bucketPut,
    bucketPutACL,
    bucketPutCors,
    bucketPutVersioning,
    bucketPutTagging,
    bucketDeleteTagging,
    bucketGetTagging,
    bucketPutWebsite,
    bucketPutReplication,
    bucketGetReplication,
    bucketDeleteReplication,
    bucketDeleteQuota,
    bucketPutLifecycle,
    bucketUpdateQuota,
    bucketGetLifecycle,
    bucketDeleteLifecycle,
    bucketPutPolicy,
    bucketGetPolicy,
    bucketGetQuota,
    bucketDeletePolicy,
    bucketPutObjectLock,
    bucketPutNotification,
    bucketGetNotification,
    bucketPutEncryption,
    corsPreflight,
    completeMultipartUpload,
    initiateMultipartUpload,
    listMultipartUploads,
    listParts,
    metadataSearch,
    multiObjectDelete,
    multipartDelete,
    objectDelete,
    objectDeleteTagging,
    objectGet,
    objectGetACL,
    objectGetLegalHold,
    objectGetRetention,
    objectGetTagging,
    objectCopy,
    objectHead,
    objectPut,
    objectPutACL,
    objectPutLegalHold,
    objectPutTagging,
    objectPutPart,
    objectPutCopyPart,
    objectPutRetention,
    objectRestore,
    serviceGet,
-    websiteGet: website,
+    websiteGet,
-    websiteHead: website,
+    websiteHead,
 };
 module.exports = api;
--- a/lib/api/apiUtils/authorization/aclChecks.js
+++ b/lib/api/apiUtils/authorization/aclChecks.js
@ -0,0 +1,124 @@
 const constants = require('../../../../constants');
 function isBucketAuthorized(bucket, requestType, canonicalID) {
    // Check to see if user is authorized to perform a
    // particular action on bucket based on ACLs.
    // TODO: Add IAM checks and bucket policy checks.
    if (bucket.getOwner() === canonicalID) {
        return true;
    } else if (requestType === 'bucketOwnerAction') {
        // only bucket owner can modify or retrieve this property of a bucket
        return false;
    }
    const bucketAcl = bucket.getAcl();
    if (requestType === 'bucketGet' || requestType === 'bucketHead') {
        if (bucketAcl.Canned === 'public-read'
            || bucketAcl.Canned === 'public-read-write'
            || (bucketAcl.Canned === 'authenticated-read'
                && canonicalID !== constants.publicId)) {
            return true;
        } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.READ.indexOf(canonicalID) > -1) {
            return true;
        }
    }
    if (requestType === 'bucketGetACL') {
        if ((bucketAcl.Canned === 'log-delivery-write'
                && canonicalID === constants.logId)
            || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
            return true;
        }
    }
    if (requestType === 'bucketPutACL') {
        if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
            return true;
        }
    }
    if (requestType === 'bucketDelete' && bucket.getOwner() === canonicalID) {
        return true;
    }
    if (requestType === 'objectDelete' || requestType === 'objectPut') {
        if (bucketAcl.Canned === 'public-read-write'
            || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.WRITE.indexOf(canonicalID) > -1) {
            return true;
        }
    }
    // Note that an account can have the ability to do objectPutACL,
    // objectGetACL, objectHead or objectGet even if the account has no rights
    // to the bucket holding the object.  So, if the request type is
    // objectPutACL, objectGetACL, objectHead or objectGet, the bucket
    // authorization check should just return true so can move on to check
    // rights at the object level.
    return (requestType === 'objectPutACL' || requestType === 'objectGetACL' ||
    requestType === 'objectGet' || requestType === 'objectHead');
 }
 function isObjAuthorized(bucket, objectMD, requestType, canonicalID) {
    const bucketOwner = bucket.getOwner();
    if (!objectMD) {
        return false;
    }
    if (objectMD['owner-id'] === canonicalID) {
        return true;
    }
    // account is authorized if:
    // - requesttype is "bucketOwnerAction" (example: for objectTagging) and
    // - account is the bucket owner
    if (requestType === 'bucketOwnerAction' && bucketOwner === canonicalID) {
        return true;
    }
    if (requestType === 'objectGet' || requestType === 'objectHead') {
        if (objectMD.acl.Canned === 'public-read'
            || objectMD.acl.Canned === 'public-read-write'
            || (objectMD.acl.Canned === 'authenticated-read'
                && canonicalID !== constants.publicId)) {
            return true;
        } else if (objectMD.acl.Canned === 'bucket-owner-read'
                && bucketOwner === canonicalID) {
            return true;
        } else if ((objectMD.acl.Canned === 'bucket-owner-full-control'
                && bucketOwner === canonicalID)
            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
            || objectMD.acl.READ.indexOf(canonicalID) > -1) {
            return true;
        }
    }
    // User is already authorized on the bucket for FULL_CONTROL or WRITE or
    // bucket has canned ACL public-read-write
    if (requestType === 'objectPut' || requestType === 'objectDelete') {
        return true;
    }
    if (requestType === 'objectPutACL') {
        if ((objectMD.acl.Canned === 'bucket-owner-full-control'
                && bucketOwner === canonicalID)
            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
            || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
            return true;
        }
    }
    if (requestType === 'objectGetACL') {
        if ((objectMD.acl.Canned === 'bucket-owner-full-control'
                && bucketOwner === canonicalID)
            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
            || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
            return true;
        }
    }
    return false;
 }
 module.exports = {
    isBucketAuthorized,
    isObjAuthorized,
 };
--- a/lib/api/apiUtils/authorization/bucketOwner.js
+++ b/lib/api/apiUtils/authorization/bucketOwner.js
@ -1,29 +0,0 @@
 const { errors } = require('arsenal');
 const vault = require('../../../auth/vault');
 function checkExpectedBucketOwner(headers, bucket, log, cb) {
    const expectedOwner = headers['x-amz-expected-bucket-owner'];
    if (expectedOwner === undefined) {
        return cb();
    }
    const bucketOwner = bucket.getOwner();
    return vault.getAccountIds([bucketOwner], log, (error, res) => {
        if (error) {
            log.error('error fetch accountId from vault', {
                method: 'checkExpectedBucketOwner',
                error,
            });
        }
        if (error || res[bucketOwner] !== expectedOwner) {
            return cb(errors.AccessDenied);
        }
        return cb();
    });
 }
 module.exports = {
    checkExpectedBucketOwner,
 };
--- a/lib/api/apiUtils/authorization/permissionChecks.js
+++ b/lib/api/apiUtils/authorization/permissionChecks.js
@ -1,641 +0,0 @@
 const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
 const { errors } = require('arsenal');
 const { parseCIDR, isValid } = require('ipaddr.js');
 const constants = require('../../../../constants');
 const { config } = require('../../../Config');
 const {
    allAuthedUsersId,
    bucketOwnerActions,
    logId,
    publicId,
    arrayOfAllowed,
    assumedRoleArnResourceType,
    backbeatLifecycleSessionName,
    actionsToConsiderAsObjectPut,
 } = constants;
 // whitelist buckets to allow public read on objects
 const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
    ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
 function getServiceAccountProperties(canonicalID) {
    const canonicalIDArray = canonicalID.split('/');
    const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
    return constants.serviceAccountProperties[serviceName];
 }
 function isServiceAccount(canonicalID) {
    return getServiceAccountProperties(canonicalID) !== undefined;
 }
 function isRequesterASessionUser(authInfo) {
    const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
    return regexpAssumedRoleArn.test(authInfo.getArn());
 }
 function isRequesterNonAccountUser(authInfo) {
    return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
 }
 /**
 * Checks the access control for a given bucket based on the request type and user's canonical ID.
 *
 * @param {Bucket} bucket - The bucket to check access control for.
 * @param {string} requestType - The list of s3 actions to check within the API call.
 * @param {string} canonicalID - The canonical ID of the user making the request.
 * @param {string} mainApiCall - The main API call (first item of the requestType).
 *
 * @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
 */
 function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
    // Same logic applies on the Versioned APIs, so let's simplify it.
    let requestTypeParsed = requestType.endsWith('Version') ?
        requestType.slice(0, 'Version'.length * -1) : requestType;
    requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
        'objectPut' : requestTypeParsed;
    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
        'objectPut' : mainApiCall;
    if (bucket.getOwner() === canonicalID) {
        return true;
    }
    if (parsedMainApiCall === 'objectGet') {
        if (requestTypeParsed === 'objectGetTagging') {
            return true;
        }
    }
    if (parsedMainApiCall === 'objectPut') {
        if (arrayOfAllowed.includes(requestTypeParsed)) {
            return true;
        }
    }
    const bucketAcl = bucket.getAcl();
    if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') {
        if (bucketAcl.Canned === 'public-read'
            || bucketAcl.Canned === 'public-read-write'
            || (bucketAcl.Canned === 'authenticated-read'
                && canonicalID !== publicId)) {
            return true;
        } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.READ.indexOf(canonicalID) > -1) {
            return true;
        } else if (bucketAcl.READ.indexOf(publicId) > -1
            || (bucketAcl.READ.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
            return true;
        }
    }
    if (requestTypeParsed === 'bucketGetACL') {
        if ((bucketAcl.Canned === 'log-delivery-write'
            && canonicalID === logId)
            || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
            return true;
        } else if (bucketAcl.READ_ACP.indexOf(publicId) > -1
            || (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
            return true;
        }
    }
    if (requestTypeParsed === 'bucketPutACL') {
        if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
            return true;
        } else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1
            || (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
            return true;
        }
    }
    if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') {
        if (bucketAcl.Canned === 'public-read-write'
            || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
            || bucketAcl.WRITE.indexOf(canonicalID) > -1) {
            return true;
        } else if (bucketAcl.WRITE.indexOf(publicId) > -1
            || (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
            return true;
        }
    }
    // Note that an account can have the ability to do objectPutACL,
    // objectGetACL, objectHead or objectGet even if the account has no rights
    // to the bucket holding the object.  So, if the request type is
    // objectPutACL, objectGetACL, objectHead or objectGet, the bucket
    // authorization check should just return true so can move on to check
    // rights at the object level.
    return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
    || requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
 }
 function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
    isUserUnauthenticated, mainApiCall) {
    const bucketOwner = bucket.getOwner();
    const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
        'objectPut' : requestType;
    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
        'objectPut' : mainApiCall;
    // acls don't distinguish between users and accounts, so both should be allowed
    if (bucketOwnerActions.includes(requestTypeParsed)
        && (bucketOwner === canonicalID)) {
        return true;
    }
    if (objectMD['owner-id'] === canonicalID) {
        return true;
    }
    // Backward compatibility
    if (parsedMainApiCall === 'objectGet') {
        if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
            && requestTypeParsed === 'objectGetTagging') {
            return true;
        }
    }
    if (!objectMD.acl) {
        return false;
    }
    if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') {
        if (objectMD.acl.Canned === 'public-read'
            || objectMD.acl.Canned === 'public-read-write'
            || (objectMD.acl.Canned === 'authenticated-read'
                && canonicalID !== publicId)) {
            return true;
        } else if (objectMD.acl.Canned === 'bucket-owner-read'
            && bucketOwner === canonicalID) {
            return true;
        } else if ((objectMD.acl.Canned === 'bucket-owner-full-control'
            && bucketOwner === canonicalID)
            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
            || objectMD.acl.READ.indexOf(canonicalID) > -1) {
            return true;
        } else if (objectMD.acl.READ.indexOf(publicId) > -1
            || (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
            return true;
        }
    }
    // User is already authorized on the bucket for FULL_CONTROL or WRITE or
    // bucket has canned ACL public-read-write
    if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') {
        return true;
    }
    if (requestTypeParsed === 'objectPutACL') {
        if ((objectMD.acl.Canned === 'bucket-owner-full-control'
            && bucketOwner === canonicalID)
            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
            || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
            return true;
        } else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1
            || (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
            return true;
        }
    }
    if (requestTypeParsed === 'objectGetACL') {
        if ((objectMD.acl.Canned === 'bucket-owner-full-control'
            && bucketOwner === canonicalID)
            || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
            || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
            return true;
        } else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1
            || (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
                && canonicalID !== publicId)
            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
            return true;
        }
    }
    // allow public reads on buckets that are whitelisted for anonymous reads
    // TODO: remove this after bucket policies are implemented
    const bucketAcl = bucket.getAcl();
    const allowPublicReads = publicReadBuckets.includes(bucket.getName())
        && bucketAcl.Canned === 'public-read'
        && (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
    if (allowPublicReads) {
        return true;
    }
    return false;
 }
 function _checkBucketPolicyActions(requestType, actions, log) {
    const mappedAction = actionMaps.actionMapBP[requestType];
    // Deny any action that isn't in list of controlled actions
    if (!mappedAction) {
        return false;
    }
    return evaluators.isActionApplicable(mappedAction, actions, log);
 }
 function _checkBucketPolicyResources(request, resource, log) {
    if (!request || (Array.isArray(resource) && resource.length === 0)) {
        return true;
    }
    // build request context from the request!
    const requestContext = new RequestContext(request.headers, request.query,
        request.bucketName, request.objectKey, null,
        request.connection.encrypted, request.resourceType, 's3');
    return evaluators.isResourceApplicable(requestContext, resource, log);
 }
 function _checkBucketPolicyConditions(request, conditions, log) {
    const ip = request ? requestUtils.getClientIp(request, config) : undefined;
    if (!conditions) {
        return true;
    }
    // build request context from the request!
    const requestContext = new RequestContext(request.headers, request.query,
        request.bucketName, request.objectKey, ip,
        request.connection.encrypted, request.resourceType, 's3', null, null,
        null, null, null, null, null, null, null, null, null,
        request.objectLockRetentionDays);
    return evaluators.meetConditions(requestContext, conditions, log);
 }
 function _getAccountId(arn) {
    // account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
    return arn.substr(13, 12);
 }
 function _isAccountId(principal) {
    return (principal.length === 12 && /^\d+$/.test(principal));
 }
 function _checkPrincipal(requester, principal) {
    if (principal === '*') {
        return true;
    }
    if (principal === requester) {
        return true;
    }
    if (_isAccountId(principal)) {
        return _getAccountId(requester) === principal;
    }
    if (principal.endsWith('root')) {
        return _getAccountId(requester) === _getAccountId(principal);
    }
    return false;
 }
 function _checkPrincipals(canonicalID, arn, principal) {
    if (principal === '*') {
        return true;
    }
    if (principal.CanonicalUser) {
        if (Array.isArray(principal.CanonicalUser)) {
            return principal.CanonicalUser.some(p => _checkPrincipal(canonicalID, p));
        }
        return _checkPrincipal(canonicalID, principal.CanonicalUser);
    }
    if (principal.AWS) {
        if (Array.isArray(principal.AWS)) {
            return principal.AWS.some(p => _checkPrincipal(arn, p));
        }
        return _checkPrincipal(arn, principal.AWS);
    }
    return false;
 }
 function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) {
    let permission = 'defaultDeny';
    // if requester is user within bucket owner account, actions should be
    // allowed unless explicitly denied (assumes allowed by IAM policy)
    if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) {
        permission = 'allow';
    }
    let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
    while (copiedStatement.length > 0) {
        const s = copiedStatement[0];
        const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
        const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
        const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
        const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') {
            // explicit deny trumps any allows, so return immediately
            return 'explicitDeny';
        }
        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') {
            permission = 'allow';
        }
        copiedStatement = copiedStatement.splice(1);
    }
    return permission;
 }
 function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log,
    request, aclPermission, results, actionImplicitDenies) {
    const bucketPolicy = bucket.getBucketPolicy();
    let processedResult = results[requestType];
    if (!bucketPolicy) {
        processedResult = actionImplicitDenies[requestType] === false && aclPermission;
    } else {
        const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
            bucketOwner, log, request, actionImplicitDenies);
        if (bucketPolicyPermission === 'explicitDeny') {
            processedResult = false;
        } else if (bucketPolicyPermission === 'allow') {
            processedResult = true;
        } else {
            processedResult = actionImplicitDenies[requestType] === false && aclPermission;
        }
    }
    return processedResult;
 }
 function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
    actionImplicitDeniesInput = {}, isWebsite = false) {
    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
    const mainApiCall = requestTypes[0];
    const results = {};
    return requestTypes.every(_requestType => {
        // By default, all missing actions are defined as allowed from IAM, to be
        // backward compatible
        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
        // Check to see if user is authorized to perform a
        // particular action on bucket based on ACLs.
        // TODO: Add IAM checks
        let requesterIsNotUser = true;
        let arn = null;
        if (authInfo) {
            requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
            arn = authInfo.getArn();
        }
        // if the bucket owner is an account, users should not have default access
        if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
            results[_requestType] = actionImplicitDenies[_requestType] === false;
            return results[_requestType];
        }
        const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
        // In case of error bucket access is checked with bucketGet
        // For website, bucket policy only uses objectGet and ignores bucketGet
        // https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
        // bucketGet should be used to check acl but switched to objectGet for bucket policy
        if (isWebsite && _requestType === 'bucketGet') {
            // eslint-disable-next-line no-param-reassign
            _requestType = 'objectGet';
            actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
        }
        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
            request, aclPermission, results, actionImplicitDenies);
    });
 }
 function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {},
    log, request) {
    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
    const results = {};
    return requestTypes.every(_requestType => {
        // By default, all missing actions are defined as allowed from IAM, to be
        // backward compatible
        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
        let arn = null;
        if (authInfo) {
            arn = authInfo.getArn();
        }
        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
        request, true, results, actionImplicitDenies);
    });
 }
 function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
    actionImplicitDeniesInput = {}, isWebsite = false) {
    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
    const results = {};
    const mainApiCall = requestTypes[0];
    return requestTypes.every(_requestType => {
        // By default, all missing actions are defined as allowed from IAM, to be
        // backward compatible
        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
        const parsedMethodName = _requestType.endsWith('Version')
            ? _requestType.slice(0, -7) : _requestType;
        const bucketOwner = bucket.getOwner();
        if (!objectMD) {
            // check bucket has read access
            // 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
            let permission = 'bucketGet';
            if (actionsToConsiderAsObjectPut.includes(_requestType)) {
                permission = 'objectPut';
            }
            results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
                actionImplicitDenies, isWebsite);
            // User is already authorized on the bucket for FULL_CONTROL or WRITE or
            // bucket has canned ACL public-read-write
            if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
                && results[_requestType] === false) {
                results[_requestType] = actionImplicitDenies[_requestType] === false;
            }
            return results[_requestType];
        }
        let requesterIsNotUser = true;
        let arn = null;
        let isUserUnauthenticated = false;
        if (authInfo) {
            requesterIsNotUser =  !isRequesterNonAccountUser(authInfo);
            arn = authInfo.getArn();
            isUserUnauthenticated = arn === undefined;
        }
        if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
            results[_requestType] = actionImplicitDenies[_requestType] === false;
            return results[_requestType];
        }
        // account is authorized if:
        // - requesttype is included in bucketOwnerActions and
        // - account is the bucket owner
        // - requester is account, not user
        if (bucketOwnerActions.includes(parsedMethodName)
        && (bucketOwner === canonicalID)
        && requesterIsNotUser) {
            results[_requestType] = actionImplicitDenies[_requestType] === false;
            return results[_requestType];
        }
        const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName,
            canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall);
        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner,
            log, request, aclPermission, results, actionImplicitDenies);
    });
 }
 function _checkResource(resource, bucketArn) {
    if (resource === bucketArn) {
        return true;
    }
    if (resource.includes('/')) {
        const rSubs = resource.split('/');
        return rSubs[0] === bucketArn;
    }
    return false;
 }
 // the resources specified in the bucket policy should contain the bucket name
 function validatePolicyResource(bucketName, policy) {
    const bucketArn = `arn:aws:s3:::${bucketName}`;
    return policy.Statement.every(s => {
        if (Array.isArray(s.Resource)) {
            return s.Resource.every(r => _checkResource(r, bucketArn));
        }
        if (typeof s.Resource === 'string') {
            return _checkResource(s.Resource, bucketArn);
        }
        return false;
    });
 }
 function checkIp(value) {
    const errString = 'Invalid IP address in Conditions';
    const values = Array.isArray(value) ? value : [value];
    for (let i = 0; i < values.length; i++) {
        // these preliminary checks are validating the provided
        // ip address against ipaddr.js, the library we use when
        // evaluating IP condition keys. It ensures compatibility,
        // but additional checks are required to enforce the right
        // notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
        // we would accept different ip formats, which is not
        // standard in an AWS use case.
        try {
            try {
                parseCIDR(values[i]);
            } catch (err) {
                isValid(values[i]);
            }
        } catch (err) {
            return errString;
        }
        // Apply the existing IP validation logic to each element
        const validateIpRegex = ip => {
            if (constants.ipv4Regex.test(ip)) {
                return ip.split('.').every(part => parseInt(part, 10) <= 255);
            }
            if (constants.ipv6Regex.test(ip)) {
                return ip.split(':').every(part => part.length <= 4);
            }
            return false;
        };
        if (validateIpRegex(values[i]) !== true) {
            return errString;
        }
    }
    // If the function hasn't returned by now, all elements are valid
    return null;
 }
 // This function checks all bucket policy conditions if the values provided
 // are valid for the condition type. If not it returns a relevant Malformed policy error string
 function validatePolicyConditions(policy) {
    const validConditions = [
        { conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
        { conditionKey: 's3:object-lock-remaining-retention-days' },
    ];
    // keys where value type does not seem to be checked by AWS:
    // - s3:object-lock-remaining-retention-days
    if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
        return null;
    }
    // there can be multiple statements in the policy, each with a Condition enclosure
    for (let i = 0; i < policy.Statement.length; i++) {
        const s = policy.Statement[i];
        if (s.Condition) {
            const conditionOperators = Object.keys(s.Condition);
            // there can be multiple condition operations in the Condition enclosure
            // eslint-disable-next-line no-restricted-syntax
            for (const conditionOperator of conditionOperators) {
                const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
                const conditionValue = s.Condition[conditionOperator][conditionKey];
                const validCondition = validConditions.find(validCondition =>
                    validCondition.conditionKey === conditionKey
                );
                // AWS returns does not return an error if the condition starts with 'aws:'
                // so we reproduce this behaviour
                if (!validCondition && !conditionKey.startsWith('aws:')) {
                    return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
                }
                if (validCondition && validCondition.conditionValueTypeChecker) {
                    const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
                    if (conditionValueTypeError) {
                        return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
                    }
                }
            }
        }
    }
    return null;
 }
 /** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
 * @param {string} arn - Amazon resource name - example:
 * arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
 * @return {boolean} true if Lifecycle assumed role session arn, false if not.
 */
 function isLifecycleSession(arn) {
    if (!arn) {
        return false;
    }
    const arnSplits = arn.split(':');
    const service = arnSplits[2];
    const resourceNames = arnSplits[arnSplits.length - 1].split('/');
    const resourceType = resourceNames[0];
    const sessionName = resourceNames[resourceNames.length - 1];
    return (service === 'sts'
        && resourceType === assumedRoleArnResourceType
        && sessionName === backbeatLifecycleSessionName);
 }
 module.exports = {
    isBucketAuthorized,
    isObjAuthorized,
    getServiceAccountProperties,
    isServiceAccount,
    isRequesterASessionUser,
    isRequesterNonAccountUser,
    checkBucketAcls,
    checkObjectAcls,
    validatePolicyResource,
    validatePolicyConditions,
    isLifecycleSession,
    evaluateBucketPolicyWithIAM,
 };
--- a/lib/api/apiUtils/authorization/prepareRequestContexts.js
+++ b/lib/api/apiUtils/authorization/prepareRequestContexts.js
@ -1,20 +1,10 @@
 const { policies } = require('arsenal');
 const { config } = require('../../../Config');
-const { RequestContext, requestUtils } = policies;
+const RequestContext = policies.RequestContext;
 let apiMethodAfterVersionCheck;
-const apiMethodWithVersion = {
+const apiMethodWithVersion = { objectGetACL: true, objectPutACL: true,
-    objectGetACL: true,
+objectGet: true, objectDelete: true, objectPutTagging: true,
-    objectPutACL: true,
+objectGetTagging: true, objectDeleteTagging: true };
    objectGet: true,
    objectDelete: true,
    objectPutTagging: true,
    objectGetTagging: true,
    objectDeleteTagging: true,
    objectGetLegalHold: true,
    objectPutLegalHold: true,
    objectPutRetention: true,
 };
 function isHeaderAcl(headers) {
    return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] ||
@ -43,16 +33,14 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
    // null as the requestContext to Vault so it will only do an authentication
    // check.
    const ip = requestUtils.getClientIp(request, config);
    function generateRequestContext(apiMethod) {
        return new RequestContext(request.headers,
            request.query, request.bucketName, request.objectKey,
-            ip, request.connection.encrypted,
+            request.socket.remoteAddress, request.connection.encrypted,
            apiMethod, 's3');
    }
-    if (apiMethod === 'bucketPut') {
+    if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
        return null;
    }
@ -65,17 +53,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
    const requestContexts = [];
-    if (apiMethod === 'multiObjectDelete') {
+    if (apiMethodAfterVersionCheck === 'objectCopy'
        // MultiObjectDelete does not require any authorization when evaluating
        // the API. Instead, we authorize each object passed.
        // But in order to get any relevant information from the authorization service
        // for example, the account quota, we must send a request context object
        // with no `specificResource`. We expect the result to be an implicit deny.
        // In the API, we then ignore these authorization results, and we can use
        // any information returned, e.g., the quota.
        const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
          requestContexts.push(requestContextMultiObjectDelete);
    } else if (apiMethodAfterVersionCheck === 'objectCopy'
        || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
        const objectGetAction = sourceVersionId ? 'objectGetVersion' :
          'objectGet';
@ -83,7 +61,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
            { versionId: sourceVersionId });
        const getRequestContext = new RequestContext(request.headers,
            reqQuery, sourceBucket, sourceObject,
-            ip, request.connection.encrypted,
+            request.socket.remoteAddress, request.connection.encrypted,
            objectGetAction, 's3');
        const putRequestContext = generateRequestContext('objectPut');
        requestContexts.push(getRequestContext, putRequestContext);
@ -107,126 +85,27 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
        const objectGetTaggingAction = (request.query &&
          request.query.versionId) ? 'objectGetTaggingVersion' :
          'objectGetTagging';
        if (request.headers['x-amz-version-id']) {
            const objectGetVersionAction = 'objectGetVersion';
            const getVersionResourceVersion =
                generateRequestContext(objectGetVersionAction);
            requestContexts.push(getVersionResourceVersion);
        }
        const getRequestContext =
          generateRequestContext(apiMethodAfterVersionCheck);
        const getTaggingRequestContext =
          generateRequestContext(objectGetTaggingAction);
        requestContexts.push(getRequestContext, getTaggingRequestContext);
    } else if (apiMethodAfterVersionCheck === 'objectGetTagging') {
        const objectGetTaggingAction = 'objectGetTagging';
        const getTaggingResourceVersion =
            generateRequestContext(objectGetTaggingAction);
        requestContexts.push(getTaggingResourceVersion);
        if (request.headers['x-amz-version-id']) {
            const objectGetTaggingVersionAction = 'objectGetTaggingVersion';
            const getTaggingVersionResourceVersion =
                generateRequestContext(objectGetTaggingVersionAction);
            requestContexts.push(getTaggingVersionResourceVersion);
        }
    } else if (apiMethodAfterVersionCheck === 'objectHead') {
        const objectHeadAction = 'objectHead';
        const headObjectAction =
            generateRequestContext(objectHeadAction);
        requestContexts.push(headObjectAction);
        if (request.headers['x-amz-version-id']) {
            const objectHeadVersionAction = 'objectGetVersion';
            const headObjectVersion =
                generateRequestContext(objectHeadVersionAction);
            requestContexts.push(headObjectVersion);
        }
    } else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
        const putObjectTaggingRequestContext =
            generateRequestContext('objectPutTagging');
        requestContexts.push(putObjectTaggingRequestContext);
        if (request.headers['x-amz-version-id']) {
            const putObjectVersionRequestContext =
                generateRequestContext('objectPutTaggingVersion');
            requestContexts.push(putObjectVersionRequestContext);
        }
    } else if (apiMethodAfterVersionCheck === 'objectPutCopyPart') {
        const putObjectRequestContext =
            generateRequestContext('objectPut');
        requestContexts.push(putObjectRequestContext);
        const getObjectRequestContext =
            generateRequestContext('objectGet');
        requestContexts.push(getObjectRequestContext);
    } else if (apiMethodAfterVersionCheck === 'objectPut') {
-        // if put object with version
+        const putRequestContext =
-        if (request.headers['x-scal-s3-version-id'] ||
+          generateRequestContext(apiMethodAfterVersionCheck);
-        request.headers['x-scal-s3-version-id'] === '') {
+        requestContexts.push(putRequestContext);
-            const putVersionRequestContext =
+        // if put object (versioning) with tag set
-              generateRequestContext('objectPutVersion');
+        if (request.headers['x-amz-tagging']) {
-            requestContexts.push(putVersionRequestContext);
+            const putTaggingRequestContext =
-        } else {
+              generateRequestContext('objectPutTagging');
-            const putRequestContext =
+            requestContexts.push(putTaggingRequestContext);
              generateRequestContext(apiMethodAfterVersionCheck);
            requestContexts.push(putRequestContext);
            // if put object (versioning) with tag set
            if (request.headers['x-amz-tagging']) {
                const putTaggingRequestContext =
                  generateRequestContext('objectPutTagging');
                requestContexts.push(putTaggingRequestContext);
            }
            if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
                const putLegalHoldStatusAction =
                    generateRequestContext('objectPutLegalHold');
                requestContexts.push(putLegalHoldStatusAction);
            }
            // if put object (versioning) with ACL
            if (isHeaderAcl(request.headers)) {
                const putAclRequestContext =
                  generateRequestContext('objectPutACL');
                requestContexts.push(putAclRequestContext);
            }
            if (request.headers['x-amz-object-lock-mode']) {
                const putObjectLockRequestContext =
                  generateRequestContext('objectPutRetention');
                requestContexts.push(putObjectLockRequestContext);
            }
            if (request.headers['x-amz-version-id']) {
                const putObjectVersionRequestContext =
                    generateRequestContext('objectPutTaggingVersion');
                requestContexts.push(putObjectVersionRequestContext);
            }
        }
    } else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
      apiMethodAfterVersionCheck === 'objectPutPart' ||
      apiMethodAfterVersionCheck === 'completeMultipartUpload'
      ) {
        if (request.headers['x-scal-s3-version-id'] ||
        request.headers['x-scal-s3-version-id'] === '') {
            const putVersionRequestContext =
              generateRequestContext('objectPutVersion');
            requestContexts.push(putVersionRequestContext);
        } else {
            const putRequestContext =
              generateRequestContext(apiMethodAfterVersionCheck);
            requestContexts.push(putRequestContext);
        }
        // if put object (versioning) with ACL
        if (isHeaderAcl(request.headers)) {
            const putAclRequestContext =
              generateRequestContext('objectPutACL');
            requestContexts.push(putAclRequestContext);
        }
        if (request.headers['x-amz-object-lock-mode']) {
            const putObjectLockRequestContext =
              generateRequestContext('objectPutRetention');
            requestContexts.push(putObjectLockRequestContext);
        }
        if (request.headers['x-amz-version-id']) {
            const putObjectVersionRequestContext =
                generateRequestContext('objectPutTaggingVersion');
            requestContexts.push(putObjectVersionRequestContext);
        }
    } else {
        const requestContext =
          generateRequestContext(apiMethodAfterVersionCheck);
--- a/lib/api/apiUtils/authorization/tagConditionKeys.js
+++ b/lib/api/apiUtils/authorization/tagConditionKeys.js
@ -1,99 +0,0 @@
 const async = require('async');
 const { auth, s3middleware } = require('arsenal');
 const metadata = require('../../../metadata/wrapper');
 const { decodeVersionId } = require('../object/versioning');
 const { parseTagXml } = s3middleware.tagging;
 function makeTagQuery(tags) {
    return Object.entries(tags)
        .map(i => i.join('='))
        .join('&');
 }
 function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) {
    async.waterfall([
        next => {
            if (request.headers['x-amz-tagging']) {
                return next(null, request.headers['x-amz-tagging']);
            }
            if (request.post && apiMethod === 'objectPutTagging') {
                return parseTagXml(request.post, log, (err, tags) => {
                    if (err) {
                        log.trace('error parsing request tags');
                        return next(err);
                    }
                    return next(null, makeTagQuery(tags));
                });
            }
            return next(null, null);
        },
        (requestTagsQuery, next) => {
            const objectKey = request.objectKey;
            const bucketName = request.bucketName;
            const decodedVidResult = decodeVersionId(request.query);
            if (decodedVidResult instanceof Error) {
                log.trace('invalid versionId query', {
                    versionId: request.query.versionId,
                    error: decodedVidResult,
                });
                return next(decodedVidResult);
            }
            const reqVersionId = decodedVidResult;
            return metadata.getObjectMD(
                bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => {
                    if (err) {
                        // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
                        if (err.NoSuchKey) {
                            return next(null, requestTagsQuery, null);
                        }
                        log.trace('error getting request object tags');
                        return next(err);
                    }
                    const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags);
                    return next(null, requestTagsQuery, existingTagsQuery);
                });
        },
    ], (err, requestTagsQuery, existingTagsQuery) => {
        if (err) {
            log.trace('error processing tag condition key evaluation');
            return cb(err);
        }
        // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
        // eslint-disable-next-line no-restricted-syntax
        for (const rc of requestContexts) {
            rc.setNeedTagEval(true);
            if (requestTagsQuery) {
                rc.setRequestObjTags(requestTagsQuery);
            }
            if (existingTagsQuery) {
                rc.setExistingObjTag(existingTagsQuery);
            }
        }
        return cb();
    });
 }
 function tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log, cb) {
    if (!authorizationResults) {
        return cb();
    }
    if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
        return cb(null, authorizationResults);
    }
    return updateRequestContextsWithTags(request, requestContexts, apiMethod, log, err => {
        if (err) {
            return cb(err);
        }
        return auth.server.doAuth(request, log,
            (err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts);
    });
 }
 module.exports = {
    tagConditionKeyAuth,
    updateRequestContextsWithTags,
    makeTagQuery,
 };
--- a/lib/api/apiUtils/bucket/bucketCors.js
+++ b/lib/api/apiUtils/bucket/bucketCors.js
@ -1,7 +1,7 @@
 const { parseString } = require('xml2js');
-const { errors, s3middleware } = require('arsenal');
+const { errors } = require('arsenal');
-const escapeForXml = s3middleware.escapeForXml;
+const escapeForXml = require('../../../utilities/escapeForXML');
 /*
    Format of xml request:
@ -81,7 +81,7 @@ const _validator = {
        }
        const parsedValue = parseInt(seconds[0], 10);
        const errMsg = `MaxAgeSeconds "${seconds[0]}" is not a valid value.`;
-        if (Number.isNaN(parsedValue) || parsedValue < 0) {
+        if (isNaN(parsedValue) || parsedValue < 0) {
            return errors.MalformedXML.customizeDescription(errMsg);
        }
        return parsedValue;
--- a/lib/api/apiUtils/bucket/bucketCreation.js
+++ b/lib/api/apiUtils/bucket/bucketCreation.js
@ -3,37 +3,31 @@ const async = require('async');
 const { errors } = require('arsenal');
 const acl = require('../../../metadata/acl');
-const BucketInfo = require('arsenal').models.BucketInfo;
+const BucketInfo = require('../../../metadata/BucketInfo');
 const constants = require('../../../../constants');
 const createKeyForUserBucket = require('./createKeyForUserBucket');
 const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
 const metadata = require('../../../metadata/wrapper');
 const kms = require('../../../kms/wrapper');
 const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
 const { isServiceAccount } = require('../authorization/permissionChecks');
 const usersBucket = constants.usersBucket;
 const oldUsersBucket = constants.oldUsersBucket;
 const zenkoSeparator = constants.zenkoSeparator;
 const userBucketOwner = 'admin';
-function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
+function addToUsersBucket(canonicalID, bucketName, log, cb) {
    // BACKWARD: Simplify once do not have to deal with old
    // usersbucket name and old splitter
    // Get new format usersBucket to see if it exists
    return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => {
-        if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) {
+        if (err && !err.NoSuchBucket && !err.BucketAlreadyExists) {
            return cb(err);
        }
        const splitter = usersBucketAttrs ?
            constants.splitter : constants.oldSplitter;
        let key = createKeyForUserBucket(canonicalID, splitter, bucketName);
-        const omVal = {
+        const omVal = { creationDate: new Date().toJSON() };
            creationDate: new Date().toJSON(),
            ingestion: bucketMD.getIngestion(),
        };
        // If the new format usersbucket does not exist, try to put the
        // key in the old usersBucket using the old splitter.
        // Otherwise put the key in the new format usersBucket
@ -41,7 +35,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
            usersBucket : oldUsersBucket;
        return metadata.putObjectMD(usersBucketBeingCalled, key,
            omVal, {}, log, err => {
-                if (err?.is?.NoSuchBucket) {
+                if (err && err.NoSuchBucket) {
                    // There must be no usersBucket so createBucket
                    // one using the new format
                    log.trace('users bucket does not exist, ' +
@ -61,8 +55,9 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
                            // from getting a BucketAlreadyExists
                            // error with respect
                            // to the usersBucket.
-                            // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
+                            if (err &&
-                            if (err && !err.BucketAlreadyExists) {
+                                err !==
                                    errors.BucketAlreadyExists) {
                                log.error('error from metadata', {
                                    error: err,
                                });
@ -96,7 +91,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
            return callback(err);
        }
        log.trace('created bucket in metadata');
-        return addToUsersBucket(canonicalID, bucketName, bucket, log, err => {
+        return addToUsersBucket(canonicalID, bucketName, log, err => {
            if (err) {
                return callback(err);
            }
@ -119,7 +114,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
 */
 function cleanUpBucket(bucketMD, canonicalID, log, callback) {
    const bucketName = bucketMD.getName();
-    return addToUsersBucket(canonicalID, bucketName, bucketMD, log, err => {
+    return addToUsersBucket(canonicalID, bucketName, log, err => {
        if (err) {
            return callback(err);
        }
@ -170,37 +165,12 @@ function createBucket(authInfo, bucketName, headers,
    const ownerDisplayName =
        authInfo.getAccountDisplayName();
    const creationDate = new Date().toJSON();
-    const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
+    const bucket = new BucketInfo(bucketName,
-    const headerObjectLock = headers['x-amz-bucket-object-lock-enabled'];
+        canonicalID, ownerDisplayName, creationDate,
-    const objectLockEnabled
+        BucketInfo.currentModelVersion());
        = headerObjectLock && headerObjectLock.toLowerCase() === 'true';
    const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName,
        creationDate, BucketInfo.currentModelVersion(), null, null, null, null,
        null, null, null, null, null, null, null, null, null, isNFSEnabled,
        null, null, objectLockEnabled);
    let locationConstraintVal = null;
-    if (locationConstraint) {
+    if (locationConstraint !== undefined) {
-        const [locationConstraintStr, ingestion] =
+        bucket.setLocationConstraint(locationConstraint);
            locationConstraint.split(zenkoSeparator);
        if (locationConstraintStr) {
            locationConstraintVal = locationConstraintStr;
            bucket.setLocationConstraint(locationConstraintStr);
        }
        if (ingestion === 'ingest') {
            bucket.enableIngestion();
            //automatically enable versioning for ingestion buckets
            bucket.setVersioningConfiguration({ Status: 'Enabled' });
        }
    }
    if (objectLockEnabled) {
        // default versioning configuration AWS sets
        // when a bucket is created with object lock
        const versioningConfiguration = {
            Status: 'Enabled',
            MfaDelete: 'Disabled',
        };
        bucket.setVersioningConfiguration(versioningConfiguration);
    }
    const parseAclParams = {
        headers,
@ -223,7 +193,6 @@ function createBucket(authInfo, bucketName, headers,
        },
        getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
            metadata.getBucket(bucketName, log, (err, data) => {
                // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
                if (err && err.NoSuchBucket) {
                    return callback(null, 'NoBucketYet');
                }
@ -241,16 +210,14 @@ function createBucket(authInfo, bucketName, headers,
        }
        const existingBucketMD = results.getAnyExistingBucketInfo;
        if (existingBucketMD instanceof BucketInfo &&
-            existingBucketMD.getOwner() !== canonicalID &&
+            existingBucketMD.getOwner() !== canonicalID) {
            !isServiceAccount(canonicalID)) {
            // return existingBucketMD to collect cors headers
            return cb(errors.BucketAlreadyExists, existingBucketMD);
        }
        const newBucketMD = results.prepareNewBucketMD;
        if (existingBucketMD === 'NoBucketYet') {
            const sseConfig = parseBucketEncryptionHeaders(headers);
            return bucketLevelServerSideEncryption(
-                bucketName, sseConfig, log,
+                bucketName, headers, log,
                (err, sseInfo) => {
                    if (err) {
                        return cb(err);
@ -273,7 +240,7 @@ function createBucket(authInfo, bucketName, headers,
        // error unless old AWS behavior (us-east-1)
        // Existing locationConstraint must have legacyAwsBehavior === true
        // New locationConstraint should have legacyAwsBehavior === true
-        if (isLegacyAWSBehavior(locationConstraintVal) &&
+        if (isLegacyAWSBehavior(locationConstraint) &&
            isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) {
            log.trace('returning 200 instead of 409 to mirror us-east-1');
            return cb(null, existingBucketMD);
--- a/lib/api/apiUtils/bucket/bucketDeletion.js
+++ b/lib/api/apiUtils/bucket/bucketDeletion.js
@ -2,21 +2,60 @@ const assert = require('assert');
 const async = require('async');
 const { errors } = require('arsenal');
-const abortMultipartUpload = require('../object/abortMultipartUpload');
+const logger = require('../../../utilities/logger');
 const { pushMetric } = require('../../../utapi/utilities');
-const { splitter, oldSplitter, mpuBucketPrefix } =
+const constants = require('../../../../constants');
-    require('../../../../constants');
+const createKeyForUserBucket = require('./createKeyForUserBucket');
 const metadata = require('../../../metadata/wrapper');
 const kms = require('../../../kms/wrapper');
-const deleteUserBucketEntry = require('./deleteUserBucketEntry');
+
 const usersBucket = constants.usersBucket;
 const oldUsersBucket = constants.oldUsersBucket;
 function _deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
    log.trace('deleting bucket name from users bucket', { method:
        '_deleteUserBucketEntry' });
    const keyForUserBucket = createKeyForUserBucket(canonicalID,
        constants.splitter, bucketName);
    metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
        // If the object representing the bucket is not in the
        // users bucket just continue
        if (error && error.NoSuchKey) {
            return cb(null);
        // BACKWARDS COMPATIBILITY: Remove this once no longer
        // have old user bucket format
        } else if (error && error.NoSuchBucket) {
            const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
                constants.oldSplitter, bucketName);
            return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
                {}, log, error => {
                    if (error && !error.NoSuchKey) {
                        log.error('from metadata while deleting user bucket',
                            { error });
                        return cb(error);
                    }
                    log.trace('deleted bucket from user bucket',
                    { method: '_deleteUserBucketEntry' });
                    return cb(null);
                });
        } else if (error) {
            log.error('from metadata while deleting user bucket', { error,
            method: '_deleteUserBucketEntry' });
            return cb(error);
        }
        log.trace('deleted bucket from user bucket', {
            method: '_deleteUserBucketEntry' });
        return cb(null);
    });
 }
 function _deleteMPUbucket(destinationBucketName, log, cb) {
    const mpuBucketName =
-        `${mpuBucketPrefix}${destinationBucketName}`;
+        `${constants.mpuBucketPrefix}${destinationBucketName}`;
    return metadata.deleteBucket(mpuBucketName, log, err => {
        // If the mpu bucket does not exist, just move on
        // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
        if (err && err.NoSuchBucket) {
            return cb();
        }
@ -24,38 +63,48 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
    });
 }
-function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) {
+/**
-    async.mapLimit(mpus, 1, (mpu, next) => {
+ * Invisibly finishes deleting a bucket that already has a deleted flag
-        const splitterChar = mpu.key.includes(oldSplitter) ?
+ * by deleting the object in the users bucket representing the created bucket
-            oldSplitter : splitter;
+ * and then deleting the bucket in metadata
-        // `overview${splitter}${objectKey}${splitter}${uploadId}
+ * @param {string} bucketName - name of bucket
-        const [, objectKey, uploadId] = mpu.key.split(splitterChar);
+ * @param {string} canonicalID - bucket owner's canonicalID
-        abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
+ * @return {undefined}
-            (err, destBucket, partSizeSum) => {
+ */
-                pushMetric('abortMultipartUpload', log, {
+function invisiblyDelete(bucketName, canonicalID) {
-                    authInfo,
+    const log = logger.newRequestLogger();
-                    canonicalID: bucketMD.getOwner(),
+    log.trace('deleting bucket with deleted flag invisibly', { bucketName });
-                    bucket: bucketName,
+    return _deleteUserBucketEntry(bucketName, canonicalID, log, err => {
-                    keys: [objectKey],
+        if (err) {
-                    byteLength: partSizeSum,
+            log.error('error invisibly deleting bucket name from user bucket',
-                });
+            { error: err });
-                next(err);
+            return log.end();
-            }, request);
+        }
-    }, cb);
+        log.trace('deleted bucket name from user bucket');
        return metadata.deleteBucket(bucketName, log, error => {
            log.trace('deleting bucket from metadata',
            { method: 'invisiblyDelete' });
            if (error) {
                log.error('error deleting bucket from metadata', { error });
                return log.end();
            }
            log.trace('invisible deletion of bucket succeeded',
            { method: 'invisiblyDelete' });
            return log.end();
        });
    });
 }
 /**
 * deleteBucket - Delete bucket from namespace
 * @param {object} authInfo - authentication info
 * @param {object} bucketMD - bucket attributes/metadata
 * @param {string} bucketName - bucket in which objectMetadata is stored
 * @param {string} canonicalID - account canonicalID of requester
 * @param {object} request - request object given by router
 *                           including normalized headers
 * @param {object} log - Werelogs logger
 * @param {function} cb - callback from async.waterfall in bucketDelete
 * @return {undefined}
 */
-function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, cb) {
+function deleteBucket(bucketMD, bucketName, canonicalID, log, cb) {
    log.trace('deleting bucket from metadata');
    assert.strictEqual(typeof bucketName, 'string');
    assert.strictEqual(typeof canonicalID, 'string');
@ -84,16 +133,18 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
                    return next();
                });
        },
-
+        // Note: This does not mirror AWS behavior.  AWS will allow a user to
        // delete a bucket even if there are ongoing multipart uploads.
        function deleteMPUbucketStep(next) {
-            const MPUBucketName = `${mpuBucketPrefix}${bucketName}`;
+            const MPUBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
            // check to see if there are any mpu overview objects (so ignore
            // any orphaned part objects)
-            return metadata.listObject(MPUBucketName, { prefix: 'overview' },
+            return metadata.listObject(MPUBucketName,
-                log, (err, objectsListRes) => {
+                { maxKeys: 1, prefix: 'overview' }, log,
                (err, objectsListRes) => {
                    // If no shadow bucket ever created, no ongoing MPU's, so
                    // continue with deletion
-                    if (err?.is.NoSuchBucket) {
+                    if (err && err.NoSuchBucket) {
                        return next();
                    }
                    if (err) {
@ -101,14 +152,12 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
                        return next(err);
                    }
                    if (objectsListRes.Contents.length) {
-                        return _deleteOngoingMPUs(authInfo, bucketName,
+                        log.debug('bucket delete failed',
-                            bucketMD, objectsListRes.Contents, request, log, err => {
+                            { error: errors.MPUinProgress });
-                                if (err) {
+                        // Return non-AWS standard error
-                                    return next(err);
+                        // regarding ongoing MPUs so user
-                                }
+                        // understands what is occurring
-                                log.trace('deleting shadow MPU bucket');
+                        return next(errors.MPUinProgress);
                                return _deleteMPUbucket(bucketName, log, next);
                            });
                    }
                    log.trace('deleting shadow MPU bucket');
                    return _deleteMPUbucket(bucketName, log, next);
@ -124,7 +173,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
        },
        function deleteUserBucketEntryStep(next) {
            log.trace('deleting bucket name from user bucket');
-            return deleteUserBucketEntry(bucketName, canonicalID, log, next);
+            return _deleteUserBucketEntry(bucketName, canonicalID, log, next);
        },
    ],
    // eslint-disable-next-line prefer-arrow-callback
@ -148,4 +197,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
    });
 }
-module.exports = deleteBucket;
+module.exports = {
    invisiblyDelete,
    deleteBucket,
 };
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Rahul Padigela	543c737f04	more controllers	2017-06-15 10:23:44 -07:00
Lauren Spiegel	d6e607f8a5	LOG: Add endpoint log	2017-06-15 10:23:44 -07:00
Lauren Spiegel	4782f57dad	FIX: Remove excess logging info	2017-06-15 10:23:44 -07:00
Alexandre Merle	0ea7145d1f	S3C-380: Get security token Send security token to vault See https://scality.atlassian.net/browse/S3C-380	2017-06-15 10:23:44 -07:00
Rahul Padigela	bbc050335c	wip: controller, middleware, models [ci skip]	2017-06-06 17:55:58 -07:00
Rahul Padigela	113d0f699b	ft: controller and router	2017-06-06 17:25:42 -07:00
		`@ -1,3 +0,0 @@`
			`#!/bin/sh`

			`python3 /usr/local/bin/run_server.py 2>&1 \| tee -a /artifacts/pykmip.log`
`@ -1,2 +1,2 @@`
	`---`	`---`
	`theme: jekyll-theme-modernist`	`theme: jekyll-theme-minimal`