Use fs.readFileSync to read config file instead of require

FIXME: Ideally, both locations and storage classes should be supported

.eslintrc

@@ -1,6 +1,10 @@
 {
     "extends": "scality",
+    "plugins": [
+        "mocha"
+    ],
     "rules": {
+        "import/extensions": "off",
         "lines-around-directive": "off",
         "no-underscore-dangle": "off",
         "indent": "off",
@@ -41,6 +45,10 @@
         "no-restricted-properties": "off",
         "new-parens": "off",
         "no-multi-spaces": "off",
-        "quote-props": "off"
+        "quote-props": "off",
+        "mocha/no-exclusive-tests": "error",
+    },
+    "parserOptions": {
+        "ecmaVersion": 2020
     }
 }

.github/actions/setup-ci/action.yaml

@@ -0,0 +1,43 @@
+---
+name: "Setup CI environment"
+description: "Setup Cloudserver CI environment"
+
+runs:
+  using: composite
+  steps:
+    - name: Setup etc/hosts
+      shell: bash
+      run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
+    - name: Setup Credentials
+      shell: bash
+      run: bash .github/scripts/credentials.bash
+    - name: Setup job artifacts directory
+      shell: bash
+      run: |-
+        set -exu;
+        mkdir -p /tmp/artifacts/${JOB_NAME}/;
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '16'
+        cache: 'yarn'
+    - name: install dependencies
+      shell: bash
+      run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
+    - uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Setup python2 test environment
+      shell: bash
+      run: |
+        sudo apt-get install -y libdigest-hmac-perl
+        pip install 's3cmd==2.3.0'
+    - name: fix sproxyd.conf permissions
+      shell: bash
+      run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
+    - name: ensure fuse kernel module is loaded (for sproxyd)
+      shell: bash
+      run: sudo modprobe fuse

.github/docker/creds.env

@@ -0,0 +1,36 @@
+azurebackend_AZURE_STORAGE_ACCESS_KEY
+azurebackend_AZURE_STORAGE_ACCOUNT_NAME
+azurebackend_AZURE_STORAGE_ENDPOINT
+azurebackend2_AZURE_STORAGE_ACCESS_KEY
+azurebackend2_AZURE_STORAGE_ACCOUNT_NAME
+azurebackend2_AZURE_STORAGE_ENDPOINT
+azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY
+azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME
+azurebackendmismatch_AZURE_STORAGE_ENDPOINT
+azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY
+azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME
+azurenonexistcontainer_AZURE_STORAGE_ENDPOINT
+azuretest_AZURE_BLOB_ENDPOINT
+b2backend_B2_ACCOUNT_ID
+b2backend_B2_STORAGE_ACCESS_KEY
+GOOGLE_SERVICE_EMAIL
+GOOGLE_SERVICE_KEY
+AWS_S3_BACKEND_ACCESS_KEY
+AWS_S3_BACKEND_SECRET_KEY
+AWS_S3_BACKEND_ACCESS_KEY_2
+AWS_S3_BACKEND_SECRET_KEY_2
+AWS_GCP_BACKEND_ACCESS_KEY
+AWS_GCP_BACKEND_SECRET_KEY
+AWS_GCP_BACKEND_ACCESS_KEY_2
+AWS_GCP_BACKEND_SECRET_KEY_2
+b2backend_B2_STORAGE_ENDPOINT
+gcpbackend2_GCP_SERVICE_EMAIL
+gcpbackend2_GCP_SERVICE_KEY
+gcpbackend2_GCP_SERVICE_KEYFILE
+gcpbackend_GCP_SERVICE_EMAIL
+gcpbackend_GCP_SERVICE_KEY
+gcpbackendmismatch_GCP_SERVICE_EMAIL
+gcpbackendmismatch_GCP_SERVICE_KEY
+gcpbackend_GCP_SERVICE_KEYFILE
+gcpbackendmismatch_GCP_SERVICE_KEYFILE
+gcpbackendnoproxy_GCP_SERVICE_KEYFILE

.github/docker/docker-compose.yaml

@@ -0,0 +1,92 @@
+services:
+  cloudserver:
+    image: ${CLOUDSERVER_IMAGE}
+    command: sh -c "yarn start > /artifacts/s3.log"
+    network_mode: "host"
+    volumes:
+      - /tmp/ssl:/ssl
+      - /tmp/ssl-kmip:/ssl-kmip
+      - ${HOME}/.aws/credentials:/root/.aws/credentials
+      - /tmp/artifacts/${JOB_NAME}:/artifacts
+    environment:
+      - CI=true
+      - ENABLE_LOCAL_CACHE=true
+      - REDIS_HOST=0.0.0.0
+      - REDIS_PORT=6379
+      - REPORT_TOKEN=report-token-1
+      - REMOTE_MANAGEMENT_DISABLE=1
+      - HEALTHCHECKS_ALLOWFROM=0.0.0.0/0
+      - DATA_HOST=0.0.0.0
+      - METADATA_HOST=0.0.0.0
+      - S3BACKEND
+      - S3DATA
+      - S3METADATA
+      - MPU_TESTING
+      - S3VAULT
+      - S3_LOCATION_FILE
+      - ENABLE_UTAPI_V2
+      - BUCKET_DENY_FILTER
+      - S3KMS
+      - S3KMIP_PORT
+      - S3KMIP_HOSTS
+      - S3KMIP-COMPOUND_CREATE
+      - S3KMIP_BUCKET_ATTRIBUTE_NAME
+      - S3KMIP_PIPELINE_DEPTH
+      - S3KMIP_KEY
+      - S3KMIP_CERT
+      - S3KMIP_CA
+      - MONGODB_HOSTS=0.0.0.0:27018
+      - MONGODB_RS=rs0
+      - DEFAULT_BUCKET_KEY_FORMAT
+      - METADATA_MAX_CACHED_BUCKETS
+      - ENABLE_NULL_VERSION_COMPAT_MODE
+      - SCUBA_HOST
+      - SCUBA_PORT
+      - SCUBA_HEALTHCHECK_FREQUENCY
+      - S3QUOTA
+      - QUOTA_ENABLE_INFLIGHTS
+    env_file:
+      - creds.env
+    depends_on:
+      - redis
+    extra_hosts:
+      - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
+      - "pykmip.local:127.0.0.1"
+  redis:
+    image: redis:alpine
+    network_mode: "host"
+  squid:
+    network_mode: "host"
+    profiles: ['ci-proxy']
+    image: scality/ci-squid
+    command: >-
+      sh -c 'mkdir -p /ssl &&
+            openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
+              -subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
+              -keyout /ssl/myca.pem  -out /ssl/myca.pem &&
+            cp /ssl/myca.pem /ssl/CA.pem &&
+            squid -f /etc/squid/squid.conf -N -z &&
+            squid -f /etc/squid/squid.conf -NYCd 1'
+    volumes:
+      - /tmp/ssl:/ssl
+  pykmip:
+    network_mode: "host"
+    profiles: ['pykmip']
+    image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
+    volumes:
+      - /tmp/artifacts/${JOB_NAME}:/artifacts
+  mongo:
+    network_mode: "host"
+    profiles: ['mongo', 'ceph']
+    image: ${MONGODB_IMAGE}
+  ceph: 
+    network_mode: "host"
+    profiles: ['ceph']
+    image: ghcr.io/scality/cloudserver/ci-ceph 
+  sproxyd:
+    network_mode: "host"
+    profiles: ['sproxyd']
+    image: sproxyd-standalone
+    build: ./sproxyd
+    user: 0:0
+    privileged: yes

.github/docker/mongodb/Dockerfile

@@ -0,0 +1,28 @@
+FROM mongo:5.0.21
+
+ENV USER=scality \
+    HOME_DIR=/home/scality \
+    CONF_DIR=/conf \
+    DATA_DIR=/data
+
+# Set up directories and permissions
+RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
+    mkdir /logs; \
+    adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
+
+# Set up environment variables and directories for scality user
+RUN mkdir ${CONF_DIR} && \
+    chown -R ${USER} ${CONF_DIR} && \
+    chown -R ${USER} ${DATA_DIR}
+
+# copy the mongo config file
+COPY /conf/mongod.conf /conf/mongod.conf
+COPY /conf/mongo-run.sh /conf/mongo-run.sh
+COPY /conf/initReplicaSet /conf/initReplicaSet.js
+
+EXPOSE 27017/tcp
+EXPOSE 27018
+
+# Set up CMD
+ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
+CMD ["bash", "/conf/mongo-run.sh"]

.github/docker/mongodb/conf/initReplicaSet

@@ -0,0 +1,4 @@
+rs.initiate({
+    _id: "rs0",
+    members: [{ _id: 0, host: "127.0.0.1:27018" }]
+});

.github/docker/mongodb/conf/mongo-run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -exo pipefail
+
+init_RS() {
+  sleep 5
+  mongo --port 27018 /conf/initReplicaSet.js
+}
+init_RS &
+
+mongod --bind_ip_all --config=/conf/mongod.conf

.github/docker/mongodb/conf/mongod.conf

@@ -0,0 +1,15 @@
+storage:
+   journal:
+      enabled: true
+   engine: wiredTiger
+   dbPath: "/data/db"
+processManagement:
+   fork: false
+net:
+   port: 27018
+   bindIp: 0.0.0.0
+replication:
+   replSetName: "rs0"
+   enableMajorityReadConcern: true
+security:
+    authorization: disabled

.github/docker/sproxyd/Dockerfile

@@ -0,0 +1,3 @@
+FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
+ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
+RUN chown root:root /conf/sproxyd0.conf

.github/docker/sproxyd/conf/fastcgi_params

@@ -0,0 +1,26 @@
+fastcgi_param  QUERY_STRING       $query_string;
+fastcgi_param  REQUEST_METHOD     $request_method;
+fastcgi_param  CONTENT_TYPE       $content_type;
+fastcgi_param  CONTENT_LENGTH     $content_length;
+
+#fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;
+fastcgi_param  SCRIPT_NAME        /var/www;
+fastcgi_param  PATH_INFO          $document_uri;
+
+fastcgi_param  REQUEST_URI        $request_uri;
+fastcgi_param  DOCUMENT_URI       $document_uri;
+fastcgi_param  DOCUMENT_ROOT      $document_root;
+fastcgi_param  SERVER_PROTOCOL    $server_protocol;
+fastcgi_param  HTTPS              $https if_not_empty;
+
+fastcgi_param  GATEWAY_INTERFACE  CGI/1.1;
+fastcgi_param  SERVER_SOFTWARE    nginx/$nginx_version;
+
+fastcgi_param  REMOTE_ADDR        $remote_addr;
+fastcgi_param  REMOTE_PORT        $remote_port;
+fastcgi_param  SERVER_ADDR        $server_addr;
+fastcgi_param  SERVER_PORT        $server_port;
+fastcgi_param  SERVER_NAME        $server_name;
+
+# PHP only, required if PHP was built with --enable-force-cgi-redirect
+fastcgi_param  REDIRECT_STATUS    200;

.github/docker/sproxyd/conf/nginx.conf

@@ -0,0 +1,88 @@
+worker_processes 1;
+error_log  /logs/error.log;
+user root root;
+events {
+    worker_connections  1000;
+    reuse_port on;
+    multi_accept on;
+}
+worker_rlimit_nofile    20000;
+http {
+    root /var/www/;
+    upstream sproxyds {
+        least_conn;
+        keepalive 40;
+        server 127.0.0.1:20000;
+    }
+    server {
+        client_max_body_size 0;
+        client_body_timeout 150;
+        client_header_timeout 150;
+        postpone_output 0;
+        client_body_postpone_size 0;
+        keepalive_requests 1100;
+        keepalive_timeout 300s;
+        server_tokens off;
+        default_type application/octet-stream;
+        gzip off;
+        tcp_nodelay on;
+        tcp_nopush  on;
+        sendfile on;
+        listen 81;
+        server_name localhost;
+        rewrite ^/arc/(.*)$ /dc1/$1 permanent;
+        location ~* ^/proxy/(.*)$ {
+            rewrite ^/proxy/(.*)$ /$1 last;
+        }
+        allow 127.0.0.1;
+
+        deny all;
+        set $usermd '-';
+        set $sentusermd '-';
+    	set $elapsed_ms '-';
+    	set $now '-';
+    	log_by_lua '
+    		if not(ngx.var.http_x_scal_usermd == nil) and  string.len(ngx.var.http_x_scal_usermd) > 2 then
+    			ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
+    		end
+                    if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
+                            ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
+                    end
+    		local elapsed_ms = tonumber(ngx.var.request_time)
+    		if not (  elapsed_ms == nil) then
+    			elapsed_ms = elapsed_ms * 1000
+    			ngx.var.elapsed_ms = tostring(elapsed_ms)
+    		end
+    		local time = tonumber(ngx.var.msec) * 1000
+    		ngx.var.now = time
+    	';
+        log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
+                       '"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
+                       '"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
+		       '"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
+		       '"contentType":"$content_type","s3Address":"$remote_addr",'
+		       '"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
+                       '"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
+		       '"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
+		       '"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
+                       '"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
+                       '"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
+        access_log /dev/stdout irm;
+    	error_log  /dev/stdout error;
+        location / {
+            proxy_request_buffering off;
+            fastcgi_request_buffering off;
+            fastcgi_no_cache 1;
+            fastcgi_cache_bypass 1;
+            fastcgi_buffering off;
+            fastcgi_ignore_client_abort on;
+            fastcgi_keep_conn on;
+            include fastcgi_params;
+            fastcgi_pass sproxyds;
+            fastcgi_next_upstream error timeout;
+            fastcgi_send_timeout 285s;
+            fastcgi_read_timeout 285s;
+        }
+    }
+}
+

.github/docker/sproxyd/conf/sproxyd0.conf

@@ -0,0 +1,12 @@
+{
+    "general": {
+        "ring": "DATA",
+        "port": 20000,
+        "syslog_facility": "local0"
+    },
+    "ring_driver:0": {
+        "alias": "dc1",
+        "type": "local",
+        "queue_path": "/tmp/ring-objs"
+    },
+}

.github/docker/sproxyd/conf/supervisord.conf

@@ -0,0 +1,43 @@
+[supervisord]
+nodaemon = true
+loglevel = info
+logfile  = %(ENV_LOG_DIR)s/supervisord.log
+pidfile  = %(ENV_SUP_RUN_DIR)s/supervisord.pid
+logfile_maxbytes = 20MB
+logfile_backups = 2
+
+[unix_http_server]
+file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[program:nginx]
+directory=%(ENV_SUP_RUN_DIR)s
+command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart=true
+autostart=true
+user=root
+
+[program:sproxyd]
+directory=%(ENV_SUP_RUN_DIR)s
+process_name=%(program_name)s-%(process_num)s
+numprocs=1
+numprocs_start=0
+command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+redirect_stderr=true
+autorestart=true
+autostart=true
+user=root

.github/pykmip/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3-alpine
+FROM python:3.10-alpine
 
 RUN apk add --no-cache \
         libressl && \
@@ -8,9 +8,13 @@ RUN apk add --no-cache \
         libressl-dev \
         sqlite-dev \
         build-base \
-        rust \
+        curl
-        cargo && \
+
-    pip install pykmip requests && \
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN pip3 install -U pip && \
+    pip3 install pykmip requests && \
     apk del .build-deps && \
     mkdir /pykmip
 

.github/pykmip/server.conf

@@ -4,7 +4,7 @@ port=5696
 certificate_path=/ssl/kmip-cert.pem
 key_path=/ssl/kmip-key.pem
 ca_path=/ssl/kmip-ca.pem
-auth_suite=Basic
+auth_suite=TLS1.2
 policy_path=/etc/pykmip/policies
 enable_tls_client_auth=True
 database_path=/pykmip/pykmip.db

.github/scripts/credentials.bash

@@ -2,9 +2,9 @@
 set -x #echo on
 set -e #exit at the first error
 
-mkdir -p ~/.aws
+mkdir -p $HOME/.aws
 
-cat >>/root/.aws/credentials <<EOF
+cat >>$HOME/.aws/credentials <<EOF
 [default]
 aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
 aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY

.github/workflows/alerts.yaml

@@ -0,0 +1,35 @@
+name: Test alerts
+
+on:
+  push:
+    branches-ignore:
+      - 'development/**'
+      - 'q/*/**'
+
+jobs:
+  run-alert-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tests:
+          - name: 1 minute interval tests
+            file: monitoring/alerts.test.yaml
+          
+          - name: 10 seconds interval tests
+            file: monitoring/alerts.10s.test.yaml
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Render and test ${{ matrix.tests.name }}
+        uses: scality/action-prom-render-test@1.0.3
+        with:
+          alert_file_path: monitoring/alerts.yaml
+          test_file_path: ${{ matrix.tests.file }}
+          alert_inputs: |
+            namespace=zenko
+            service=artesca-data-connector-s3api-metrics
+            reportJob=artesca-data-ops-report-handler
+            replicas=3
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql.yaml

@@ -0,0 +1,25 @@
+---
+name: codeQL
+
+on:
+  push:
+    branches: [w/**, q/*]
+  pull_request:
+    branches: [development/*, stabilization/*, hotfix/*]
+  workflow_dispatch:
+
+jobs:
+  analyze:
+    name: Static analysis with CodeQL
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: javascript, python, ruby
+
+      - name: Build and analyze
+        uses: github/codeql-action/analyze@v3

.github/workflows/dependency-review.yaml

@@ -0,0 +1,16 @@
+---
+name: dependency review
+
+on:
+  pull_request:
+    branches: [development/*, stabilization/*, hotfix/*]
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v4
+
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v4

.github/workflows/release.yaml

@@ -0,0 +1,80 @@
+---
+name: release
+run-name: release ${{ inputs.tag }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to be released'
+        required: true
+
+env:
+  PROJECT_NAME: ${{ github.event.repository.name }}
+
+jobs:
+  build-federation-image:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push image for federation
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          file: images/svc-base/Dockerfile
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
+          cache-from: type=gha,scope=federation
+          cache-to: type=gha,mode=max,scope=federation
+
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildk
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Push dashboards into the production namespace
+        run: |
+          oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
+            dashboard.json:application/grafana-dashboard+json \
+            alerts.yaml:application/prometheus-alerts+yaml
+        working-directory: monitoring
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Create Release
+        uses: softprops/action-gh-release@v2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          name: Release ${{ github.event.inputs.tag }}
+          tag_name: ${{ github.event.inputs.tag }}
+          generate_release_notes: true
+          target_commitish: ${{ github.sha }}

.github/workflows/tests.yaml

@@ -0,0 +1,533 @@
+---
+name: tests
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches-ignore:
+      - 'development/**'
+      - 'q/*/**'
+
+env:
+  # Secrets
+  azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
+  azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
+  azurebackend_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
+  azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
+  azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
+  azurebackend2_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
+  azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
+  azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
+  azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
+  azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
+  azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
+    ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
+  azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
+    ${{ secrets.AZURE_STORAGE_ENDPOINT }}
+  azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
+  b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
+  b2backend_B2_STORAGE_ACCESS_KEY: >-
+    ${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
+  GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
+  GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
+  AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
+  AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
+  AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
+  AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
+  AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
+  AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
+  AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
+  AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
+  b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
+  gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
+  gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
+  gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
+  gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
+  gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
+    ${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
+  gcpbackendmismatch_GCP_SERVICE_KEY: >-
+    ${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
+  gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
+  # Configs
+  ENABLE_LOCAL_CACHE: "true"
+  REPORT_TOKEN: "report-token-1"
+  REMOTE_MANAGEMENT_DISABLE: "1"
+  # https://github.com/git-lfs/git-lfs/issues/5749
+  GIT_CLONE_PROTECTION_ACTIVE: 'false'
+jobs:
+  linting-coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '16'
+        cache: yarn
+    - name: install dependencies
+      run: yarn install --frozen-lockfile --network-concurrency 1
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip
+    - name: Install python deps
+      run: pip install flake8
+    - name: Lint Javascript
+      run: yarn run --silent lint -- --max-warnings 0
+    - name: Lint Markdown
+      run: yarn run --silent lint_md
+    - name: Lint python
+      run: flake8 $(git ls-files "*.py")
+    - name: Lint Yaml
+      run: yamllint -c yamllint.yml $(git ls-files "*.yml")
+    - name: Unit Coverage
+      run: |
+        set -ex
+        mkdir -p $CIRCLE_TEST_REPORTS/unit
+        yarn test
+        yarn run test_legacy_location
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+        CIRCLE_TEST_REPORTS: /tmp
+        CIRCLE_ARTIFACTS: /tmp
+        CI_REPORTS: /tmp
+    - name: Unit Coverage logs
+      run: find /tmp/unit -exec cat {} \;
+    - name: preparing junit files for upload
+      run: |
+        mkdir -p artifacts/junit
+        find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
+      if: always()
+    - name: Upload files to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: artifacts
+      if: always()
+
+  build:
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Build and push cloudserver image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          provenance: false
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.sha }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=cloudserver
+          cache-to: type=gha,mode=max,scope=cloudserver
+      - name: Build and push pykmip image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .github/pykmip
+          tags: |
+            ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=pykmip
+          cache-to: type=gha,mode=max,scope=pykmip
+      - name: Build and push MongoDB
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .github/docker/mongodb
+          tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+          cache-from: type=gha,scope=mongodb
+          cache-to: type=gha,mode=max,scope=mongodb
+
+  multiple-backend:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      S3BACKEND: mem
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      S3DATA: multiple
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Login to Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile sproxyd up -d
+      working-directory: .github/docker
+    - name: Run multiple backend test
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        bash wait_for_local_port.bash 81 40
+        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+ 
+  mongo-v0-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v0
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  mongo-v1-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      MPU_TESTING: "yes" 
+      S3METADATA: mongodb
+      S3KMS: file
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      DEFAULT_BUCKET_KEY_FORMAT: v1
+      METADATA_MAX_CACHED_BUCKETS: 1
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run functional tests
+      run: |-
+        set -o pipefail;        
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
+        yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  file-ft-tests:
+    strategy:
+      matrix:
+        include:
+          - job-name: file-ft-tests
+    name: ${{ matrix.job-name }}
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: file
+      S3VAULT: mem
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      MPU_TESTING: "yes"
+      JOB_NAME: ${{ matrix.job-name }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup matrix job artifacts directory
+      shell: bash
+      run: |
+        set -exu
+        mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
+    - name: Setup CI services
+      run: docker compose up -d
+      working-directory: .github/docker
+    - name: Run file ft tests
+      run: |-
+        set -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  utapi-v2-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      ENABLE_UTAPI_V2: t
+      S3BACKEND: mem
+      BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose up -d
+      working-directory: .github/docker
+    - name: Run file utapi v2 tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  quota-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        inflights:
+          - name: "With Inflights"
+            value: "true"
+          - name: "Without Inflights"
+            value: "false"
+    env:
+      S3METADATA: mongodb
+      S3BACKEND: mem
+      S3QUOTA: scuba
+      QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
+      SCUBA_HOST: localhost
+      SCUBA_PORT: 8100
+      SCUBA_HEALTHCHECK_FREQUENCY: 100
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Setup CI services
+      run: docker compose --profile mongo up -d
+      working-directory: .github/docker
+    - name: Run quota tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  kmip-ft-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: file
+      S3VAULT: mem
+      MPU_TESTING: "yes" 
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - name: Copy KMIP certs
+      run: cp -r ./certs /tmp/ssl-kmip
+      working-directory: .github/pykmip
+    - name: Setup CI services
+      run: docker compose --profile pykmip up -d
+      working-directory: .github/docker
+    - name: Run file KMIP tests
+      run: |-
+        set -ex -o pipefail;
+        bash wait_for_local_port.bash 8000 40
+        bash wait_for_local_port.bash 5696 40
+        yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()
+
+  ceph-backend-test:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      S3BACKEND: mem
+      S3DATA: multiple
+      S3KMS: file
+      CI_CEPH: 'true'
+      MPU_TESTING: "yes" 
+      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
+      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
+      JOB_NAME: ${{ github.job }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Login to GitHub Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+    - name: Setup CI environment
+      uses: ./.github/actions/setup-ci
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '2.5.9'
+    - name: Install Ruby dependencies
+      run: |
+        gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
+    - name: Install Java dependencies
+      run: |
+        sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
+    - name: Setup CI services
+      run: docker compose --profile ceph up -d
+      working-directory: .github/docker
+      env: 
+        S3METADATA: mongodb
+    - name: Run Ceph multiple backend tests
+      run: |-
+        set -ex -o pipefail;
+        bash .github/ceph/wait_for_ceph.sh
+        bash wait_for_local_port.bash 27018 40
+        bash wait_for_local_port.bash 8000 40
+        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+        S3METADATA: mem
+    - name: Run Java tests
+      run: |-
+        set -ex -o pipefail;
+        mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
+      working-directory: tests/functional/jaws
+    - name: Run Ruby tests
+      run: |-
+        set -ex -o pipefail;
+        rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
+      working-directory: tests/functional/fog
+    - name: Run Javascript AWS SDK tests
+      run: |-
+        set -ex -o pipefail;
+        yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
+        yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
+      env:
+        S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
+        S3BACKEND: file
+        S3VAULT: mem
+        S3METADATA: mongodb
+    - name: Upload logs to artifacts
+      uses: scality/action-artifacts@v4
+      with:
+        method: upload
+        url: https://artifacts.scality.net
+        user: ${{ secrets.ARTIFACTS_USER }}
+        password: ${{ secrets.ARTIFACTS_PASSWORD }}
+        source: /tmp/artifacts
+      if: always()

Dockerfile

@@ -1,40 +1,60 @@
-FROM node:10.22.0-slim
+ARG NODE_VERSION=16.20-bullseye-slim
-MAINTAINER Giorgio Regni <gr@scality.com>
+
+FROM node:${NODE_VERSION} as builder
+
+WORKDIR /usr/src/app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        curl \
+        git \
+        gnupg2 \
+        jq \
+        python3 \
+        ssh \
+        wget \
+        libffi-dev \
+        zlib1g-dev \
+    && apt-get clean \
+    && mkdir -p /root/ssh \
+    && ssh-keyscan -H github.com > /root/ssh/known_hosts
+
+ENV PYTHON=python3
+COPY package.json yarn.lock /usr/src/app/
+RUN npm install typescript -g
+RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
+
+################################################################################
+FROM node:${NODE_VERSION}
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        jq \
+    && rm -rf /var/lib/apt/lists/*
 
 ENV NO_PROXY localhost,127.0.0.1
 ENV no_proxy localhost,127.0.0.1
 
 EXPOSE 8000
+EXPOSE 8002
 
-COPY ./package.json /usr/src/app/
+RUN apt-get update && \
-COPY ./yarn.lock /usr/src/app/
+    apt-get install -y --no-install-recommends \
+        jq \
+        tini \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /usr/src/app
 
-RUN apt-get update \
+# Keep the .git directory in order to properly report version
-    && apt-get install -y \
-    curl \
-    gnupg2
-
-RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
-
-RUN apt-get update \
-    && apt-get install -y jq python git build-essential ssh --no-install-recommends yarn \
-    && mkdir -p /root/ssh \
-    && ssh-keyscan -H github.com > /root/ssh/known_hosts \
-    && yarn cache clean \
-    && yarn install --frozen-lockfile --production --ignore-optional \
-    && apt-get autoremove --purge -y python git build-essential \
-    && rm -rf /var/lib/apt/lists/* \
-    && yarn cache clean \
-    && rm -rf ~/.node-gyp \
-    && rm -rf /tmp/yarn-*
-
 COPY . /usr/src/app
+COPY --from=builder /usr/src/app/node_modules ./node_modules/
+
 
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
-ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
+ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
 
 CMD [ "yarn", "start" ]

Healthchecks.md

@@ -1,6 +1,7 @@
 # S3 Healthcheck
 
-Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
+Scality S3 exposes a healthcheck route `/live` on the port used
+for the metrics (defaults to port 8002) which returns a
 response with HTTP code
 
 - 200 OK

README.md

@@ -1,10 +1,7 @@
-# Zenko CloudServer
+# Zenko CloudServer with Vitastor Backend
 
 ![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
 
-[![Docker Pulls][badgedocker]](https://hub.docker.com/r/zenko/cloudserver)
-[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
-
 ## Overview
 
 CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
@@ -14,137 +11,71 @@ Scality’s Open Source Multi-Cloud Data Controller.
 CloudServer provides a single AWS S3 API interface to access multiple
 backend data storage both on-premise or public in the cloud.
 
-CloudServer is useful for Developers, either to run as part of a
+This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
-continous integration test environment to emulate the AWS S3 service locally
+backend support.
-or as an abstraction layer to develop object storage enabled
-application on the go.
 
-## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
+## Quick Start with Vitastor
 
-## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
+Vitastor Backend is in experimental status, however you can already try to
+run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
+it works too 😊.
 
-## Docker
+Installation instructions:
 
-[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/zenko/cloudserver/)
+### Install Vitastor
 
-## Contributing
+Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
 
-In order to contribute, please follow the
+### Install Zenko with Vitastor Backend
-[Contributing Guidelines](
-https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
 
-## Installation
+- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
+- Install dependencies: `npm install --omit dev` or just `npm install`
+- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
+- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
+  You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
+- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
 
-### Dependencies
+### Install and Configure MongoDB
 
-Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
+Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
-. Up-to-date versions can be found at
-[Nodesource](https://github.com/nodesource/distributions).
 
-### Clone source code
+### Setup Zenko
 
-```shell
+- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
-git clone https://github.com/scality/S3.git
+- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
+- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
+- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
+- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
+  and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
+  access keys, but it's not published, so let's use a file for now.
+- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
+  You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
+  in this file.
+
+Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
+instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
+
+### Start Zenko
+
+Start the S3 server with: `node index.js`
+
+If you use default settings, Zenko CloudServer starts on port 8000.
+The default access key is `accessKey1` with a secret key of `verySecretKey1`.
+
+Now you can access your S3 with `s3cmd` or `geesefs`:
+
+```
+s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
 ```
 
-### Install js dependencies
+```
-
+AWS_ACCESS_KEY_ID=accessKey1 \
-Go to the ./S3 folder,
+    AWS_SECRET_ACCESS_KEY=verySecretKey1 \
-
+    geesefs --endpoint http://localhost:8000 testbucket mountdir
-```shell
-yarn install --frozen-lockfile
 ```
 
-If you get an error regarding installation of the diskUsage module,
+# Author & License
-please install g++.
 
-If you get an error regarding level-down bindings, try clearing your yarn cache:
+- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
-
+- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
-```shell
+  (a "network copyleft" license based on AGPL/SSPL, but worded in a better way)
-yarn cache clean
-```
-
-## Run it with a file backend
-
-```shell
-yarn start
-```
-
-This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
-9991 are also open locally for internal transfer of metadata and data,
-respectively.
-
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-By default the metadata files will be saved in the
-localMetadata directory and the data files will be saved
-in the localData directory within the ./S3 directory on your
-machine.  These directories have been pre-created within the
-repository.  If you would like to save the data or metadata in
-different locations of your choice, you must specify them with absolute paths.
-So, when starting the server:
-
-```shell
-mkdir -m 700 $(pwd)/myFavoriteDataPath
-mkdir -m 700 $(pwd)/myFavoriteMetadataPath
-export S3DATAPATH="$(pwd)/myFavoriteDataPath"
-export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
-yarn start
-```
-
-## Run it with multiple data backends
-
-```shell
-export S3DATA='multiple'
-yarn start
-```
-
-This starts a Zenko CloudServer on port 8000.
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-With multiple backends, you have the ability to
-choose where each object will be saved by setting
-the following header with a locationConstraint on
-a PUT request:
-
-```shell
-'x-amz-meta-scal-location-constraint':'myLocationConstraint'
-```
-
-If no header is sent with a PUT object request, the
-location constraint of the bucket will determine
-where the data is saved. If the bucket has no location
-constraint, the endpoint of the PUT request will be
-used to determine location.
-
-See the Configuration section in our documentation
-[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
-to learn how to set location constraints.
-
-## Run it with an in-memory backend
-
-```shell
-yarn run mem_backend
-```
-
-This starts a Zenko CloudServer on port 8000.
-The default access key is accessKey1 with
-a secret key of verySecretKey1.
-
-## Run it with Vault user management
-
-Note: Vault is proprietary and must be accessed separately.
-
-```shell
-export S3VAULT=vault
-yarn start
-```
-
-This starts a Zenko CloudServer using Vault for user management.
-
-[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
-[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
-[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
-[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae

_config.yml

@@ -1,2 +1,2 @@
 ---
-theme: jekyll-theme-minimal
+theme: jekyll-theme-modernist

bin/metrics_server.js

@@ -1,46 +0,0 @@
-#!/usr/bin/env node
-'use strict'; // eslint-disable-line strict
-
-const {
-    startWSManagementClient,
-    startPushConnectionHealthCheckServer,
-} = require('../lib/management/push');
-
-const logger = require('../lib/utilities/logger');
-
-const {
-    PUSH_ENDPOINT: pushEndpoint,
-    INSTANCE_ID: instanceId,
-    MANAGEMENT_TOKEN: managementToken,
-} = process.env;
-
-if (!pushEndpoint) {
-    logger.error('missing push endpoint env var');
-    process.exit(1);
-}
-
-if (!instanceId) {
-    logger.error('missing instance id env var');
-    process.exit(1);
-}
-
-if (!managementToken) {
-    logger.error('missing management token env var');
-    process.exit(1);
-}
-
-startPushConnectionHealthCheckServer(err => {
-    if (err) {
-        logger.error('could not start healthcheck server', { error: err });
-        process.exit(1);
-    }
-    const url = `${pushEndpoint}/${instanceId}/ws?metrics=1`;
-    startWSManagementClient(url, managementToken, err => {
-        if (err) {
-            logger.error('connection failed, exiting', { error: err });
-            process.exit(1);
-        }
-        logger.info('no more connection, exiting');
-        process.exit(0);
-    });
-});

bin/search_bucket.js

@@ -13,20 +13,26 @@ function _performSearch(host,
                         port,
                         bucketName,
                         query,
+                        listVersions,
                         accessKey,
                         secretKey,
+                        sessionToken,
                         verbose, ssl) {
     const escapedSearch = encodeURIComponent(query);
     const options = {
         host,
         port,
         method: 'GET',
-        path: `/${bucketName}/?search=${escapedSearch}`,
+        path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`,
         headers: {
             'Content-Length': 0,
         },
         rejectUnauthorized: false,
+        versions: '',
     };
+    if (sessionToken) {
+        options.headers['x-amz-security-token'] = sessionToken;
+    }
     const transport = ssl ? https : http;
     const request = transport.request(options, response => {
         if (verbose) {
@@ -55,9 +61,9 @@ function _performSearch(host,
     // generateV4Headers exepects request object with path that does not
     // include query
     request.path = `/${bucketName}`;
-    auth.client.generateV4Headers(request, { search: query },
+    const requestData = listVersions ? { search: query, versions: '' } : { search: query };
-        accessKey, secretKey, 's3');
+    auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3');
-    request.path = `/${bucketName}?search=${escapedSearch}`;
+    request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`;
     if (verbose) {
         logger.info('request headers', { headers: request._headers });
     }
@@ -76,15 +82,17 @@ function searchBucket() {
         .version('0.0.1')
         .option('-a, --access-key <accessKey>', 'Access key id')
         .option('-k, --secret-key <secretKey>', 'Secret access key')
+        .option('-t, --session-token <sessionToken>', 'Session token')
         .option('-b, --bucket <bucket>', 'Name of the bucket')
         .option('-q, --query <query>', 'Search query')
         .option('-h, --host <host>', 'Host of the server')
         .option('-p, --port <port>', 'Port of the server')
         .option('-s', '--ssl', 'Enable ssl')
+        .option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
+            'otherwise only list the latest version')
         .option('-v, --verbose')
         .parse(process.argv);
-
+    const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
-    const { host, port, accessKey, secretKey, bucket, query, verbose, ssl } =
         commander;
 
     if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
@@ -93,7 +101,7 @@ function searchBucket() {
         process.exit(1);
     }
 
-    _performSearch(host, port, bucket, query, accessKey, secretKey, verbose,
+    _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose,
         ssl);
 }
 

bin/secure_channel_proxy.js

@@ -1,46 +0,0 @@
-#!/usr/bin/env node
-'use strict'; // eslint-disable-line strict
-
-const {
-    startWSManagementClient,
-    startPushConnectionHealthCheckServer,
-} = require('../lib/management/push');
-
-const logger = require('../lib/utilities/logger');
-
-const {
-    PUSH_ENDPOINT: pushEndpoint,
-    INSTANCE_ID: instanceId,
-    MANAGEMENT_TOKEN: managementToken,
-} = process.env;
-
-if (!pushEndpoint) {
-    logger.error('missing push endpoint env var');
-    process.exit(1);
-}
-
-if (!instanceId) {
-    logger.error('missing instance id env var');
-    process.exit(1);
-}
-
-if (!managementToken) {
-    logger.error('missing management token env var');
-    process.exit(1);
-}
-
-startPushConnectionHealthCheckServer(err => {
-    if (err) {
-        logger.error('could not start healthcheck server', { error: err });
-        process.exit(1);
-    }
-    const url = `${pushEndpoint}/${instanceId}/ws?proxy=1`;
-    startWSManagementClient(url, managementToken, err => {
-        if (err) {
-            logger.error('connection failed, exiting', { error: err });
-            process.exit(1);
-        }
-        logger.info('no more connection, exiting');
-        process.exit(0);
-    });
-});

config.json.example

@@ -1,7 +1,10 @@
 {
     "port": 8000,
     "listenOn": [],
+    "metricsPort": 8002,
+    "metricsListenOn": [],
     "replicationGroupId": "RG001",
+    "workers": 4,
     "restEndpoints": {
         "localhost": "us-east-1",
         "127.0.0.1": "us-east-1",
@@ -99,6 +102,14 @@
         "readPreference": "primary",
         "database": "metadata"
     },
+    "authdata": "authdata.json",
+    "backends": {
+        "auth": "file",
+        "data": "file",
+        "metadata": "mongodb",
+        "kms": "file",
+        "quota": "none"
+    },
     "externalBackends": {
         "aws_s3": {
             "httpAgent": {

config.json.vitastor

@@ -0,0 +1,71 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "metricsPort": 8002,
+    "metricsListenOn": [],
+    "replicationGroupId": "RG001",
+    "restEndpoints": {
+        "localhost": "STANDARD",
+        "127.0.0.1": "STANDARD",
+        "yourhostname.ru": "STANDARD"
+    },
+    "websiteEndpoints": [
+        "static.yourhostname.ru"
+    ],
+    "replicationEndpoints": [ {
+        "site": "zenko",
+        "servers": ["127.0.0.1:8000"],
+        "default": true
+    } ],
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "backends": {
+        "metadata": "mongodb"
+    },
+    "mongodb": {
+        "replicaSetHosts": "127.0.0.1:27017",
+        "writeConcern": "majority",
+        "replicaSet": "rs0",
+        "readPreference": "primary",
+        "database": "s3",
+        "authCredentials": {
+            "username": "s3",
+            "password": ""
+        }
+    },
+    "externalBackends": {
+        "aws_s3": {
+            "httpAgent": {
+                "keepAlive": false,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        },
+        "gcp": {
+            "httpAgent": {
+                "keepAlive": true,
+                "keepAliveMsecs": 1000,
+                "maxFreeSockets": 256,
+                "maxSockets": null
+            }
+        }
+    },
+    "requests": {
+        "viaProxy": false,
+        "trustedProxyCIDRs": [],
+        "extractClientIPFromHeader": ""
+    },
+    "bucketNotificationDestinations": [
+        {
+            "resource": "target1",
+            "type": "dummy",
+            "host": "localhost:6000"
+        }
+    ]
+}

constants.js

@@ -86,6 +86,9 @@ const constants = {
     // In testing, AWS seems to allow up to 88 more bytes, so we do the same.
     maximumMetaHeadersSize: 2136,
 
+    // Maximum HTTP headers size allowed
+    maxHttpHeadersSize: 14122,
+
     // hex digest of sha256 hash of empty string:
     emptyStringHash: crypto.createHash('sha256')
         .update('', 'binary').digest('hex'),
@@ -95,28 +98,26 @@ const constants = {
     unsupportedQueries: [
         'accelerate',
         'analytics',
-        'encryption',
         'inventory',
         'logging',
         'metrics',
         'policyStatus',
         'publicAccessBlock',
         'requestPayment',
-        'restore',
         'torrent',
     ],
 
     // Headers supported by AWS that we do not currently support.
     unsupportedHeaders: [
         'x-amz-server-side-encryption-customer-algorithm',
-        'x-amz-server-side-encryption-aws-kms-key-id',
         'x-amz-server-side-encryption-context',
         'x-amz-server-side-encryption-customer-key',
         'x-amz-server-side-encryption-customer-key-md5',
     ],
 
     // user metadata header to set object locationConstraint
-    objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
+    objectLocationConstraintHeader: 'x-amz-storage-class',
+    lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
     legacyLocations: ['sproxyd', 'legacy'],
     // declare here all existing service accounts and their properties
     // (if any, otherwise an empty object)
@@ -129,7 +130,7 @@ const constants = {
         },
     },
     /* eslint-disable camelcase */
-    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true },
+    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
     // some of the available data backends  (if called directly rather
     // than through the multiple backend gateway) need a key provided
     // as a string as first parameter of the get/delete methods.
@@ -175,6 +176,8 @@ const constants = {
         'objectDeleteTagging',
         'objectGetTagging',
         'objectPutTagging',
+        'objectPutLegalHold',
+        'objectPutRetention',
     ],
     // response header to be sent when there are invalid
     // user metadata in the object's metadata
@@ -182,13 +185,64 @@ const constants = {
     // Bucket specific queries supported by AWS that we do not currently support
     // these queries may or may not be supported at object level
     unsupportedBucketQueries: [
-        'tagging',
     ],
     suppressedUtapiEventFields: [
         'object',
         'location',
         'versionId',
     ],
+    allowedUtapiEventFilterFields: [
+        'operationId',
+        'location',
+        'account',
+        'user',
+        'bucket',
+    ],
+    arrayOfAllowed: [
+        'objectPutTagging',
+        'objectPutLegalHold',
+        'objectPutRetention',
+    ],
+    allowedUtapiEventFilterStates: ['allow', 'deny'],
+    allowedRestoreObjectRequestTierValues: ['Standard'],
+    lifecycleListing: {
+        CURRENT_TYPE: 'current',
+        NON_CURRENT_TYPE: 'noncurrent',
+        ORPHAN_DM_TYPE: 'orphan',
+    },
+    multiObjectDeleteConcurrency: 50,
+    maxScannedLifecycleListingEntries: 10000,
+    overheadField: [
+        'content-length',
+        'owner-id',
+        'versionId',
+        'isNull',
+        'isDeleteMarker',
+    ],
+    unsupportedSignatureChecksums: new Set([
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
+        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
+        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
+        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
+    ]),
+    supportedSignatureChecksums: new Set([
+        'UNSIGNED-PAYLOAD',
+        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
+    ]),
+    ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
+    ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
+    // The AWS assumed Role resource type
+    assumedRoleArnResourceType: 'assumed-role',
+    // Session name of the backbeat lifecycle assumed role session.
+    backbeatLifecycleSessionName: 'backbeat-lifecycle',
+    actionsToConsiderAsObjectPut: [
+        'initiateMultipartUpload',
+        'objectPutPart',
+        'completeMultipartUpload',
+    ],
+    // if requester is not bucket owner, bucket policy actions should be denied with
+    // MethodNotAllowed error
+    onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
 };
 
 module.exports = constants;

docker-entrypoint.sh

@@ -195,6 +195,14 @@ if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
     JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
 fi
 
+if [[ -n "$BUCKET_DENY_FILTER" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
+fi
+
+if [[ "$TESTING_MODE" ]]; then
+    JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
+fi
+
 if [[ $JQ_FILTERS_CONFIG != "." ]]; then
     jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
     mv config.json.tmp config.json

docs/CLIENTS.rst

@@ -295,3 +295,51 @@ Should force path-style requests even though v3 advertises it does by default.
     $client->createBucket(array(
         'Bucket' => 'bucketphp',
     ));
+
+Go
+~~
+
+`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: go
+
+   package main
+
+   import (
+      "context"
+      "fmt"
+      "log"
+      "os"
+      "time"
+
+      "github.com/aws/aws-sdk-go/aws"
+      "github.com/aws/aws-sdk-go/aws/endpoints"
+      "github.com/aws/aws-sdk-go/aws/session"
+      "github.com/aws/aws-sdk-go/service/s3"
+   )
+
+   func main() {
+      os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
+      os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
+      endpoint := "http://localhost:8000"
+      timeout := time.Duration(10) * time.Second
+      sess := session.Must(session.NewSession())
+
+      // Create a context with a timeout that will abort the upload if it takes
+      // more than the passed in timeout.
+      ctx, cancel := context.WithTimeout(context.Background(), timeout)
+      defer cancel()
+
+      svc := s3.New(sess, &aws.Config{
+         Region:   aws.String(endpoints.UsEast1RegionID),
+         Endpoint: &endpoint,
+      })
+
+      out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
+      if err != nil {
+         log.Fatal(err)
+      } else {
+         fmt.Println(out)
+      }
+   }

docs/RELEASE.md

@@ -2,11 +2,12 @@
 
 ## Docker Image Generation
 
-Docker images are hosted on [registry.scality.com](registry.scality.com).
+Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
-CloudServer has two namespaces there:
+CloudServer has a few images there:
 
-* Production Namespace: registry.scality.com/cloudserver
+* Cloudserver container image: ghcr.io/scality/cloudserver
-* Dev Namespace: registry.scality.com/cloudserver-dev
+* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
+* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
 
 With every CI build, the CI will push images, tagging the
 content with the developer branch's short SHA-1 commit hash.
@@ -18,26 +19,55 @@ Tagged versions of cloudserver will be stored in the production namespace.
 ## How to Pull Docker Images
 
 ```sh
-docker pull registry.scality.com/cloudserver-dev/cloudserver:<commit hash>
+docker pull ghcr.io/scality/cloudserver:<commit hash>
-docker pull registry.scality.com/cloudserver/cloudserver:<tag>
+docker pull ghcr.io/scality/cloudserver:<tag>
 ```
 
 ## Release Process
 
 To release a production image:
 
-* Name the tag for the repository and Docker image.
+* Create a PR to bump the package version
+  Update Cloudserver's `package.json` by bumping it to the relevant next
+  version in a new PR. Per example if the last released version was
+  `8.4.7`, the next version would be `8.4.8`.
 
-* Use the `yarn version` command with the same tag to update `package.json`.
+```js
+{
+  "name": "cloudserver",
+  "version": "8.4.8", <--- Here
+  [...]
+}
+```
 
-* Create a PR and merge the `package.json` change.
+* Review & merge the PR
 
-* Tag the repository using the same tag.
+* Create the release on GitHub
   
-* [Force a build] using:
+  * Go the Release tab (https://github.com/scality/cloudserver/releases);
-  * A given branch that ideally matches the tag.
+  * Click on the `Draft new release button`;
-  * The `release` stage.
+  * In the `tag` field, type the name of the release (`8.4.8`), and confirm
-  * An extra property with the name `tag` and its value being the actual tag.
+    to create the tag on publish;
+  * Click on `Generate release notes` button to fill the fields;
+  * Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
+  * Click to `Publish the release` to create the GitHub release and git tag
 
-[Force a build]:
+  Notes:
-https://eve.devsca.com/github/scality/cloudserver/#/builders/bootstrap/force/force
+  * the Git tag will be created automatically.
+  * this should be done as soon as the PR is merged, so that the tag
+    is put on the "version bump" commit.
+
+* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
+
+  * Branch Name: The one used for the tag earlier. In this example `development/8.4`
+  * Override Stage: 'release'
+  * Extra properties:
+    * name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
+
+* Release the release version on Jira
+
+  * Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
+  * Create a next version
+    * Name: `[next version]`, in this example `8.4.9`
+  * Click `...` and select `Release` on the recently released version (`8.4.8`)
+  * Fill in the field to move incomplete version to the next one

eve/get_product_version.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-script_full_path=$(readlink -f "$0")
-file_dir=$(dirname "$script_full_path")/..
-
-PACKAGE_VERSION=$(cat $file_dir/package.json \
-  | grep version \
-  | head -1 \
-  | awk -F: '{ print $2 }' \
-  | sed 's/[",]//g' \
-  | tr -d '[[:space:]]')
-
-echo $PACKAGE_VERSION

eve/main.yml

@@ -1,529 +0,0 @@
----
-version: 0.2
-
-branches:
-  feature/*, documentation/*, improvement/*, bugfix/*, w/*, q/*, hotfix/*:
-    stage: pre-merge
-
-models:
-  - env: &global-env
-      azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
-        %(secret:azure_storage_access_key)s
-      azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
-        %(secret:azure_storage_account_name)s
-      azurebackend_AZURE_STORAGE_ENDPOINT: >-
-        %(secret:azure_storage_endpoint)s
-      azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
-        %(secret:azure_storage_access_key_2)s
-      azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
-        %(secret:azure_storage_account_name_2)s
-      azurebackend2_AZURE_STORAGE_ENDPOINT: >-
-        %(secret:azure_storage_endpoint_2)s
-      azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
-        %(secret:azure_storage_access_key)s
-      azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
-        %(secret:azure_storage_account_name)s
-      azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
-        %(secret:azure_storage_endpoint)s
-      azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
-        %(secret:azure_storage_access_key)s
-      azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
-        %(secret:azure_storage_account_name)s
-      azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
-        %(secret:azure_storage_endpoint)s
-      azuretest_AZURE_BLOB_ENDPOINT: "%(secret:azure_storage_endpoint)s"
-      b2backend_B2_ACCOUNT_ID: "%(secret:b2backend_b2_account_id)s"
-      b2backend_B2_STORAGE_ACCESS_KEY: >-
-        %(secret:b2backend_b2_storage_access_key)s
-      GOOGLE_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
-      GOOGLE_SERVICE_KEY: "%(secret:gcp_service_key)s"
-      AWS_S3_BACKEND_ACCESS_KEY: "%(secret:aws_s3_backend_access_key)s"
-      AWS_S3_BACKEND_SECRET_KEY: "%(secret:aws_s3_backend_secret_key)s"
-      AWS_S3_BACKEND_ACCESS_KEY_2: "%(secret:aws_s3_backend_access_key_2)s"
-      AWS_S3_BACKEND_SECRET_KEY_2: "%(secret:aws_s3_backend_secret_key_2)s"
-      AWS_GCP_BACKEND_ACCESS_KEY: "%(secret:aws_gcp_backend_access_key)s"
-      AWS_GCP_BACKEND_SECRET_KEY: "%(secret:aws_gcp_backend_secret_key)s"
-      AWS_GCP_BACKEND_ACCESS_KEY_2: "%(secret:aws_gcp_backend_access_key_2)s"
-      AWS_GCP_BACKEND_SECRET_KEY_2: "%(secret:aws_gcp_backend_secret_key_2)s"
-      b2backend_B2_STORAGE_ENDPOINT: "%(secret:b2backend_b2_storage_endpoint)s"
-      gcpbackend2_GCP_SERVICE_EMAIL: "%(secret:gcp2_service_email)s"
-      gcpbackend2_GCP_SERVICE_KEY: "%(secret:gcp2_service_key)s"
-      gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
-      gcpbackend_GCP_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
-      gcpbackend_GCP_SERVICE_KEY: "%(secret:gcp_service_key)s"
-      gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
-        %(secret:gcpbackendmismatch_gcp_service_email)s
-      gcpbackendmismatch_GCP_SERVICE_KEY: >-
-        %(secret:gcpbackendmismatch_gcp_service_key)s
-      gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
-      gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
-      gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
-      gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
-  - env: &mongo-vars
-      S3BACKEND: "mem"
-      MPU_TESTING: "yes"
-      S3METADATA: mongodb
-  - env: &multiple-backend-vars
-      S3BACKEND: "mem"
-      S3DATA: "multiple"
-      MPU_TESTING: "yes"
-  - env: &file-mem-mpu
-      S3BACKEND: "file"
-      S3VAULT: "mem"
-      MPU_TESTING: "yes"
-  - Git: &clone
-      name: Pull repo
-      repourl: '%(prop:git_reference)s'
-      shallow: true
-      retryFetch: true
-      haltOnFailure: true
-  - ShellCommand: &credentials
-      name: Setup Credentials
-      command: bash eve/workers/build/credentials.bash
-      haltOnFailure: true
-      env: *global-env
-  - ShellCommand: &yarn-install
-      name: install modules
-      command: yarn install
-      haltOnFailure: true
-  - ShellCommand: &check-s3-action-logs
-      name: Check s3 action logs
-      command: |
-        LOGS=`cat /artifacts/s3.log | grep 'No actionLog'`
-        test `echo -n ${LOGS} | wc -l` -eq 0 || (echo $LOGS && false)
-  - Upload: &upload-artifacts
-      source: /artifacts
-      urls:
-        - "*"
-  - ShellCommand: &follow-s3-log
-      logfiles:
-        s3:
-          filename: /artifacts/s3.log
-          follow: true
-  - ShellCommand: &follow-s3-ceph-logs
-      logfiles:
-        ceph:
-          filename: /artifacts/ceph.log
-          follow: true
-        s3:
-          filename: /artifacts/s3.log
-          follow: true
-  - ShellCommand: &add-hostname
-      name: add hostname
-      command: |
-        echo "127.0.0.1 testrequestbucket.localhost" >> /etc/hosts
-        echo \
-        "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" \
-        >> /etc/hosts
-      haltOnFailure: true
-  - ShellCommand: &setup-junit-upload
-      name: preparing junit files for upload
-      command: |
-        mkdir -p artifacts/junit
-        find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
-      alwaysRun: true
-  - Upload: &upload-junits
-      source: artifacts
-      urls:
-        - "*"
-      alwaysRun: true
-  - env: &docker_env
-      DEVELOPMENT_DOCKER_IMAGE_NAME: >-
-        registry.scality.com/%(prop:git_slug)s-dev/%(prop:git_slug)s
-      PRODUCTION_DOCKER_IMAGE_NAME: >-
-        registry.scality.com/%(prop:git_slug)s/%(prop:git_slug)s
-  - ShellCommand: &docker_login
-      name: Login to docker registry
-      command: >
-        docker login
-        -u "${HARBOR_LOGIN}"
-        -p "${HARBOR_PASSWORD}"
-        registry.scality.com
-      usePTY: true
-      env:
-        HARBOR_LOGIN: '%(secret:harbor_login)s'
-        HARBOR_PASSWORD: '%(secret:harbor_password)s'
-  - ShellCommand: &wait_docker_daemon
-      name: Wait for Docker daemon to be ready
-      command: |
-        bash -c '
-        for i in {1..150}
-        do
-          docker info &> /dev/null && exit
-          sleep 2
-        done
-        echo "Could not reach Docker daemon from buildbot worker" >&2
-        exit 1'
-      haltOnFailure: true
-
-
-stages:
-  pre-merge:
-    worker:
-      type: local
-    steps:
-      - TriggerStages:
-          name: Launch all workers
-          stage_names:
-            - docker-build
-            - linting-coverage
-            - file-ft-tests
-            - multiple-backend-test
-            - mongo-ft-tests
-            - ceph-backend-tests
-            - kmip-ft-tests
-            - utapi-v2-tests
-          waitForFinish: true
-          haltOnFailure: true
-
-  linting-coverage:
-    worker:
-      type: docker
-      path: eve/workers/build
-      volumes: &default_volumes
-        - '/home/eve/workspace'
-    steps:
-      - Git: *clone
-      - ShellCommand: *yarn-install
-      - ShellCommand: *add-hostname
-      - ShellCommand: *credentials
-      - ShellCommand:
-          name: Unit Coverage mandatory file
-          command: |
-            set -ex
-            test -f .git/HEAD
-      - ShellCommand:
-          name: Linting
-          command: |
-            set -ex
-            yarn run --silent lint -- --max-warnings 0
-            yarn run --silent lint_md
-            flake8 $(git ls-files "*.py")
-            yamllint $(git ls-files "*.yml")
-      - ShellCommand:
-          name: Unit Coverage
-          command: |
-            set -ex
-            unset HTTP_PROXY HTTPS_PROXY NO_PROXY
-            unset http_proxy https_proxy no_proxy
-            mkdir -p $CIRCLE_TEST_REPORTS/unit
-            yarn test
-            yarn run test_legacy_location
-          env: &shared-vars
-            <<: *global-env
-            S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
-            CIRCLE_TEST_REPORTS: /tmp
-            CIRCLE_ARTIFACTS: /tmp
-            CI_REPORTS: /tmp
-      - ShellCommand:
-          name: Unit Coverage logs
-          command: find /tmp/unit -exec cat {} \;
-      - ShellCommand: *setup-junit-upload
-      - Upload: *upload-junits
-
-  multiple-backend-test:
-    worker:
-      type: kube_pod
-      path: eve/workers/pod.yaml
-      images:
-        aggressor: eve/workers/build
-        s3: "."
-      vars:
-        aggressorMem: "2560Mi"
-        s3Mem: "2560Mi"
-        env:
-          <<: *multiple-backend-vars
-          <<: *global-env
-    steps:
-      - Git: *clone
-      - ShellCommand: *credentials
-      - ShellCommand: *yarn-install
-      - ShellCommand:
-          command: |
-            bash -c "
-            source /root/.aws/exports &> /dev/null
-            set -ex
-            bash wait_for_local_port.bash 8000 40
-            yarn run multiple_backend_test
-            yarn run ft_awssdk_external_backends"
-          <<: *follow-s3-log
-          env:
-            <<: *multiple-backend-vars
-            <<: *global-env
-            S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
-      - ShellCommand:
-          command: mvn test
-          workdir: build/tests/functional/jaws
-          <<: *follow-s3-log
-          env:
-            <<: *multiple-backend-vars
-      - ShellCommand:
-          command: rspec tests.rb
-          workdir: build/tests/functional/fog
-          <<: *follow-s3-log
-          env:
-            <<: *multiple-backend-vars
-      - ShellCommand: *check-s3-action-logs
-      - ShellCommand: *setup-junit-upload
-      - Upload: *upload-artifacts
-      - Upload: *upload-junits
-
-  ceph-backend-tests:
-    worker:
-      type: kube_pod
-      path: eve/workers/pod.yaml
-      images:
-        aggressor: eve/workers/build
-        s3: "."
-        ceph: eve/workers/ceph
-      vars:
-        aggressorMem: "2500Mi"
-        s3Mem: "2560Mi"
-        redis: enabled
-        env:
-          <<: *multiple-backend-vars
-          <<: *global-env
-          CI_CEPH: "true"
-          MPU_TESTING: "yes"
-          S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
-    steps:
-      - Git: *clone
-      - ShellCommand: *credentials
-      - ShellCommand: *yarn-install
-      - ShellCommand:
-          command: |
-            bash -c "
-            source /root/.aws/exports &> /dev/null
-            set -ex
-            bash eve/workers/ceph/wait_for_ceph.sh
-            bash wait_for_local_port.bash 8000 40
-            yarn run multiple_backend_test"
-          env:
-            <<: *multiple-backend-vars
-            <<: *global-env
-          <<: *follow-s3-ceph-logs
-      - ShellCommand:
-          command: mvn test
-          workdir: build/tests/functional/jaws
-          <<: *follow-s3-ceph-logs
-          env:
-            <<: *multiple-backend-vars
-      - ShellCommand:
-          command: rspec tests.rb
-          workdir: build/tests/functional/fog
-          <<: *follow-s3-ceph-logs
-          env:
-            <<: *multiple-backend-vars
-      - ShellCommand:
-          command: |
-            yarn run ft_awssdk &&
-            yarn run ft_s3cmd
-          env:
-            <<: *file-mem-mpu
-            <<: *global-env
-            S3_LOCATION_FILE: "/kube_pod-prod-cloudserver-backend-0/\
-              build/tests/locationConfig/locationConfigCeph.json"
-          <<: *follow-s3-ceph-logs
-      - ShellCommand: *setup-junit-upload
-      - Upload: *upload-artifacts
-      - Upload: *upload-junits
-
-  mongo-ft-tests:
-    worker: &s3-pod
-      type: kube_pod
-      path: eve/workers/pod.yaml
-      images:
-        aggressor: eve/workers/build
-        s3: "."
-      vars:
-        aggressorMem: "2Gi"
-        s3Mem: "1664Mi"
-        redis: enabled
-        env:
-          <<: *mongo-vars
-          <<: *global-env
-    steps:
-      - Git: *clone
-      - ShellCommand: *credentials
-      - ShellCommand: *yarn-install
-      - ShellCommand:
-          command: |
-            set -ex
-            bash wait_for_local_port.bash 8000 40
-            yarn run ft_test
-          <<: *follow-s3-log
-          env:
-            <<: *mongo-vars
-            <<: *global-env
-      - ShellCommand: *setup-junit-upload
-      - Upload: *upload-artifacts
-      - Upload: *upload-junits
-
-  file-ft-tests:
-    worker:
-      type: kube_pod
-      path: eve/workers/pod.yaml
-      images:
-        aggressor: eve/workers/build
-        s3: "."
-      vars:
-        aggressorMem: "3Gi"
-        s3Mem: "2560Mi"
-        redis: enabled
-        env:
-          <<: *file-mem-mpu
-          <<: *global-env
-    steps:
-      - Git: *clone
-      - ShellCommand: *credentials
-      - ShellCommand: *yarn-install
-      - ShellCommand:
-          command: |
-            set -ex
-            bash wait_for_local_port.bash 8000 40
-            yarn run ft_test
-          <<: *follow-s3-log
-          env:
-            <<: *file-mem-mpu
-            <<: *global-env
-      - ShellCommand: *check-s3-action-logs
-      - ShellCommand: *setup-junit-upload
-      - Upload: *upload-artifacts
-      - Upload: *upload-junits
-
-  kmip-ft-tests:
-    worker:
-      type: kube_pod
-      path: eve/workers/pod.yaml
-      images:
-        aggressor: eve/workers/build
-        s3: "."
-        pykmip: eve/workers/pykmip
-      vars:
-        aggressorMem: "2Gi"
-        s3Mem: "1664Mi"
-        redis: enabled
-        pykmip: enabled
-        env:
-          <<: *file-mem-mpu
-          <<: *global-env
-    steps:
-      - Git: *clone
-      - ShellCommand: *credentials
-      - ShellCommand: *yarn-install
-      - ShellCommand:
-          command: |
-            set -ex
-            bash wait_for_local_port.bash 8000 40
-            bash wait_for_local_port.bash 5696 40
-            yarn run ft_kmip
-          logfiles:
-            pykmip:
-              filename: /artifacts/pykmip.log
-              follow: true
-            s3:
-              filename: /artifacts/s3.log
-              follow: true
-          env:
-            <<: *file-mem-mpu
-            <<: *global-env
-      - ShellCommand: *setup-junit-upload
-      - Upload: *upload-artifacts
-      - Upload: *upload-junits
-
-  utapi-v2-tests:
-    worker:
-      type: kube_pod
-      path: eve/workers/pod.yaml
-      images:
-        aggressor: eve/workers/build
-        s3: "."
-      vars:
-        aggressorMem: "2Gi"
-        s3Mem: "2Gi"
-        env:
-          ENABLE_UTAPI_V2: t
-          S3BACKEND: mem
-    steps:
-      - Git: *clone
-      - ShellCommand: *credentials
-      - ShellCommand: *yarn-install
-      - ShellCommand:
-          command: |
-            bash -c "
-            source /root/.aws/exports &> /dev/null
-            set -ex
-            bash wait_for_local_port.bash 8000 40
-            yarn run test_utapi_v2"
-          <<: *follow-s3-log
-          env:
-            ENABLE_UTAPI_V2: t
-            S3BACKEND: mem
-      - ShellCommand: *check-s3-action-logs
-      - ShellCommand: *setup-junit-upload
-      - Upload: *upload-artifacts
-      - Upload: *upload-junits
-
-  # The docker-build stage ensures that your images are built on every commit
-  # and also hosted on the registry to help you pull it up and
-  # test it in a real environment if needed.
-  # It also allows us to pull and rename it when performing a release.
-  docker-build:
-    worker: &docker_worker
-      type: kube_pod
-      path: eve/workers/docker/pod.yaml
-      images:
-        worker: eve/workers/docker
-    steps:
-      - Git: *clone
-      - ShellCommand: *wait_docker_daemon
-      - ShellCommand: *docker_login
-      - ShellCommand:
-          name: docker build
-          command: >-
-            docker build .
-            --tag=${DEVELOPMENT_DOCKER_IMAGE_NAME}:%(prop:commit_short_revision)s
-          env: *docker_env
-          haltOnFailure: true
-      - ShellCommand:
-          name: push docker image into the development namespace
-          command: docker push ${DEVELOPMENT_DOCKER_IMAGE_NAME}
-          haltOnFailure: true
-          env: *docker_env
-
-  # This stage can be used to release your Docker image.
-  # To use this stage:
-  # 1. Tag the repository
-  # 2. Force a build using:
-  #    * A branch that ideally matches the tag
-  #    * The release stage
-  #    * An extra property with the name tag and its value being the actual tag
-  release:
-    worker:
-      type: local
-    steps:
-      - TriggerStages:
-          stage_names:
-            - docker-release
-          haltOnFailure: true
-  docker-release:
-    worker: *docker_worker
-    steps:
-      - Git: *clone
-      - ShellCommand: *wait_docker_daemon
-      - ShellCommand: *docker_login
-      - EvePropertyFromCommand:
-          name: get tag short revision
-          property: tag_revision
-          command: |
-            git checkout refs/tags/%(prop:tag)s 2&> /dev/null
-            git rev-parse --short HEAD
-          haltOnFailure: true
-      - ShellCommand:
-          name: publish docker image to Scality Production OCI registry
-          command: |
-            docker pull ${DEVELOPMENT_DOCKER_IMAGE_NAME}:%(prop:tag_revision)s
-            docker tag ${DEVELOPMENT_DOCKER_IMAGE_NAME}:%(prop:tag_revision)s \
-            ${PRODUCTION_DOCKER_IMAGE_NAME}:%(prop:tag)s
-            docker push ${PRODUCTION_DOCKER_IMAGE_NAME}:%(prop:tag)s
-          env: *docker_env

eve/workers/build/Dockerfile

@@ -1,58 +0,0 @@
-FROM buildpack-deps:xenial-curl
-
-#
-# Install packages needed by the buildchain
-#
-ENV LANG C.UTF-8
-COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
-RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
-    && apt-get update \
-    && apt-get install -y yarn \
-    && cat /tmp/*packages.list | xargs apt-get install -y \
-    && git clone https://github.com/tj/n.git \
-    && make -C ./n \
-    && n 10.22.0 \
-    && pip install pip==9.0.1 \
-    && rm -rf ./n \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm -f /tmp/packages.list
-
-#
-# Add user eve
-#
-
-RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
-    && adduser eve sudo \
-    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
-#
-# Install Dependencies
-#
-
-# Install RVM and gems
-ENV RUBY_VERSION="2.4.1"
-COPY ./gems.list /tmp/
-RUN cat /tmp/gems.list | xargs gem install
-#RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
-#    && curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
-#    && usermod -a -G rvm eve
-#RUN /bin/bash -l -c "\
-#    source /usr/local/rvm/scripts/rvm \
-#    && cat /tmp/gems.list | xargs gem install \
-#    && rm /tmp/gems.list"
-
-# Install Pip packages
-COPY ./pip_packages.list /tmp/
-RUN cat /tmp/pip_packages.list | xargs pip install \
-    && rm -f /tmp/pip_packages.list \
-    && mkdir /home/eve/.aws \
-    && chown eve /home/eve/.aws
-
-#
-# Run buildbot-worker on startup
-#
-
-ARG BUILDBOT_VERSION
-RUN pip install buildbot-worker==$BUILDBOT_VERSION
-
-CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS   && buildbot-worker start --nodaemon"]

eve/workers/build/buildbot_worker_packages.list

@@ -1,13 +0,0 @@
-ca-certificates
-git
-gnupg
-libffi-dev
-libssl-dev
-python-pip
-python2.7
-python2.7-dev
-software-properties-common
-sudo
-tcl
-wget
-procps

eve/workers/build/gems.list

@@ -1,4 +0,0 @@
-fog-aws:1.3.0
-json
-mime-types:3.1
-rspec:3.5

eve/workers/build/pip_packages.list

@@ -1,3 +0,0 @@
-flake8
-s3cmd==1.6.1
-yamllint

eve/workers/build/s3_packages.list

@@ -1,11 +0,0 @@
-build-essential
-curl
-default-jdk
-libdigest-hmac-perl
-lsof
-maven
-netcat
-redis-server
-ruby-full
-yarn
-zlib1g-dev

eve/workers/docker/Dockerfile

@@ -1,25 +0,0 @@
-FROM centos:7
-
-ARG BUILDBOT_VERSION=0.9.12
-
-VOLUME /home/eve/workspace
-
-WORKDIR /home/eve/workspace
-
-RUN yum install -y epel-release \
-    && yum-config-manager \
-    --add-repo \
-    https://download.docker.com/linux/centos/docker-ce.repo \
-    && yum install -y \
-    python-devel \
-    python-pip \
-    python36 \
-    python36-devel \
-    python36-pip \
-    git \
-    docker-ce-cli-18.09.6 \
-    && adduser -u 1042 --home /home/eve eve --groups docker \
-    && chown -R eve:eve /home/eve \
-    && pip3 install buildbot-worker==${BUILDBOT_VERSION}
-
-CMD buildbot-worker create-worker . ${BUILDMASTER}:${BUILDMASTER_PORT} ${WORKERNAME} ${WORKERPASS} && buildbot-worker start --nodaemon

eve/workers/docker/pod.yaml

@@ -1,43 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: worker
-spec:
-  containers:
-    - name: build-worker
-      image: "{{ images.worker }}"
-      resources:
-        requests:
-          cpu: "250m"
-          memory: 1Gi
-        limits:
-          cpu: "1"
-          memory: 1Gi
-      env:
-        - name: DOCKER_HOST
-          value: localhost:2375
-      volumeMounts:
-        - name: worker-workspace
-          mountPath: /home/eve/workspace
-    - name: dind-daemon
-      image: docker:18.09.6-dind
-      resources:
-        requests:
-          cpu: "500m"
-          memory: 1Gi
-        limits:
-          cpu: "1"
-          memory: 1Gi
-      securityContext:
-        privileged: true
-      volumeMounts:
-        - name: docker-storage
-          mountPath: /var/lib/docker
-        - name: worker-workspace
-          mountPath: /home/eve/workspace
-  volumes:
-    - name: docker-storage
-      emptyDir: {}
-    - name: worker-workspace
-      emptyDir: {}

eve/workers/pod.yaml

@@ -1,233 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "proxy-ci-test-pod"
-spec:
-  restartPolicy: Never
-  terminationGracePeriodSeconds: 10
-  hostAliases:
-  - ip: "127.0.0.1"
-    hostnames:
-    - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com"
-    - "testrequestbucket.localhost"
-    - "pykmip.local"
-  {% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
-  initContainers:
-    - name: kmip-certs-installer
-      image: {{ images.pykmip }}
-      command: [ 'sh', '-c', 'cp /ssl/* /ssl-kmip/']
-      volumeMounts:
-        - name: kmip-certs
-          readOnly: false
-          mountPath: /ssl-kmip
-  {%- endif %}
-  containers:
-    {% if vars.env.S3METADATA is defined and vars.env.S3METADATA == "mongodb" -%}
-    - name: mongo
-      image: scality/ci-mongo:3.6.8
-      imagePullPolicy: IfNotPresent
-      resources:
-        requests:
-          cpu: 500m
-          memory: 1Gi
-        limits:
-          cpu: 500m
-          memory: 1Gi
-    {%- endif %}
-    - name: aggressor
-      image: {{ images.aggressor }}
-      imagePullPolicy: IfNotPresent
-      resources:
-        requests:
-          cpu: "1"
-          memory: {{ vars.aggressorMem }}
-        limits:
-          cpu: "1"
-          memory: {{ vars.aggressorMem }}
-      volumeMounts:
-        - name: creds
-          readOnly: false
-          mountPath: /root/.aws
-        - name: artifacts
-          readOnly: true
-          mountPath: /artifacts
-      command:
-        - bash
-        - -lc
-        - |
-          buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS
-          buildbot-worker start --nodaemon
-      env:
-        - name: CI
-          value: "true"
-        - name: ENABLE_LOCAL_CACHE
-          value: "true"
-        - name: REPORT_TOKEN
-          value: "report-token-1"
-        - name: REMOTE_MANAGEMENT_DISABLE
-          value: "1"
-        {% for key, value in vars.env.items() %}
-        - name: {{ key }}
-          value: "{{ value }}"
-        {% endfor %}
-    - name: s3
-      image: {{ images.s3 }}
-      imagePullPolicy: IfNotPresent
-      resources:
-        requests:
-          cpu: "2"
-          memory: {{ vars.s3Mem }}
-        limits:
-          cpu: "2"
-          memory: {{ vars.s3Mem }}
-      volumeMounts:
-        - name: creds
-          readOnly: false
-          mountPath: /root/.aws
-        - name: certs
-          readOnly: false
-          mountPath: /tmp
-        - name: artifacts
-          readOnly: false
-          mountPath: /artifacts
-        - name: kmip-certs
-          readOnly: false
-          mountPath: /ssl-kmip
-      command:
-        - bash
-        - -ec
-        - |
-          sleep 10 # wait for mongo
-          /usr/src/app/docker-entrypoint.sh yarn start | tee -a /artifacts/s3.log
-      env:
-        {% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" and vars.env.CI_CEPH is not defined -%}
-        - name: S3_LOCATION_FILE
-          value: "/usr/src/app/tests/locationConfig/locationConfigTests.json"
-        {%- endif %}
-        {% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" and vars.env.CI_CEPH is defined and vars.env.CI_CEPH == "true" -%}
-        - name: S3_LOCATION_FILE
-          value: "/usr/src/app/tests/locationConfig/locationConfigCeph.json"
-        {%- endif %}
-        {% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
-        - name: S3KMS
-          value: kmip
-        - name: S3KMIP_PORT
-          value: "5696"
-        - name: S3KMIP_HOSTS
-          value: "pykmip.local"
-        - name: S3KMIP_COMPOUND_CREATE
-          value: "false"
-        - name: S3KMIP_BUCKET_ATTRIBUTE_NAME
-          value: ''
-        - name: S3KMIP_PIPELINE_DEPTH
-          value: "8"
-        - name: S3KMIP_KEY
-          value: /ssl-kmip/kmip-client-key.pem
-        - name: S3KMIP_CERT
-          value: /ssl-kmip/kmip-client-cert.pem
-        - name: S3KMIP_CA
-          value: /ssl-kmip/kmip-ca.pem
-        {%- endif %}
-        - name: CI
-          value: "true"
-        - name: ENABLE_LOCAL_CACHE
-          value: "true"
-        - name: MONGODB_HOSTS
-          value: "localhost:27018"
-        - name: MONGODB_RS
-          value: "rs0"
-        - name: REDIS_HOST
-          value: "localhost"
-        - name: REDIS_PORT
-          value: "6379"
-        - name: REPORT_TOKEN
-          value: "report-token-1"
-        - name: REMOTE_MANAGEMENT_DISABLE
-          value: "1"
-        - name: HEALTHCHECKS_ALLOWFROM
-          value: "0.0.0.0/0"
-        {% for key, value in vars.env.items() %}
-        - name: {{ key }}
-          value: "{{ value }}"
-        {% endfor %}
-    {% if vars.redis is defined and vars.redis == "enabled" -%}
-    - name: redis
-      image: redis:alpine
-      imagePullPolicy: IfNotPresent
-      resources:
-        requests:
-          cpu: 200m
-          memory: 128Mi
-        limits:
-          cpu: 200m
-          memory: 128Mi
-    {%- endif %}
-    {% if vars.env.CI_PROXY is defined and vars.env.CI_PROXY == "true" -%}
-    - name: squid
-      image: scality/ci-squid
-      imagePullPolicy: IfNotPresent
-      resources:
-        requests:
-          cpu: 250m
-          memory: 128Mi
-        limits:
-          cpu: 250m
-          memory: 128Mi
-      volumeMounts:
-        - name: certs
-          readOnly: false
-          mountPath: /ssl
-      command:
-        - sh
-        - -exc
-        - |
-          mkdir -p /ssl
-          openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-           -subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-           -keyout /ssl/myca.pem  -out /ssl/myca.pem
-          cp /ssl/myca.pem /ssl/CA.pem
-          squid -f /etc/squid/squid.conf -N -z
-          squid -f /etc/squid/squid.conf -NYCd 1
-    {%- endif %}
-    {% if vars.env.CI_CEPH is defined and vars.env.CI_CEPH == "true" -%}
-    - name: ceph
-      image: {{ images.ceph }}
-      imagePullPolicy: IfNotPresent
-      resources:
-        requests:
-          cpu: 500m
-          memory: 1536Mi
-        limits:
-          cpu: 500m
-          memory: 1536Mi
-      volumeMounts:
-        - name: artifacts
-          readOnly: false
-          mountPath: /artifacts
-    {%- endif %}
-    {% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
-    - name: pykmip
-      image: {{ images.pykmip }}
-      imagePullPolicy: IfNotPresent
-      volumeMounts:
-        - name: artifacts
-          readOnly: false
-          mountPath: /artifacts
-      resources:
-        requests:
-          cpu: 100m
-          memory: 128Mi
-        limits:
-          cpu: 100m
-          memory: 128Mi
-    {%- endif %}
-  volumes:
-    - name: creds
-      emptyDir: {}
-    - name: certs
-      emptyDir: {}
-    - name: artifacts
-      emptyDir: {}
-    - name: kmip-certs
-      emptyDir: {}

examples/python-md-search.py

@@ -48,7 +48,7 @@ signed_headers = 'host;x-amz-content-sha256;x-amz-date'
 canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
     .format(method, canonical_uri, canonical_querystring, canonical_headers,
             signed_headers, payload_hash)
-print canonical_request
+print(canonical_request)
 
 credential_scope = '{0}/{1}/{2}/aws4_request' \
     .format(date_stamp, region, service)
@@ -76,4 +76,4 @@ headers = {
 endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
 
 r = requests.get(endpoint, headers=headers)
-print (r.text)
+print(r.text)

images/svc-base/Dockerfile

@@ -0,0 +1,28 @@
+FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
+
+ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
+ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
+
+COPY . ${HOME_DIR}/s3
+RUN chown -R ${USER} ${HOME_DIR}
+RUN pip3 install redis===3.5.3 requests==2.27.1 && \
+    apt-get install -y git-lfs
+
+USER ${USER}
+WORKDIR ${HOME_DIR}/s3
+RUN rm -f ~/.gitconfig && \
+    git config --global --add safe.directory . && \
+    git lfs install && \
+    GIT_LFS_SKIP_SMUDGE=1 && \
+    yarn global add typescript && \
+    yarn install --frozen-lockfile --production --network-concurrency 1 && \
+    yarn cache clean --all && \
+    yarn global remove typescript
+
+# run symlinking separately to avoid yarn installation errors
+# we might have to check if the symlinking is really needed!
+RUN ln -sf /scality-kms node_modules
+
+EXPOSE 8000
+
+CMD bash -c "source ${CONF_DIR}/env && export && supervisord -c ${CONF_DIR}/supervisord.conf"

index.js

@@ -1,3 +1,10 @@
 'use strict'; // eslint-disable-line strict
 
+require('werelogs').stderrUtils.catchAndTimestampStderr(
+    undefined,
+    // Do not exit as workers have their own listener that will exit
+    // But primary don't have another listener
+    require('cluster').isPrimary ? 1 : null,
+);
+
 require('./lib/server.js')();

lib/Config.js

@@ -5,16 +5,21 @@ const path = require('path');
 const url = require('url');
 const crypto = require('crypto');
 
-const uuid = require('node-uuid');
+const { v4: uuidv4 } = require('uuid');
 const cronParser = require('cron-parser');
 const joi = require('@hapi/joi');
-
+const { s3routes, auth: arsenalAuth, s3middleware } = require('arsenal');
-const { isValidBucketName } = require('arsenal').s3routes.routesUtils;
+const { isValidBucketName } = s3routes.routesUtils;
-const validateAuthConfig = require('arsenal').auth.inMemory.validateAuthConfig;
+const validateAuthConfig = arsenalAuth.inMemory.validateAuthConfig;
 const { buildAuthDataAccount } = require('./auth/in_memory/builder');
 const validExternalBackends = require('../constants').externalBackends;
-const { azureAccountNameRegex, base64Regex } = require('../constants');
+const { azureAccountNameRegex, base64Regex,
+    allowedUtapiEventFilterFields, allowedUtapiEventFilterStates,
+} = require('../constants');
 const { utapiVersion } = require('utapi');
+const { scaleMsPerDay } = s3middleware.objectUtils;
+
+const constants = require('../constants');
 
 // config paths
 const configSearchPaths = [
@@ -102,6 +107,47 @@ function parseSproxydConfig(configSproxyd) {
     return joi.attempt(configSproxyd, joiSchema, 'bad config');
 }
 
+function parseRedisConfig(redisConfig) {
+    const joiSchema = joi.object({
+        password: joi.string().allow(''),
+        host: joi.string(),
+        port: joi.number(),
+        retry: joi.object({
+            connectBackoff: joi.object({
+                min: joi.number().required(),
+                max: joi.number().required(),
+                jitter: joi.number().required(),
+                factor: joi.number().required(),
+                deadline: joi.number().required(),
+            }),
+        }),
+        // sentinel config
+        sentinels: joi.alternatives().try(
+            joi.string()
+                .pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
+                .custom(hosts => hosts.split(',').map(item => {
+                    const [host, port] = item.split(':');
+                    return { host, port: Number.parseInt(port, 10) };
+                })),
+            joi.array().items(
+                joi.object({
+                    host: joi.string().required(),
+                    port: joi.number().required(),
+                })
+            ).min(1),
+        ),
+        name: joi.string(),
+        sentinelPassword: joi.string().allow(''),
+    })
+          .and('host', 'port')
+          .and('sentinels', 'name')
+          .xor('host', 'sentinels')
+          .without('sentinels', ['host', 'port'])
+          .without('host', ['sentinels', 'sentinelPassword']);
+
+    return joi.attempt(redisConfig, joiSchema, 'bad config');
+}
+
 function restEndpointsAssert(restEndpoints, locationConstraints) {
     assert(typeof restEndpoints === 'object',
         'bad config: restEndpoints must be an object of endpoints');
@@ -132,26 +178,71 @@ function gcpLocationConstraintAssert(location, locationObj) {
     });
 }
 
-function azureLocationConstraintAssert(location, locationObj) {
+function azureGetStorageAccountName(location, locationDetails) {
-    const {
+    const { azureStorageAccountName } = locationDetails;
-        azureStorageEndpoint,
-        azureStorageAccountName,
-        azureStorageAccessKey,
-        azureContainerName,
-    } = locationObj.details;
-    const storageEndpointFromEnv =
-        process.env[`${location}_AZURE_STORAGE_ENDPOINT`];
     const storageAccountNameFromEnv =
         process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`];
-    const storageAccessKeyFromEnv =
+    return storageAccountNameFromEnv || azureStorageAccountName;
-        process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`];
+}
+
+function azureGetLocationCredentials(location, locationDetails) {
+    const storageAccessKey =
+        process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`] ||
+        locationDetails.azureStorageAccessKey;
+    const sasToken =
+        process.env[`${location}_AZURE_SAS_TOKEN`] ||
+        locationDetails.sasToken;
+    const clientKey =
+        process.env[`${location}_AZURE_CLIENT_KEY`] ||
+        locationDetails.clientKey;
+
+    const authMethod =
+        process.env[`${location}_AZURE_AUTH_METHOD`] ||
+        locationDetails.authMethod ||
+        (storageAccessKey && 'shared-key') ||
+        (sasToken && 'shared-access-signature') ||
+        (clientKey && 'client-secret') ||
+        'shared-key';
+
+    switch (authMethod) {
+    case 'shared-key':
+    default:
+        return {
+            authMethod,
+            storageAccountName:
+                azureGetStorageAccountName(location, locationDetails),
+            storageAccessKey,
+        };
+
+    case 'shared-access-signature':
+        return {
+            authMethod,
+            sasToken,
+        };
+
+    case 'client-secret':
+        return {
+            authMethod,
+            tenantId:
+                process.env[`${location}_AZURE_TENANT_ID`] ||
+                locationDetails.tenantId,
+            clientId:
+                process.env[`${location}_AZURE_CLIENT_ID`] ||
+                locationDetails.clientId,
+            clientKey,
+        };
+    }
+}
+
+function azureLocationConstraintAssert(location, locationObj) {
     const locationParams = {
-        azureStorageEndpoint: storageEndpointFromEnv || azureStorageEndpoint,
+        ...azureGetLocationCredentials(location, locationObj.details),
-        azureStorageAccountName:
+        azureStorageEndpoint:
-            storageAccountNameFromEnv || azureStorageAccountName,
+            process.env[`${location}_AZURE_STORAGE_ENDPOINT`] ||
-        azureStorageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
+            locationObj.details.azureStorageEndpoint,
-        azureContainerName,
+        azureContainerName: locationObj.details.azureContainerName,
     };
+
     Object.keys(locationParams).forEach(param => {
         const value = locationParams[param];
         assert.notEqual(value, undefined,
@@ -161,13 +252,16 @@ function azureLocationConstraintAssert(location, locationObj) {
             `bad location constraint: "${location}" ${param} ` +
             `"${value}" must be a string`);
     });
-    assert(azureAccountNameRegex.test(locationParams.azureStorageAccountName),
+
-        `bad location constraint: "${location}" azureStorageAccountName ` +
+    if (locationParams.authMethod === 'shared-key') {
-        `"${locationParams.storageAccountName}" is an invalid value`);
+        assert(azureAccountNameRegex.test(locationParams.storageAccountName),
-    assert(base64Regex.test(locationParams.azureStorageAccessKey),
+            `bad location constraint: "${location}" azureStorageAccountName ` +
-        `bad location constraint: "${location}" ` +
+            `"${locationParams.storageAccountName}" is an invalid value`);
-        'azureStorageAccessKey is not a valid base64 string');
+        assert(base64Regex.test(locationParams.storageAccessKey),
-    assert(isValidBucketName(azureContainerName, []),
+            `bad location constraint: "${location}" ` +
+            'azureStorageAccessKey is not a valid base64 string');
+    }
+    assert(isValidBucketName(locationParams.azureContainerName, []),
         `bad location constraint: "${location}" ` +
         'azureContainerName is an invalid container name');
 }
@@ -186,10 +280,104 @@ function hdClientLocationConstraintAssert(configHd) {
     return hdclientFields;
 }
 
+function azureArchiveLocationConstraintAssert(locationObj) {
+    const checkedFields = [
+        'azureContainerName',
+        'azureStorageEndpoint',
+    ];
+    if (Object.keys(locationObj.details).length === 0 ||
+        !checkedFields.every(field => field in locationObj.details)) {
+        return;
+    }
+    const {
+        azureContainerName,
+        azureStorageEndpoint,
+    } = locationObj.details;
+    const stringFields = [
+        azureContainerName,
+        azureStorageEndpoint,
+    ];
+    stringFields.forEach(field => {
+        assert(typeof field === 'string',
+            `bad config: ${field} must be a string`);
+    });
+
+    let hasAuthMethod = false;
+    if (locationObj.details.sasToken !== undefined) {
+        assert(typeof locationObj.details.sasToken === 'string',
+            `bad config: ${locationObj.details.sasToken} must be a string`);
+        hasAuthMethod = true;
+    }
+
+    if (locationObj.details.azureStorageAccountName !== undefined &&
+        locationObj.details.azureStorageAccessKey !== undefined) {
+        assert(typeof locationObj.details.azureStorageAccountName === 'string',
+            `bad config: ${locationObj.details.azureStorageAccountName} must be a string`);
+        assert(typeof locationObj.details.azureStorageAccessKey === 'string',
+            `bad config: ${locationObj.details.azureStorageAccessKey} must be a string`);
+        assert(!hasAuthMethod, 'Multiple authentication methods are not allowed');
+        hasAuthMethod = true;
+    }
+
+    if (locationObj.details.tenantId !== undefined &&
+        locationObj.details.clientId !== undefined &&
+        locationObj.details.clientKey !== undefined) {
+        assert(typeof locationObj.details.tenantId === 'string',
+            `bad config: ${locationObj.details.tenantId} must be a string`);
+        assert(typeof locationObj.details.clientId === 'string',
+            `bad config: ${locationObj.details.clientId} must be a string`);
+        assert(typeof locationObj.details.clientKey === 'string',
+            `bad config: ${locationObj.details.clientKey} must be a string`);
+        assert(!hasAuthMethod, 'Multiple authentication methods are not allowed');
+        hasAuthMethod = true;
+    }
+    assert(hasAuthMethod, 'Missing authentication method');
+}
+
+function dmfLocationConstraintAssert(locationObj) {
+    const checkedFields = [
+        'endpoint',
+        'username',
+        'password',
+        'repoId',
+        'nsId',
+    ];
+    if (Object.keys(locationObj.details).length === 0 ||
+        !checkedFields.every(field => field in locationObj.details)) {
+        return;
+    }
+    const {
+        endpoint,
+        username,
+        password,
+        repoId,
+        nsId,
+    } = locationObj.details;
+    const stringFields = [
+        endpoint,
+        username,
+        password,
+        nsId,
+    ];
+    stringFields.forEach(field => {
+        assert(typeof field === 'string',
+            `bad config: ${field} must be a string`);
+    });
+    assert.strictEqual(Array.isArray(repoId), true);
+    repoId.forEach(rId => {
+        assert(typeof rId === 'string',
+            `bad config: ${rId} must be a string`);
+    });
+    assert(repoId.every(
+        r => typeof r === 'string'),
+        'bad config: each repoId must be a string',
+    );
+}
+
 function locationConstraintAssert(locationConstraints) {
     const supportedBackends =
           ['mem', 'file', 'scality',
-           'mongodb'].concat(Object.keys(validExternalBackends));
+           'mongodb', 'dmf', 'azure_archive', 'vitastor'].concat(Object.keys(validExternalBackends));
     assert(typeof locationConstraints === 'object',
         'bad config: locationConstraints must be an object');
     Object.keys(locationConstraints).forEach(l => {
@@ -297,6 +485,12 @@ function locationConstraintAssert(locationConstraints) {
         if (locationConstraints[l].type === 'gcp') {
             gcpLocationConstraintAssert(l, locationConstraints[l]);
         }
+        if (locationConstraints[l].type === 'dmf') {
+            dmfLocationConstraintAssert(locationConstraints[l]);
+        }
+        if (locationConstraints[l].type === 'azure_archive') {
+            azureArchiveLocationConstraintAssert(locationConstraints[l]);
+        }
         if (locationConstraints[l].type === 'pfs') {
             assert(typeof details.pfsDaemonEndpoint === 'object',
             'bad config: pfsDaemonEndpoint is mandatory and must be an object');
@@ -308,26 +502,33 @@ function locationConstraintAssert(locationConstraints) {
                 locationConstraints[l].details.connector.hdclient);
         }
     });
-    assert(Object.keys(locationConstraints)
-        .includes('us-east-1'), 'bad locationConfig: must ' +
-        'include us-east-1 as a locationConstraint');
 }
 
-function parseUtapiReindex({ enabled, schedule, sentinel, bucketd }) {
+function parseUtapiReindex(config) {
+    const {
+        enabled,
+        schedule,
+        redis,
+        bucketd,
+        onlyCountLatestWhenObjectLocked,
+    } = config;
     assert(typeof enabled === 'boolean',
-        'bad config: utapi.reindex.enabled must be a boolean');
+           'bad config: utapi.reindex.enabled must be a boolean');
-    assert(typeof sentinel === 'object',
+
-        'bad config: utapi.reindex.sentinel must be an object');
+    const parsedRedis = parseRedisConfig(redis);
-    assert(typeof sentinel.port === 'number',
+    assert(Array.isArray(parsedRedis.sentinels),
-        'bad config: utapi.reindex.sentinel.port must be a number');
+           'bad config: utapi reindex redis config requires a list of sentinels');
-    assert(typeof sentinel.name === 'string',
+
-        'bad config: utapi.reindex.sentinel.name must be a string');
     assert(typeof bucketd === 'object',
         'bad config: utapi.reindex.bucketd must be an object');
     assert(typeof bucketd.port === 'number',
         'bad config: utapi.reindex.bucketd.port must be a number');
     assert(typeof schedule === 'string',
         'bad config: utapi.reindex.schedule must be a string');
+    if (onlyCountLatestWhenObjectLocked !== undefined) {
+        assert(typeof onlyCountLatestWhenObjectLocked === 'boolean',
+            'bad config: utapi.reindex.onlyCountLatestWhenObjectLocked must be a boolean');
+    }
     try {
         cronParser.parseExpression(schedule);
     } catch (e) {
@@ -335,6 +536,13 @@ function parseUtapiReindex({ enabled, schedule, sentinel, bucketd }) {
             'bad config: utapi.reindex.schedule must be a valid ' +
             `cron schedule. ${e.message}.`);
     }
+    return {
+        enabled,
+        schedule,
+        redis: parsedRedis,
+        bucketd,
+        onlyCountLatestWhenObjectLocked,
+    };
 }
 
 function requestsConfigAssert(requestsConfig) {
@@ -422,7 +630,6 @@ class Config extends EventEmitter {
         // Read config automatically
         this._getLocationConfig();
         this._getConfig();
-        this._configureBackends();
     }
 
     _getLocationConfig() {
@@ -474,6 +681,35 @@ class Config extends EventEmitter {
         return tlsFileContent;
     }
 
+    /**
+     * Parse list of endpoints.
+     * @param {string[] | undefined} listenOn - List of string of the form "ip:port"
+     * @param {string} fieldName - Name of field being parsed
+     * @returns {{ip: string, port: number}[]} - List of ip/port pairs
+     */
+    _parseEndpoints(listenOn, fieldName) {
+        let result = [];
+        if (listenOn !== undefined) {
+            assert(Array.isArray(listenOn)
+                && listenOn.every(e => typeof e === 'string'),
+                `bad config: ${fieldName} must be a list of strings`);
+            result = listenOn.map(item => {
+                const lastColon = item.lastIndexOf(':');
+                // if address is IPv6 format, it includes brackets
+                // that have to be removed from the final IP address
+                const ipAddress = item.indexOf(']') > 0 ?
+                    item.substr(1, lastColon - 2) :
+                    item.substr(0, lastColon);
+                // the port should not include the colon
+                const port = item.substr(lastColon + 1);
+                assert(Number.parseInt(port, 10),
+                    `bad config: ${fieldName} port must be a positive integer`);
+                return { ip: ipAddress, port };
+            });
+        }
+        return result;
+    }
+
     _getConfig() {
         let config;
         try {
@@ -502,26 +738,17 @@ class Config extends EventEmitter {
             this.port = config.port;
         }
 
-        this.listenOn = [];
+        this.listenOn = this._parseEndpoints(config.listenOn, 'listenOn');
-        if (config.listenOn !== undefined) {
+
-            assert(Array.isArray(config.listenOn)
+        this.metricsPort = 8002;
-                && config.listenOn.every(e => typeof e === 'string'),
+        if (config.metricsPort !== undefined) {
-                'bad config: listenOn must be a list of strings');
+            assert(Number.isInteger(config.metricsPort) && config.metricsPort > 0,
-            config.listenOn.forEach(item => {
+                'bad config: metricsPort must be a positive integer');
-                const lastColon = item.lastIndexOf(':');
+            this.metricsPort = config.metricsPort;
-                // if address is IPv6 format, it includes brackets
-                // that have to be removed from the final IP address
-                const ipAddress = item.indexOf(']') > 0 ?
-                    item.substr(1, lastColon - 2) :
-                    item.substr(0, lastColon);
-                // the port should not include the colon
-                const port = item.substr(lastColon + 1);
-                assert(Number.parseInt(port, 10),
-                    'bad config: listenOn port must be a positive integer');
-                this.listenOn.push({ ip: ipAddress, port });
-            });
         }
 
+        this.metricsListenOn = this._parseEndpoints(config.metricsListenOn, 'metricsListenOn');
+
         if (config.replicationGroupId) {
             assert(typeof config.replicationGroupId === 'string',
                 'bad config: replicationGroupId must be a string');
@@ -535,12 +762,6 @@ class Config extends EventEmitter {
             const { replicationEndpoints } = config;
             assert(replicationEndpoints instanceof Array, 'bad config: ' +
                 '`replicationEndpoints` property must be an array');
-            if (replicationEndpoints.length > 1) {
-                const hasDefault = replicationEndpoints.some(
-                    replicationEndpoint => replicationEndpoint.default);
-                assert(hasDefault, 'bad config: `replicationEndpoints` must ' +
-                    'contain a default endpoint');
-            }
             replicationEndpoints.forEach(replicationEndpoint => {
                 assert.strictEqual(typeof replicationEndpoint, 'object',
                     'bad config: `replicationEndpoints` property must be an ' +
@@ -558,7 +779,7 @@ class Config extends EventEmitter {
                     assert(validExternalBackends[type], 'bad config: `type` ' +
                         'property of `replicationEndpoints` object must be ' +
                         'a valid external backend (one of: "' +
-                        `${Object.keys(validExternalBackends).join('", "')})`);
+                        `${Object.keys(validExternalBackends).join('", "')}")`);
                 } else {
                     assert.notStrictEqual(servers, undefined, 'bad config: ' +
                         'each object of `replicationEndpoints` array that is ' +
@@ -620,11 +841,11 @@ class Config extends EventEmitter {
             this.websiteEndpoints = config.websiteEndpoints;
         }
 
-        this.clusters = false;
+        this.workers = false;
-        if (config.clusters !== undefined) {
+        if (config.workers !== undefined) {
-            assert(Number.isInteger(config.clusters) && config.clusters > 0,
+            assert(Number.isInteger(config.workers) && config.workers > 0,
-                   'bad config: clusters must be a positive integer');
+                   'bad config: workers must be a positive integer');
-            this.clusters = config.clusters;
+            this.workers = config.workers;
         }
 
         if (config.usEastBehavior !== undefined) {
@@ -862,8 +1083,7 @@ class Config extends EventEmitter {
                 assert(typeof config.localCache.port === 'number',
                       'config: bad port for localCache. port must be a number');
                 if (config.localCache.password !== undefined) {
-                    assert(
+                    assert(typeof config.localCache.password === 'string',
-                        this._verifyRedisPassword(config.localCache.password),
                         'config: vad password for localCache. password must' +
                     ' be a string');
                 }
@@ -889,56 +1109,46 @@ class Config extends EventEmitter {
         }
 
         if (config.redis) {
-            if (config.redis.sentinels) {
+            this.redis = parseRedisConfig(config.redis);
-                this.redis = { sentinels: [], name: null };
+        }
-
+        if (config.scuba) {
-                assert(typeof config.redis.name === 'string',
+            this.scuba = {};
-                    'bad config: redis sentinel name must be a string');
+            if (config.scuba.host) {
-                this.redis.name = config.redis.name;
+                assert(typeof config.scuba.host === 'string',
-                assert(Array.isArray(config.redis.sentinels) ||
+                    'bad config: scuba host must be a string');
-                    typeof config.redis.sentinels === 'string',
+                this.scuba.host = config.scuba.host;
-                    'bad config: redis sentinels must be an array or string');
-
-                if (typeof config.redis.sentinels === 'string') {
-                    config.redis.sentinels.split(',').forEach(item => {
-                        const [host, port] = item.split(':');
-                        this.redis.sentinels.push({ host,
-                            port: Number.parseInt(port, 10) });
-                    });
-                } else if (Array.isArray(config.redis.sentinels)) {
-                    config.redis.sentinels.forEach(item => {
-                        const { host, port } = item;
-                        assert(typeof host === 'string',
-                            'bad config: redis sentinel host must be a string');
-                        assert(typeof port === 'number',
-                            'bad config: redis sentinel port must be a number');
-                        this.redis.sentinels.push({ host, port });
-                    });
-                }
-
-                if (config.redis.sentinelPassword !== undefined) {
-                    assert(
-                    this._verifyRedisPassword(config.redis.sentinelPassword));
-                    this.redis.sentinelPassword = config.redis.sentinelPassword;
-                }
-            } else {
-                // check for standalone configuration
-                this.redis = {};
-                assert(typeof config.redis.host === 'string',
-                    'bad config: redis.host must be a string');
-                assert(typeof config.redis.port === 'number',
-                    'bad config: redis.port must be a number');
-                this.redis.host = config.redis.host;
-                this.redis.port = config.redis.port;
             }
-            if (config.redis.password !== undefined) {
+            if (config.scuba.port) {
-                assert(
+                assert(Number.isInteger(config.scuba.port)
-                    this._verifyRedisPassword(config.redis.password),
+                    && config.scuba.port > 0,
-                    'bad config: invalid password for redis. password must ' +
+                    'bad config: scuba port must be a positive integer');
-                    'be a string');
+                this.scuba.port = config.scuba.port;
-                this.redis.password = config.redis.password;
             }
         }
+        if (process.env.SCUBA_HOST && process.env.SCUBA_PORT) {
+            assert(typeof process.env.SCUBA_HOST === 'string',
+                'bad config: scuba host must be a string');
+            assert(Number.isInteger(Number(process.env.SCUBA_PORT))
+                && Number(process.env.SCUBA_PORT) > 0,
+                'bad config: scuba port must be a positive integer');
+            this.scuba = {
+                host: process.env.SCUBA_HOST,
+                port: Number(process.env.SCUBA_PORT),
+            };
+        }
+        if (this.scuba) {
+            this.quotaEnabled = true;
+        }
+        const maxStaleness = Number(process.env.QUOTA_MAX_STALENESS_MS) ||
+            config.quota?.maxStatenessMS ||
+            24 * 60 * 60 * 1000;
+        assert(Number.isInteger(maxStaleness), 'bad config: maxStalenessMS must be an integer');
+        const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' ||
+            config.quota?.enableInflights || false;
+        this.quota = {
+            maxStaleness,
+            enableInflights,
+        };
         if (config.utapi) {
             this.utapi = { component: 's3' };
             if (config.utapi.host) {
@@ -967,48 +1177,17 @@ class Config extends EventEmitter {
                 assert(config.redis, 'missing required property of utapi ' +
                     'configuration: redis');
                 if (config.utapi.redis) {
-                    if (config.utapi.redis.sentinels) {
+                    this.utapi.redis = parseRedisConfig(config.utapi.redis);
-                        this.utapi.redis = { sentinels: [], name: null };
+                    if (this.utapi.redis.retry === undefined) {
-
+                        this.utapi.redis.retry = {
-                        assert(typeof config.utapi.redis.name === 'string',
+                            connectBackoff: {
-                            'bad config: redis sentinel name must be a string');
+                                min: 10,
-                        this.utapi.redis.name = config.utapi.redis.name;
+                                max: 1000,
-
+                                jitter: 0.1,
-                        assert(Array.isArray(config.utapi.redis.sentinels),
+                                factor: 1.5,
-                            'bad config: redis sentinels must be an array');
+                                deadline: 10000,
-                        config.utapi.redis.sentinels.forEach(item => {
+                            },
-                            const { host, port } = item;
+                        };
-                            assert(typeof host === 'string',
-                                'bad config: redis sentinel host must be a string');
-                            assert(typeof port === 'number',
-                                'bad config: redis sentinel port must be a number');
-                            this.utapi.redis.sentinels.push({ host, port });
-                        });
-                    } else {
-                        // check for standalone configuration
-                        this.utapi.redis = {};
-                        assert(typeof config.utapi.redis.host === 'string',
-                            'bad config: redis.host must be a string');
-                        assert(typeof config.utapi.redis.port === 'number',
-                            'bad config: redis.port must be a number');
-                        this.utapi.redis.host = config.utapi.redis.host;
-                        this.utapi.redis.port = config.utapi.redis.port;
-                    }
-                    if (config.utapi.redis.password !== undefined) {
-                        assert(
-                            this._verifyRedisPassword(config.utapi.redis.password),
-                            'config: invalid password for utapi redis. password' +
-                            ' must be a string');
-                        this.utapi.redis.password = config.utapi.redis.password;
-                    }
-                        if (config.utapi.redis.sentinelPassword !== undefined) {
-                            assert(
-                        this._verifyRedisPassword(
-                            config.utapi.redis.sentinelPassword),
-                            'config: invalid password for utapi redis. password' +
-                            ' must be a string');
-                        this.utapi.redis.sentinelPassword =
-                            config.utapi.redis.sentinelPassword;
                     }
                 }
                 if (config.utapi.metrics) {
@@ -1079,10 +1258,28 @@ class Config extends EventEmitter {
                 }
 
                 if (config.utapi && config.utapi.reindex) {
-                    parseUtapiReindex(config.utapi.reindex);
+                    this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
-                    this.utapi.reindex = config.utapi.reindex;
                 }
             }
+
+            if (utapiVersion === 2 && config.utapi.filter) {
+                const { filter: filterConfig } = config.utapi;
+                const utapiResourceFilters = {};
+                allowedUtapiEventFilterFields.forEach(
+                    field => allowedUtapiEventFilterStates.forEach(
+                        state => {
+                            const resources = (filterConfig[state] && filterConfig[state][field]) || null;
+                            if (resources) {
+                                assert.strictEqual(utapiResourceFilters[field], undefined,
+                                    `bad config: utapi.filter.${state}.${field} can't define an allow and a deny list`);
+                                assert(resources.every(r => typeof r === 'string'),
+                                    `bad config: utapi.filter.${state}.${field} must be an array of strings`);
+                                utapiResourceFilters[field] = { [state]: new Set(resources) };
+                            }
+                        }
+                ));
+                this.utapi.filter = utapiResourceFilters;
+            }
         }
         if (Object.keys(this.locationConstraints).some(
         loc => this.locationConstraints[loc].sizeLimitGB)) {
@@ -1106,6 +1303,8 @@ class Config extends EventEmitter {
             }
         }
 
+        this.authdata = config.authdata || 'authdata.json';
+
         this.kms = {};
         if (config.kms) {
             assert(typeof config.kms.userName === 'string');
@@ -1325,32 +1524,13 @@ class Config extends EventEmitter {
             this.outboundProxy.certs = certObj.certs;
         }
 
-        this.managementAgent = {};
-        this.managementAgent.port = 8010;
-        this.managementAgent.host = 'localhost';
-        if (config.managementAgent !== undefined) {
-            if (config.managementAgent.port !== undefined) {
-                assert(Number.isInteger(config.managementAgent.port)
-                       && config.managementAgent.port > 0,
-                       'bad config: managementAgent port must be a positive ' +
-                       'integer');
-                this.managementAgent.port = config.managementAgent.port;
-            }
-            if (config.managementAgent.host !== undefined) {
-                assert.strictEqual(typeof config.managementAgent.host, 'string',
-                                   'bad config: management agent host must ' +
-                                   'be a string');
-                this.managementAgent.host = config.managementAgent.host;
-            }
-        }
-
         // Ephemeral token to protect the reporting endpoint:
         // try inherited from parent first, then hardcoded in conf file,
         // then create a fresh one as last resort.
         this.reportToken =
             process.env.REPORT_TOKEN ||
             config.reportToken ||
-            uuid.v4().toString();
+            uuidv4();
 
         // External backends
         // Currently supports configuring httpAgent(s) for keepAlive
@@ -1396,52 +1576,135 @@ class Config extends EventEmitter {
             requestsConfigAssert(config.requests);
             this.requests = config.requests;
         }
+        // CLDSRV-378: on 8.x branches, null version compatibility
+        // mode is enforced because null keys are not supported by the
+        // MongoDB backend.
+        this.nullVersionCompatMode = true;
         if (config.bucketNotificationDestinations) {
             this.bucketNotificationDestinations = bucketNotifAssert(config.bucketNotificationDestinations);
         }
+
+        this.lifecycleRoleName = config.lifecycleRoleName || null;
+
+        // Version of the configuration we're running under
+        this.overlayVersion = config.overlayVersion || 0;
+
+        this._setTimeOptions();
+        this.multiObjectDeleteConcurrency = constants.multiObjectDeleteConcurrency;
+        const extractedNumber = Number.parseInt(config.multiObjectDeleteConcurrency, 10);
+        if (!isNaN(extractedNumber) && extractedNumber > 0 && extractedNumber < 1000) {
+            this.multiObjectDeleteConcurrency = extractedNumber;
+        }
+
+        this.multiObjectDeleteEnableOptimizations = true;
+        if (config.multiObjectDeleteEnableOptimizations === false) {
+            this.multiObjectDeleteEnableOptimizations = false;
+        }
+
+        this.testingMode = config.testingMode || false;
+
+        this.maxScannedLifecycleListingEntries = constants.maxScannedLifecycleListingEntries;
+        if (config.maxScannedLifecycleListingEntries !== undefined) {
+            // maxScannedLifecycleListingEntries > 2 is required as a minimum because we must
+            // scan at least three entries to determine version eligibility.
+            // Two entries representing the master key and the following one representing the non-current version.
+            assert(Number.isInteger(config.maxScannedLifecycleListingEntries) &&
+                config.maxScannedLifecycleListingEntries > 2,
+                'bad config: maxScannedLifecycleListingEntries must be greater than 2');
+            this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
+        }
+
+        this._configureBackends(config);
     }
 
-    _configureBackends() {
+    _setTimeOptions() {
+        // NOTE: EXPIRE_ONE_DAY_EARLIER and TRANSITION_ONE_DAY_EARLIER are deprecated in favor of
+        // TIME_PROGRESSION_FACTOR which decreases the weight attributed to a day in order to among other things
+        // expedite the lifecycle of objects.
+
+        // moves lifecycle expiration deadlines 1 day earlier, mostly for testing
+        const expireOneDayEarlier = process.env.EXPIRE_ONE_DAY_EARLIER === 'true';
+        // moves lifecycle transition deadlines 1 day earlier, mostly for testing
+        const transitionOneDayEarlier = process.env.TRANSITION_ONE_DAY_EARLIER === 'true';
+        // decreases the weight attributed to a day in order to expedite the lifecycle of objects.
+        const timeProgressionFactor = Number.parseInt(process.env.TIME_PROGRESSION_FACTOR, 10) || 1;
+
+        const isIncompatible = (expireOneDayEarlier || transitionOneDayEarlier) && (timeProgressionFactor > 1);
+        assert(!isIncompatible, 'The environment variables "EXPIRE_ONE_DAY_EARLIER" or ' +
+        '"TRANSITION_ONE_DAY_EARLIER" are not compatible with the "TIME_PROGRESSION_FACTOR" variable.');
+
+        // The scaledMsPerDay value is initially set to the number of milliseconds per day
+        // (24 * 60 * 60 * 1000)  as the default value.
+        // However, during testing, if the timeProgressionFactor is defined and greater than 1,
+        // the scaledMsPerDay value is decreased. This adjustment allows for simulating actions occurring
+        // earlier in time.
+        const scaledMsPerDay = scaleMsPerDay(timeProgressionFactor);
+
+        this.timeOptions = {
+            expireOneDayEarlier,
+            transitionOneDayEarlier,
+            timeProgressionFactor,
+            scaledMsPerDay,
+        };
+    }
+
+    getTimeOptions() {
+        return this.timeOptions;
+    }
+
+    _getAuthData() {
+        return JSON.parse(fs.readFileSync(findConfigFile(process.env.S3AUTH_CONFIG || this.authdata), { encoding: 'utf-8' }));
+    }
+
+    _configureBackends(config) {
+        const backends = config.backends || {};
         /**
          * Configure the backends for Authentication, Data and Metadata.
          */
-        let auth = 'mem';
+        let auth = backends.auth || 'mem';
-        let data = 'multiple';
+        let data = backends.data || 'multiple';
-        let metadata = 'file';
+        let metadata = backends.metadata || 'file';
-        let kms = 'file';
+        let kms = backends.kms || 'file';
+        let quota = backends.quota || 'none';
         if (process.env.S3BACKEND) {
             const validBackends = ['mem', 'file', 'scality', 'cdmi'];
             assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
                 'bad environment variable: S3BACKEND environment variable ' +
                 'should be one of mem/file/scality/cdmi'
             );
-            auth = process.env.S3BACKEND;
+            auth = process.env.S3BACKEND == 'scality' ? 'scality' : 'mem';
             data = process.env.S3BACKEND;
             metadata = process.env.S3BACKEND;
             kms = process.env.S3BACKEND;
         }
         if (process.env.S3VAULT) {
             auth = process.env.S3VAULT;
+            auth = (auth === 'file' || auth === 'mem' || auth === 'cdmi' ? 'mem' : auth);
         }
+
         if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
             // Auth only checks for 'mem' since mem === file
-            auth = 'mem';
-            const authfile = findConfigFile(process.env.S3AUTH_CONFIG ||
-                                            'authdata.json');
             let authData;
             if (process.env.SCALITY_ACCESS_KEY_ID &&
-            process.env.SCALITY_SECRET_ACCESS_KEY) {
+                process.env.SCALITY_SECRET_ACCESS_KEY) {
                 authData = buildAuthDataAccount(
-                  process.env.SCALITY_ACCESS_KEY_ID,
+                    process.env.SCALITY_ACCESS_KEY_ID,
-                  process.env.SCALITY_SECRET_ACCESS_KEY);
+                    process.env.SCALITY_SECRET_ACCESS_KEY);
             } else {
-                authData = require(authfile);
+                authData = this._getAuthData();
             }
             if (validateAuthConfig(authData)) {
                 throw new Error('bad config: invalid auth config file.');
             }
             this.authData = authData;
+        } else if (auth === 'multiple') {
+            const authData = this._getAuthData();
+            if (validateAuthConfig(authData)) {
+                throw new Error('bad config: invalid auth config file.');
+            }
+            this.authData = authData;
         }
+
         if (process.env.S3DATA) {
             const validData = ['mem', 'file', 'scality', 'multiple'];
             assert(validData.indexOf(process.env.S3DATA) > -1,
@@ -1449,9 +1712,9 @@ class Config extends EventEmitter {
                 'should be one of mem/file/scality/multiple'
             );
             data = process.env.S3DATA;
-        }
+            if (data === 'scality' || data === 'multiple') {
-        if (data === 'scality' || data === 'multiple') {
+                data = 'multiple';
-            data = 'multiple';
+            }
         }
         assert(this.locationConstraints !== undefined &&
             this.restEndpoints !== undefined,
@@ -1464,18 +1727,18 @@ class Config extends EventEmitter {
         if (process.env.S3KMS) {
             kms = process.env.S3KMS;
         }
+        if (process.env.S3QUOTA) {
+            quota = process.env.S3QUOTA;
+        }
         this.backends = {
             auth,
             data,
             metadata,
             kms,
+            quota,
         };
     }
 
-    _verifyRedisPassword(password) {
-        return typeof password === 'string';
-    }
-
     setAuthDataAccounts(accounts) {
         this.authData.accounts = accounts;
         this.emit('authdata-update');
@@ -1522,9 +1785,8 @@ class Config extends EventEmitter {
 
     getAzureEndpoint(locationConstraint) {
         let azureStorageEndpoint =
-        process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] ||
+            process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] ||
-        this.locationConstraints[locationConstraint]
+            this.locationConstraints[locationConstraint].details.azureStorageEndpoint;
-            .details.azureStorageEndpoint;
         if (!azureStorageEndpoint.endsWith('/')) {
             // append the trailing slash
             azureStorageEndpoint = `${azureStorageEndpoint}/`;
@@ -1533,23 +1795,40 @@ class Config extends EventEmitter {
     }
 
     getAzureStorageAccountName(locationConstraint) {
-        const { azureStorageAccountName } =
+        const accountName = azureGetStorageAccountName(
-            this.locationConstraints[locationConstraint].details;
+            locationConstraint,
-        const storageAccountNameFromEnv =
+            this.locationConstraints[locationConstraint].details
-            process.env[`${locationConstraint}_AZURE_STORAGE_ACCOUNT_NAME`];
+        );
-        return storageAccountNameFromEnv || azureStorageAccountName;
+        if (accountName) {
+            return accountName;
+        }
+
+        // For SAS & ServicePrincipal, retrieve the accountName from the endpoint
+        const endpoint = this.getAzureEndpoint(locationConstraint);
+        const url = new URL(endpoint);
+        const fragments = url.hostname.split('.', 3);
+        if (fragments.length === 3 && fragments[1] === 'blob') {
+            return fragments[0];
+        }
+
+        // We should always reach here, though it may not be the case for "mock" servers,
+        // where the accoutName is in the path
+        const path = url.pathname.replace(/^\//, '').replace(/\/$/, '');
+        if (path) {
+            return path;
+        }
+
+        // We should not reach here; if that happens, use the endpoint itself, which
+        // should be close-enough since this function is used for detecting when two
+        // locations actually point to the same account
+        return endpoint;
     }
 
     getAzureStorageCredentials(locationConstraint) {
-        const { azureStorageAccessKey } =
+        return azureGetLocationCredentials(
-            this.locationConstraints[locationConstraint].details;
+            locationConstraint,
-        const storageAccessKeyFromEnv =
+            this.locationConstraints[locationConstraint].details
-            process.env[`${locationConstraint}_AZURE_STORAGE_ACCESS_KEY`];
+        );
-        return {
-            storageAccountName:
-                this.getAzureStorageAccountName(locationConstraint),
-            storageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
-        };
     }
 
     getPfsDaemonEndpoint(locationConstraint) {
@@ -1568,7 +1847,7 @@ class Config extends EventEmitter {
         return azureSrcAccount === azureDestAccount;
     }
 
-    isAWSServerSideEncrytion(locationConstraint) {
+    isAWSServerSideEncryption(locationConstraint) {
         return this.locationConstraints[locationConstraint].details
         .serverSideEncryption === true;
     }
@@ -1582,13 +1861,25 @@ class Config extends EventEmitter {
                                 .update(instanceId)
                                 .digest('hex');
     }
+
+    isQuotaEnabled() {
+        return !!this.quotaEnabled;
+    }
+
+    isQuotaInflightEnabled() {
+        return this.quota.enableInflights;
+    }
 }
 
 module.exports = {
     parseSproxydConfig,
+    parseRedisConfig,
     locationConstraintAssert,
     ConfigObject: Config,
     config: new Config(),
     requestsConfigAssert,
     bucketNotifAssert,
+    azureGetStorageAccountName,
+    azureGetLocationCredentials,
+    azureArchiveLocationConstraintAssert,
 };

lib/api/api.js

@@ -3,10 +3,12 @@ const async = require('async');
 
 const bucketDelete = require('./bucketDelete');
 const bucketDeleteCors = require('./bucketDeleteCors');
+const bucketDeleteEncryption = require('./bucketDeleteEncryption');
 const bucketDeleteWebsite = require('./bucketDeleteWebsite');
 const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
 const bucketDeletePolicy = require('./bucketDeletePolicy');
-const bucketGet = require('./bucketGet');
+const bucketDeleteQuota = require('./bucketDeleteQuota');
+const { bucketGet } = require('./bucketGet');
 const bucketGetACL = require('./bucketGetACL');
 const bucketGetCors = require('./bucketGetCors');
 const bucketGetVersioning = require('./bucketGetVersioning');
@@ -16,17 +18,24 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
 const bucketGetNotification = require('./bucketGetNotification');
 const bucketGetObjectLock = require('./bucketGetObjectLock');
 const bucketGetPolicy = require('./bucketGetPolicy');
+const bucketGetQuota = require('./bucketGetQuota');
+const bucketGetEncryption = require('./bucketGetEncryption');
 const bucketHead = require('./bucketHead');
 const { bucketPut } = require('./bucketPut');
 const bucketPutACL = require('./bucketPutACL');
 const bucketPutCors = require('./bucketPutCors');
 const bucketPutVersioning = require('./bucketPutVersioning');
+const bucketPutTagging = require('./bucketPutTagging');
+const bucketDeleteTagging = require('./bucketDeleteTagging');
+const bucketGetTagging = require('./bucketGetTagging');
 const bucketPutWebsite = require('./bucketPutWebsite');
 const bucketPutReplication = require('./bucketPutReplication');
 const bucketPutLifecycle = require('./bucketPutLifecycle');
 const bucketPutNotification = require('./bucketPutNotification');
+const bucketPutEncryption = require('./bucketPutEncryption');
 const bucketPutPolicy = require('./bucketPutPolicy');
 const bucketPutObjectLock = require('./bucketPutObjectLock');
+const bucketUpdateQuota = require('./bucketUpdateQuota');
 const bucketGetReplication = require('./bucketGetReplication');
 const bucketDeleteReplication = require('./bucketDeleteReplication');
 const corsPreflight = require('./corsPreflight');
@@ -34,10 +43,11 @@ const completeMultipartUpload = require('./completeMultipartUpload');
 const initiateMultipartUpload = require('./initiateMultipartUpload');
 const listMultipartUploads = require('./listMultipartUploads');
 const listParts = require('./listParts');
+const metadataSearch = require('./metadataSearch');
 const { multiObjectDelete } = require('./multiObjectDelete');
 const multipartDelete = require('./multipartDelete');
 const objectCopy = require('./objectCopy');
-const objectDelete = require('./objectDelete');
+const { objectDelete } = require('./objectDelete');
 const objectDeleteTagging = require('./objectDeleteTagging');
 const objectGet = require('./objectGet');
 const objectGetACL = require('./objectGetACL');
@@ -52,16 +62,18 @@ const objectPutTagging = require('./objectPutTagging');
 const objectPutPart = require('./objectPutPart');
 const objectPutCopyPart = require('./objectPutCopyPart');
 const objectPutRetention = require('./objectPutRetention');
+const objectRestore = require('./objectRestore');
 const prepareRequestContexts
     = require('./apiUtils/authorization/prepareRequestContexts');
 const serviceGet = require('./serviceGet');
 const vault = require('../auth/vault');
-const websiteGet = require('./websiteGet');
+const website = require('./website');
-const websiteHead = require('./websiteHead');
 const writeContinue = require('../utilities/writeContinue');
 const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
 const parseCopySource = require('./apiUtils/object/parseCopySource');
 const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
+const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
+const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
 
 const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
 
@@ -70,6 +82,14 @@ auth.setHandler(vault);
 /* eslint-disable no-param-reassign */
 const api = {
     callApiMethod(apiMethod, request, response, log, callback) {
+        // Attach the apiMethod method to the request, so it can used by monitoring in the server
+        // eslint-disable-next-line no-param-reassign
+        request.apiMethod = apiMethod;
+        // Array of end of API callbacks, used to perform some logic
+        // at the end of an API.
+        // eslint-disable-next-line no-param-reassign
+        request.finalizerHooks = [];
+
         const actionLog = monitoringMap[apiMethod];
         if (!actionLog &&
             apiMethod !== 'websiteGet' &&
@@ -103,6 +123,7 @@ const api = {
         // no need to check auth on website or cors preflight requests
         if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
         apiMethod === 'corsPreflight') {
+            request.actionImplicitDenies = false;
             return this[apiMethod](request, log, callback);
         }
 
@@ -115,17 +136,35 @@ const api = {
             return process.nextTick(callback, parsingError);
         }
 
+        const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers);
+        if (httpHeadersSizeError) {
+            log.debug('http header size limit exceeded', {
+                error: httpHeadersSizeError,
+            });
+            return process.nextTick(callback, httpHeadersSizeError);
+        }
+
         const requestContexts = prepareRequestContexts(apiMethod, request,
             sourceBucket, sourceObject, sourceVersionId);
+        // Extract all the _apiMethods and store them in an array
+        const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : [];
+        // Attach the names to the current request
+        // eslint-disable-next-line no-param-reassign
+        request.apiMethods = apiMethods;
 
         function checkAuthResults(authResults) {
             let returnTagCount = true;
+            const isImplicitDeny = {};
+            let isOnlyImplicitDeny = true;
             if (apiMethod === 'objectGet') {
                 // first item checks s3:GetObject(Version) action
-                if (!authResults[0].isAllowed) {
+                if (!authResults[0].isAllowed && !authResults[0].isImplicit) {
                     log.trace('get object authorization denial from Vault');
                     return errors.AccessDenied;
                 }
+                // TODO add support for returnTagCount in the bucket policy
+                // checks
+                isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
                 // second item checks s3:GetObject(Version)Tagging action
                 if (!authResults[1].isAllowed) {
                     log.trace('get tagging authorization denial ' +
@@ -134,114 +173,143 @@ const api = {
                 }
             } else {
                 for (let i = 0; i < authResults.length; i++) {
-                    if (!authResults[i].isAllowed) {
+                    isImplicitDeny[authResults[i].action] = true;
+                    if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
+                        // Any explicit deny rejects the current API call
                         log.trace('authorization denial from Vault');
                         return errors.AccessDenied;
                     }
+                    if (authResults[i].isAllowed) {
+                        // If the action is allowed, the result is not implicit
+                        // Deny.
+                        isImplicitDeny[authResults[i].action] = false;
+                        isOnlyImplicitDeny = false;
+                    }
                 }
             }
-            return returnTagCount;
+            // These two APIs cannot use ACLs or Bucket Policies, hence, any
+            // implicit deny from vault must be treated as an explicit deny.
+            if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
+                return errors.AccessDenied;
+            }
+            return { returnTagCount, isImplicitDeny };
         }
 
         return async.waterfall([
-            next => auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) =>
+            next => auth.server.doAuth(
-                next(err, userInfo, authorizationResults, streamingV4Params), 's3', requestContexts),
+                request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
-            (userInfo, authorizationResults, streamingV4Params, next) => {
+                    if (err) {
-                if (authorizationResults) {
+                        // VaultClient returns standard errors, but the route requires
-                    const checkedResults = checkAuthResults(authorizationResults);
+                        // Arsenal errors
-                    if (checkedResults instanceof Error) {
+                        const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
-                        return next(checkedResults);
+                        log.trace('authentication error', { error: err });
+                        return next(arsenalError);
                     }
-                    returnTagCount = checkedResults;
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                }, 's3', requestContexts),
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => {
+                const authNames = { accountName: userInfo.getAccountDisplayName() };
+                if (userInfo.isRequesterAnIAMUser()) {
+                    authNames.userName = userInfo.getIAMdisplayName();
                 }
-                return tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log,
+                if (isRequesterASessionUser(userInfo)) {
-                (err, tagAuthResults, updatedContexts) =>
+                    authNames.sessionName = userInfo.getShortid().split(':')[1];
-                    next(err, tagAuthResults, authorizationResults, userInfo, streamingV4Params, updatedContexts));
+                }
+                log.addDefaultFields(authNames);
+                if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                }
+                // issue 100 Continue to the client
+                writeContinue(request, response);
+                const MAX_POST_LENGTH = request.method === 'POST' ?
+                      1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
+                const post = [];
+                let postLength = 0;
+                request.on('data', chunk => {
+                    postLength += chunk.length;
+                    // Sanity check on post length
+                    if (postLength <= MAX_POST_LENGTH) {
+                        post.push(chunk);
+                    }
+                });
+
+                request.on('error', err => {
+                    log.trace('error receiving request', {
+                        error: err,
+                    });
+                    return next(errors.InternalError);
+                });
+
+                request.on('end', () => {
+                    if (postLength > MAX_POST_LENGTH) {
+                        log.error('body length is too long for request type',
+                                  { postLength });
+                        return next(errors.InvalidRequest);
+                    }
+                    // Convert array of post buffers into one string
+                    request.post = Buffer.concat(post, postLength).toString();
+                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                });
+                return undefined;
             },
-        ], (err, tagAuthResults, authorizationResults, userInfo, streamingV4Params, updatedContexts) => {
+            // Tag condition keys require information from CloudServer for evaluation
+            (userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
+                authorizationResults,
+                request,
+                requestContexts,
+                apiMethod,
+                log,
+                (err, authResultsWithTags) => {
+                    if (err) {
+                        log.trace('tag authentication error', { error: err });
+                        return next(err);
+                    }
+                    return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
+                },
+            ),
+        ], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
             if (err) {
-                log.trace('authentication error', { error: err });
                 return callback(err);
             }
-            const authNames = { accountName: userInfo.getAccountDisplayName() };
+            request.accountQuotas = infos?.accountQuota;
-            if (userInfo.isRequesterAnIAMUser()) {
+            if (authorizationResults) {
-                authNames.userName = userInfo.getIAMdisplayName();
+                const checkedResults = checkAuthResults(authorizationResults);
-            }
-            log.addDefaultFields(authNames);
-            if (tagAuthResults) {
-                const checkedResults = checkAuthResults(tagAuthResults);
                 if (checkedResults instanceof Error) {
                     return callback(checkedResults);
                 }
-                returnTagCount = checkedResults;
+                returnTagCount = checkedResults.returnTagCount;
+                request.actionImplicitDenies = checkedResults.isImplicitDeny;
+            } else {
+                // create an object of keys apiMethods with all values to false:
+                // for backward compatibility, all apiMethods are allowed by default
+                // thus it is explicitly allowed, so implicit deny is false
+                request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
+                    acc[curr] = false;
+                    return acc;
+                }, {});
             }
+            const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
+                    (hook, done) => hook(err, done),
+                    () => callback(err, ...results));
+
             if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
                 request._response = response;
                 return this[apiMethod](userInfo, request, streamingV4Params,
-                    log, callback, authorizationResults);
+                    log, methodCallback, authorizationResults);
             }
-            // issue 100 Continue to the client
+            if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
-            writeContinue(request, response);
+                return this[apiMethod](userInfo, request, sourceBucket,
-            const MAX_POST_LENGTH = request.method.toUpperCase() === 'POST' ?
+                    sourceObject, sourceVersionId, log, methodCallback);
-                1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
+            }
-            const post = [];
+            if (apiMethod === 'objectGet') {
-            let postLength = 0;
+                return this[apiMethod](userInfo, request, returnTagCount, log, callback);
-            request.on('data', chunk => {
+            }
-                postLength += chunk.length;
+            return this[apiMethod](userInfo, request, log, methodCallback);
-                // Sanity check on post length
-                if (postLength <= MAX_POST_LENGTH) {
-                    post.push(chunk);
-                }
-                return undefined;
-            });
-
-            request.on('error', err => {
-                log.trace('error receiving request', {
-                    error: err,
-                });
-                return callback(errors.InternalError);
-            });
-
-            request.on('end', () => {
-                if (postLength > MAX_POST_LENGTH) {
-                    log.error('body length is too long for request type',
-                        { postLength });
-                    return callback(errors.InvalidRequest);
-                }
-                // Convert array of post buffers into one string
-                request.post = Buffer.concat(post, postLength).toString();
-
-                // IAM policy -Tag condition keys require information from CloudServer for evaluation
-                return tagConditionKeyAuth(authorizationResults, request, (updatedContexts || requestContexts),
-                apiMethod, log, (err, tagAuthResults) => {
-                    if (err) {
-                        log.trace('tag authentication error', { error: err });
-                        return callback(err);
-                    }
-                    if (tagAuthResults) {
-                        const checkedResults = checkAuthResults(tagAuthResults);
-                        if (checkedResults instanceof Error) {
-                            return callback(checkedResults);
-                        }
-                        returnTagCount = checkedResults;
-                    }
-                    if (apiMethod === 'objectCopy' ||
-                    apiMethod === 'objectPutCopyPart') {
-                        return this[apiMethod](userInfo, request, sourceBucket,
-                            sourceObject, sourceVersionId, log, callback);
-                    }
-                    if (apiMethod === 'objectGet') {
-                        return this[apiMethod](userInfo, request,
-                        returnTagCount, log, callback);
-                    }
-                    return this[apiMethod](userInfo, request, log, callback);
-                });
-            });
-            return undefined;
         });
     },
     bucketDelete,
     bucketDeleteCors,
+    bucketDeleteEncryption,
     bucketDeleteWebsite,
     bucketGet,
     bucketGetACL,
@@ -250,29 +318,38 @@ const api = {
     bucketGetVersioning,
     bucketGetWebsite,
     bucketGetLocation,
+    bucketGetEncryption,
     bucketHead,
     bucketPut,
     bucketPutACL,
     bucketPutCors,
     bucketPutVersioning,
+    bucketPutTagging,
+    bucketDeleteTagging,
+    bucketGetTagging,
     bucketPutWebsite,
     bucketPutReplication,
     bucketGetReplication,
     bucketDeleteReplication,
+    bucketDeleteQuota,
     bucketPutLifecycle,
+    bucketUpdateQuota,
     bucketGetLifecycle,
     bucketDeleteLifecycle,
     bucketPutPolicy,
     bucketGetPolicy,
+    bucketGetQuota,
     bucketDeletePolicy,
     bucketPutObjectLock,
     bucketPutNotification,
     bucketGetNotification,
+    bucketPutEncryption,
     corsPreflight,
     completeMultipartUpload,
     initiateMultipartUpload,
     listMultipartUploads,
     listParts,
+    metadataSearch,
     multiObjectDelete,
     multipartDelete,
     objectDelete,
@@ -291,9 +368,10 @@ const api = {
     objectPutPart,
     objectPutCopyPart,
     objectPutRetention,
+    objectRestore,
     serviceGet,
-    websiteGet,
+    websiteGet: website,
-    websiteHead,
+    websiteHead: website,
 };
 
 module.exports = api;

lib/api/apiUtils/authorization/bucketOwner.js

@@ -0,0 +1,29 @@
+const { errors } = require('arsenal');
+const vault = require('../../../auth/vault');
+
+function checkExpectedBucketOwner(headers, bucket, log, cb) {
+    const expectedOwner = headers['x-amz-expected-bucket-owner'];
+    if (expectedOwner === undefined) {
+        return cb();
+    }
+
+    const bucketOwner = bucket.getOwner();
+    return vault.getAccountIds([bucketOwner], log, (error, res) => {
+        if (error) {
+            log.error('error fetch accountId from vault', {
+                method: 'checkExpectedBucketOwner',
+                error,
+            });
+        }
+
+        if (error || res[bucketOwner] !== expectedOwner) {
+            return cb(errors.AccessDenied);
+        }
+
+        return cb();
+    });
+}
+
+module.exports = {
+    checkExpectedBucketOwner,
+};

lib/api/apiUtils/authorization/permissionChecks.js

@@ -1,49 +1,23 @@
-const { evaluators } = require('arsenal').policies;
+const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
+const { errors } = require('arsenal');
+const { parseCIDR, isValid } = require('ipaddr.js');
 const constants = require('../../../../constants');
+const { config } = require('../../../Config');
 
-const actionMap = {
+const {
-    multipartDelete: 's3:AbortMultipartUpload',
+    allAuthedUsersId,
-    bucketDelete: 's3:DeleteBucket',
+    bucketOwnerActions,
-    bucketDeletePolicy: 's3:DeleteBucketPolicy',
+    logId,
-    bucketDeleteWebsite: 's3:DeleteBucketWebsite',
+    publicId,
-    objectDelete: 's3:DeleteObject',
+    arrayOfAllowed,
-    objectDeleteTagging: 's3:DeleteObjectTagging',
+    assumedRoleArnResourceType,
-    bucketGetACL: 's3:GetBucketAcl',
+    backbeatLifecycleSessionName,
-    bucketGetCors: 's3:GetBucketCORS',
+    actionsToConsiderAsObjectPut,
-    bucketGetLocation: 's3:GetBucketLocation',
+} = constants;
-    bucketGetObjectLock: 's3:GetBucketObjectLockConfiguration',
-    bucketGetPolicy: 's3:GetBucketPolicy',
-    bucketGetVersioning: 's3:GetBucketVersioning',
-    bucketGetWebsite: 's3:GetBucketWebsite',
-    bucketGetLifecycle: 's3:GetLifecycleConfiguration',
-    objectGet: 's3:GetObject',
-    objectGetACL: 's3:GetObjectAcl',
-    objectGetLegalHold: 's3:GetObjectLegalHold',
-    objectGetRetention: 's3:GetObjectRetention',
-    objectGetTagging: 's3:GetObjectTagging',
-    bucketGetReplication: 's3:GetReplicationConfiguration',
-    bucketHead: 's3:ListBucket',
-    bucketGet: 's3:ListBucket',
-    listMultipartUploads: 's3:ListBucketMultipartUploads',
-    listParts: 's3:ListMultipartUploadParts',
-    bucketPutACL: 's3:PutBucketAcl',
-    bucketPutCors: 's3:PutBucketCORS',
-    bucketPutObjectLock: 's3:PutBucketObjectLockConfiguration',
-    bucketPutPolicy: 's3:PutBucketPolicy',
-    bucketPutVersioning: 's3:PutBucketVersioning',
-    bucketPutWebsite: 's3:PutBucketWebsite',
-    bucketPutLifecycle: 's3:PutLifecycleConfiguration',
-    objectPut: 's3:PutObject',
-    objectPutACL: 's3:PutObjectAcl',
-    objectPutTagging: 's3:PutObjectTagging',
-    objectPutLegalHold: 's3:PutObjectLegalHold',
-    objectPutRetention: 's3:PutObjectRetention',
-    bucketPutReplication: 's3:PutReplicationConfiguration',
-};
 
 // whitelist buckets to allow public read on objects
-const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ?
+const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
-    process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
+    ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
 
 function getServiceAccountProperties(canonicalID) {
     const canonicalIDArray = canonicalID.split('/');
@@ -55,51 +29,109 @@ function isServiceAccount(canonicalID) {
     return getServiceAccountProperties(canonicalID) !== undefined;
 }
 
-function checkBucketAcls(bucket, requestType, canonicalID) {
+function isRequesterASessionUser(authInfo) {
+    const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
+    return regexpAssumedRoleArn.test(authInfo.getArn());
+}
+
+function isRequesterNonAccountUser(authInfo) {
+    return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
+}
+
+/**
+ * Checks the access control for a given bucket based on the request type and user's canonical ID.
+ *
+ * @param {Bucket} bucket - The bucket to check access control for.
+ * @param {string} requestType - The list of s3 actions to check within the API call.
+ * @param {string} canonicalID - The canonical ID of the user making the request.
+ * @param {string} mainApiCall - The main API call (first item of the requestType).
+ *
+ * @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
+ */
+
+function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
+    // Same logic applies on the Versioned APIs, so let's simplify it.
+    let requestTypeParsed = requestType.endsWith('Version') ?
+        requestType.slice(0, 'Version'.length * -1) : requestType;
+    requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
+        'objectPut' : requestTypeParsed;
+    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
+        'objectPut' : mainApiCall;
     if (bucket.getOwner() === canonicalID) {
         return true;
     }
+    if (parsedMainApiCall === 'objectGet') {
+        if (requestTypeParsed === 'objectGetTagging') {
+            return true;
+        }
+    }
+    if (parsedMainApiCall === 'objectPut') {
+        if (arrayOfAllowed.includes(requestTypeParsed)) {
+            return true;
+        }
+    }
 
-    // any requestType outside of those checked in this function is
-    // outside the scope of ACL permissions and will be denied unless
-    // allowed by a different permissions granter
     const bucketAcl = bucket.getAcl();
-    if (requestType === 'bucketGet' || requestType === 'bucketHead') {
+    if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') {
         if (bucketAcl.Canned === 'public-read'
             || bucketAcl.Canned === 'public-read-write'
             || (bucketAcl.Canned === 'authenticated-read'
-                && canonicalID !== constants.publicId)) {
+                && canonicalID !== publicId)) {
             return true;
         } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
             || bucketAcl.READ.indexOf(canonicalID) > -1) {
             return true;
+        } else if (bucketAcl.READ.indexOf(publicId) > -1
+            || (bucketAcl.READ.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
         }
     }
-    if (requestType === 'bucketGetACL') {
+    if (requestTypeParsed === 'bucketGetACL') {
         if ((bucketAcl.Canned === 'log-delivery-write'
-                && canonicalID === constants.logId)
+            && canonicalID === logId)
             || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
             || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
             return true;
-        }
+        } else if (bucketAcl.READ_ACP.indexOf(publicId) > -1
-    }
+            || (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1
-
+                && canonicalID !== publicId)
-    if (requestType === 'bucketPutACL') {
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
-        if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
+                && canonicalID !== publicId)
-            || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
             return true;
         }
     }
 
-    if (requestType === 'bucketDelete' && bucket.getOwner() === canonicalID) {
+    if (requestTypeParsed === 'bucketPutACL') {
-        return true;
+        if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
+            || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
+            return true;
+        } else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1
+            || (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
+        }
     }
 
-    if (requestType === 'objectDelete' || requestType === 'objectPut') {
+    if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') {
         if (bucketAcl.Canned === 'public-read-write'
             || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
             || bucketAcl.WRITE.indexOf(canonicalID) > -1) {
             return true;
+        } else if (bucketAcl.WRITE.indexOf(publicId) > -1
+            || (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
         }
     }
     // Note that an account can have the ability to do objectPutACL,
@@ -108,86 +140,146 @@ function checkBucketAcls(bucket, requestType, canonicalID) {
     // objectPutACL, objectGetACL, objectHead or objectGet, the bucket
     // authorization check should just return true so can move on to check
     // rights at the object level.
-    return (requestType === 'objectPutACL' || requestType === 'objectGetACL' ||
+    return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
-    requestType === 'objectGet' || requestType === 'objectHead');
+    || requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
 }
 
-function checkObjectAcls(bucket, objectMD, requestType, canonicalID) {
+function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
+    isUserUnauthenticated, mainApiCall) {
     const bucketOwner = bucket.getOwner();
+    const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
+        'objectPut' : requestType;
+    const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
+        'objectPut' : mainApiCall;
     // acls don't distinguish between users and accounts, so both should be allowed
-    if (constants.bucketOwnerActions.includes(requestType)
+    if (bucketOwnerActions.includes(requestTypeParsed)
-    && (bucketOwner === canonicalID)) {
+        && (bucketOwner === canonicalID)) {
         return true;
     }
     if (objectMD['owner-id'] === canonicalID) {
         return true;
     }
+
+    // Backward compatibility
+    if (parsedMainApiCall === 'objectGet') {
+        if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
+            && requestTypeParsed === 'objectGetTagging') {
+            return true;
+        }
+    }
+
     if (!objectMD.acl) {
         return false;
     }
 
-    if (requestType === 'objectGet' || requestType === 'objectHead') {
+    if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') {
         if (objectMD.acl.Canned === 'public-read'
             || objectMD.acl.Canned === 'public-read-write'
             || (objectMD.acl.Canned === 'authenticated-read'
-                && canonicalID !== constants.publicId)) {
+                && canonicalID !== publicId)) {
             return true;
         } else if (objectMD.acl.Canned === 'bucket-owner-read'
-                && bucketOwner === canonicalID) {
+            && bucketOwner === canonicalID) {
             return true;
         } else if ((objectMD.acl.Canned === 'bucket-owner-full-control'
-                && bucketOwner === canonicalID)
+            && bucketOwner === canonicalID)
             || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
             || objectMD.acl.READ.indexOf(canonicalID) > -1) {
             return true;
+        } else if (objectMD.acl.READ.indexOf(publicId) > -1
+            || (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
         }
     }
 
     // User is already authorized on the bucket for FULL_CONTROL or WRITE or
     // bucket has canned ACL public-read-write
-    if (requestType === 'objectPut' || requestType === 'objectDelete') {
+    if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') {
         return true;
     }
 
-    if (requestType === 'objectPutACL') {
+    if (requestTypeParsed === 'objectPutACL') {
         if ((objectMD.acl.Canned === 'bucket-owner-full-control'
-                && bucketOwner === canonicalID)
+            && bucketOwner === canonicalID)
             || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
             || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
             return true;
+        } else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1
+            || (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
         }
     }
 
-    if (requestType === 'objectGetACL') {
+    if (requestTypeParsed === 'objectGetACL') {
         if ((objectMD.acl.Canned === 'bucket-owner-full-control'
-                && bucketOwner === canonicalID)
+            && bucketOwner === canonicalID)
             || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
             || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
             return true;
+        } else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1
+            || (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
+                && canonicalID !== publicId)
+            || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
+            return true;
         }
     }
 
     // allow public reads on buckets that are whitelisted for anonymous reads
     // TODO: remove this after bucket policies are implemented
     const bucketAcl = bucket.getAcl();
-    const allowPublicReads = publicReadBuckets.includes(bucket.getName()) &&
+    const allowPublicReads = publicReadBuckets.includes(bucket.getName())
-        bucketAcl.Canned === 'public-read' &&
+        && bucketAcl.Canned === 'public-read'
-        (requestType === 'objectGet' || requestType === 'objectHead');
+        && (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
     if (allowPublicReads) {
         return true;
     }
     return false;
 }
 
-function _checkActions(requestType, actions, log) {
+function _checkBucketPolicyActions(requestType, actions, log) {
-    const mappedAction = actionMap[requestType];
+    const mappedAction = actionMaps.actionMapBP[requestType];
-    // if requestType isn't in list of controlled actions
+    // Deny any action that isn't in list of controlled actions
     if (!mappedAction) {
-        return true;
+        return false;
     }
     return evaluators.isActionApplicable(mappedAction, actions, log);
 }
 
+function _checkBucketPolicyResources(request, resource, log) {
+    if (!request || (Array.isArray(resource) && resource.length === 0)) {
+        return true;
+    }
+    // build request context from the request!
+    const requestContext = new RequestContext(request.headers, request.query,
+        request.bucketName, request.objectKey, null,
+        request.connection.encrypted, request.resourceType, 's3');
+    return evaluators.isResourceApplicable(requestContext, resource, log);
+}
+
+function _checkBucketPolicyConditions(request, conditions, log) {
+    const ip = request ? requestUtils.getClientIp(request, config) : undefined;
+    if (!conditions) {
+        return true;
+    }
+    // build request context from the request!
+    const requestContext = new RequestContext(request.headers, request.query,
+        request.bucketName, request.objectKey, ip,
+        request.connection.encrypted, request.resourceType, 's3', null, null,
+        null, null, null, null, null, null, null, null, null,
+        request.objectLockRetentionDays);
+    return evaluators.meetConditions(requestContext, conditions, log);
+}
+
 function _getAccountId(arn) {
     // account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
     return arn.substr(13, 12);
@@ -232,24 +324,26 @@ function _checkPrincipals(canonicalID, arn, principal) {
     return false;
 }
 
-function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log) {
+function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) {
     let permission = 'defaultDeny';
     // if requester is user within bucket owner account, actions should be
     // allowed unless explicitly denied (assumes allowed by IAM policy)
-    if (bucketOwner === canonicalID) {
+    if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) {
         permission = 'allow';
     }
     let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
     while (copiedStatement.length > 0) {
         const s = copiedStatement[0];
         const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
-        const actionMatch = _checkActions(requestType, s.Action, log);
+        const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
+        const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
+        const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
 
-        if (principalMatch && actionMatch && s.Effect === 'Deny') {
+        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') {
             // explicit deny trumps any allows, so return immediately
             return 'explicitDeny';
         }
-        if (principalMatch && actionMatch && s.Effect === 'Allow') {
+        if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') {
             permission = 'allow';
         }
         copiedStatement = copiedStatement.splice(1);
@@ -257,73 +351,141 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l
     return permission;
 }
 
-function isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log) {
+function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log,
-    // Check to see if user is authorized to perform a
+    request, aclPermission, results, actionImplicitDenies) {
-    // particular action on bucket based on ACLs.
-    // TODO: Add IAM checks
-    let requesterIsNotUser = true;
-    let arn = null;
-    if (authInfo) {
-        requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
-        arn = authInfo.getArn();
-    }
-    // if the bucket owner is an account, users should not have default access
-    if (((bucket.getOwner() === canonicalID) && requesterIsNotUser)
-    || isServiceAccount(canonicalID)) {
-        return true;
-    }
-    const aclPermission = checkBucketAcls(bucket, requestType, canonicalID);
     const bucketPolicy = bucket.getBucketPolicy();
+    let processedResult = results[requestType];
     if (!bucketPolicy) {
-        return aclPermission;
+        processedResult = actionImplicitDenies[requestType] === false && aclPermission;
+    } else {
+        const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
+            bucketOwner, log, request, actionImplicitDenies);
+
+        if (bucketPolicyPermission === 'explicitDeny') {
+            processedResult = false;
+        } else if (bucketPolicyPermission === 'allow') {
+            processedResult = true;
+        } else {
+            processedResult = actionImplicitDenies[requestType] === false && aclPermission;
+        }
     }
-    const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
+    return processedResult;
-        canonicalID, arn, bucket.getOwner(), log);
-    if (bucketPolicyPermission === 'explicitDeny') {
-        return false;
-    }
-    return (aclPermission || (bucketPolicyPermission === 'allow'));
 }
 
-function isObjAuthorized(bucket, objectMD, requestType, canonicalID, authInfo, log) {
+function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
-    const bucketOwner = bucket.getOwner();
+    actionImplicitDeniesInput = {}, isWebsite = false) {
-    if (!objectMD) {
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
-        return false;
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
-    }
+    const mainApiCall = requestTypes[0];
-    let requesterIsNotUser = true;
+    const results = {};
-    let arn = null;
+    return requestTypes.every(_requestType => {
-    if (authInfo) {
+        // By default, all missing actions are defined as allowed from IAM, to be
-        requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
+        // backward compatible
-        arn = authInfo.getArn();
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
-    }
+        // Check to see if user is authorized to perform a
-    if (objectMD['owner-id'] === canonicalID && requesterIsNotUser) {
+        // particular action on bucket based on ACLs.
-        return true;
+        // TODO: Add IAM checks
-    }
+        let requesterIsNotUser = true;
+        let arn = null;
+        if (authInfo) {
+            requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
+            arn = authInfo.getArn();
+        }
+        // if the bucket owner is an account, users should not have default access
+        if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
+            results[_requestType] = actionImplicitDenies[_requestType] === false;
+            return results[_requestType];
+        }
+        const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
+        // In case of error bucket access is checked with bucketGet
+        // For website, bucket policy only uses objectGet and ignores bucketGet
+        // https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
+        // bucketGet should be used to check acl but switched to objectGet for bucket policy
+        if (isWebsite && _requestType === 'bucketGet') {
+            // eslint-disable-next-line no-param-reassign
+            _requestType = 'objectGet';
+            actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
+        }
+        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
+            request, aclPermission, results, actionImplicitDenies);
+    });
+}
 
-    if (isServiceAccount(canonicalID)) {
+function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {},
-        return true;
+    log, request) {
-    }
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
-    // account is authorized if:
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
-    // - requesttype is included in bucketOwnerActions and
+    const results = {};
-    // - account is the bucket owner
+    return requestTypes.every(_requestType => {
-    // - requester is account, not user
+        // By default, all missing actions are defined as allowed from IAM, to be
-    if (constants.bucketOwnerActions.includes(requestType)
+        // backward compatible
-    && (bucketOwner === canonicalID)
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
-    && requesterIsNotUser) {
+        let arn = null;
-        return true;
+        if (authInfo) {
-    }
+            arn = authInfo.getArn();
-    const aclPermission = checkObjectAcls(bucket, objectMD, requestType,
+        }
-        canonicalID);
+        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
-    const bucketPolicy = bucket.getBucketPolicy();
+        request, true, results, actionImplicitDenies);
-    if (!bucketPolicy) {
+    });
-        return aclPermission;
+}
-    }
+
-    const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
+function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
-        canonicalID, arn, bucket.getOwner(), log);
+    actionImplicitDeniesInput = {}, isWebsite = false) {
-    if (bucketPolicyPermission === 'explicitDeny') {
+    const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
-        return false;
+    const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
-    }
+    const results = {};
-    return (aclPermission || (bucketPolicyPermission === 'allow'));
+    const mainApiCall = requestTypes[0];
+    return requestTypes.every(_requestType => {
+        // By default, all missing actions are defined as allowed from IAM, to be
+        // backward compatible
+        actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
+        const parsedMethodName = _requestType.endsWith('Version')
+            ? _requestType.slice(0, -7) : _requestType;
+        const bucketOwner = bucket.getOwner();
+        if (!objectMD) {
+            // check bucket has read access
+            // 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
+            let permission = 'bucketGet';
+            if (actionsToConsiderAsObjectPut.includes(_requestType)) {
+                permission = 'objectPut';
+            }
+            results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
+                actionImplicitDenies, isWebsite);
+            // User is already authorized on the bucket for FULL_CONTROL or WRITE or
+            // bucket has canned ACL public-read-write
+            if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
+                && results[_requestType] === false) {
+                results[_requestType] = actionImplicitDenies[_requestType] === false;
+            }
+            return results[_requestType];
+        }
+        let requesterIsNotUser = true;
+        let arn = null;
+        let isUserUnauthenticated = false;
+        if (authInfo) {
+            requesterIsNotUser =  !isRequesterNonAccountUser(authInfo);
+            arn = authInfo.getArn();
+            isUserUnauthenticated = arn === undefined;
+        }
+        if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
+            results[_requestType] = actionImplicitDenies[_requestType] === false;
+            return results[_requestType];
+        }
+        // account is authorized if:
+        // - requesttype is included in bucketOwnerActions and
+        // - account is the bucket owner
+        // - requester is account, not user
+        if (bucketOwnerActions.includes(parsedMethodName)
+        && (bucketOwner === canonicalID)
+        && requesterIsNotUser) {
+            results[_requestType] = actionImplicitDenies[_requestType] === false;
+            return results[_requestType];
+        }
+        const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName,
+            canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall);
+        return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner,
+            log, request, aclPermission, results, actionImplicitDenies);
+    });
 }
 
 function _checkResource(resource, bucketArn) {
@@ -352,12 +514,128 @@ function validatePolicyResource(bucketName, policy) {
     });
 }
 
+function checkIp(value) {
+    const errString = 'Invalid IP address in Conditions';
+
+    const values = Array.isArray(value) ? value : [value];
+
+    for (let i = 0; i < values.length; i++) {
+        // these preliminary checks are validating the provided
+        // ip address against ipaddr.js, the library we use when
+        // evaluating IP condition keys. It ensures compatibility,
+        // but additional checks are required to enforce the right
+        // notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
+        // we would accept different ip formats, which is not
+        // standard in an AWS use case.
+        try {
+            try {
+                parseCIDR(values[i]);
+            } catch (err) {
+                isValid(values[i]);
+            }
+        } catch (err) {
+            return errString;
+        }
+
+        // Apply the existing IP validation logic to each element
+        const validateIpRegex = ip => {
+            if (constants.ipv4Regex.test(ip)) {
+                return ip.split('.').every(part => parseInt(part, 10) <= 255);
+            }
+            if (constants.ipv6Regex.test(ip)) {
+                return ip.split(':').every(part => part.length <= 4);
+            }
+            return false;
+        };
+
+        if (validateIpRegex(values[i]) !== true) {
+            return errString;
+        }
+    }
+
+    // If the function hasn't returned by now, all elements are valid
+    return null;
+}
+
+// This function checks all bucket policy conditions if the values provided
+// are valid for the condition type. If not it returns a relevant Malformed policy error string
+function validatePolicyConditions(policy) {
+    const validConditions = [
+        { conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
+        { conditionKey: 's3:object-lock-remaining-retention-days' },
+    ];
+    // keys where value type does not seem to be checked by AWS:
+    // - s3:object-lock-remaining-retention-days
+
+    if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
+        return null;
+    }
+
+    // there can be multiple statements in the policy, each with a Condition enclosure
+    for (let i = 0; i < policy.Statement.length; i++) {
+        const s = policy.Statement[i];
+        if (s.Condition) {
+            const conditionOperators = Object.keys(s.Condition);
+            // there can be multiple condition operations in the Condition enclosure
+            // eslint-disable-next-line no-restricted-syntax
+            for (const conditionOperator of conditionOperators) {
+                const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
+                const conditionValue = s.Condition[conditionOperator][conditionKey];
+                const validCondition = validConditions.find(validCondition =>
+                    validCondition.conditionKey === conditionKey
+                );
+                // AWS returns does not return an error if the condition starts with 'aws:'
+                // so we reproduce this behaviour
+                if (!validCondition && !conditionKey.startsWith('aws:')) {
+                    return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
+                }
+                if (validCondition && validCondition.conditionValueTypeChecker) {
+                    const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
+                    if (conditionValueTypeError) {
+                        return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
+                    }
+                }
+            }
+        }
+    }
+    return null;
+}
+
+
+/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
+ * @param {string} arn - Amazon resource name - example:
+ * arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
+ * @return {boolean} true if Lifecycle assumed role session arn, false if not.
+ */
+function isLifecycleSession(arn) {
+    if (!arn) {
+        return false;
+    }
+
+    const arnSplits = arn.split(':');
+    const service = arnSplits[2];
+
+    const resourceNames = arnSplits[arnSplits.length - 1].split('/');
+
+    const resourceType = resourceNames[0];
+    const sessionName = resourceNames[resourceNames.length - 1];
+
+    return (service === 'sts'
+        && resourceType === assumedRoleArnResourceType
+        && sessionName === backbeatLifecycleSessionName);
+}
+
 module.exports = {
     isBucketAuthorized,
     isObjAuthorized,
     getServiceAccountProperties,
     isServiceAccount,
+    isRequesterASessionUser,
+    isRequesterNonAccountUser,
     checkBucketAcls,
     checkObjectAcls,
     validatePolicyResource,
+    validatePolicyConditions,
+    isLifecycleSession,
+    evaluateBucketPolicyWithIAM,
 };

lib/api/apiUtils/authorization/prepareRequestContexts.js

@@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
             apiMethod, 's3');
     }
 
-    if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
+    if (apiMethod === 'bucketPut') {
         return null;
     }
 
@@ -65,7 +65,17 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
 
     const requestContexts = [];
 
-    if (apiMethodAfterVersionCheck === 'objectCopy'
+    if (apiMethod === 'multiObjectDelete') {
+        // MultiObjectDelete does not require any authorization when evaluating
+        // the API. Instead, we authorize each object passed.
+        // But in order to get any relevant information from the authorization service
+        // for example, the account quota, we must send a request context object
+        // with no `specificResource`. We expect the result to be an implicit deny.
+        // In the API, we then ignore these authorization results, and we can use
+        // any information returned, e.g., the quota.
+        const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
+          requestContexts.push(requestContextMultiObjectDelete);
+    } else if (apiMethodAfterVersionCheck === 'objectCopy'
         || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
         const objectGetAction = sourceVersionId ? 'objectGetVersion' :
           'objectGet';
@@ -97,27 +107,126 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
         const objectGetTaggingAction = (request.query &&
           request.query.versionId) ? 'objectGetTaggingVersion' :
           'objectGetTagging';
+        if (request.headers['x-amz-version-id']) {
+            const objectGetVersionAction = 'objectGetVersion';
+            const getVersionResourceVersion =
+                generateRequestContext(objectGetVersionAction);
+            requestContexts.push(getVersionResourceVersion);
+        }
         const getRequestContext =
           generateRequestContext(apiMethodAfterVersionCheck);
         const getTaggingRequestContext =
           generateRequestContext(objectGetTaggingAction);
         requestContexts.push(getRequestContext, getTaggingRequestContext);
-    } else if (apiMethodAfterVersionCheck === 'objectPut') {
+    } else if (apiMethodAfterVersionCheck === 'objectGetTagging') {
-        const putRequestContext =
+        const objectGetTaggingAction = 'objectGetTagging';
-          generateRequestContext(apiMethodAfterVersionCheck);
+        const getTaggingResourceVersion =
-        requestContexts.push(putRequestContext);
+            generateRequestContext(objectGetTaggingAction);
-        // if put object (versioning) with tag set
+        requestContexts.push(getTaggingResourceVersion);
-        if (request.headers['x-amz-tagging']) {
+        if (request.headers['x-amz-version-id']) {
-            const putTaggingRequestContext =
+            const objectGetTaggingVersionAction = 'objectGetTaggingVersion';
-              generateRequestContext('objectPutTagging');
+            const getTaggingVersionResourceVersion =
-            requestContexts.push(putTaggingRequestContext);
+                generateRequestContext(objectGetTaggingVersionAction);
+            requestContexts.push(getTaggingVersionResourceVersion);
         }
+    } else if (apiMethodAfterVersionCheck === 'objectHead') {
+        const objectHeadAction = 'objectHead';
+        const headObjectAction =
+            generateRequestContext(objectHeadAction);
+        requestContexts.push(headObjectAction);
+        if (request.headers['x-amz-version-id']) {
+            const objectHeadVersionAction = 'objectGetVersion';
+            const headObjectVersion =
+                generateRequestContext(objectHeadVersionAction);
+            requestContexts.push(headObjectVersion);
+        }
+    } else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
+        const putObjectTaggingRequestContext =
+            generateRequestContext('objectPutTagging');
+        requestContexts.push(putObjectTaggingRequestContext);
+        if (request.headers['x-amz-version-id']) {
+            const putObjectVersionRequestContext =
+                generateRequestContext('objectPutTaggingVersion');
+            requestContexts.push(putObjectVersionRequestContext);
+        }
+    } else if (apiMethodAfterVersionCheck === 'objectPutCopyPart') {
+        const putObjectRequestContext =
+            generateRequestContext('objectPut');
+        requestContexts.push(putObjectRequestContext);
+        const getObjectRequestContext =
+            generateRequestContext('objectGet');
+        requestContexts.push(getObjectRequestContext);
+    } else if (apiMethodAfterVersionCheck === 'objectPut') {
+        // if put object with version
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
+            const putRequestContext =
+              generateRequestContext(apiMethodAfterVersionCheck);
+            requestContexts.push(putRequestContext);
+            // if put object (versioning) with tag set
+            if (request.headers['x-amz-tagging']) {
+                const putTaggingRequestContext =
+                  generateRequestContext('objectPutTagging');
+                requestContexts.push(putTaggingRequestContext);
+            }
+            if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
+                const putLegalHoldStatusAction =
+                    generateRequestContext('objectPutLegalHold');
+                requestContexts.push(putLegalHoldStatusAction);
+            }
+            // if put object (versioning) with ACL
+            if (isHeaderAcl(request.headers)) {
+                const putAclRequestContext =
+                  generateRequestContext('objectPutACL');
+                requestContexts.push(putAclRequestContext);
+            }
+            if (request.headers['x-amz-object-lock-mode']) {
+                const putObjectLockRequestContext =
+                  generateRequestContext('objectPutRetention');
+                requestContexts.push(putObjectLockRequestContext);
+            }
+            if (request.headers['x-amz-version-id']) {
+                const putObjectVersionRequestContext =
+                    generateRequestContext('objectPutTaggingVersion');
+                requestContexts.push(putObjectVersionRequestContext);
+            }
+        }
+    } else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
+      apiMethodAfterVersionCheck === 'objectPutPart' ||
+      apiMethodAfterVersionCheck === 'completeMultipartUpload'
+      ) {
+        if (request.headers['x-scal-s3-version-id'] ||
+        request.headers['x-scal-s3-version-id'] === '') {
+            const putVersionRequestContext =
+              generateRequestContext('objectPutVersion');
+            requestContexts.push(putVersionRequestContext);
+        } else {
+            const putRequestContext =
+              generateRequestContext(apiMethodAfterVersionCheck);
+            requestContexts.push(putRequestContext);
+        }
+
         // if put object (versioning) with ACL
         if (isHeaderAcl(request.headers)) {
             const putAclRequestContext =
               generateRequestContext('objectPutACL');
             requestContexts.push(putAclRequestContext);
         }
+
+        if (request.headers['x-amz-object-lock-mode']) {
+            const putObjectLockRequestContext =
+              generateRequestContext('objectPutRetention');
+            requestContexts.push(putObjectLockRequestContext);
+        }
+        if (request.headers['x-amz-version-id']) {
+            const putObjectVersionRequestContext =
+                generateRequestContext('objectPutTaggingVersion');
+            requestContexts.push(putObjectVersionRequestContext);
+        }
     } else {
         const requestContext =
           generateRequestContext(apiMethodAfterVersionCheck);

lib/api/apiUtils/authorization/tagConditionKeys.js

@@ -1,5 +1,5 @@
-const assert = require('assert');
 const async = require('async');
+
 const { auth, s3middleware } = require('arsenal');
 const metadata = require('../../../metadata/wrapper');
 const { decodeVersionId } = require('../object/versioning');
@@ -12,63 +12,66 @@ function makeTagQuery(tags) {
         .join('&');
 }
 
-function updateRequestContexts(request, requestContexts, apiMethod, log, cb) {
+function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) {
-    requestContexts.forEach(rc => {
+    async.waterfall([
-        rc.setNeedTagEval(true);
+        next => {
-
+            if (request.headers['x-amz-tagging']) {
-        async.series([
+                return next(null, request.headers['x-amz-tagging']);
-            next => {
+            }
-                if (request.headers['x-amz-tagging']) {
+            if (request.post && apiMethod === 'objectPutTagging') {
-                    rc.setRequestObjTags(request.headers['x-amz-tagging']);
+                return parseTagXml(request.post, log, (err, tags) => {
-                    process.nextTick(() => next());
-                } else if (request.post && apiMethod === 'objectPutTagging') {
-                    parseTagXml(request.post, log, (err, tags) => {
-                        if (err) {
-                            log.trace('error parsing request tags');
-                            return next(err);
-                        }
-                        rc.setRequestObjTags(makeTagQuery(tags));
-                        return next();
-                    });
-                } else {
-                    process.nextTick(() => next());
-                }
-            },
-            next => {
-                const objectKey = request.objectKey;
-                const bucketName = request.bucketName;
-                const decodedVidResult = decodeVersionId(request.query);
-                if (decodedVidResult instanceof Error) {
-                    log.trace('invalid versionId query', {
-                        versionId: request.query.versionId,
-                        error: decodedVidResult,
-                    });
-                    return process.nextTick(() => next(decodedVidResult));
-                }
-                const reqVersionId = decodedVidResult;
-                return metadata.getObjectMD(bucketName, objectKey, { versionId: reqVersionId }, log,
-                (err, objMD) => {
                     if (err) {
+                        log.trace('error parsing request tags');
+                        return next(err);
+                    }
+                    return next(null, makeTagQuery(tags));
+                });
+            }
+            return next(null, null);
+        },
+        (requestTagsQuery, next) => {
+            const objectKey = request.objectKey;
+            const bucketName = request.bucketName;
+            const decodedVidResult = decodeVersionId(request.query);
+            if (decodedVidResult instanceof Error) {
+                log.trace('invalid versionId query', {
+                    versionId: request.query.versionId,
+                    error: decodedVidResult,
+                });
+                return next(decodedVidResult);
+            }
+            const reqVersionId = decodedVidResult;
+            return metadata.getObjectMD(
+                bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => {
+                    if (err) {
+                        // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
                         if (err.NoSuchKey) {
-                            return next();
+                            return next(null, requestTagsQuery, null);
                         }
                         log.trace('error getting request object tags');
                         return next(err);
                     }
-                    const existingTags = objMD.tags;
+                    const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags);
-                    if (existingTags) {
+                    return next(null, requestTagsQuery, existingTagsQuery);
-                        rc.setExistingObjTag(makeTagQuery(existingTags));
-                    }
-                    return next();
                 });
-            },
+        },
-        ], err => {
+    ], (err, requestTagsQuery, existingTagsQuery) => {
-            if (err) {
+        if (err) {
-                log.trace('error processing tag condition key evaluation');
+            log.trace('error processing tag condition key evaluation');
-                return cb(err);
+            return cb(err);
+        }
+        // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
+        // eslint-disable-next-line no-restricted-syntax
+        for (const rc of requestContexts) {
+            rc.setNeedTagEval(true);
+            if (requestTagsQuery) {
+                rc.setRequestObjTags(requestTagsQuery);
             }
-            return cb(null, requestContexts);
+            if (existingTagsQuery) {
-        });
+                rc.setExistingObjTag(existingTagsQuery);
+            }
+        }
+        return cb();
     });
 }
 
@@ -77,23 +80,20 @@ function tagConditionKeyAuth(authorizationResults, request, requestContexts, api
         return cb();
     }
     if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
-        return cb();
+        return cb(null, authorizationResults);
     }
 
-    return updateRequestContexts(request, requestContexts, apiMethod, log, (err, updatedContexts) => {
+    return updateRequestContextsWithTags(request, requestContexts, apiMethod, log, err => {
         if (err) {
             return cb(err);
         }
-        if (assert.deepStrictEqual(requestContexts, updatedContexts)) {
-            return cb();
-        }
         return auth.server.doAuth(request, log,
-            (err, userInfo, tagAuthResults) => cb(err, tagAuthResults), 's3', updatedContexts);
+            (err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts);
     });
 }
 
 module.exports = {
     tagConditionKeyAuth,
-    updateRequestContexts,
+    updateRequestContextsWithTags,
     makeTagQuery,
 };

lib/api/apiUtils/bucket/bucketCreation.js

@@ -6,6 +6,7 @@ const acl = require('../../../metadata/acl');
 const BucketInfo = require('arsenal').models.BucketInfo;
 const constants = require('../../../../constants');
 const createKeyForUserBucket = require('./createKeyForUserBucket');
+const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
 const metadata = require('../../../metadata/wrapper');
 const kms = require('../../../kms/wrapper');
 const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
@@ -23,7 +24,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
 
     // Get new format usersBucket to see if it exists
     return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => {
-        if (err && !err.NoSuchBucket && !err.BucketAlreadyExists) {
+        if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) {
             return cb(err);
         }
         const splitter = usersBucketAttrs ?
@@ -40,7 +41,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
             usersBucket : oldUsersBucket;
         return metadata.putObjectMD(usersBucketBeingCalled, key,
             omVal, {}, log, err => {
-                if (err && err.NoSuchBucket) {
+                if (err?.is?.NoSuchBucket) {
                     // There must be no usersBucket so createBucket
                     // one using the new format
                     log.trace('users bucket does not exist, ' +
@@ -60,9 +61,8 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
                             // from getting a BucketAlreadyExists
                             // error with respect
                             // to the usersBucket.
-                            if (err &&
+                            // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
-                                err !==
+                            if (err && !err.BucketAlreadyExists) {
-                                    errors.BucketAlreadyExists) {
                                 log.error('error from metadata', {
                                     error: err,
                                 });
@@ -223,6 +223,7 @@ function createBucket(authInfo, bucketName, headers,
         },
         getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
             metadata.getBucket(bucketName, log, (err, data) => {
+                // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
                 if (err && err.NoSuchBucket) {
                     return callback(null, 'NoBucketYet');
                 }
@@ -247,8 +248,9 @@ function createBucket(authInfo, bucketName, headers,
         }
         const newBucketMD = results.prepareNewBucketMD;
         if (existingBucketMD === 'NoBucketYet') {
+            const sseConfig = parseBucketEncryptionHeaders(headers);
             return bucketLevelServerSideEncryption(
-                bucketName, headers, log,
+                bucketName, sseConfig, log,
                 (err, sseInfo) => {
                     if (err) {
                         return cb(err);

lib/api/apiUtils/bucket/bucketDeletion.js

@@ -16,6 +16,7 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
         `${mpuBucketPrefix}${destinationBucketName}`;
     return metadata.deleteBucket(mpuBucketName, log, err => {
         // If the mpu bucket does not exist, just move on
+        // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
         if (err && err.NoSuchBucket) {
             return cb();
         }
@@ -23,7 +24,7 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
     });
 }
 
-function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, log, cb) {
+function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) {
     async.mapLimit(mpus, 1, (mpu, next) => {
         const splitterChar = mpu.key.includes(oldSplitter) ?
             oldSplitter : splitter;
@@ -39,7 +40,7 @@ function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, log, cb) {
                     byteLength: partSizeSum,
                 });
                 next(err);
-            });
+            }, request);
     }, cb);
 }
 /**
@@ -48,11 +49,13 @@ function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, log, cb) {
  * @param {object} bucketMD - bucket attributes/metadata
  * @param {string} bucketName - bucket in which objectMetadata is stored
  * @param {string} canonicalID - account canonicalID of requester
+ * @param {object} request - request object given by router
+ *                           including normalized headers
  * @param {object} log - Werelogs logger
  * @param {function} cb - callback from async.waterfall in bucketDelete
  * @return {undefined}
  */
-function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, log, cb) {
+function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, cb) {
     log.trace('deleting bucket from metadata');
     assert.strictEqual(typeof bucketName, 'string');
     assert.strictEqual(typeof canonicalID, 'string');
@@ -90,7 +93,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, log, cb) {
                 log, (err, objectsListRes) => {
                     // If no shadow bucket ever created, no ongoing MPU's, so
                     // continue with deletion
-                    if (err && err.NoSuchBucket) {
+                    if (err?.is.NoSuchBucket) {
                         return next();
                     }
                     if (err) {
@@ -99,7 +102,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, log, cb) {
                     }
                     if (objectsListRes.Contents.length) {
                         return _deleteOngoingMPUs(authInfo, bucketName,
-                            bucketMD, objectsListRes.Contents, log, err => {
+                            bucketMD, objectsListRes.Contents, request, log, err => {
                                 if (err) {
                                     return next(err);
                                 }

lib/api/apiUtils/bucket/bucketEncryption.js

@@ -0,0 +1,255 @@
+const { errors } = require('arsenal');
+const metadata = require('../../../metadata/wrapper');
+const kms = require('../../../kms/wrapper');
+const { parseString } = require('xml2js');
+
+/**
+ * ServerSideEncryptionInfo - user configuration for server side encryption
+ * @typedef {Object} ServerSideEncryptionInfo
+ * @property {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
+ * @property {string} masterKeyId - Key id for the kms key used to encrypt data keys.
+ * @property {string} configuredMasterKeyId - User configured master key id.
+ * @property {boolean} mandatory - Whether a default encryption policy has been enabled.
+*/
+
+/**
+ * @callback ServerSideEncryptionInfo~callback
+ * @param {Object} error - Instance of Arsenal error
+ * @param {ServerSideEncryptionInfo} - SSE configuration
+ */
+
+/**
+ * parseEncryptionXml - Parses and validates a ServerSideEncryptionConfiguration xml document
+ * @param {object} xml - ServerSideEncryptionConfiguration doc
+ * @param {object} log - logger
+ * @param {ServerSideEncryptionInfo~callback} cb - callback
+ * @returns {undefined}
+ */
+function parseEncryptionXml(xml, log, cb) {
+    return parseString(xml, (err, parsed) => {
+        if (err) {
+            log.trace('xml parsing failed', {
+                error: err,
+                method: 'parseEncryptionXml',
+            });
+            log.debug('invalid xml', { xml });
+            return cb(errors.MalformedXML);
+        }
+
+        if (!parsed
+            || !parsed.ServerSideEncryptionConfiguration
+            || !parsed.ServerSideEncryptionConfiguration.Rule) {
+            log.trace('error in sse config, invalid ServerSideEncryptionConfiguration section', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const { Rule } = parsed.ServerSideEncryptionConfiguration;
+
+        if (!Array.isArray(Rule)
+            || Rule.length > 1
+            || !Rule[0]
+            || !Rule[0].ApplyServerSideEncryptionByDefault
+            || !Rule[0].ApplyServerSideEncryptionByDefault[0]) {
+            log.trace('error in sse config, invalid ApplyServerSideEncryptionByDefault section', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const [encConfig] = Rule[0].ApplyServerSideEncryptionByDefault;
+
+        if (!encConfig.SSEAlgorithm || !encConfig.SSEAlgorithm[0]) {
+            log.trace('error in sse config, no SSEAlgorithm provided', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const [algorithm] = encConfig.SSEAlgorithm;
+
+        if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
+            log.trace('error in sse config, unknown SSEAlgorithm', {
+                method: 'parseEncryptionXml',
+            });
+            return cb(errors.MalformedXML);
+        }
+
+        const result = { algorithm, mandatory: true };
+
+        if (encConfig.KMSMasterKeyID) {
+            if (algorithm === 'AES256') {
+                log.trace('error in sse config, can not specify KMSMasterKeyID when using AES256', {
+                    method: 'parseEncryptionXml',
+                });
+                return cb(errors.InvalidArgument.customizeDescription(
+                    'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'));
+            }
+
+            if (!encConfig.KMSMasterKeyID[0] || typeof encConfig.KMSMasterKeyID[0] !== 'string') {
+                log.trace('error in sse config, invalid KMSMasterKeyID', {
+                    method: 'parseEncryptionXml',
+                });
+                return cb(errors.MalformedXML);
+            }
+
+            result.configuredMasterKeyId = encConfig.KMSMasterKeyID[0];
+        }
+        return cb(null, result);
+    });
+}
+
+/**
+ * hydrateEncryptionConfig - Constructs a ServerSideEncryptionInfo object from arguments
+ * ensuring no invalid or undefined keys are added
+ *
+ * @param {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
+ * @param {string} configuredMasterKeyId - User configured master key id.
+ * @param {boolean} [mandatory] - Whether a default encryption policy has been enabled.
+ * @returns {ServerSideEncryptionInfo} - SSE configuration
+ */
+function hydrateEncryptionConfig(algorithm, configuredMasterKeyId, mandatory = null) {
+    if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
+        return {
+            algorithm: null,
+        };
+    }
+
+    const sseConfig = { algorithm, mandatory };
+
+    if (algorithm === 'aws:kms' && configuredMasterKeyId) {
+        sseConfig.configuredMasterKeyId = configuredMasterKeyId;
+    }
+
+    if (mandatory !== null) {
+        sseConfig.mandatory = mandatory;
+    }
+
+    return sseConfig;
+}
+
+/**
+ * parseBucketEncryptionHeaders - retrieves bucket level sse configuration from request headers
+ * @param {object} headers - Request headers
+ * @returns {ServerSideEncryptionInfo} - SSE configuration
+ */
+function parseBucketEncryptionHeaders(headers) {
+    const sseAlgorithm = headers['x-amz-scal-server-side-encryption'];
+    const configuredMasterKeyId = headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] || null;
+    return hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId, true);
+}
+
+/**
+ * parseObjectEncryptionHeaders - retrieves bucket level sse configuration from request headers
+ * @param {object} headers - Request headers
+ * @returns {ServerSideEncryptionInfo} - SSE configuration
+ */
+function parseObjectEncryptionHeaders(headers) {
+    const sseAlgorithm = headers['x-amz-server-side-encryption'];
+    const configuredMasterKeyId = headers['x-amz-server-side-encryption-aws-kms-key-id'] || null;
+
+    if (sseAlgorithm && sseAlgorithm !== 'AES256' && sseAlgorithm !== 'aws:kms') {
+        return {
+            error: errors.InvalidArgument.customizeDescription('The encryption method specified is not supported'),
+        };
+    }
+
+    if (sseAlgorithm !== 'aws:kms' && configuredMasterKeyId) {
+        return {
+            error: errors.InvalidArgument.customizeDescription(
+                'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'),
+        };
+    }
+    return { objectSSE: hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId) };
+}
+
+/**
+ * createDefaultBucketEncryptionMetadata - Creates master key and sets up default server side encryption configuration
+ * @param {BucketInfo} bucket - bucket metadata
+ * @param {object} log - werelogs logger
+ * @param {ServerSideEncryptionInfo~callback} cb - callback
+ * @returns {undefined}
+ */
+function createDefaultBucketEncryptionMetadata(bucket, log, cb) {
+    return kms.bucketLevelEncryption(
+        bucket.getName(),
+        { algorithm: 'AES256', mandatory: false },
+        log,
+        (error, sseConfig) => {
+            if (error) {
+                return cb(error);
+            }
+            bucket.setServerSideEncryption(sseConfig);
+            return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig));
+        });
+}
+
+/**
+ *
+ * @param {object} headers - request headers
+ * @param {BucketInfo} bucket - BucketInfo model
+ * @param {*} log - werelogs logger
+ * @param {ServerSideEncryptionInfo~callback} cb - callback
+ * @returns {undefined}
+ */
+function getObjectSSEConfiguration(headers, bucket, log, cb) {
+    const bucketSSE = bucket.getServerSideEncryption();
+    const { error, objectSSE } = parseObjectEncryptionHeaders(headers);
+    if (error) {
+        return cb(error);
+    }
+
+    // If a per object sse algo has been passed through
+    // x-amz-server-side-encryption
+    if (objectSSE.algorithm) {
+        // If aws:kms and a custom key id
+        // pass it through without updating the bucket md
+        if (objectSSE.algorithm === 'aws:kms' && objectSSE.configuredMasterKeyId) {
+            return cb(null, objectSSE);
+        }
+
+        // If the client has not specified a key id,
+        // and we have a default config, then we reuse
+        // it and pass it through
+        if (!objectSSE.configuredMasterKeyId && bucketSSE) {
+            // The default configs algo is overridden with the one passed in the
+            // request headers. Our implementations of AES256 and aws:kms are the
+            // same underneath so this is only cosmetic change.
+            const sseConfig = Object.assign({}, bucketSSE, { algorithm: objectSSE.algorithm });
+            return cb(null, sseConfig);
+        }
+
+        // If the client has not specified a key id, and we
+        // don't have a default config, generate it
+        if (!objectSSE.configuredMasterKeyId && !bucketSSE) {
+            return createDefaultBucketEncryptionMetadata(bucket, log, (error, sseConfig) => {
+                if (error) {
+                    return cb(error);
+                }
+                // Override the algorithm, for the same reasons as above.
+                Object.assign(sseConfig, { algorithm: objectSSE.algorithm });
+                return cb(null, sseConfig);
+            });
+        }
+    }
+
+    // If the bucket has a default encryption config, and it is mandatory
+    // (created with putBucketEncryption or legacy headers)
+    // pass it through
+    if (bucketSSE && bucketSSE.mandatory) {
+        return cb(null, bucketSSE);
+    }
+
+    // No encryption config
+    return cb(null, null);
+}
+
+module.exports = {
+    createDefaultBucketEncryptionMetadata,
+    getObjectSSEConfiguration,
+    hydrateEncryptionConfig,
+    parseEncryptionXml,
+    parseBucketEncryptionHeaders,
+    parseObjectEncryptionHeaders,
+};

lib/api/apiUtils/bucket/bucketShield.js

@@ -30,6 +30,9 @@ function bucketShield(bucket, requestType) {
      // Otherwise return an error to the client
     if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
         (requestType !== 'objectPut' &&
+        requestType !== 'initiateMultipartUpload' &&
+        requestType !== 'objectPutPart' &&
+        requestType !== 'completeMultipartUpload' &&
         requestType !== 'bucketPutACL' &&
         requestType !== 'bucketDelete')) {
         return true;

lib/api/apiUtils/bucket/deleteUserBucketEntry.js

@@ -11,15 +11,16 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
     metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
         // If the object representing the bucket is not in the
         // users bucket just continue
-        if (error && error.NoSuchKey) {
+        if (error?.is.NoSuchKey) {
             return cb(null);
         // BACKWARDS COMPATIBILITY: Remove this once no longer
         // have old user bucket format
-        } else if (error && error.NoSuchBucket) {
+        } else if (error?.is.NoSuchBucket) {
             const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
                 oldSplitter, bucketName);
             return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
                 {}, log, error => {
+                    // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
                     if (error && !error.NoSuchKey) {
                         log.error('from metadata while deleting user bucket',
                             { error });

lib/api/apiUtils/bucket/getNotificationConfiguration.js

@@ -15,10 +15,21 @@ function getNotificationConfiguration(parsedXml) {
     }
     const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource));
     const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]);
-    if (!notifConfigTargets.every(t => targets.has(t))) {
+    // getting invalid targets
-        return { error: errors.MalformedXML.customizeDescription(
+    const invalidTargets = [];
-            'queue arn target is not included in Cloudserver bucket notification config'),
+    notifConfigTargets.forEach((t, i) => {
-        };
+        if (!targets.has(t)) {
+            invalidTargets.push({
+                ArgumentName: notifConfig.queueConfig[i].queueArn,
+                ArgumentValue: 'The destination queue does not exist',
+            });
+        }
+    });
+    if (invalidTargets.length > 0) {
+        const errDesc = 'Unable to validate the following destination configurations';
+        let error = errors.InvalidArgument.customizeDescription(errDesc);
+        error = error.addMetadataEntry('invalidArguments', invalidTargets);
+        return { error };
     }
     return notifConfig;
 }

lib/api/apiUtils/object/abortMultipartUpload.js

@@ -3,7 +3,7 @@ const async = require('async');
 const constants = require('../../../../constants');
 const { data } = require('../../../data/wrapper');
 const locationConstraintCheck = require('../object/locationConstraintCheck');
-const { metadataValidateBucketAndObj } =
+const { standardMetadataValidateBucketAndObj } =
     require('../../../metadata/metadataUtils');
 const services = require('../../../services');
 
@@ -14,17 +14,19 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
         bucketName,
         objectKey,
         uploadId,
-        preciseRequestType: 'multipartDelete',
+        preciseRequestType: request.apiMethods || 'multipartDelete',
+        request,
     };
     // For validating the request at the destinationBucket level
     // params are the same as validating at the MPU level
     // but the requestType is the more general 'objectDelete'
     const metadataValParams = Object.assign({}, metadataValMPUparams);
     metadataValParams.requestType = 'objectPut';
+    const authzIdentityResult = request ? request.actionImplicitDenies : false;
 
     async.waterfall([
         function checkDestBucketVal(next) {
-            metadataValidateBucketAndObj(metadataValParams, log,
+            standardMetadataValidateBucketAndObj(metadataValParams, authzIdentityResult, log,
                 (err, destinationBucket) => {
                     if (err) {
                         return next(err, destinationBucket);
@@ -55,9 +57,14 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
         function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket,
         next) {
             const location = mpuOverviewObj.controllingLocationConstraint;
+            const originalIdentityAuthzResults = request.actionImplicitDenies;
+            // eslint-disable-next-line no-param-reassign
+            delete request.actionImplicitDenies;
             return data.abortMPU(objectKey, uploadId, location, bucketName,
             request, destBucket, locationConstraintCheck, log,
             (err, skipDataDelete) => {
+                // eslint-disable-next-line no-param-reassign
+                request.actionImplicitDenies = originalIdentityAuthzResults;
                 if (err) {
                     return next(err, destBucket);
                 }

lib/api/apiUtils/object/checkEncryption.js

@@ -1,29 +0,0 @@
-/**
- * Checks the object encryption request against bucket encryption for matching
- * SSE-S3 configuration
- * @param {object} objectRequest - http request
- * @param {object} serverSideEncryption - bucket encryption info
- * @return {boolean} returns true if the object request has correct SSE-S3
- * configuration
- */
-function isValidSSES3(objectRequest, serverSideEncryption) {
-    // x-amz-server-side-encryption is allowed only if bucket
-    // encryption is enabled and if the value is AES256
-    // NOTE: object level encryption is not supported, but we allow
-    // encryption headers in the object request headers!
-    const sseHeader = objectRequest.headers['x-amz-server-side-encryption'];
-    const encryptionAlgorithm = 'AES256';
-
-    const result = ((!serverSideEncryption && sseHeader) ||
-        (serverSideEncryption && sseHeader
-            && sseHeader === encryptionAlgorithm
-            && serverSideEncryption.algorithm !== encryptionAlgorithm) ||
-        (serverSideEncryption && sseHeader
-            && sseHeader !== encryptionAlgorithm
-            && serverSideEncryption.algorithm === encryptionAlgorithm));
-    return !result;
-}
-
-module.exports = {
-    isValidSSES3,
-};

lib/api/apiUtils/object/checkHttpHeadersSize.js

@@ -0,0 +1,25 @@
+const { errors } = require('arsenal');
+const { maxHttpHeadersSize } = require('../../../../constants');
+
+/**
+ * Checks the size of the HTTP headers
+ * @param {object} requestHeaders - HTTP request headers
+ * @return {object} object with error or null
+ */
+function checkHttpHeadersSize(requestHeaders) {
+    let httpHeadersSize = 0;
+
+    Object.keys(requestHeaders).forEach(header => {
+        httpHeadersSize += Buffer.byteLength(header, 'utf8') +
+            Buffer.byteLength(requestHeaders[header], 'utf8');
+    });
+
+    if (httpHeadersSize > maxHttpHeadersSize) {
+        return {
+            httpHeadersSizeError: errors.HttpHeadersTooLarge,
+        };
+    }
+    return {};
+}
+
+module.exports = checkHttpHeadersSize;

lib/api/apiUtils/object/coldStorage.js

@@ -0,0 +1,247 @@
+/*
+ * Code based on Yutaka Oishi (Fujifilm) contributions
+ * Date: 11 Sep 2020
+ */
+const { ObjectMDArchive } = require('arsenal').models;
+const errors = require('arsenal').errors;
+const { config } = require('../../../Config');
+const { locationConstraints } = config;
+
+const { scaledMsPerDay } = config.getTimeOptions();
+
+/**
+ * Get response header "x-amz-restore"
+ * Be called by objectHead.js
+ * @param {object} objMD - object's metadata
+ * @returns {string|undefined} x-amz-restore
+ */
+function getAmzRestoreResHeader(objMD) {
+    if (objMD.archive &&
+        objMD.archive.restoreRequestedAt &&
+        !objMD.archive.restoreCompletedAt) {
+        // Avoid race condition by relying on the `archive` MD of the object
+        // and return the right header after a RESTORE request.
+        // eslint-disable-next-line
+        return `ongoing-request="true"`;
+    }
+    if (objMD['x-amz-restore']) {
+        if (objMD['x-amz-restore']['expiry-date']) {
+            const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString();
+            // eslint-disable-next-line
+            return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
+        }
+    }
+    return undefined;
+}
+
+/**
+ * Check if restore can be done.
+ *
+ * @param {ObjectMD} objectMD - object metadata
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
+ */
+function _validateStartRestore(objectMD, log) {
+    if (objectMD.archive?.restoreCompletedAt) {
+        if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
+            // return InvalidObjectState error if the restored object is expired
+            // but restore info md of this object has not yet been cleared
+            log.debug('The restored object already expired.',
+                {
+                    archive: objectMD.archive,
+                    method: '_validateStartRestore',
+                });
+            return errors.InvalidObjectState;
+        }
+
+        // If object is already restored, no further check is needed
+        // Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have
+        // been reset.
+        return undefined;
+    }
+    const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
+    if (!isLocationCold) {
+        // return InvalidObjectState error if the object is not in cold storage,
+        // not in cold storage means either location cold flag not exists or cold flag is explicit false
+        log.debug('The bucket of the object is not in a cold storage location.',
+            {
+                isLocationCold,
+                method: '_validateStartRestore',
+            });
+        return errors.InvalidObjectState;
+    }
+    if (objectMD.archive?.restoreRequestedAt) {
+        // return RestoreAlreadyInProgress error if the object is currently being restored
+        // check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
+        log.debug('The object is currently being restored.',
+            {
+                archive: objectMD.archive,
+                method: '_validateStartRestore',
+            });
+        return errors.RestoreAlreadyInProgress;
+    }
+    return undefined;
+}
+
+/**
+ * Check if "put version id" is allowed
+ *
+ * @param {ObjectMD} objMD - object metadata
+ * @param {string} versionId - object's version id
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} - undefined if "put version id" is allowed
+ */
+function validatePutVersionId(objMD, versionId, log) {
+    if (!objMD) {
+        const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
+        log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
+        return err;
+    }
+
+    if (objMD.isDeleteMarker) {
+        log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
+        return errors.MethodNotAllowed;
+    }
+
+    const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
+    if (!isLocationCold) {
+        log.error('The object data is not stored in a cold storage location.',
+            {
+                isLocationCold,
+                dataStoreName: objMD.dataStoreName,
+                method: 'validatePutVersionId',
+            });
+        return errors.InvalidObjectState;
+    }
+
+    // make sure object archive restoration is in progress
+    // NOTE: we do not use putObjectVersion to update the restoration period.
+    if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
+        || objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
+        log.error('object archive restoration is not in progress',
+            { method: 'validatePutVersionId', versionId });
+        return errors.InvalidObjectState;
+    }
+
+    return undefined;
+}
+
+/**
+ * Check if the object is already restored, and update the expiration date accordingly:
+ * > After restoring an archived object, you can update the restoration period by reissuing the
+ * > request with a new period. Amazon S3 updates the restoration period relative to the current
+ * > time.
+ *
+ * @param {ObjectMD} objectMD - object metadata
+ * @param {object} log - werelogs logger
+ * @return {boolean} - true if the object is already restored
+ */
+function _updateObjectExpirationDate(objectMD, log) {
+    // Check if restoreCompletedAt field exists
+    // Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
+    // checked earlier in the process, so checking again here would create weird states
+    const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
+    log.debug('The restore status of the object.', {
+        isObjectAlreadyRestored,
+        method: 'isObjectAlreadyRestored'
+    });
+    if (isObjectAlreadyRestored) {
+        const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
+        expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
+
+        /* eslint-disable no-param-reassign */
+        objectMD.archive.restoreWillExpireAt = expiryDate;
+        objectMD['x-amz-restore'] = {
+            'ongoing-request': false,
+            'expiry-date': expiryDate,
+        };
+        /* eslint-enable no-param-reassign */
+    }
+    return isObjectAlreadyRestored;
+}
+
+/**
+ * update restore expiration date.
+ *
+ * @param {ObjectMD} objectMD - objectMD instance
+ * @param {object} restoreParam - restore param
+ * @param {object} log - werelogs logger
+ * @return {ArsenalError|undefined} internal error if object MD is not valid
+ *
+ */
+function _updateRestoreInfo(objectMD, restoreParam, log) {
+    if (!objectMD.archive) {
+        log.debug('objectMD.archive doesn\'t exits', {
+            objectMD,
+            method: '_updateRestoreInfo'
+        });
+        return errors.InternalError.customizeDescription('Archive metadata is missing.');
+    }
+    /* eslint-disable no-param-reassign */
+    objectMD.archive.restoreRequestedAt = new Date();
+    objectMD.archive.restoreRequestedDays = restoreParam.days;
+    objectMD.originOp = 's3:ObjectRestore:Post';
+    /* eslint-enable no-param-reassign */
+    if (!ObjectMDArchive.isValid(objectMD.archive)) {
+        log.debug('archive is not valid', {
+            archive: objectMD.archive,
+            method: '_updateRestoreInfo'
+        });
+        return errors.InternalError.customizeDescription('Invalid archive metadata.');
+    }
+    return undefined;
+}
+
+/**
+ * start to restore object.
+ * If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
+ * calculate restore expiry-date and add it to objectMD.
+ * Be called by objectRestore.js
+ *
+ * @param {ObjectMD} objectMD - objectMd instance
+ * @param {object} restoreParam - bucket name
+ * @param {object} log - werelogs logger
+ * @param {function} cb - bucket name
+ * @return {undefined}
+ *
+ */
+function startRestore(objectMD, restoreParam, log, cb) {
+    log.info('Validating if restore can be done or not.');
+    const checkResultError = _validateStartRestore(objectMD, log);
+    if (checkResultError) {
+        return cb(checkResultError);
+    }
+    log.info('Updating restore information.');
+    const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
+    if (updateResultError) {
+        return cb(updateResultError);
+    }
+    const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
+    return cb(null, isObjectAlreadyRestored);
+}
+
+/**
+ * checks if object data is available or if it's in cold storage
+ * @param {ObjectMD} objMD Object metadata
+ * @returns {ArsenalError|null} error if object data is not available
+ */
+function verifyColdObjectAvailable(objMD) {
+    // return error when object is cold
+    if (objMD.archive &&
+        // Object is in cold backend
+        (!objMD.archive.restoreRequestedAt ||
+            // Object is being restored
+            (objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
+        const err = errors.InvalidObjectState
+            .customizeDescription('The operation is not valid for the object\'s storage class');
+        return err;
+    }
+    return null;
+}
+
+module.exports = {
+    startRestore,
+    getAmzRestoreResHeader,
+    validatePutVersionId,
+    verifyColdObjectAvailable,
+};

lib/api/apiUtils/object/createAndStoreObject.js

@@ -5,10 +5,9 @@ const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
 const constants = require('../../../../constants');
 const { data } = require('../../../data/wrapper');
 const services = require('../../../services');
-const logger = require('../../../utilities/logger');
 const { dataStore } = require('./storeObject');
 const locationConstraintCheck = require('./locationConstraintCheck');
-const { versioningPreprocessing } = require('./versioning');
+const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
 const removeAWSChunked = require('./removeAWSChunked');
 const getReplicationInfo = require('./getReplicationInfo');
 const { config } = require('../../../Config');
@@ -21,7 +20,7 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' +
 'a versioned object to a location-constraint of type Azure or GCP.';
 
 function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
-    metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
+    metadataStoreParams, dataToDelete, log, requestMethod, callback) {
     services.metadataStoreObject(bucketName, dataGetInfo,
         cipherBundle, metadataStoreParams, (err, result) => {
             if (err) {
@@ -31,7 +30,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
                 const newDataStoreName = Array.isArray(dataGetInfo) ?
                     dataGetInfo[0].dataStoreName : null;
                 return data.batchDelete(dataToDelete, requestMethod,
-                    newDataStoreName, deleteLog, err => callback(err, result));
+                    newDataStoreName, log, err => callback(err, result));
             }
             return callback(null, result);
         });
@@ -51,7 +50,9 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
  * @param {(object|null)} streamingV4Params - if v4 auth, object containing
  * accessKey, signatureFromRequest, region, scopeDate, timestamp, and
  * credentialScope (to be used for streaming v4 auth if applicable)
+ * @param {(object|null)} overheadField - fields to be included in metadata overhead
  * @param {RequestLogger} log - logger instance
+ * @param {string} originOp - Origin operation
  * @param {function} callback - callback function
  * @return {undefined} and call callback with (err, result) -
  * result.contentMD5 - content md5 of new object or version
@@ -59,7 +60,10 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
  */
 function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
-        log, callback) {
+        overheadField, log, originOp, callback) {
+    const putVersionId = request.headers['x-scal-s3-version-id'];
+    const isPutVersion = putVersionId || putVersionId === '';
+
     const size = isDeleteMarker ? 0 : request.parsedContentLength;
     // although the request method may actually be 'DELETE' if creating a
     // delete marker, for our purposes we consider this to be a 'PUT'
@@ -112,6 +116,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         isDeleteMarker,
         replicationInfo: getReplicationInfo(
             objectKey, bucketMD, false, size, null, null, authInfo),
+        overheadField,
         log,
     };
 
@@ -129,7 +134,6 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         }
     }
 
-
     if (!isDeleteMarker) {
         metadataStoreParams.contentType = request.headers['content-type'];
         metadataStoreParams.cacheControl = request.headers['cache-control'];
@@ -139,7 +143,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             removeAWSChunked(request.headers['content-encoding']);
         metadataStoreParams.expires = request.headers.expires;
         metadataStoreParams.tagging = request.headers['x-amz-tagging'];
-        metadataStoreParams.originOp = 's3:ObjectCreated:Put';
+        metadataStoreParams.originOp = originOp;
         const defaultObjectLockConfiguration
             = bucketMD.getObjectLockConfiguration();
         if (defaultObjectLockConfiguration) {
@@ -154,7 +158,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         // eslint-disable-next-line no-param-reassign
         request.headers[constants.objectLocationConstraintHeader] =
             objMD[constants.objectLocationConstraintHeader];
-        metadataStoreParams.originOp = 's3:ObjectRemoved:DeleteMarkerCreated';
+        metadataStoreParams.originOp = originOp;
     }
 
     const backendInfoObj =
@@ -185,14 +189,17 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
         }
     }
 
+    if (objMD && objMD.uploadId) {
+        metadataStoreParams.oldReplayId = objMD.uploadId;
+    }
+
     /* eslint-disable camelcase */
     const dontSkipBackend = externalBackends;
     /* eslint-enable camelcase */
 
-    const requestLogger =
-        logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
     const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
     const mdOnlySize = request.headers['x-amz-meta-size'];
+
     return async.waterfall([
         function storeData(next) {
             if (size === 0) {
@@ -257,12 +264,17 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             return next(null, dataGetInfoArr);
         },
         function getVersioningInfo(infoArr, next) {
+            // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
+            if (isPutVersion) {
+                const options = overwritingVersioning(objMD, metadataStoreParams);
+                return process.nextTick(() => next(null, options, infoArr));
+            }
             return versioningPreprocessing(bucketName, bucketMD,
                 metadataStoreParams.objectKey, objMD, log, (err, options) => {
                     if (err) {
                         // TODO: check AWS error when user requested a specific
                         // version before any versions have been put
-                        const logLvl = err === errors.BadRequest ?
+                        const logLvl = err.is.BadRequest ?
                             'debug' : 'error';
                         log[logLvl]('error getting versioning info', {
                             error: err,
@@ -276,10 +288,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             metadataStoreParams.versionId = options.versionId;
             metadataStoreParams.versioning = options.versioning;
             metadataStoreParams.isNull = options.isNull;
-            metadataStoreParams.nullVersionId = options.nullVersionId;
+            metadataStoreParams.deleteNullKey = options.deleteNullKey;
+            if (options.extraMD) {
+                Object.assign(metadataStoreParams, options.extraMD);
+            }
             return _storeInMDandDeleteData(bucketName, infoArr,
                 cipherBundle, metadataStoreParams,
-                options.dataToDelete, requestLogger, requestMethod, next);
+                options.dataToDelete, log, requestMethod, next);
         },
     ], callback);
 }

lib/api/apiUtils/object/deleteObject.js

@@ -0,0 +1,18 @@
+/**
+ * _bucketRequiresOplogUpdate - DELETE an object from a bucket
+ * @param {BucketInfo} bucket - bucket object
+ * @return {boolean} whether objects require oplog updates on deletion, or not
+ */
+function _bucketRequiresOplogUpdate(bucket) {
+    // Default behavior is to require an oplog update
+    if (!bucket || !bucket.getLifecycleConfiguration || !bucket.getNotificationConfiguration) {
+        return true;
+    }
+    // If the bucket has lifecycle configuration or notification configuration
+    // set, we also require an oplog update
+    return bucket.getLifecycleConfiguration() || bucket.getNotificationConfiguration();
+}
+
+module.exports = {
+    _bucketRequiresOplogUpdate,
+};

lib/api/apiUtils/object/expirationHeaders.js

@@ -0,0 +1,140 @@
+const { supportedLifecycleRules } = require('arsenal').constants;
+const { LifecycleConfiguration } = require('arsenal').models;
+const {
+    LifecycleDateTime,
+    LifecycleUtils,
+} = require('arsenal').s3middleware.lifecycleHelpers;
+const { config } = require('../../../Config');
+
+const {
+    expireOneDayEarlier,
+    transitionOneDayEarlier,
+    timeProgressionFactor,
+    scaledMsPerDay,
+} = config.getTimeOptions();
+
+const lifecycleDateTime = new LifecycleDateTime({
+    transitionOneDayEarlier,
+    expireOneDayEarlier,
+    timeProgressionFactor,
+});
+
+const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor);
+
+function calculateDate(objDate, expDays, datetime) {
+    return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay));
+}
+
+function formatExpirationHeader(date, id) {
+    return `expiry-date="${date}", rule-id="${encodeURIComponent(id)}"`;
+}
+
+// format: x-amz-expiration: expiry-date="Fri, 21 Dec 2012 00:00:00 GMT", rule-id="id"
+const AMZ_EXP_HEADER = 'x-amz-expiration';
+// format: x-amz-abort-date: "Fri, 21 Dec 2012 00:00:00 GMT"
+const AMZ_ABORT_DATE_HEADER = 'x-amz-abort-date';
+// format: x-amz-abort-rule-id: "rule id"
+const AMZ_ABORT_ID_HEADER = 'x-amz-abort-rule-id';
+
+
+function _generateExpHeadersObjects(rules, params, datetime) {
+    const tags = {
+        TagSet: params.tags
+            ? Object.keys(params.tags)
+                .map(key => ({ Key: key, Value: params.tags[key] }))
+            : [],
+    };
+
+    const objectInfo = { Key: params.key };
+    const filteredRules = lifecycleUtils.filterRules(rules, objectInfo, tags);
+    const applicable = lifecycleUtils.getApplicableRules(filteredRules, objectInfo, datetime);
+
+    if (applicable.Expiration) {
+        const rule = applicable.Expiration;
+
+        if (rule.Days === undefined && rule.Date === undefined) {
+            return {};
+        }
+
+        if (rule.Date) {
+            return {
+                [AMZ_EXP_HEADER]: formatExpirationHeader(rule.Date, rule.ID),
+            };
+        }
+
+        const date = calculateDate(params.date, rule.Days, datetime);
+        return {
+            [AMZ_EXP_HEADER]: formatExpirationHeader(date.toUTCString(), rule.ID),
+        };
+    }
+
+    return {};
+}
+
+function _generateExpHeadresMPU(rules, params, datetime) {
+    const noTags = { TagSet: [] };
+
+    const objectInfo = { Key: params.key };
+
+    const filteredRules = lifecycleUtils.filterRules(rules, objectInfo, noTags);
+    const applicable = lifecycleUtils.getApplicableRules(filteredRules, {}, datetime);
+
+    if (applicable.AbortIncompleteMultipartUpload) {
+        const rule = applicable.AbortIncompleteMultipartUpload;
+        const date = calculateDate(
+            params.date,
+            rule.DaysAfterInitiation,
+            datetime
+        );
+
+        return {
+            [AMZ_ABORT_ID_HEADER]: encodeURIComponent(rule.ID),
+            [AMZ_ABORT_DATE_HEADER]: date.toUTCString(),
+        };
+    }
+
+    return {};
+}
+
+/**
+ * generate response expiration headers
+ * @param {object} params - params
+ * @param {LifecycleDateTime} datetime - lifecycle datetime object
+ * @returns {object} - expiration response headers
+ */
+function generateExpirationHeaders(params, datetime) {
+    const { lifecycleConfig, objectParams, mpuParams, isVersionedReq } = params;
+
+    if (!lifecycleConfig || isVersionedReq) {
+        return {};
+    }
+
+    const lcfg = LifecycleConfiguration.getConfigJson(lifecycleConfig);
+
+    if (objectParams) {
+        return _generateExpHeadersObjects(lcfg.Rules, objectParams, datetime);
+    }
+
+    if (mpuParams) {
+        return _generateExpHeadresMPU(lcfg.Rules, mpuParams, datetime);
+    }
+
+    return {};
+}
+
+/**
+ * set response expiration headers to target header object
+ * @param {object} headers - target header object
+ * @param {object} params - params
+ * @returns {undefined}
+ */
+function setExpirationHeaders(headers, params) {
+    const expHeaders = generateExpirationHeaders(params, lifecycleDateTime);
+    Object.assign(headers, expHeaders);
+}
+
+module.exports = {
+    lifecycleDateTime,
+    generateExpirationHeaders,
+    setExpirationHeaders,
+};

lib/api/apiUtils/object/getReplicationInfo.js

@@ -23,12 +23,12 @@ function _getStorageClasses(rule) {
     }
     const { replicationEndpoints } = s3config;
     // If no storage class, use the given default endpoint or the sole endpoint
-    if (replicationEndpoints.length > 1) {
+    if (replicationEndpoints.length > 0) {
         const endPoint =
-            replicationEndpoints.find(endpoint => endpoint.default);
+            replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
         return [endPoint.site];
     }
-    return [replicationEndpoints[0].site];
+    return undefined;
 }
 
 function _getReplicationInfo(rule, replicationConfig, content, operationType,
@@ -36,6 +36,9 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
     const storageTypes = [];
     const backends = [];
     const storageClasses = _getStorageClasses(rule);
+    if (!storageClasses) {
+        return undefined;
+    }
     storageClasses.forEach(storageClass => {
         const storageClassName =
               storageClass.endsWith(':preferred_read') ?

lib/api/apiUtils/object/lifecycle.js

@@ -0,0 +1,190 @@
+const { versioning } = require('arsenal');
+const versionIdUtils = versioning.VersionID;
+
+const { lifecycleListing } = require('../../../../constants');
+const { CURRENT_TYPE, NON_CURRENT_TYPE, ORPHAN_DM_TYPE } = lifecycleListing;
+
+function _makeTags(tags) {
+    const res = [];
+    Object.entries(tags).forEach(([key, value]) =>
+        res.push(
+            {
+                Key: key,
+                Value: value,
+            }
+        ));
+    return res;
+}
+
+function processCurrents(bucketName, listParams, isBucketVersioned, list) {
+    const data = {
+        Name: bucketName,
+        Prefix: listParams.prefix,
+        MaxKeys: listParams.maxKeys,
+        MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
+        IsTruncated: !!list.IsTruncated,
+        Marker: listParams.marker,
+        BeforeDate: listParams.beforeDate,
+        NextMarker: list.NextMarker,
+        Contents: [],
+    };
+
+    list.Contents.forEach(item => {
+        const v = item.value;
+
+        const content = {
+            Key: item.key,
+            LastModified: v.LastModified,
+            ETag: `"${v.ETag}"`,
+            Size: v.Size,
+            Owner: {
+                ID: v.Owner.ID,
+                DisplayName: v.Owner.DisplayName,
+            },
+            StorageClass: v.StorageClass,
+            TagSet: _makeTags(v.tags),
+            IsLatest: true, // for compatibility with AWS ListObjectVersions.
+            DataStoreName: v.dataStoreName,
+            ListType: CURRENT_TYPE,
+        };
+
+        // NOTE: The current versions listed to be lifecycle should include version id
+        // if the bucket is versioned.
+        if (isBucketVersioned) {
+            const versionId = (v.IsNull || v.VersionId === undefined) ?
+                'null' : versionIdUtils.encode(v.VersionId);
+            content.VersionId = versionId;
+        }
+
+        data.Contents.push(content);
+    });
+
+    return data;
+}
+
+function _encodeVersionId(vid) {
+    let versionId = vid;
+    if (versionId && versionId !== 'null') {
+        versionId = versionIdUtils.encode(versionId);
+    }
+    return versionId;
+}
+
+function processNonCurrents(bucketName, listParams, list) {
+    const nextVersionIdMarker = _encodeVersionId(list.NextVersionIdMarker);
+    const versionIdMarker = _encodeVersionId(listParams.versionIdMarker);
+
+    const data = {
+        Name: bucketName,
+        Prefix: listParams.prefix,
+        MaxKeys: listParams.maxKeys,
+        MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
+        IsTruncated: !!list.IsTruncated,
+        KeyMarker: listParams.keyMarker,
+        VersionIdMarker: versionIdMarker,
+        BeforeDate: listParams.beforeDate,
+        NextKeyMarker: list.NextKeyMarker,
+        NextVersionIdMarker: nextVersionIdMarker,
+        Contents: [],
+    };
+
+    list.Contents.forEach(item => {
+        const v = item.value;
+        const versionId = (v.IsNull || v.VersionId === undefined) ?
+            'null' : versionIdUtils.encode(v.VersionId);
+
+        const content = {
+            Key: item.key,
+            LastModified: v.LastModified,
+            ETag: `"${v.ETag}"`,
+            Size: v.Size,
+            Owner: {
+                ID: v.Owner.ID,
+                DisplayName: v.Owner.DisplayName,
+            },
+            StorageClass: v.StorageClass,
+            TagSet: _makeTags(v.tags),
+            staleDate: v.staleDate, // lowerCamelCase to be compatible with existing lifecycle.
+            VersionId: versionId,
+            DataStoreName: v.dataStoreName,
+            ListType: NON_CURRENT_TYPE,
+        };
+
+        data.Contents.push(content);
+    });
+
+    return data;
+}
+
+function processOrphans(bucketName, listParams, list) {
+    const data = {
+        Name: bucketName,
+        Prefix: listParams.prefix,
+        MaxKeys: listParams.maxKeys,
+        MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
+        IsTruncated: !!list.IsTruncated,
+        Marker: listParams.marker,
+        BeforeDate: listParams.beforeDate,
+        NextMarker: list.NextMarker,
+        Contents: [],
+    };
+
+    list.Contents.forEach(item => {
+        const v = item.value;
+        const versionId = (v.IsNull || v.VersionId === undefined) ?
+            'null' : versionIdUtils.encode(v.VersionId);
+        data.Contents.push({
+            Key: item.key,
+            LastModified: v.LastModified,
+            Owner: {
+                ID: v.Owner.ID,
+                DisplayName: v.Owner.DisplayName,
+            },
+            VersionId: versionId,
+            IsLatest: true, // for compatibility with AWS ListObjectVersions.
+            ListType: ORPHAN_DM_TYPE,
+        });
+    });
+
+    return data;
+}
+
+function getLocationConstraintErrorMessage(locationName) {
+    return 'value of the location you are attempting to set ' +
+        `- ${locationName} - is not listed in the locationConstraint config`;
+}
+
+/**
+ * validateMaxScannedEntries - Validates and returns the maximum scanned entries value.
+ *
+ * @param {object} params - Query parameters
+ * @param {object} config - CloudServer configuration
+ * @param {number} min - Minimum number of entries to be scanned
+ * @returns {Object} - An object indicating the validation result:
+ *   - isValid (boolean): Whether the validation is successful.
+ *   - maxScannedLifecycleListingEntries (number): The validated maximum scanned entries value if isValid is true.
+ */
+function validateMaxScannedEntries(params, config, min) {
+    let maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
+
+    if (params['max-scanned-lifecycle-listing-entries']) {
+        const maxEntriesParams = Number.parseInt(params['max-scanned-lifecycle-listing-entries'], 10);
+
+        if (Number.isNaN(maxEntriesParams) || maxEntriesParams < min ||
+            maxEntriesParams > maxScannedLifecycleListingEntries) {
+            return { isValid: false };
+        }
+
+        maxScannedLifecycleListingEntries = maxEntriesParams;
+    }
+
+    return { isValid: true, maxScannedLifecycleListingEntries };
+}
+
+module.exports = {
+    processCurrents,
+    processNonCurrents,
+    processOrphans,
+    getLocationConstraintErrorMessage,
+    validateMaxScannedEntries,
+};

lib/api/apiUtils/object/locationKeysHaveChanged.js

@@ -0,0 +1,34 @@
+/**
+* Check if all keys that exist in the current list which will be used
+* in composing object are not present in the old object's list.
+*
+* This method can be used to check against accidentally removing data
+* keys due to instability from the metadata layer, or for replay
+* detection in general.
+*
+* @param {array|string|null} prev - list of keys from the object being
+* overwritten
+* @param {array|null} curr - list of keys to be used in composing
+* current object
+* @returns {boolean} true if no key in `curr` is present in `prev`,
+* false otherwise
+*/
+function locationKeysHaveChanged(prev, curr) {
+    if (!prev || prev.length === 0 || !curr) {
+        return true;
+    }
+    // backwards compatibility check if object is of model version 2
+    if (typeof prev === 'string') {
+        return curr.every(v => v.key !== prev);
+    }
+    const keysMap = {};
+    prev.forEach(v => {
+        if (!keysMap[v.dataStoreType]) {
+            keysMap[v.dataStoreType] = {};
+        }
+        keysMap[v.dataStoreType][v.key] = true;
+    });
+    return curr.every(v => !(keysMap[v.dataStoreType] && keysMap[v.dataStoreType][v.key]));
+}
+
+module.exports = locationKeysHaveChanged;

lib/api/apiUtils/object/locationKeysSanityCheck.js

@@ -1,29 +0,0 @@
-/**
-* Check keys that exist in the current list which will be used in composing
-* object. This method checks against accidentally removing data keys due to
-* instability from the metadata layer. The check returns true if there was no
-* match and false if at least one key from the previous list exists in the
-* current list
-* @param {array|string} prev - list of keys from the object being overwritten
-* @param {array} curr - list of keys to be used in composing current object
-* @returns {array} list of keys that can be deleted
-*/
-function locationKeysSanityCheck(prev, curr) {
-    if (!prev || prev.length === 0) {
-        return true;
-    }
-    // backwards compatibility check if object is of model version 2
-    if (typeof prev === 'string') {
-        return curr.every(v => v.key !== prev);
-    }
-    const keysMap = {};
-    prev.forEach(v => {
-        if (!keysMap[v.dataStoreType]) {
-            keysMap[v.dataStoreType] = {};
-        }
-        keysMap[v.dataStoreType][v.key] = true;
-    });
-    return curr.every(v => !(keysMap[v.dataStoreType] && keysMap[v.dataStoreType][v.key]));
-}
-
-module.exports = locationKeysSanityCheck;

lib/api/apiUtils/object/objectLockHelpers.js

@@ -1,5 +1,11 @@
-const { errors } = require('arsenal');
+const { errors, auth, policies } = require('arsenal');
 const moment = require('moment');
+
+const { config } = require('../../../Config');
+const vault = require('../../../auth/vault');
+const { evaluateBucketPolicyWithIAM } = require('../authorization/permissionChecks');
+
+const { scaledMsPerDay } = config.getTimeOptions();
 /**
  * Calculates retain until date for the locked object version
  * @param {object} retention - includes days or years retention period
@@ -7,11 +13,17 @@ const moment = require('moment');
  */
 function calculateRetainUntilDate(retention) {
     const { days, years } = retention;
+
+    if (!days && !years) {
+        return undefined;
+    }
+
     const date = moment();
     // Calculate the number of days to retain the lock on the object
     const retainUntilDays = days || years * 365;
+    const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay;
     const retainUntilDate
-        = date.add(retainUntilDays, 'days');
+        = date.add(retainUntilDaysInMs, 'ms');
     return retainUntilDate.toISOString();
 }
 /**
@@ -38,7 +50,7 @@ function validateHeaders(bucket, headers, log) {
         !(objectLockMode && objectLockDate)) {
         return errors.InvalidArgument.customizeDescription(
             'x-amz-object-lock-retain-until-date and ' +
-            'x-amz-object-lock-mode must both be supplied'
+            'x-amz-object-lock-mode must both be supplied',
         );
     }
     const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']);
@@ -67,17 +79,25 @@ function validateHeaders(bucket, headers, log) {
  */
 function compareObjectLockInformation(headers, defaultRetention) {
     const objectLockInfoToSave = {};
+
+    if (defaultRetention && defaultRetention.rule) {
+        const defaultMode = defaultRetention.rule.mode;
+        const defaultTime = calculateRetainUntilDate(defaultRetention.rule);
+        if (defaultMode && defaultTime) {
+            objectLockInfoToSave.retentionInfo = {
+                mode: defaultMode,
+                date: defaultTime,
+            };
+        }
+    }
+
     if (headers) {
         const headerMode = headers['x-amz-object-lock-mode'];
         const headerDate = headers['x-amz-object-lock-retain-until-date'];
-        const objectRetention = headerMode && headerDate;
+        if (headerMode && headerDate) {
-        if (objectRetention || defaultRetention) {
-            const mode = headerMode || defaultRetention.rule.mode;
-            const date = headerDate
-                || calculateRetainUntilDate(defaultRetention.rule);
             objectLockInfoToSave.retentionInfo = {
-                mode,
+                mode: headerMode,
-                date,
+                date: headerDate,
             };
         }
         const headerLegalHold = headers['x-amz-object-lock-legal-hold'];
@@ -86,6 +106,7 @@ function compareObjectLockInformation(headers, defaultRetention) {
             objectLockInfoToSave.legalHold = legalHold;
         }
     }
+
     return objectLockInfoToSave;
 }
 
@@ -112,69 +133,216 @@ function setObjectLockInformation(headers, md, defaultRetention) {
 }
 
 /**
- * isObjectLocked - checks whether object is locked or not
+ *  Helper class for check object lock state checks
- * @param {obect} bucket - bucket metadata
- * @param {object} objectMD - object metadata
- * @param {array} headers - request headers
- * @return {boolean} - indicates whether object is locked or not
  */
-function isObjectLocked(bucket, objectMD, headers) {
+class ObjectLockInfo {
-    if (bucket.isObjectLockEnabled()) {
+    /**
-        const objectLegalHold = objectMD.legalHold;
+     *
-        if (objectLegalHold) {
+     * @param {object} retentionInfo - The object lock retention policy
+     * @param {"GOVERNANCE" | "COMPLIANCE" | null} retentionInfo.mode - Retention policy mode.
+     * @param {string} retentionInfo.date - Expiration date of retention policy. A string in ISO-8601 format
+     * @param {bool} retentionInfo.legalHold - Whether a legal hold is enable for the object
+     */
+    constructor(retentionInfo) {
+        this.mode = retentionInfo.mode || null;
+        this.date = retentionInfo.date || null;
+        this.legalHold = retentionInfo.legalHold || false;
+    }
+
+    /**
+     * ObjectLockInfo.isLocked
+     * @returns {bool} - Whether the retention policy is active and protecting the object
+     */
+    isLocked() {
+        if (this.legalHold) {
             return true;
         }
-        const retentionMode = objectMD.retentionMode;
+
-        const retentionDate = objectMD.retentionDate;
+        if (!this.mode || !this.date) {
-        if (!retentionMode || !retentionDate) {
             return false;
         }
-        if (retentionMode === 'GOVERNANCE' &&
+
-        headers['x-amz-bypass-governance-retention']) {
+        return !this.isExpired();
-            return false;
+    }
-        }
+
-        const objectDate = moment(retentionDate);
+    /**
-        const now = moment();
+     * ObjectLockInfo.isGovernanceMode
-        // indicates retain until date has expired
+     * @returns {bool} - true if retention mode is GOVERNANCE
-        if (now.isSameOrAfter(objectDate)) {
+     */
-            return false;
+    isGovernanceMode() {
-        }
+        return this.mode === 'GOVERNANCE';
-        return true;
+    }
+
+    /**
+     * ObjectLockInfo.isComplianceMode
+     * @returns {bool} - True if retention mode is COMPLIANCE
+     */
+    isComplianceMode() {
+        return this.mode === 'COMPLIANCE';
+    }
+
+    /**
+     * ObjectLockInfo.isExpired
+     * @returns {bool} - True if the retention policy has expired
+     */
+    isExpired() {
+        const now = moment();
+        return this.date === null || now.isSameOrAfter(this.date);
+    }
+
+    /**
+     * ObjectLockInfo.isExtended
+     * @param {string} timestamp - Timestamp in ISO-8601 format
+     * @returns {bool} - True if the given timestamp is after the policy expiration date or if no expiration date is set
+     */
+    isExtended(timestamp) {
+        return timestamp !== undefined && (this.date === null || moment(timestamp).isSameOrAfter(this.date));
+    }
+
+    /**
+     * ObjectLockInfo.canModifyObject
+     * @param {bool} hasGovernanceBypass - Whether to bypass governance retention policies
+     * @returns {bool} - True if the retention policy allows the objects data to be modified (overwritten/deleted)
+     */
+    canModifyObject(hasGovernanceBypass) {
+        // can modify object if object is not locked
+        // cannot modify object in any cases if legal hold is enabled
+        // if no legal hold, can only modify object if bypassing governance when locked
+        if (!this.isLocked()) {
+            return true;
+        }
+        return !this.legalHold && this.isGovernanceMode() && !!hasGovernanceBypass;
+    }
+
+    /**
+     * ObjectLockInfo.canModifyPolicy
+     * @param {object} policyChanges - Proposed changes to the retention policy
+     * @param {"GOVERNANCE" | "COMPLIANCE" | undefined} policyChanges.mode - Retention policy mode.
+     * @param {string} policyChanges.date - Expiration date of retention policy. A string in ISO-8601 format
+     * @param {bool} hasGovernanceBypass - Whether to bypass governance retention policies
+     * @returns {bool} - True if the changes are allowed to be applied to the retention policy
+     */
+    canModifyPolicy(policyChanges, hasGovernanceBypass) {
+        // If an object does not have a retention policy or it is expired then all changes are allowed
+        if (!this.isLocked()) {
+            return true;
+        }
+
+        // The only allowed change in compliance mode is extending the retention period
+        if (this.isComplianceMode()) {
+            if (policyChanges.mode === 'COMPLIANCE' && this.isExtended(policyChanges.date)) {
+                return true;
+            }
+        }
+
+        if (this.isGovernanceMode()) {
+            // Extensions are always allowed in governance mode
+            if (policyChanges.mode === 'GOVERNANCE' && this.isExtended(policyChanges.date)) {
+                return true;
+            }
+
+            // All other changes in governance mode require a bypass
+            if (hasGovernanceBypass) {
+                return true;
+            }
+        }
+
+        return false;
     }
-    return false;
 }
 
-function validateObjectLockUpdate(objectMD, retentionInfo, bypassGovernance) {
+/**
-    const { retentionMode: existingMode, retentionDate: existingDateISO } = objectMD;
+ *
-    if (!existingMode) {
+ * @param {object} headers - s3 request headers
-        return null;
+ * @returns {bool} - True if the headers is present and === "true"
-    }
+ */
+function hasGovernanceBypassHeader(headers) {
+    const bypassHeader = headers['x-amz-bypass-governance-retention'] || '';
+    return bypassHeader.toLowerCase() === 'true';
+}
 
-    const existingDate = new Date(existingDateISO);
-    const isExpired = existingDate < Date.now();
 
-    if (existingMode === 'GOVERNANCE' && !isExpired && !bypassGovernance) {
+/**
-        return errors.AccessDenied;
+ * checkUserGovernanceBypass
-    }
+ *
+ *  Checks for the presence of the s3:BypassGovernanceRetention permission for a given user
+ *
+ * @param {object} request - Incoming s3 request
+ * @param {object} authInfo - s3 authentication info
+ * @param {object} bucketMD - bucket metadata
+ * @param {string} objectKey - object key
+ * @param {object} log - Werelogs logger
+ * @param {function} cb - callback returns errors.AccessDenied if the authorization fails
+ * @returns {undefined} -
+ */
+function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log, cb) {
+    log.trace(
+        'object in GOVERNANCE mode and is user, checking for attached policies',
+        { method: 'checkUserPolicyGovernanceBypass' },
+    );
 
-    if (existingMode === 'COMPLIANCE') {
+    const authParams = auth.server.extractParams(request, log, 's3', request.query);
-        if (retentionInfo.mode === 'GOVERNANCE' && !isExpired) {
+    const ip = policies.requestUtils.getClientIp(request, config);
-            return errors.AccessDenied;
+    const requestContextParams = {
-        }
+        constantParams: {
+            headers: request.headers,
+            query: request.query,
+            generalResource: bucketMD.getName(),
+            specificResource: { key: objectKey },
+            requesterIp: ip,
+            sslEnabled: request.connection.encrypted,
+            apiMethod: 'bypassGovernanceRetention',
+            awsService: 's3',
+            locationConstraint: bucketMD.getLocationConstraint(),
+            requesterInfo: authInfo,
+            signatureVersion: authParams.params.data.signatureVersion,
+            authType: authParams.params.data.authType,
+            signatureAge: authParams.params.data.signatureAge,
+        },
+    };
+    return vault.checkPolicies(requestContextParams,
+        authInfo.getArn(), log, (err, authorizationResults) => {
+            if (err) {
+                return cb(err);
+            }
+            const explicitDenyExists = authorizationResults.some(
+                authzResult => authzResult.isAllowed === false && !authzResult.isImplicit);
+            if (explicitDenyExists) {
+                log.trace('authorization check failed for user',
+                    {
+                        'method': 'checkUserPolicyGovernanceBypass',
+                        's3:BypassGovernanceRetention': false,
+                    });
+                return cb(errors.AccessDenied);
+            }
+            // Convert authorization results into an easier to handle format
+            const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => {
+                const apiMethod = authorizationResults[idx].action;
+                // eslint-disable-next-line no-param-reassign
+                acc[apiMethod] = curr.isImplicit;
+                return acc;
+            }, {});
 
-        if (new Date(retentionInfo.date) < existingDate) {
+            // Evaluate against the bucket policies
-            return errors.AccessDenied;
+            const areAllActionsAllowed = evaluateBucketPolicyWithIAM(
-        }
+                bucketMD,
-    }
+                Object.keys(actionImplicitDenies),
+                authInfo.getCanonicalID(),
+                authInfo,
+                actionImplicitDenies,
+                log,
+                request);
 
-    return null;
+            return cb(areAllActionsAllowed === true ? null : errors.AccessDenied);
+        });
 }
 
 module.exports = {
     calculateRetainUntilDate,
     compareObjectLockInformation,
     setObjectLockInformation,
-    isObjectLocked,
     validateHeaders,
-    validateObjectLockUpdate,
+    hasGovernanceBypassHeader,
+    checkUserGovernanceBypass,
+    ObjectLockInfo,
 };

lib/api/apiUtils/object/objectRestore.js

@@ -0,0 +1,172 @@
+const async = require('async');
+const { errors, s3middleware } = require('arsenal');
+
+const { allowedRestoreObjectRequestTierValues } = require('../../../../constants');
+const coldStorage = require('./coldStorage');
+const monitoring = require('../../../utilities/monitoringHandler');
+const { pushMetric } = require('../../../utapi/utilities');
+const { decodeVersionId } = require('./versioning');
+const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
+const { parseRestoreRequestXml } = s3middleware.objectRestore;
+const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
+
+/**
+ * Check if tier is supported
+ * @param {object} restoreInfo - restore information
+ * @returns {ArsenalError|undefined} return NotImplemented error if tier not support
+ */
+function checkTierSupported(restoreInfo) {
+    if (!allowedRestoreObjectRequestTierValues.includes(restoreInfo.tier)) {
+        return errors.NotImplemented;
+    }
+    return undefined;
+}
+
+/**
+ * POST Object restore process
+ *
+ * @param {MetadataWrapper} metadata - metadata wrapper
+ * @param {object} mdUtils - utility object to treat metadata
+ * @param {AuthInfo} userInfo - Instance of AuthInfo class with requester's info
+ * @param {IncomingMessage} request - request info
+ * @param {object} log - Werelogs logger
+ * @param {function} callback callback function
+ * @return {undefined}
+ */
+function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
+    const METHOD = 'objectRestore';
+
+    const { bucketName, objectKey } = request;
+
+    log.debug('processing request', { method: METHOD });
+
+    const decodedVidResult = decodeVersionId(request.query);
+    if (decodedVidResult instanceof Error) {
+        log.trace('invalid versionId query',
+            {
+                method: METHOD,
+                versionId: request.query.versionId,
+                error: decodedVidResult,
+            });
+        return process.nextTick(() => callback(decodedVidResult));
+    }
+
+    let isObjectRestored = false;
+
+    const mdValueParams = {
+        authInfo: userInfo,
+        bucketName,
+        objectKey,
+        versionId: decodedVidResult,
+        requestType: request.apiMethods || 'restoreObject',
+        /**
+         * Restoring an object might not cause any impact on
+         * the storage, if the object is already restored: in
+         * this case, the duration is extended. We disable the
+         * quota evaluation and trigger it manually.
+         */
+        checkQuota: false,
+        request,
+    };
+
+    return async.waterfall([
+            // get metadata of bucket and object
+            function validateBucketAndObject(next) {
+                return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies,
+                log, (err, bucketMD, objectMD) => {
+                    if (err) {
+                        log.trace('request authorization failed', { method: METHOD, error: err });
+                        return next(err);
+                    }
+                    // Call back error if object metadata could not be obtained
+                    if (!objectMD) {
+                        const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey;
+                        log.trace('error no object metadata found', { method: METHOD, error: err });
+                        return next(err, bucketMD);
+                    }
+                    // If object metadata is delete marker,
+                    // call back NoSuchKey or MethodNotAllowed depending on specifying versionId
+                    if (objectMD.isDeleteMarker) {
+                        let err = errors.NoSuchKey;
+                        if (decodedVidResult) {
+                            err = errors.MethodNotAllowed;
+                        }
+                        log.trace('version is a delete marker', { method: METHOD, error: err });
+                        return next(err, bucketMD, objectMD);
+                    }
+                    log.info('it acquired the object metadata.', {
+                        'method': METHOD,
+                    });
+                    return next(null, bucketMD, objectMD);
+                });
+            },
+
+            // generate restore param obj from xml of request body and check tier validity
+            function parseRequestXmlAndCheckTier(bucketMD, objectMD, next) {
+                log.trace('parsing object restore information');
+                return parseRestoreRequestXml(request.post, log, (err, restoreInfo) => {
+                    if (err) {
+                        return next(err, bucketMD, objectMD, restoreInfo);
+                    }
+                    log.info('it parsed xml of the request body.', { method: METHOD, value: restoreInfo });
+                    const checkTierResult = checkTierSupported(restoreInfo);
+                    if (checkTierResult instanceof Error) {
+                        return next(checkTierResult);
+                    }
+                    return next(null, bucketMD, objectMD, restoreInfo);
+                });
+            },
+            // start restore process
+            function startRestore(bucketMD, objectMD, restoreInfo, next) {
+                return coldStorage.startRestore(objectMD, restoreInfo, log,
+                    (err, _isObjectRestored) => {
+                        isObjectRestored = _isObjectRestored;
+                        return next(err, bucketMD, objectMD);
+                    });
+            },
+            function evaluateQuotas(bucketMD, objectMD, next) {
+                if (isObjectRestored) {
+                    return next(null, bucketMD, objectMD);
+                }
+                const actions = Array.isArray(mdValueParams.requestType) ?
+                    mdValueParams.requestType : [mdValueParams.requestType];
+                const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
+                return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
+                    false, log, err => next(err, bucketMD, objectMD));
+            },
+            function updateObjectMD(bucketMD, objectMD, next) {
+                const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
+                metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
+                    log, err => next(err, bucketMD, objectMD));
+            },
+        ],
+        (err, bucketMD) => {
+            // generate CORS response header
+            const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD);
+            if (err) {
+                log.trace('error processing request',
+                    {
+                        method: METHOD,
+                        error: err,
+                    });
+                monitoring.promMetrics(
+                    'POST', bucketName, err.code, 'restoreObject');
+                return callback(err, err.code, responseHeaders);
+            }
+            pushMetric('restoreObject', log, {
+                userInfo,
+                bucket: bucketName,
+            });
+            if (isObjectRestored) {
+                monitoring.promMetrics(
+                    'POST', bucketName, '200', 'restoreObject');
+                return callback(null, 200, responseHeaders);
+            }
+            monitoring.promMetrics(
+                'POST', bucketName, '202', 'restoreObject');
+            return callback(null, 202, responseHeaders);
+        });
+}
+
+
+module.exports = objectRestore;

lib/api/apiUtils/object/setUpCopyLocator.js

@@ -2,7 +2,7 @@ const { errors } = require('arsenal');
 const {
     parseRangeSpec,
     parseRange,
-} = require('arsenal/lib/network/http/utils');
+} = require('arsenal').network.http.utils;
 
 const constants = require('../../../../constants');
 const setPartRanges = require('./setPartRanges');

lib/api/apiUtils/object/validateChecksumHeaders.js

@@ -0,0 +1,32 @@
+const { errors } = require('arsenal');
+
+const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants');
+
+function validateChecksumHeaders(headers) {
+    // If the x-amz-trailer header is present the request is using one of the
+    // trailing checksum algorithms, which are not supported.
+    if (headers['x-amz-trailer'] !== undefined) {
+        return errors.BadRequest.customizeDescription('trailing checksum is not supported');
+    }
+
+    const signatureChecksum = headers['x-amz-content-sha256'];
+    if (signatureChecksum === undefined) {
+        return null;
+    }
+
+    if (supportedSignatureChecksums.has(signatureChecksum)) {
+        return null;
+    }
+
+    // If the value is not one of the possible checksum algorithms
+    // the only other valid value is the actual sha256 checksum of the payload.
+    // Do a simple sanity check of the length to guard against future algos.
+    // If the value is an unknown algo, then it will fail checksum validation.
+    if (!unsupportedSignatureChecksums.has(signatureChecksum) && signatureChecksum.length === 64) {
+        return null;
+    }
+
+    return errors.BadRequest.customizeDescription('unsupported checksum algorithm');
+}
+
+module.exports = validateChecksumHeaders;

lib/api/apiUtils/object/versioning.js

@@ -4,13 +4,40 @@ const async = require('async');
 const metadata = require('../../../metadata/wrapper');
 const { config } = require('../../../Config');
 
+const { scaledMsPerDay } = config.getTimeOptions();
+
 const versionIdUtils = versioning.VersionID;
 // Use Arsenal function to generate a version ID used internally by metadata
 // for null versions that are created before bucket versioning is configured
 const nonVersionedObjId =
     versionIdUtils.getInfVid(config.replicationGroupId);
 
-/** decodedVidResult - decode the version id from a query object
+/** decodeVID - decode the version id
+ * @param {string} versionId - version ID
+ * @return {(Error|string|undefined)} - return Invalid Argument if decryption
+ * fails due to improper format, otherwise undefined or the decoded version id
+ */
+function decodeVID(versionId) {
+    if (versionId === 'null') {
+        return versionId;
+    }
+
+    let decoded;
+    const invalidErr = errors.InvalidArgument.customizeDescription('Invalid version id specified');
+    try {
+        decoded = versionIdUtils.decode(versionId);
+    } catch (err) {
+        return invalidErr;
+    }
+
+    if (decoded instanceof Error) {
+        return invalidErr;
+    }
+
+    return decoded;
+}
+
+/** decodeVersionId - decode the version id from a query object
  * @param {object} [reqQuery] - request query object
  * @param {string} [reqQuery.versionId] - version ID sent in request query
  * @return {(Error|string|undefined)} - return Invalid Argument if decryption
@@ -20,16 +47,7 @@ function decodeVersionId(reqQuery) {
     if (!reqQuery || !reqQuery.versionId) {
         return undefined;
     }
-    let versionId = reqQuery.versionId;
+    return decodeVID(reqQuery.versionId);
-    if (versionId === 'null') {
-        return versionId;
-    }
-    versionId = versionIdUtils.decode(versionId);
-    if (versionId instanceof Error) {
-        return errors.InvalidArgument
-            .customizeDescription('Invalid version id specified');
-    }
-    return versionId;
 }
 
 /** getVersionIdResHeader - return encrypted version ID if appropriate
@@ -40,7 +58,7 @@ function decodeVersionId(reqQuery) {
  */
 function getVersionIdResHeader(verCfg, objectMD) {
     if (verCfg) {
-        if (objectMD.isNull || (objectMD && !objectMD.versionId)) {
+        if (objectMD.isNull || !objectMD.versionId) {
             return 'null';
         }
         return versionIdUtils.encode(objectMD.versionId);
@@ -61,17 +79,34 @@ function checkQueryVersionId(query) {
     return undefined;
 }
 
-function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
+function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb) {
-    metadata.putObjectMD(bucketName, objKey, objMD, options, log, err => {
+    // In compatibility mode, create null versioned keys instead of null keys
+    let versionId;
+    let nullVersionMD;
+    if (config.nullVersionCompatMode) {
+        versionId = nullVersionId;
+        nullVersionMD = Object.assign({}, objMD, {
+            versionId: nullVersionId,
+            isNull: true,
+        });
+    } else {
+        versionId = 'null';
+        nullVersionMD = Object.assign({}, objMD, {
+            versionId: nullVersionId,
+            isNull: true,
+            isNull2: true,
+        });
+    }
+    metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => {
         if (err) {
             log.debug('error from metadata storing null version as new version',
             { error: err });
         }
-        cb(err, options);
+        cb(err);
     });
 }
 
-/** get location of null version data for deletion
+/** check existence and get location of null version data for deletion
 * @param {string} bucketName - name of bucket
 * @param {string} objKey - name of object key
 * @param {object} options - metadata options for getting object MD
@@ -82,100 +117,179 @@ function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
 * @param {function} cb - callback
 * @return {undefined} - and call callback with (err, dataToDelete)
 */
-function _getNullVersionsToDelete(bucketName, objKey, options, mst, log, cb) {
+function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) {
+    const nullOptions = {};
+    if (!options.deleteData) {
+        return process.nextTick(cb, null, nullOptions);
+    }
     if (options.versionId === mst.versionId) {
-        // no need to get delete location, we already have the master's metadata
+        // no need to get another key as the master is the target
-        const dataToDelete = mst.objLocation;
+        nullOptions.dataToDelete = mst.objLocation;
-        return process.nextTick(cb, null, dataToDelete);
+        return process.nextTick(cb, null, nullOptions);
+    }
+    if (options.versionId === 'null') {
+        // deletion of the null key will be done by the main metadata
+        // PUT via this option
+        nullOptions.deleteNullKey = true;
     }
     return metadata.getObjectMD(bucketName, objKey, options, log,
         (err, versionMD) => {
             if (err) {
-                log.debug('err from metadata getting specified version', {
+                // the null key may not exist, hence it's a normal
-                    error: err,
+                // situation to have a NoSuchKey error, in which case
-                    method: '_getNullVersionsToDelete',
+                // there is nothing to delete
-                });
+                if (err.is.NoSuchKey) {
+                    log.debug('null version does not exist', {
+                        method: '_prepareNullVersionDeletion',
+                    });
+                } else {
+                    log.warn('could not get null version metadata', {
+                        error: err,
+                        method: '_prepareNullVersionDeletion',
+                    });
+                }
                 return cb(err);
             }
-            if (!versionMD.location) {
+            if (versionMD.location) {
-                return cb();
+                const dataToDelete = Array.isArray(versionMD.location) ?
+                      versionMD.location : [versionMD.location];
+                nullOptions.dataToDelete = dataToDelete;
             }
-            const dataToDelete = Array.isArray(versionMD.location) ?
+            return cb(null, nullOptions);
-                versionMD.location : [versionMD.location];
-            return cb(null, dataToDelete);
         });
 }
 
-function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
+function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
-    return _getNullVersionsToDelete(bucketName, objKey, options, mst, log,
+    return metadata.deleteObjectMD(bucketName, objKey, options, log, err => {
-        (err, nullDataToDelete) => {
+        if (err) {
-            if (err) {
+            log.warn('metadata error deleting null versioned key',
-                log.warn('could not find null version metadata', {
+                     { bucketName, objKey, error: err, method: '_deleteNullVersionMD' });
-                    error: err,
+        }
-                    method: '_deleteNullVersionMD',
+        return cb(err);
-                });
+    });
-                return cb(err);
-            }
-            return metadata.deleteObjectMD(bucketName, objKey, options, log,
-                err => {
-                    if (err) {
-                        log.warn('metadata error deleting null version',
-                        { error: err, method: '_deleteNullVersionMD' });
-                        return cb(err);
-                    }
-                    return cb(null, nullDataToDelete);
-                });
-        });
 }
 
-function processVersioningState(mst, vstat, cb) {
+/**
-    const options = {};
+ * Process state from the master version of an object and the bucket
-    const storeOptions = {};
+ * versioning configuration, return a set of options objects
-    const delOptions = {};
+ *
-    // object does not exist or is not versioned (before versioning)
+ * @param {object} mst - state of master version, as returned by
-    if (mst.versionId === undefined || mst.isNull) {
+ * getMasterState()
-        // versioning is suspended, overwrite existing master version
+ * @param {string} vstat - bucket versioning status: 'Enabled' or 'Suspended'
-        if (vstat === 'Suspended') {
+ * @param {boolean} nullVersionCompatMode - if true, behaves in null
-            options.versionId = '';
+ * version compatibility mode and return appropriate values: this mode
-            options.isNull = true;
+ * does not attempt to create null keys but create null versioned keys
-            options.dataToDelete = mst.objLocation;
+ * instead
-            // if null version exists, clean it up prior to put
+ *
-            if (mst.isNull) {
+ * @return {object} result object with the following attributes:
-                delOptions.versionId = mst.versionId;
+ * - {object} options: versioning-related options to pass to the
-                return cb(null, options, null, delOptions);
+     services.metadataStoreObject() call
+ * - {object} [options.extraMD]: extra attributes to set in object metadata
+ * - {string} [nullVersionId]: null version key to create, if needed
+ * - {object} [delOptions]: options for metadata to delete the null
+     version key, if needed
+ */
+function processVersioningState(mst, vstat, nullVersionCompatMode) {
+    const versioningSuspended = (vstat === 'Suspended');
+    const masterIsNull = mst.exists && (mst.isNull || !mst.versionId);
+
+    if (versioningSuspended) {
+        // versioning is suspended: overwrite the existing null version
+        const options = { versionId: '', isNull: true };
+        if (masterIsNull) {
+            // if the null version exists, clean it up prior to put
+            if (mst.objLocation) {
+                options.dataToDelete = mst.objLocation;
             }
-            return cb(null, options);
+            // backward-compat: a null version key may exist even with
+            // a null master (due to S3C-7526), if so, delete it (its
+            // data will be deleted as part of the master cleanup, so
+            // no "deleteData" param is needed)
+            //
+            // "isNull2" attribute is set in master metadata when
+            // null keys are used, which is used as an optimization to
+            // avoid having to check the versioned key since there can
+            // be no more versioned key to clean up
+            if (mst.isNull && mst.versionId && !mst.isNull2) {
+                const delOptions = { versionId: mst.versionId };
+                return { options, delOptions };
+            }
+            return { options };
         }
-        // versioning is enabled, create a new version
+        if (mst.nullVersionId) {
-        options.versioning = true;
+            // backward-compat: delete the null versioned key and data
-        if (mst.exists) {
+            const delOptions = { versionId: mst.nullVersionId, deleteData: true };
-            // store master version in a new key
+            if (mst.nullUploadId) {
-            const versionId = mst.isNull ? mst.versionId : nonVersionedObjId;
+                delOptions.replayId = mst.nullUploadId;
-            storeOptions.versionId = versionId;
+            }
-            storeOptions.isNull = true;
+            return { options, delOptions };
-            options.nullVersionId = versionId;
-            return cb(null, options, storeOptions);
         }
-        return cb(null, options);
+        // clean up the eventual null key's location data prior to put
+
+        // NOTE: due to metadata v1 internal format, we cannot guess
+        // from the master key whether there is an associated null
+        // key, because the master key may be removed whenever the
+        // latest version becomes a delete marker. Hence we need to
+        // pessimistically try to get the null key metadata and delete
+        // it if it exists.
+        const delOptions = { versionId: 'null', deleteData: true };
+        return { options, delOptions };
     }
-    // master is versioned and is not a null version
+
-    const nullVersionId = mst.nullVersionId;
+    // versioning is enabled: create a new version
-    if (vstat === 'Suspended') {
+    const options = { versioning: true };
-        // versioning is suspended, overwrite the existing master version
+    if (masterIsNull) {
-        options.versionId = '';
+        // if master is a null version or a non-versioned key,
-        options.isNull = true;
+        // copy it to a new null key
-        if (nullVersionId === undefined) {
+        const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
-            return cb(null, options);
+        if (nullVersionCompatMode) {
+            options.extraMD = {
+                nullVersionId,
+            };
+            if (mst.uploadId) {
+                options.extraMD.nullUploadId = mst.uploadId;
+            }
+            return { options, nullVersionId };
         }
-        delOptions.versionId = nullVersionId;
+        if (mst.isNull && !mst.isNull2) {
-        return cb(null, options, null, delOptions);
+            // if master null version was put with an older
+            // Cloudserver (or in compat mode), there is a
+            // possibility that it also has a null versioned key
+            // associated, so we need to delete it as we write the
+            // null key
+            const delOptions = {
+                versionId: nullVersionId,
+            };
+            return { options, nullVersionId, delOptions };
+        }
+        return { options, nullVersionId };
     }
-    // versioning is enabled, put the new version
+    // backward-compat: keep a reference to the existing null
-    options.versioning = true;
+    // versioned key
-    options.nullVersionId = nullVersionId;
+    if (mst.nullVersionId) {
-    return cb(null, options);
+        options.extraMD = {
+            nullVersionId: mst.nullVersionId,
+        };
+        if (mst.nullUploadId) {
+            options.extraMD.nullUploadId = mst.nullUploadId;
+        }
+    }
+    return { options };
 }
 
+/**
+ * Build the state of the master version from its object metadata
+ *
+ * @param {object} objMD - object metadata parsed from JSON
+ *
+ * @return {object} state of master version, with the following attributes:
+ * - {boolean} exists - true if the object exists (i.e. if `objMD` is truish)
+ * - {string} versionId - version ID of the master key
+ * - {boolean} isNull - whether the master version is a null version
+ * - {string} nullVersionId - if not a null version, reference to the
+ *   null version ID
+ * - {array} objLocation - array of data locations
+ */
 function getMasterState(objMD) {
     if (!objMD) {
         return {};
@@ -183,8 +297,11 @@ function getMasterState(objMD) {
     const mst = {
         exists: true,
         versionId: objMD.versionId,
+        uploadId: objMD.uploadId,
         isNull: objMD.isNull,
+        isNull2: objMD.isNull2,
         nullVersionId: objMD.nullVersionId,
+        nullUploadId: objMD.nullUploadId,
     };
     if (objMD.location) {
         mst.objLocation = Array.isArray(objMD.location) ?
@@ -206,62 +323,113 @@ function getMasterState(objMD) {
  *  ('' overwrites the master version)
  * options.versioning - (true/undefined) metadata instruction to create new ver
  * options.isNull - (true/undefined) whether new version is null or not
- * options.nullVersionId - if storing a null version in version history, the
- *  version id of the null version
- * options.deleteNullVersionData - whether to delete the data of the null ver
  */
 function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
     log, callback) {
-    const options = {};
     const mst = getMasterState(objMD);
     const vCfg = bucketMD.getVersioningConfiguration();
     // bucket is not versioning configured
     if (!vCfg) {
-        options.dataToDelete = mst.objLocation;
+        const options = { dataToDelete: mst.objLocation };
         return process.nextTick(callback, null, options);
     }
     // bucket is versioning configured
-    return async.waterfall([
+    const { options, nullVersionId, delOptions } =
-        function processState(next) {
+          processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode);
-            processVersioningState(mst, vCfg.Status,
+    return async.series([
-                (err, options, storeOptions, delOptions) => {
+        function storeNullVersionMD(next) {
-                    process.nextTick(next, err, options, storeOptions,
+            if (!nullVersionId) {
-                        delOptions);
+                return process.nextTick(next);
-                });
-        },
-        function storeVersion(options, storeOptions, delOptions, next) {
-            if (!storeOptions) {
-                return process.nextTick(next, null, options, delOptions);
             }
-            const versionMD = Object.assign({}, objMD, storeOptions);
+            return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
-            const params = { versionId: storeOptions.versionId };
-            return _storeNullVersionMD(bucketName, objectKey, versionMD,
-                params, log, err => next(err, options, delOptions));
         },
-        function deleteNullVersion(options, delOptions, next) {
+        function prepareNullVersionDeletion(next) {
             if (!delOptions) {
-                return process.nextTick(next, null, options);
+                return process.nextTick(next);
             }
-            return _deleteNullVersionMD(bucketName, objectKey, delOptions, mst,
+            return _prepareNullVersionDeletion(
-                log, (err, nullDataToDelete) => {
+                bucketName, objectKey, delOptions, mst, log,
+                (err, nullOptions) => {
                     if (err) {
-                        log.warn('unexpected error deleting null version md', {
+                        return next(err);
-                            error: err,
-                            method: 'versioningPreprocessing',
-                        });
-                        // it's possible there was a concurrent request to
-                        // delete the null version, so proceed with putting a
-                        // new version
-                        if (err === errors.NoSuchKey) {
-                            return next(null, options);
-                        }
-                        return next(errors.InternalError);
                     }
-                    Object.assign(options, { dataToDelete: nullDataToDelete });
+                    Object.assign(options, nullOptions);
-                    return next(null, options);
+                    return next();
                 });
         },
-    ], (err, options) => callback(err, options));
+        function deleteNullVersionMD(next) {
+            if (delOptions &&
+                delOptions.versionId &&
+                delOptions.versionId !== 'null') {
+                // backward-compat: delete old null versioned key
+                return _deleteNullVersionMD(
+                    bucketName, objectKey, { versionId: delOptions.versionId }, log, next);
+            }
+            return process.nextTick(next);
+        },
+    ], err => {
+        // it's possible there was a prior request that deleted the
+        // null version, so proceed with putting a new version
+        if (err && err.is.NoSuchKey) {
+            return callback(null, options);
+        }
+        return callback(err, options);
+    });
+}
+
+/** Return options to pass to Metadata layer for version-specific
+ * operations with the given requested version ID
+ *
+ * @param {object} objectMD - object metadata
+ * @param {boolean} nullVersionCompatMode - if true, behaves in null
+ * version compatibility mode
+ * @return {object} options object with params:
+ * {string} [options.versionId] - specific versionId to update
+ * {boolean} [options.isNull=true|false|undefined] - if set, tells the
+ * Metadata backend if we're updating or deleting a new-style null
+ * version (stored in master or null key), or not a null version.
+ */
+function getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode) {
+    // Use the internal versionId if it is a "real" null version (not
+    // non-versioned)
+    //
+    // If the target object is non-versioned: do not specify a
+    // "versionId" attribute nor "isNull"
+    //
+    // If the target version is a null version, i.e. has the "isNull"
+    // attribute:
+    //
+    // - send the "isNull=true" param to Metadata if the version is
+    //   already a null key put by a non-compat mode Cloudserver, to
+    //   let Metadata know that the null key is to be updated or
+    //   deleted. This is the case if the "isNull2" metadata attribute
+    //   exists
+    //
+    // - otherwise, do not send the "isNull" parameter to hint
+    //   Metadata that it is a legacy null version
+    //
+    // If the target version is not a null version and is versioned:
+    //
+    // - send the "isNull=false" param to Metadata in non-compat
+    //   mode (mandatory for v1 format)
+    //
+    // - otherwise, do not send the "isNull" parameter to hint
+    //   Metadata that an existing null version may not be stored in a
+    //   null key
+    //
+    //
+    if (objectMD.versionId === undefined) {
+        return {};
+    }
+    const options = { versionId: objectMD.versionId };
+    if (objectMD.isNull) {
+        if (objectMD.isNull2) {
+            options.isNull = true;
+        }
+    } else if (!nullVersionCompatMode) {
+        options.isNull = false;
+    }
+    return options;
 }
 
 /** preprocessingVersioningDelete - return versioning information for S3 to
@@ -270,59 +438,124 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
  * @param {object} bucketMD - bucket metadata
  * @param {object} objectMD - obj metadata
  * @param {string} [reqVersionId] - specific version ID sent as part of request
- * @param {RequestLogger} log - logger instance
+ * @param {boolean} nullVersionCompatMode - if true, behaves in null version compatibility mode
- * @param {function} callback - callback
+ * @return {object} options object with params:
- * @return {undefined} and call callback with params (err, options):
+ * {boolean} [options.deleteData=true|undefined] - whether to delete data (if undefined
- * options.deleteData - (true/undefined) whether to delete data (if undefined
  *  means creating a delete marker instead)
- * options.versionId - specific versionId to delete
+ * {string} [options.versionId] - specific versionId to delete
+ * {boolean} [options.isNull=true|false|undefined] - if set, tells the
+ * Metadata backend if we're deleting a new-style null version (stored
+ * in master or null key), or not a null version.
  */
-function preprocessingVersioningDelete(bucketName, bucketMD, objectMD,
+function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersionId, nullVersionCompatMode) {
-    reqVersionId, log, callback) {
+    let options = {};
-    const options = {};
+    if (bucketMD.getVersioningConfiguration() && reqVersionId) {
-    // bucket is not versioning enabled
+        options = getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode);
-    if (!bucketMD.getVersioningConfiguration()) {
+    }
+    if (!bucketMD.getVersioningConfiguration() || reqVersionId) {
+        // delete data if bucket is non-versioned or the request
+        // deletes a specific version
         options.deleteData = true;
-        return callback(null, options);
     }
-    // bucket is versioning enabled
+    return options;
-    if (reqVersionId && reqVersionId !== 'null') {
+}
-        // deleting a specific version
+
-        options.deleteData = true;
+/**
-        options.versionId = reqVersionId;
+ * Keep metadatas when the object is restored from cold storage
-        return callback(null, options);
+ * but remove the specific ones we don't want to keep
+ * @param {object} objMD - obj metadata
+ * @param {object} metadataStoreParams - custom built object containing resource details.
+ * @return {undefined}
+ */
+function restoreMetadata(objMD, metadataStoreParams) {
+    /* eslint-disable no-param-reassign */
+    const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt'];
+    // We need to keep user metadata and tags
+    Object.keys(objMD).forEach(key => {
+        if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) {
+            metadataStoreParams.metaHeaders[key] = objMD[key];
+        }
+    });
+
+    if (objMD['x-amz-website-redirect-location']) {
+        if (!metadataStoreParams.headers) {
+            metadataStoreParams.headers = {};
+        }
+        metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location'];
     }
-    if (reqVersionId) {
+
-        // deleting the 'null' version if it exists
+    if (objMD.replicationInfo) {
-        if (objectMD.versionId === undefined) {
+        metadataStoreParams.replicationInfo = objMD.replicationInfo;
-            // object is not versioned, deleting it
-            options.deleteData = true;
-            return callback(null, options);
-        }
-        if (objectMD.isNull) {
-            // master is the null version
-            options.deleteData = true;
-            options.versionId = objectMD.versionId;
-            return callback(null, options);
-        }
-        if (objectMD.nullVersionId) {
-            // null version exists, deleting it
-            options.deleteData = true;
-            options.versionId = objectMD.nullVersionId;
-            return callback(null, options);
-        }
-        // null version does not exist, no deletion
-        // TODO check AWS behaviour for no deletion (seems having no error)
-        return callback(errors.NoSuchKey);
     }
-    // not deleting any specific version, making a delete marker instead
+
-    return callback(null, options);
+    if (objMD.legalHold) {
+        metadataStoreParams.legalHold = objMD.legalHold;
+    }
+
+    if (objMD.acl) {
+        metadataStoreParams.acl = objMD.acl;
+    }
+
+    metadataStoreParams.creationTime = objMD['creation-time'];
+    metadataStoreParams.lastModifiedDate = objMD['last-modified'];
+    metadataStoreParams.taggingCopy = objMD.tags;
+}
+
+/** overwritingVersioning - return versioning information for S3 to handle
+ * storing version metadata with a specific version id.
+ * @param {object} objMD - obj metadata
+ * @param {object} metadataStoreParams - custom built object containing resource details.
+ * @return {object} options
+ * options.versionId - specific versionId to overwrite in metadata
+ * options.isNull - (true/undefined) whether new version is null or not
+ * options.nullVersionId - if storing a null version in version history, the
+ *  version id of the null version
+ */
+function overwritingVersioning(objMD, metadataStoreParams) {
+    metadataStoreParams.updateMicroVersionId = true;
+    metadataStoreParams.amzStorageClass = objMD['x-amz-storage-class'];
+
+    // set correct originOp
+    metadataStoreParams.originOp = 's3:ObjectRestore:Completed';
+
+    // update restore
+    const days = objMD.archive?.restoreRequestedDays;
+    const now = Date.now();
+    metadataStoreParams.archive = {
+        archiveInfo: objMD.archive?.archiveInfo,
+        restoreRequestedAt: objMD.archive?.restoreRequestedAt,
+        restoreRequestedDays: objMD.archive?.restoreRequestedDays,
+        restoreCompletedAt: new Date(now),
+        restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
+    };
+
+    /* eslint-enable no-param-reassign */
+
+    const versionId = objMD.versionId || undefined;
+    const options = {
+        versionId,
+        isNull: objMD.isNull,
+    };
+    if (objMD.nullVersionId) {
+        options.extraMD = {
+            nullVersionId: objMD.nullVersionId,
+        };
+    }
+
+    restoreMetadata(objMD, metadataStoreParams);
+
+    return options;
 }
 
 module.exports = {
     decodeVersionId,
     getVersionIdResHeader,
     checkQueryVersionId,
+    processVersioningState,
+    getMasterState,
     versioningPreprocessing,
+    getVersionSpecificMetadataOptions,
     preprocessingVersioningDelete,
+    overwritingVersioning,
+    decodeVID,
 };

lib/api/apiUtils/object/websiteServing.js

@@ -101,8 +101,33 @@ function validateWebsiteHeader(header) {
     header.startsWith('http://') || header.startsWith('https://'));
 }
 
+/**
+ * appendWebsiteIndexDocument - append index to objectKey if necessary
+ * @param {object} request - normalized request object
+ * @param {string} indexDocumentSuffix - index document from website config
+ * @param {boolean} force - flag to force append index
+ * @return {undefined}
+ */
+function appendWebsiteIndexDocument(request, indexDocumentSuffix, force = false) {
+    const reqObjectKey = request.objectKey ? request.objectKey : '';
+    /* eslint-disable no-param-reassign */
+
+    // find index document if "directory" sent in request
+    if (reqObjectKey.endsWith('/')) {
+        request.objectKey += indexDocumentSuffix;
+    // find index document if no key provided
+    } else if (reqObjectKey === '') {
+        request.objectKey = indexDocumentSuffix;
+    // force for redirect 302 on folder without trailing / that has an index
+    } else if (force) {
+        request.objectKey += `/${indexDocumentSuffix}`;
+    }
+    /* eslint-enable no-param-reassign */
+}
+
 module.exports = {
     findRoutingRule,
     extractRedirectInfo,
     validateWebsiteHeader,
+    appendWebsiteIndexDocument,
 };

lib/api/apiUtils/quotas/quotaUtils.js

@@ -0,0 +1,314 @@
+const async = require('async');
+const { errors } = require('arsenal');
+const monitoring = require('../../../utilities/monitoringHandler');
+const {
+    actionNeedQuotaCheckCopy,
+    actionNeedQuotaCheck,
+    actionWithDataDeletion,
+} = require('arsenal').policies;
+const { config } = require('../../../Config');
+const QuotaService = require('../../../quotas/quotas');
+
+/**
+ * Process the bytes to write based on the request and object metadata
+ * @param {string} apiMethod - api method
+ * @param {BucketInfo} bucket - bucket info
+ * @param {string} versionId - version id of the object
+ * @param {number} contentLength - content length of the object
+ * @param {object} objMD - object metadata
+ * @param {object} destObjMD - destination object metadata
+ * @return {number} processed content length
+ */
+function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
+    let bytes = contentLength;
+    if (apiMethod === 'objectRestore') {
+        // object is being restored
+        bytes = Number.parseInt(objMD['content-length'], 10);
+    } else if (!bytes && objMD?.['content-length']) {
+        if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
+            if (!destObjMD || bucket.isVersioningEnabled()) {
+                // object is being copied
+                bytes = Number.parseInt(objMD['content-length'], 10);
+            } else if (!bucket.isVersioningEnabled()) {
+                // object is being copied and replaces the target
+                bytes = Number.parseInt(objMD['content-length'], 10) -
+                    Number.parseInt(destObjMD['content-length'], 10);
+            }
+        } else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
+            // object is being deleted
+            bytes = -Number.parseInt(objMD['content-length'], 10);
+        }
+    } else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
+        // object is being replaced: store the diff, if the bucket is not versioned
+        bytes = bytes - Number.parseInt(objMD['content-length'], 10);
+    }
+    return bytes || 0;
+}
+
+/**
+ * Checks if a metric is stale based on the provided parameters.
+ *
+ * @param {Object} metric - The metric object to check.
+ * @param {string} resourceType - The type of the resource.
+ * @param {string} resourceName - The name of the resource.
+ * @param {string} action - The action being performed.
+ * @param {number} inflight - The number of inflight requests.
+ * @param {Object} log - The logger object.
+ * @returns {boolean} Returns true if the metric is stale, false otherwise.
+ */
+function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
+    if (metric.date && Date.now() - new Date(metric.date).getTime() >
+        QuotaService.maxStaleness) {
+        log.warn('Stale metrics from the quota service, allowing the request', {
+            resourceType,
+            resourceName,
+            action,
+            inflight,
+        });
+        monitoring.requestWithQuotaMetricsUnavailable.inc();
+        return true;
+    }
+    return false;
+}
+
+/**
+ * Evaluates quotas for a bucket and an account and update inflight count.
+ *
+ * @param {number} bucketQuota - The quota limit for the bucket.
+ * @param {number} accountQuota - The quota limit for the account.
+ * @param {object} bucket - The bucket object.
+ * @param {object} account - The account object.
+ * @param {number} inflight - The number of inflight requests.
+ * @param {number} inflightForCheck - The number of inflight requests for checking quotas.
+ * @param {string} action - The action being performed.
+ * @param {object} log - The logger object.
+ * @param {function} callback - The callback function to be called when evaluation is complete.
+ * @returns {object} - The result of the evaluation.
+ */
+function _evaluateQuotas(
+    bucketQuota,
+    accountQuota,
+    bucket,
+    account,
+    inflight,
+    inflightForCheck,
+    action,
+    log,
+    callback,
+) {
+    let bucketQuotaExceeded = false;
+    let accountQuotaExceeded = false;
+    const creationDate = new Date(bucket.getCreationDate()).getTime();
+    return async.parallel({
+        bucketQuota: parallelDone => {
+            if (bucketQuota > 0) {
+                return QuotaService.getUtilizationMetrics('bucket',
+                    `${bucket.getName()}_${creationDate}`, null, {
+                    action,
+                    inflight,
+                }, (err, bucketMetrics) => {
+                    if (err || inflight < 0) {
+                        return parallelDone(err);
+                    }
+                    if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
+                        bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
+                        log.debug('Bucket quota exceeded', {
+                            bucket: bucket.getName(),
+                            action,
+                            inflight,
+                            quota: bucketQuota,
+                            bytesTotal: bucketMetrics.bytesTotal,
+                        });
+                        bucketQuotaExceeded = true;
+                    }
+                    return parallelDone();
+                });
+            }
+            return parallelDone();
+        },
+        accountQuota: parallelDone => {
+            if (accountQuota > 0 && account?.account) {
+                return QuotaService.getUtilizationMetrics('account',
+                    account.account, null, {
+                    action,
+                    inflight,
+                }, (err, accountMetrics) => {
+                    if (err || inflight < 0) {
+                        return parallelDone(err);
+                    }
+                    if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
+                        accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
+                        log.debug('Account quota exceeded', {
+                            accountId: account.account,
+                            action,
+                            inflight,
+                            quota: accountQuota,
+                            bytesTotal: accountMetrics.bytesTotal,
+                        });
+                        accountQuotaExceeded = true;
+                    }
+                    return parallelDone();
+                });
+            }
+            return parallelDone();
+        },
+    }, err => {
+        if (err) {
+            log.warn('Error evaluating quotas', {
+                error: err.name,
+                description: err.message,
+                isInflightDeletion: inflight < 0,
+            });
+        }
+        return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
+    });
+}
+
+/**
+ * Monitors the duration of quota evaluation for a specific API method.
+ *
+ * @param {string} apiMethod - The name of the API method being monitored.
+ * @param {string} type - The type of quota being evaluated.
+ * @param {string} code - The code associated with the quota being evaluated.
+ * @param {number} duration - The duration of the quota evaluation in nanoseconds.
+ * @returns {undefined} - Returns nothing.
+ */
+function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
+    monitoring.quotaEvaluationDuration.labels({
+        action: apiMethod,
+        type,
+        code,
+    }).observe(duration / 1e9);
+}
+
+/**
+ *
+ * @param {Request} request - request object
+ * @param {BucketInfo} bucket - bucket object
+ * @param {Account} account - account object
+ * @param {array} apiNames - action names: operations to authorize
+ * @param {string} apiMethod - the main API call
+ * @param {number} inflight - inflight bytes
+ * @param {boolean} isStorageReserved - Flag to check if the current quota, minus
+ * the incoming bytes, are under the limit.
+ * @param {Logger} log - logger
+ * @param {function} callback - callback function
+ * @returns {boolean} - true if the quota is valid, false otherwise
+ */
+function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
+    if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
+        return callback(null);
+    }
+    let type;
+    let bucketQuotaExceeded = false;
+    let accountQuotaExceeded = false;
+    let quotaEvaluationDuration;
+    const requestStartTime = process.hrtime.bigint();
+    const bucketQuota = bucket.getQuota();
+    const accountQuota = account?.quota || 0;
+    const shouldSendInflights = config.isQuotaInflightEnabled();
+
+    if (bucketQuota && accountQuota) {
+        type = 'bucket+account';
+    } else if (bucketQuota) {
+        type = 'bucket';
+    } else {
+        type = 'account';
+    }
+
+    if (actionWithDataDeletion[apiMethod]) {
+        type = 'delete';
+    }
+
+    if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
+        if (bucketQuota > 0 || accountQuota > 0) {
+            log.warn('quota is set for a bucket, but the quota service is disabled', {
+                bucketName: bucket.getName(),
+            });
+            monitoring.requestWithQuotaMetricsUnavailable.inc();
+        }
+        return callback(null);
+    }
+
+    if (isStorageReserved) {
+        // eslint-disable-next-line no-param-reassign
+        inflight = 0;
+    }
+
+    return async.forEach(apiNames, (apiName, done) => {
+        // Object copy operations first check the target object,
+        // meaning the source object, containing the current bytes,
+        // is checked second. This logic handles these APIs calls by
+        // ensuring the bytes are positives (i.e., not an object
+        // replacement).
+        if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
+            // eslint-disable-next-line no-param-reassign
+            inflight = Math.abs(inflight);
+        } else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
+            return done();
+        }
+        // When inflights are disabled, the sum of the current utilization metrics
+        // and the current bytes are compared with the quota. The current bytes
+        // are not sent to the utilization service. When inflights are enabled,
+        // the sum of the current utilization metrics only are compared with the
+        // quota. They include the current inflight bytes sent in the request.
+        let _inflights = shouldSendInflights ? inflight : undefined;
+        const inflightForCheck = shouldSendInflights ? 0 : inflight;
+        return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
+            inflightForCheck, apiName, log,
+            (err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
+                if (err) {
+                    return done(err);
+                }
+
+                bucketQuotaExceeded = _bucketQuotaExceeded;
+                accountQuotaExceeded = _accountQuotaExceeded;
+
+                // Inflights are inverted: in case of cleanup, we just re-issue
+                // the same API call.
+                if (_inflights) {
+                    _inflights = -_inflights;
+                }
+
+                request.finalizerHooks.push((errorFromAPI, _done) => {
+                    const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
+                    const quotaCleanUpStartTime = process.hrtime.bigint();
+                    // Quotas are cleaned only in case of error in the API
+                    async.waterfall([
+                        cb => {
+                            if (errorFromAPI) {
+                                return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
+                                    null, apiName, log, cb);
+                            }
+                            return cb();
+                        },
+                    ], () => {
+                        monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
+                            Number(process.hrtime.bigint() - quotaCleanUpStartTime));
+                        return _done();
+                    });
+                });
+
+                return done();
+            });
+    }, err => {
+        quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
+        if (err) {
+            log.warn('Error getting metrics from the quota service, allowing the request', {
+                error: err.name,
+                description: err.message,
+            });
+        }
+        if (!actionWithDataDeletion[apiMethod] &&
+            (bucketQuotaExceeded || accountQuotaExceeded)) {
+            return callback(errors.QuotaExceeded);
+        }
+        return callback();
+    });
+}
+
+module.exports = {
+    processBytesToWrite,
+    isMetricStale,
+    validateQuotas,
+};

lib/api/backbeat/listLifecycleCurrents.js

@@ -0,0 +1,117 @@
+const { errors } = require('arsenal');
+const constants = require('../../../constants');
+const services = require('../../services');
+const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
+const { pushMetric } = require('../../utapi/utilities');
+const monitoring = require('../../utilities/monitoringHandler');
+const { getLocationConstraintErrorMessage, processCurrents,
+    validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
+const { config } = require('../../Config');
+
+function handleResult(listParams, requestMaxKeys, authInfo,
+    bucketName, list, isBucketVersioned, log, callback) {
+    // eslint-disable-next-line no-param-reassign
+    listParams.maxKeys = requestMaxKeys;
+    const res = processCurrents(bucketName, listParams, isBucketVersioned, list);
+
+    pushMetric('listLifecycleCurrents', log, { authInfo, bucket: bucketName });
+    monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleCurrents');
+    return callback(null, res);
+}
+
+/**
+ * listLifecycleCurrents - Return list of current versions/masters in bucket
+ * @param {AuthInfo} authInfo            - Instance of AuthInfo class with
+ *                                          requester's info
+ * @param {array} locationConstraints    - array of location contraint
+ * @param {object} request               - http request object
+ * @param {function} log                 - Werelogs request logger
+ * @param {function} callback            - callback to respond to http request
+ *                                          with either error code or xml response body
+ * @return {undefined}
+ */
+function listLifecycleCurrents(authInfo, locationConstraints, request, log, callback) {
+    const params = request.query;
+    const bucketName = request.bucketName;
+
+    log.debug('processing request', { method: 'listLifecycleCurrents' });
+    const requestMaxKeys = params['max-keys'] ?
+        Number.parseInt(params['max-keys'], 10) : 1000;
+    if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listLifecycleCurrents');
+        return callback(errors.InvalidArgument);
+    }
+    const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
+
+    const minEntriesToBeScanned = 1;
+    const { isValid, maxScannedLifecycleListingEntries } =
+        validateMaxScannedEntries(params, config, minEntriesToBeScanned);
+    if (!isValid) {
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
+        return callback(errors.InvalidArgument);
+    }
+
+    const excludedDataStoreName = params['excluded-data-store-name'];
+    if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
+        const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
+        log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
+
+        return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
+    }
+
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: 'listLifecycleCurrents',
+        request,
+    };
+    const listParams = {
+        listingType: 'DelimiterCurrent',
+        maxKeys: actualMaxKeys,
+        prefix: params.prefix,
+        beforeDate: params['before-date'],
+        marker: params.marker,
+        excludedDataStoreName,
+        maxScannedLifecycleListingEntries,
+    };
+
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
+        if (err) {
+            log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'listLifecycleCurrents');
+            return callback(err, null);
+        }
+
+        const vcfg = bucket.getVersioningConfiguration();
+        const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
+
+        if (!requestMaxKeys) {
+            const emptyList = {
+                Contents: [],
+                IsTruncated: false,
+            };
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, emptyList, isBucketVersioned, log, callback);
+        }
+
+        return services.getLifecycleListing(bucketName, listParams, log,
+        (err, list) => {
+            if (err) {
+                log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'listLifecycleCurrents');
+                return callback(err, null);
+            }
+
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, list, isBucketVersioned, log, callback);
+        });
+    });
+}
+
+module.exports = {
+    listLifecycleCurrents,
+};

lib/api/backbeat/listLifecycleNonCurrents.js

@@ -0,0 +1,127 @@
+const { errors, versioning } = require('arsenal');
+const constants = require('../../../constants');
+const services = require('../../services');
+const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
+const { pushMetric } = require('../../utapi/utilities');
+const versionIdUtils = versioning.VersionID;
+const monitoring = require('../../utilities/monitoringHandler');
+const { getLocationConstraintErrorMessage, processNonCurrents,
+    validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
+const { config } = require('../../Config');
+
+function handleResult(listParams, requestMaxKeys, authInfo,
+    bucketName, list, log, callback) {
+    // eslint-disable-next-line no-param-reassign
+    listParams.maxKeys = requestMaxKeys;
+    const res = processNonCurrents(bucketName, listParams, list);
+
+    pushMetric('listLifecycleNonCurrents', log, { authInfo, bucket: bucketName });
+    monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleNonCurrents');
+    return callback(null, res);
+}
+
+/**
+ * listLifecycleNonCurrents - Return list of non-current versions in bucket
+ * @param {AuthInfo} authInfo           - Instance of AuthInfo class with
+ *                                         requester's info
+ * @param {array} locationConstraints    - array of location contraint
+ * @param {object} request              - http request object
+ * @param {function} log                - Werelogs request logger
+ * @param {function} callback           - callback to respond to http request
+ *                                          with either error code or xml response body
+ * @return {undefined}
+ */
+function listLifecycleNonCurrents(authInfo, locationConstraints, request, log, callback) {
+    const params = request.query;
+    const bucketName = request.bucketName;
+
+    log.debug('processing request', { method: 'listLifecycleNonCurrents' });
+    const requestMaxKeys = params['max-keys'] ?
+        Number.parseInt(params['max-keys'], 10) : 1000;
+    if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listLifecycleNonCurrents');
+        return callback(errors.InvalidArgument);
+    }
+    const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
+
+    // 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
+    // Two entries representing the master key and the following one representing the non-current version.
+    const minEntriesToBeScanned = 3;
+    const { isValid, maxScannedLifecycleListingEntries } =
+        validateMaxScannedEntries(params, config, minEntriesToBeScanned);
+    if (!isValid) {
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleNonCurrents');
+        return callback(errors.InvalidArgument);
+    }
+
+    const excludedDataStoreName = params['excluded-data-store-name'];
+    if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
+        const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
+        log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
+
+        return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
+    }
+
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: 'listLifecycleNonCurrents',
+        request,
+    };
+    const listParams = {
+        listingType: 'DelimiterNonCurrent',
+        maxKeys: actualMaxKeys,
+        prefix: params.prefix,
+        beforeDate: params['before-date'],
+        keyMarker: params['key-marker'],
+        excludedDataStoreName,
+        maxScannedLifecycleListingEntries,
+    };
+
+    listParams.versionIdMarker = params['version-id-marker'] ?
+        versionIdUtils.decode(params['version-id-marker']) : undefined;
+
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
+        if (err) {
+            log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'listLifecycleNonCurrents');
+            return callback(err, null);
+        }
+
+        const vcfg = bucket.getVersioningConfiguration();
+        const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
+        if (!isBucketVersioned) {
+            log.debug('bucket is not versioned');
+            return callback(errors.InvalidRequest.customizeDescription(
+                'bucket is not versioned'), null);
+        }
+
+        if (!requestMaxKeys) {
+            const emptyList = {
+                Contents: [],
+                IsTruncated: false,
+            };
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, emptyList, log, callback);
+        }
+
+        return services.getLifecycleListing(bucketName, listParams, log,
+        (err, list) => {
+            if (err) {
+                log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'listLifecycleNonCurrents');
+                return callback(err, null);
+            }
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, list, log, callback);
+        });
+    });
+}
+
+module.exports = {
+    listLifecycleNonCurrents,
+};

lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js

@@ -0,0 +1,112 @@
+const { errors } = require('arsenal');
+const constants = require('../../../constants');
+const services = require('../../services');
+const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
+const { pushMetric } = require('../../utapi/utilities');
+const monitoring = require('../../utilities/monitoringHandler');
+const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
+const { config } = require('../../Config');
+
+function handleResult(listParams, requestMaxKeys, authInfo,
+    bucketName, list, log, callback) {
+    // eslint-disable-next-line no-param-reassign
+    listParams.maxKeys = requestMaxKeys;
+    const res = processOrphans(bucketName, listParams, list);
+
+    pushMetric('listLifecycleOrphanDeleteMarkers', log, { authInfo, bucket: bucketName });
+    monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleOrphanDeleteMarkers');
+    return callback(null, res);
+}
+
+/**
+ * listLifecycleOrphanDeleteMarkers - Return list of expired object delete marker in bucket
+ * @param {AuthInfo} authInfo            - Instance of AuthInfo class with
+ *                                         requester's info
+ * @param {array} locationConstraints    - array of location contraint
+ * @param {object} request               - http request object
+ * @param {function} log                 - Werelogs request logger
+ * @param {function} callback            - callback to respond to http request
+ *                                          with either error code or xml response body
+ * @return {undefined}
+ */
+function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request, log, callback) {
+    const params = request.query;
+    const bucketName = request.bucketName;
+
+    log.debug('processing request', { method: 'listLifecycleOrphanDeleteMarkers' });
+    const requestMaxKeys = params['max-keys'] ?
+        Number.parseInt(params['max-keys'], 10) : 1000;
+    if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
+        monitoring.promMetrics(
+            'GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
+        return callback(errors.InvalidArgument);
+    }
+    const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
+
+    // 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
+    // Two entries representing the master key and the following one representing the non-current version.
+    const minEntriesToBeScanned = 3;
+    const { isValid, maxScannedLifecycleListingEntries } =
+        validateMaxScannedEntries(params, config, minEntriesToBeScanned);
+    if (!isValid) {
+        monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
+        return callback(errors.InvalidArgument);
+    }
+
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: 'listLifecycleOrphanDeleteMarkers',
+        request,
+    };
+    const listParams = {
+        listingType: 'DelimiterOrphanDeleteMarker',
+        maxKeys: actualMaxKeys,
+        prefix: params.prefix,
+        beforeDate: params['before-date'],
+        marker: params.marker,
+        maxScannedLifecycleListingEntries,
+    };
+
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
+        if (err) {
+            log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
+            monitoring.promMetrics(
+                'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
+            return callback(err, null);
+        }
+
+        const vcfg = bucket.getVersioningConfiguration();
+        const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
+        if (!isBucketVersioned) {
+            log.debug('bucket is not versioned or suspended');
+            return callback(errors.InvalidRequest.customizeDescription(
+                'bucket is not versioned'), null);
+        }
+
+        if (!requestMaxKeys) {
+            const emptyList = {
+                Contents: [],
+                IsTruncated: false,
+            };
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, emptyList, log, callback);
+        }
+
+        return services.getLifecycleListing(bucketName, listParams, log,
+        (err, list) => {
+            if (err) {
+                log.debug('error processing request', { error: err });
+                monitoring.promMetrics(
+                    'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
+                return callback(err, null);
+            }
+            return handleResult(listParams, requestMaxKeys, authInfo,
+                bucketName, list, log, callback);
+        });
+    });
+}
+
+module.exports = {
+    listLifecycleOrphanDeleteMarkers,
+};

lib/api/bucketDelete.js

@@ -2,7 +2,7 @@ const { errors } = require('arsenal');
 
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const deleteBucket = require('./apiUtils/bucket/bucketDeletion');
-const { metadataValidateBucket } = require('../metadata/metadataUtils');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const monitoring = require('../utilities/monitoringHandler');
 
@@ -31,9 +31,10 @@ function bucketDelete(authInfo, request, log, cb) {
         authInfo,
         bucketName,
         requestType: 'bucketDelete',
+        request,
     };
 
-    return metadataValidateBucket(metadataValParams, log,
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
         (err, bucketMD) => {
             const corsHeaders = collectCorsHeaders(request.headers.origin,
                 request.method, bucketMD);
@@ -47,7 +48,7 @@ function bucketDelete(authInfo, request, log, cb) {
             log.trace('passed checks',
                 { method: 'metadataValidateBucket' });
             return deleteBucket(authInfo, bucketMD, bucketName,
-                authInfo.getCanonicalID(), log, err => {
+                authInfo.getCanonicalID(), request, log, err => {
                     if (err) {
                         monitoring.promMetrics(
                             'DELETE', bucketName, err.code, 'deleteBucket');

lib/api/bucketDeleteCors.js

@@ -38,8 +38,8 @@ function bucketDeleteCors(authInfo, request, log, callback) {
         }
         log.trace('found bucket in metadata');
 
-        if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo,
+        if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
-        log)) {
+            authInfo, log, request, request.actionImplicitDenies)) {
             log.debug('access denied for user on bucket', {
                 requestType,
                 method: 'bucketDeleteCors',

lib/api/bucketDeleteEncryption.js

@@ -0,0 +1,62 @@
+const async = require('async');
+
+const metadata = require('../metadata/wrapper');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
+const { pushMetric } = require('../utapi/utilities');
+const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const { checkExpectedBucketOwner } = require('./apiUtils/authorization/bucketOwner');
+
+/**
+ * Bucket Delete Encryption - Delete bucket SSE configuration
+ * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
+ * @param {object} request - http request object
+ * @param {object} log - Werelogs logger
+ * @param {function} callback - callback to server
+ * @return {undefined}
+ */
+
+function bucketDeleteEncryption(authInfo, request, log, callback) {
+    const bucketName = request.bucketName;
+
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: request.apiMethods || 'bucketDeleteEncryption',
+        request,
+    };
+
+    return async.waterfall([
+        next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next),
+        (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)),
+        (bucket, next) => {
+            const sseConfig = bucket.getServerSideEncryption();
+            if (sseConfig === null) {
+                return next(null, bucket);
+            }
+
+            const updatedConfig = {
+                mandatory: false,
+                algorithm: sseConfig.algorithm,
+                cryptoScheme: sseConfig.cryptoScheme,
+                masterKeyId: sseConfig.masterKeyId,
+            };
+
+            bucket.setServerSideEncryption(updatedConfig);
+            return metadata.updateBucket(bucketName, bucket, log, err => next(err, bucket));
+        },
+    ],
+    (err, bucket) => {
+        const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket);
+        if (err) {
+            log.trace('error processing request', { error: err, method: 'bucketDeleteEncryption' });
+            return callback(err, corsHeaders);
+        }
+        pushMetric('deleteBucketEncryption', log, {
+            authInfo,
+            bucket: bucketName,
+        });
+        return callback(null, corsHeaders);
+    });
+}
+
+module.exports = bucketDeleteEncryption;

lib/api/bucketDeleteLifecycle.js

@@ -1,5 +1,5 @@
 const metadata = require('../metadata/wrapper');
-const { metadataValidateBucket } = require('../metadata/metadataUtils');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const monitoring = require('../utilities/monitoringHandler');
@@ -18,9 +18,10 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) {
     const metadataValParams = {
         authInfo,
         bucketName,
-        requestType: 'bucketDeleteLifecycle',
+        requestType: request.apiMethods || 'bucketDeleteLifecycle',
+        request,
     };
-    return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
         const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
         if (err) {
             log.debug('error processing request', {

lib/api/bucketDeletePolicy.js

@@ -1,5 +1,5 @@
 const metadata = require('../metadata/wrapper');
-const { metadataValidateBucket } = require('../metadata/metadataUtils');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 
 /**
@@ -16,9 +16,10 @@ function bucketDeletePolicy(authInfo, request, log, callback) {
     const metadataValParams = {
         authInfo,
         bucketName,
-        requestType: 'bucketDeletePolicy',
+        requestType: request.apiMethods || 'bucketDeletePolicy',
+        request,
     };
-    return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
         const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
         if (err) {
             log.debug('error processing request', {

lib/api/bucketDeleteQuota.js

@@ -0,0 +1,58 @@
+const { waterfall } = require('async');
+const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
+const metadata = require('../metadata/wrapper');
+const { pushMetric } = require('../utapi/utilities');
+const monitoring = require('../utilities/monitoringHandler');
+
+const requestType = 'bucketDeleteQuota';
+
+/**
+ * Bucket Update Quota - Update bucket quota
+ * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
+ * @param {object} request - http request object
+ * @param {object} log - Werelogs logger
+ * @param {function} callback - callback to server
+ * @return {undefined}
+ */
+function bucketDeleteQuota(authInfo, request, log, callback) {
+    log.debug('processing request', { method: 'bucketDeleteQuota' });
+
+    const { bucketName } = request;
+    const metadataValParams = {
+        authInfo,
+        bucketName,
+        requestType: request.apiMethods || requestType,
+        request,
+    };
+    return waterfall([
+        next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
+            (err, bucket) => next(err, bucket)),
+        (bucket, next) => {
+            bucket.setQuota(0);
+            metadata.updateBucket(bucket.getName(), bucket, log, err =>
+                next(err, bucket));
+        },
+    ], (err, bucket) => {
+        const corsHeaders = collectCorsHeaders(request.headers.origin,
+            request.method, bucket);
+        if (err) {
+            log.debug('error processing request', {
+                error: err,
+                method: 'bucketDeleteQuota'
+            });
+            monitoring.promMetrics('DELETE', bucketName, err.code,
+                'bucketDeleteQuota');
+            return callback(err, err.code, corsHeaders);
+        }
+        monitoring.promMetrics(
+            'DELETE', bucketName, '204', 'bucketDeleteQuota');
+        pushMetric('bucketDeleteQuota', log, {
+            authInfo,
+            bucket: bucketName,
+        });
+        return callback(null, 204, corsHeaders);
+    });
+}
+
+module.exports = bucketDeleteQuota;

lib/api/bucketDeleteReplication.js

@@ -1,5 +1,5 @@
 const metadata = require('../metadata/wrapper');
-const { metadataValidateBucket } = require('../metadata/metadataUtils');
+const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
 const { pushMetric } = require('../utapi/utilities');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const monitoring = require('../utilities/monitoringHandler');
@@ -18,9 +18,10 @@ function bucketDeleteReplication(authInfo, request, log, callback) {
     const metadataValParams = {
         authInfo,
         bucketName,
-        requestType: 'bucketDeleteReplication',
+        requestType: request.apiMethods || 'bucketDeleteReplication',
+        request,
     };
-    return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
+    return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
         const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
         if (err) {
             log.debug('error processing request', {