Compare commits
1 Commits
developmen
...
feature/AR
Author | SHA1 | Date |
---|---|---|
Francois Ferrand | 547a700611 |
|
@ -1,8 +1,5 @@
|
||||||
{
|
{
|
||||||
"extends": "scality",
|
"extends": "scality",
|
||||||
"plugins": [
|
|
||||||
"mocha"
|
|
||||||
],
|
|
||||||
"rules": {
|
"rules": {
|
||||||
"import/extensions": "off",
|
"import/extensions": "off",
|
||||||
"lines-around-directive": "off",
|
"lines-around-directive": "off",
|
||||||
|
@ -45,8 +42,7 @@
|
||||||
"no-restricted-properties": "off",
|
"no-restricted-properties": "off",
|
||||||
"new-parens": "off",
|
"new-parens": "off",
|
||||||
"no-multi-spaces": "off",
|
"no-multi-spaces": "off",
|
||||||
"quote-props": "off",
|
"quote-props": "off"
|
||||||
"mocha/no-exclusive-tests": "error",
|
|
||||||
},
|
},
|
||||||
"parserOptions": {
|
"parserOptions": {
|
||||||
"ecmaVersion": 2020
|
"ecmaVersion": 2020
|
||||||
|
|
|
@ -1,43 +0,0 @@
|
||||||
---
|
|
||||||
name: "Setup CI environment"
|
|
||||||
description: "Setup Cloudserver CI environment"
|
|
||||||
|
|
||||||
runs:
|
|
||||||
using: composite
|
|
||||||
steps:
|
|
||||||
- name: Setup etc/hosts
|
|
||||||
shell: bash
|
|
||||||
run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
|
|
||||||
- name: Setup Credentials
|
|
||||||
shell: bash
|
|
||||||
run: bash .github/scripts/credentials.bash
|
|
||||||
- name: Setup job artifacts directory
|
|
||||||
shell: bash
|
|
||||||
run: |-
|
|
||||||
set -exu;
|
|
||||||
mkdir -p /tmp/artifacts/${JOB_NAME}/;
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: '16'
|
|
||||||
cache: 'yarn'
|
|
||||||
- name: install dependencies
|
|
||||||
shell: bash
|
|
||||||
run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
|
|
||||||
- uses: actions/cache@v3
|
|
||||||
with:
|
|
||||||
path: ~/.cache/pip
|
|
||||||
key: ${{ runner.os }}-pip
|
|
||||||
- uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: 3.9
|
|
||||||
- name: Setup python2 test environment
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
sudo apt-get install -y libdigest-hmac-perl
|
|
||||||
pip install 's3cmd==2.3.0'
|
|
||||||
- name: fix sproxyd.conf permissions
|
|
||||||
shell: bash
|
|
||||||
run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
|
|
||||||
- name: ensure fuse kernel module is loaded (for sproxyd)
|
|
||||||
shell: bash
|
|
||||||
run: sudo modprobe fuse
|
|
|
@ -1,36 +0,0 @@
|
||||||
azurebackend_AZURE_STORAGE_ACCESS_KEY
|
|
||||||
azurebackend_AZURE_STORAGE_ACCOUNT_NAME
|
|
||||||
azurebackend_AZURE_STORAGE_ENDPOINT
|
|
||||||
azurebackend2_AZURE_STORAGE_ACCESS_KEY
|
|
||||||
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME
|
|
||||||
azurebackend2_AZURE_STORAGE_ENDPOINT
|
|
||||||
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY
|
|
||||||
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME
|
|
||||||
azurebackendmismatch_AZURE_STORAGE_ENDPOINT
|
|
||||||
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY
|
|
||||||
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME
|
|
||||||
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT
|
|
||||||
azuretest_AZURE_BLOB_ENDPOINT
|
|
||||||
b2backend_B2_ACCOUNT_ID
|
|
||||||
b2backend_B2_STORAGE_ACCESS_KEY
|
|
||||||
GOOGLE_SERVICE_EMAIL
|
|
||||||
GOOGLE_SERVICE_KEY
|
|
||||||
AWS_S3_BACKEND_ACCESS_KEY
|
|
||||||
AWS_S3_BACKEND_SECRET_KEY
|
|
||||||
AWS_S3_BACKEND_ACCESS_KEY_2
|
|
||||||
AWS_S3_BACKEND_SECRET_KEY_2
|
|
||||||
AWS_GCP_BACKEND_ACCESS_KEY
|
|
||||||
AWS_GCP_BACKEND_SECRET_KEY
|
|
||||||
AWS_GCP_BACKEND_ACCESS_KEY_2
|
|
||||||
AWS_GCP_BACKEND_SECRET_KEY_2
|
|
||||||
b2backend_B2_STORAGE_ENDPOINT
|
|
||||||
gcpbackend2_GCP_SERVICE_EMAIL
|
|
||||||
gcpbackend2_GCP_SERVICE_KEY
|
|
||||||
gcpbackend2_GCP_SERVICE_KEYFILE
|
|
||||||
gcpbackend_GCP_SERVICE_EMAIL
|
|
||||||
gcpbackend_GCP_SERVICE_KEY
|
|
||||||
gcpbackendmismatch_GCP_SERVICE_EMAIL
|
|
||||||
gcpbackendmismatch_GCP_SERVICE_KEY
|
|
||||||
gcpbackend_GCP_SERVICE_KEYFILE
|
|
||||||
gcpbackendmismatch_GCP_SERVICE_KEYFILE
|
|
||||||
gcpbackendnoproxy_GCP_SERVICE_KEYFILE
|
|
|
@ -1,92 +0,0 @@
|
||||||
services:
|
|
||||||
cloudserver:
|
|
||||||
image: ${CLOUDSERVER_IMAGE}
|
|
||||||
command: sh -c "yarn start > /artifacts/s3.log"
|
|
||||||
network_mode: "host"
|
|
||||||
volumes:
|
|
||||||
- /tmp/ssl:/ssl
|
|
||||||
- /tmp/ssl-kmip:/ssl-kmip
|
|
||||||
- ${HOME}/.aws/credentials:/root/.aws/credentials
|
|
||||||
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
|
||||||
environment:
|
|
||||||
- CI=true
|
|
||||||
- ENABLE_LOCAL_CACHE=true
|
|
||||||
- REDIS_HOST=0.0.0.0
|
|
||||||
- REDIS_PORT=6379
|
|
||||||
- REPORT_TOKEN=report-token-1
|
|
||||||
- REMOTE_MANAGEMENT_DISABLE=1
|
|
||||||
- HEALTHCHECKS_ALLOWFROM=0.0.0.0/0
|
|
||||||
- DATA_HOST=0.0.0.0
|
|
||||||
- METADATA_HOST=0.0.0.0
|
|
||||||
- S3BACKEND
|
|
||||||
- S3DATA
|
|
||||||
- S3METADATA
|
|
||||||
- MPU_TESTING
|
|
||||||
- S3VAULT
|
|
||||||
- S3_LOCATION_FILE
|
|
||||||
- ENABLE_UTAPI_V2
|
|
||||||
- BUCKET_DENY_FILTER
|
|
||||||
- S3KMS
|
|
||||||
- S3KMIP_PORT
|
|
||||||
- S3KMIP_HOSTS
|
|
||||||
- S3KMIP-COMPOUND_CREATE
|
|
||||||
- S3KMIP_BUCKET_ATTRIBUTE_NAME
|
|
||||||
- S3KMIP_PIPELINE_DEPTH
|
|
||||||
- S3KMIP_KEY
|
|
||||||
- S3KMIP_CERT
|
|
||||||
- S3KMIP_CA
|
|
||||||
- MONGODB_HOSTS=0.0.0.0:27018
|
|
||||||
- MONGODB_RS=rs0
|
|
||||||
- DEFAULT_BUCKET_KEY_FORMAT
|
|
||||||
- METADATA_MAX_CACHED_BUCKETS
|
|
||||||
- ENABLE_NULL_VERSION_COMPAT_MODE
|
|
||||||
- SCUBA_HOST
|
|
||||||
- SCUBA_PORT
|
|
||||||
- SCUBA_HEALTHCHECK_FREQUENCY
|
|
||||||
- S3QUOTA
|
|
||||||
- QUOTA_ENABLE_INFLIGHTS
|
|
||||||
env_file:
|
|
||||||
- creds.env
|
|
||||||
depends_on:
|
|
||||||
- redis
|
|
||||||
extra_hosts:
|
|
||||||
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
|
|
||||||
- "pykmip.local:127.0.0.1"
|
|
||||||
redis:
|
|
||||||
image: redis:alpine
|
|
||||||
network_mode: "host"
|
|
||||||
squid:
|
|
||||||
network_mode: "host"
|
|
||||||
profiles: ['ci-proxy']
|
|
||||||
image: scality/ci-squid
|
|
||||||
command: >-
|
|
||||||
sh -c 'mkdir -p /ssl &&
|
|
||||||
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
|
|
||||||
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
|
|
||||||
-keyout /ssl/myca.pem -out /ssl/myca.pem &&
|
|
||||||
cp /ssl/myca.pem /ssl/CA.pem &&
|
|
||||||
squid -f /etc/squid/squid.conf -N -z &&
|
|
||||||
squid -f /etc/squid/squid.conf -NYCd 1'
|
|
||||||
volumes:
|
|
||||||
- /tmp/ssl:/ssl
|
|
||||||
pykmip:
|
|
||||||
network_mode: "host"
|
|
||||||
profiles: ['pykmip']
|
|
||||||
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
|
|
||||||
volumes:
|
|
||||||
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
|
||||||
mongo:
|
|
||||||
network_mode: "host"
|
|
||||||
profiles: ['mongo', 'ceph']
|
|
||||||
image: ${MONGODB_IMAGE}
|
|
||||||
ceph:
|
|
||||||
network_mode: "host"
|
|
||||||
profiles: ['ceph']
|
|
||||||
image: ghcr.io/scality/cloudserver/ci-ceph
|
|
||||||
sproxyd:
|
|
||||||
network_mode: "host"
|
|
||||||
profiles: ['sproxyd']
|
|
||||||
image: sproxyd-standalone
|
|
||||||
build: ./sproxyd
|
|
||||||
user: 0:0
|
|
||||||
privileged: yes
|
|
|
@ -1,28 +0,0 @@
|
||||||
FROM mongo:5.0.21
|
|
||||||
|
|
||||||
ENV USER=scality \
|
|
||||||
HOME_DIR=/home/scality \
|
|
||||||
CONF_DIR=/conf \
|
|
||||||
DATA_DIR=/data
|
|
||||||
|
|
||||||
# Set up directories and permissions
|
|
||||||
RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
|
|
||||||
mkdir /logs; \
|
|
||||||
adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
|
|
||||||
|
|
||||||
# Set up environment variables and directories for scality user
|
|
||||||
RUN mkdir ${CONF_DIR} && \
|
|
||||||
chown -R ${USER} ${CONF_DIR} && \
|
|
||||||
chown -R ${USER} ${DATA_DIR}
|
|
||||||
|
|
||||||
# copy the mongo config file
|
|
||||||
COPY /conf/mongod.conf /conf/mongod.conf
|
|
||||||
COPY /conf/mongo-run.sh /conf/mongo-run.sh
|
|
||||||
COPY /conf/initReplicaSet /conf/initReplicaSet.js
|
|
||||||
|
|
||||||
EXPOSE 27017/tcp
|
|
||||||
EXPOSE 27018
|
|
||||||
|
|
||||||
# Set up CMD
|
|
||||||
ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
|
|
||||||
CMD ["bash", "/conf/mongo-run.sh"]
|
|
|
@ -1,4 +0,0 @@
|
||||||
rs.initiate({
|
|
||||||
_id: "rs0",
|
|
||||||
members: [{ _id: 0, host: "127.0.0.1:27018" }]
|
|
||||||
});
|
|
|
@ -1,10 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -exo pipefail
|
|
||||||
|
|
||||||
init_RS() {
|
|
||||||
sleep 5
|
|
||||||
mongo --port 27018 /conf/initReplicaSet.js
|
|
||||||
}
|
|
||||||
init_RS &
|
|
||||||
|
|
||||||
mongod --bind_ip_all --config=/conf/mongod.conf
|
|
|
@ -1,15 +0,0 @@
|
||||||
storage:
|
|
||||||
journal:
|
|
||||||
enabled: true
|
|
||||||
engine: wiredTiger
|
|
||||||
dbPath: "/data/db"
|
|
||||||
processManagement:
|
|
||||||
fork: false
|
|
||||||
net:
|
|
||||||
port: 27018
|
|
||||||
bindIp: 0.0.0.0
|
|
||||||
replication:
|
|
||||||
replSetName: "rs0"
|
|
||||||
enableMajorityReadConcern: true
|
|
||||||
security:
|
|
||||||
authorization: disabled
|
|
|
@ -1,3 +0,0 @@
|
||||||
FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
|
|
||||||
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
|
|
||||||
RUN chown root:root /conf/sproxyd0.conf
|
|
|
@ -1,26 +0,0 @@
|
||||||
fastcgi_param QUERY_STRING $query_string;
|
|
||||||
fastcgi_param REQUEST_METHOD $request_method;
|
|
||||||
fastcgi_param CONTENT_TYPE $content_type;
|
|
||||||
fastcgi_param CONTENT_LENGTH $content_length;
|
|
||||||
|
|
||||||
#fastcgi_param SCRIPT_NAME $fastcgi_script_name;
|
|
||||||
fastcgi_param SCRIPT_NAME /var/www;
|
|
||||||
fastcgi_param PATH_INFO $document_uri;
|
|
||||||
|
|
||||||
fastcgi_param REQUEST_URI $request_uri;
|
|
||||||
fastcgi_param DOCUMENT_URI $document_uri;
|
|
||||||
fastcgi_param DOCUMENT_ROOT $document_root;
|
|
||||||
fastcgi_param SERVER_PROTOCOL $server_protocol;
|
|
||||||
fastcgi_param HTTPS $https if_not_empty;
|
|
||||||
|
|
||||||
fastcgi_param GATEWAY_INTERFACE CGI/1.1;
|
|
||||||
fastcgi_param SERVER_SOFTWARE nginx/$nginx_version;
|
|
||||||
|
|
||||||
fastcgi_param REMOTE_ADDR $remote_addr;
|
|
||||||
fastcgi_param REMOTE_PORT $remote_port;
|
|
||||||
fastcgi_param SERVER_ADDR $server_addr;
|
|
||||||
fastcgi_param SERVER_PORT $server_port;
|
|
||||||
fastcgi_param SERVER_NAME $server_name;
|
|
||||||
|
|
||||||
# PHP only, required if PHP was built with --enable-force-cgi-redirect
|
|
||||||
fastcgi_param REDIRECT_STATUS 200;
|
|
|
@ -1,88 +0,0 @@
|
||||||
worker_processes 1;
|
|
||||||
error_log /logs/error.log;
|
|
||||||
user root root;
|
|
||||||
events {
|
|
||||||
worker_connections 1000;
|
|
||||||
reuse_port on;
|
|
||||||
multi_accept on;
|
|
||||||
}
|
|
||||||
worker_rlimit_nofile 20000;
|
|
||||||
http {
|
|
||||||
root /var/www/;
|
|
||||||
upstream sproxyds {
|
|
||||||
least_conn;
|
|
||||||
keepalive 40;
|
|
||||||
server 127.0.0.1:20000;
|
|
||||||
}
|
|
||||||
server {
|
|
||||||
client_max_body_size 0;
|
|
||||||
client_body_timeout 150;
|
|
||||||
client_header_timeout 150;
|
|
||||||
postpone_output 0;
|
|
||||||
client_body_postpone_size 0;
|
|
||||||
keepalive_requests 1100;
|
|
||||||
keepalive_timeout 300s;
|
|
||||||
server_tokens off;
|
|
||||||
default_type application/octet-stream;
|
|
||||||
gzip off;
|
|
||||||
tcp_nodelay on;
|
|
||||||
tcp_nopush on;
|
|
||||||
sendfile on;
|
|
||||||
listen 81;
|
|
||||||
server_name localhost;
|
|
||||||
rewrite ^/arc/(.*)$ /dc1/$1 permanent;
|
|
||||||
location ~* ^/proxy/(.*)$ {
|
|
||||||
rewrite ^/proxy/(.*)$ /$1 last;
|
|
||||||
}
|
|
||||||
allow 127.0.0.1;
|
|
||||||
|
|
||||||
deny all;
|
|
||||||
set $usermd '-';
|
|
||||||
set $sentusermd '-';
|
|
||||||
set $elapsed_ms '-';
|
|
||||||
set $now '-';
|
|
||||||
log_by_lua '
|
|
||||||
if not(ngx.var.http_x_scal_usermd == nil) and string.len(ngx.var.http_x_scal_usermd) > 2 then
|
|
||||||
ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
|
|
||||||
end
|
|
||||||
if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
|
|
||||||
ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
|
|
||||||
end
|
|
||||||
local elapsed_ms = tonumber(ngx.var.request_time)
|
|
||||||
if not ( elapsed_ms == nil) then
|
|
||||||
elapsed_ms = elapsed_ms * 1000
|
|
||||||
ngx.var.elapsed_ms = tostring(elapsed_ms)
|
|
||||||
end
|
|
||||||
local time = tonumber(ngx.var.msec) * 1000
|
|
||||||
ngx.var.now = time
|
|
||||||
';
|
|
||||||
log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
|
|
||||||
'"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
|
|
||||||
'"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
|
|
||||||
'"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
|
|
||||||
'"contentType":"$content_type","s3Address":"$remote_addr",'
|
|
||||||
'"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
|
|
||||||
'"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
|
|
||||||
'"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
|
|
||||||
'"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
|
|
||||||
'"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
|
|
||||||
'"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
|
|
||||||
access_log /dev/stdout irm;
|
|
||||||
error_log /dev/stdout error;
|
|
||||||
location / {
|
|
||||||
proxy_request_buffering off;
|
|
||||||
fastcgi_request_buffering off;
|
|
||||||
fastcgi_no_cache 1;
|
|
||||||
fastcgi_cache_bypass 1;
|
|
||||||
fastcgi_buffering off;
|
|
||||||
fastcgi_ignore_client_abort on;
|
|
||||||
fastcgi_keep_conn on;
|
|
||||||
include fastcgi_params;
|
|
||||||
fastcgi_pass sproxyds;
|
|
||||||
fastcgi_next_upstream error timeout;
|
|
||||||
fastcgi_send_timeout 285s;
|
|
||||||
fastcgi_read_timeout 285s;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,12 +0,0 @@
|
||||||
{
|
|
||||||
"general": {
|
|
||||||
"ring": "DATA",
|
|
||||||
"port": 20000,
|
|
||||||
"syslog_facility": "local0"
|
|
||||||
},
|
|
||||||
"ring_driver:0": {
|
|
||||||
"alias": "dc1",
|
|
||||||
"type": "local",
|
|
||||||
"queue_path": "/tmp/ring-objs"
|
|
||||||
},
|
|
||||||
}
|
|
|
@ -1,43 +0,0 @@
|
||||||
[supervisord]
|
|
||||||
nodaemon = true
|
|
||||||
loglevel = info
|
|
||||||
logfile = %(ENV_LOG_DIR)s/supervisord.log
|
|
||||||
pidfile = %(ENV_SUP_RUN_DIR)s/supervisord.pid
|
|
||||||
logfile_maxbytes = 20MB
|
|
||||||
logfile_backups = 2
|
|
||||||
|
|
||||||
[unix_http_server]
|
|
||||||
file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
|
|
||||||
|
|
||||||
[rpcinterface:supervisor]
|
|
||||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
|
||||||
|
|
||||||
[supervisorctl]
|
|
||||||
serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
|
|
||||||
|
|
||||||
[program:nginx]
|
|
||||||
directory=%(ENV_SUP_RUN_DIR)s
|
|
||||||
command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
|
|
||||||
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
|
|
||||||
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
|
|
||||||
stdout_logfile_maxbytes=100MB
|
|
||||||
stdout_logfile_backups=7
|
|
||||||
stderr_logfile_maxbytes=100MB
|
|
||||||
stderr_logfile_backups=7
|
|
||||||
autorestart=true
|
|
||||||
autostart=true
|
|
||||||
user=root
|
|
||||||
|
|
||||||
[program:sproxyd]
|
|
||||||
directory=%(ENV_SUP_RUN_DIR)s
|
|
||||||
process_name=%(program_name)s-%(process_num)s
|
|
||||||
numprocs=1
|
|
||||||
numprocs_start=0
|
|
||||||
command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
|
|
||||||
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
|
|
||||||
stdout_logfile_maxbytes=100MB
|
|
||||||
stdout_logfile_backups=7
|
|
||||||
redirect_stderr=true
|
|
||||||
autorestart=true
|
|
||||||
autostart=true
|
|
||||||
user=root
|
|
|
@ -1,10 +1,7 @@
|
||||||
name: Test alerts
|
name: Test alerts
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push
|
||||||
branches-ignore:
|
|
||||||
- 'development/**'
|
|
||||||
- 'q/*/**'
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
run-alert-tests:
|
run-alert-tests:
|
||||||
|
@ -20,16 +17,13 @@ jobs:
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
- name: Render and test ${{ matrix.tests.name }}
|
- name: Render and test ${{ matrix.tests.name }}
|
||||||
uses: scality/action-prom-render-test@1.0.3
|
uses: scality/action-prom-render-test@1.0.1
|
||||||
with:
|
with:
|
||||||
alert_file_path: monitoring/alerts.yaml
|
alert_file_path: monitoring/alerts.yaml
|
||||||
test_file_path: ${{ matrix.tests.file }}
|
test_file_path: ${{ matrix.tests.file }}
|
||||||
alert_inputs: |
|
alert_inputs: >-
|
||||||
namespace=zenko
|
namespace=zenko,service=artesca-data-connector-s3api-metrics,replicas=3
|
||||||
service=artesca-data-connector-s3api-metrics
|
|
||||||
reportJob=artesca-data-ops-report-handler
|
|
||||||
replicas=3
|
|
||||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
|
@ -1,25 +0,0 @@
|
||||||
---
|
|
||||||
name: codeQL
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [w/**, q/*]
|
|
||||||
pull_request:
|
|
||||||
branches: [development/*, stabilization/*, hotfix/*]
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
analyze:
|
|
||||||
name: Static analysis with CodeQL
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Initialize CodeQL
|
|
||||||
uses: github/codeql-action/init@v3
|
|
||||||
with:
|
|
||||||
languages: javascript, python, ruby
|
|
||||||
|
|
||||||
- name: Build and analyze
|
|
||||||
uses: github/codeql-action/analyze@v3
|
|
|
@ -1,16 +0,0 @@
|
||||||
---
|
|
||||||
name: dependency review
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
branches: [development/*, stabilization/*, hotfix/*]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
dependency-review:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: 'Checkout Repository'
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: 'Dependency Review'
|
|
||||||
uses: actions/dependency-review-action@v4
|
|
|
@ -1,80 +0,0 @@
|
||||||
---
|
|
||||||
name: release
|
|
||||||
run-name: release ${{ inputs.tag }}
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
tag:
|
|
||||||
description: 'Tag to be released'
|
|
||||||
required: true
|
|
||||||
|
|
||||||
env:
|
|
||||||
PROJECT_NAME: ${{ github.event.repository.name }}
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-federation-image:
|
|
||||||
runs-on: ubuntu-20.04
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
- name: Login to GitHub Registry
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.repository_owner }}
|
|
||||||
password: ${{ github.token }}
|
|
||||||
- name: Build and push image for federation
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
context: .
|
|
||||||
file: images/svc-base/Dockerfile
|
|
||||||
tags: |
|
|
||||||
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
|
|
||||||
cache-from: type=gha,scope=federation
|
|
||||||
cache-to: type=gha,mode=max,scope=federation
|
|
||||||
|
|
||||||
release:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Docker Buildk
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
|
|
||||||
- name: Login to Registry
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.repository_owner }}
|
|
||||||
password: ${{ github.token }}
|
|
||||||
|
|
||||||
- name: Push dashboards into the production namespace
|
|
||||||
run: |
|
|
||||||
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
|
|
||||||
dashboard.json:application/grafana-dashboard+json \
|
|
||||||
alerts.yaml:application/prometheus-alerts+yaml
|
|
||||||
working-directory: monitoring
|
|
||||||
|
|
||||||
- name: Build and push
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
push: true
|
|
||||||
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
|
|
||||||
cache-from: type=gha
|
|
||||||
cache-to: type=gha,mode=max
|
|
||||||
|
|
||||||
- name: Create Release
|
|
||||||
uses: softprops/action-gh-release@v2
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ github.token }}
|
|
||||||
with:
|
|
||||||
name: Release ${{ github.event.inputs.tag }}
|
|
||||||
tag_name: ${{ github.event.inputs.tag }}
|
|
||||||
generate_release_notes: true
|
|
||||||
target_commitish: ${{ github.sha }}
|
|
|
@ -1,533 +0,0 @@
|
||||||
---
|
|
||||||
name: tests
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
push:
|
|
||||||
branches-ignore:
|
|
||||||
- 'development/**'
|
|
||||||
- 'q/*/**'
|
|
||||||
|
|
||||||
env:
|
|
||||||
# Secrets
|
|
||||||
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
|
|
||||||
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
|
|
||||||
azurebackend_AZURE_STORAGE_ENDPOINT: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ENDPOINT }}
|
|
||||||
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
|
|
||||||
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
|
|
||||||
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
|
|
||||||
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
|
|
||||||
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
|
|
||||||
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ENDPOINT }}
|
|
||||||
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
|
|
||||||
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
|
|
||||||
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
|
|
||||||
${{ secrets.AZURE_STORAGE_ENDPOINT }}
|
|
||||||
azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
|
|
||||||
b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
|
|
||||||
b2backend_B2_STORAGE_ACCESS_KEY: >-
|
|
||||||
${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
|
|
||||||
GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
|
|
||||||
GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
|
|
||||||
AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
|
|
||||||
AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
|
|
||||||
AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
|
|
||||||
AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
|
|
||||||
AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
|
|
||||||
AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
|
|
||||||
AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
|
|
||||||
AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
|
|
||||||
b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
|
|
||||||
gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
|
|
||||||
gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
|
|
||||||
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
|
||||||
gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
|
|
||||||
gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
|
|
||||||
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
|
|
||||||
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
|
|
||||||
gcpbackendmismatch_GCP_SERVICE_KEY: >-
|
|
||||||
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
|
|
||||||
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
|
||||||
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
|
||||||
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
|
||||||
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
|
||||||
# Configs
|
|
||||||
ENABLE_LOCAL_CACHE: "true"
|
|
||||||
REPORT_TOKEN: "report-token-1"
|
|
||||||
REMOTE_MANAGEMENT_DISABLE: "1"
|
|
||||||
# https://github.com/git-lfs/git-lfs/issues/5749
|
|
||||||
GIT_CLONE_PROTECTION_ACTIVE: 'false'
|
|
||||||
jobs:
|
|
||||||
linting-coverage:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: '16'
|
|
||||||
cache: yarn
|
|
||||||
- name: install dependencies
|
|
||||||
run: yarn install --frozen-lockfile --network-concurrency 1
|
|
||||||
- uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: '3.9'
|
|
||||||
- uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: ~/.cache/pip
|
|
||||||
key: ${{ runner.os }}-pip
|
|
||||||
- name: Install python deps
|
|
||||||
run: pip install flake8
|
|
||||||
- name: Lint Javascript
|
|
||||||
run: yarn run --silent lint -- --max-warnings 0
|
|
||||||
- name: Lint Markdown
|
|
||||||
run: yarn run --silent lint_md
|
|
||||||
- name: Lint python
|
|
||||||
run: flake8 $(git ls-files "*.py")
|
|
||||||
- name: Lint Yaml
|
|
||||||
run: yamllint -c yamllint.yml $(git ls-files "*.yml")
|
|
||||||
- name: Unit Coverage
|
|
||||||
run: |
|
|
||||||
set -ex
|
|
||||||
mkdir -p $CIRCLE_TEST_REPORTS/unit
|
|
||||||
yarn test
|
|
||||||
yarn run test_legacy_location
|
|
||||||
env:
|
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
|
||||||
CIRCLE_TEST_REPORTS: /tmp
|
|
||||||
CIRCLE_ARTIFACTS: /tmp
|
|
||||||
CI_REPORTS: /tmp
|
|
||||||
- name: Unit Coverage logs
|
|
||||||
run: find /tmp/unit -exec cat {} \;
|
|
||||||
- name: preparing junit files for upload
|
|
||||||
run: |
|
|
||||||
mkdir -p artifacts/junit
|
|
||||||
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
|
|
||||||
if: always()
|
|
||||||
- name: Upload files to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
build:
|
|
||||||
runs-on: ubuntu-20.04
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
packages: write
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
- name: Login to GitHub Registry
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.repository_owner }}
|
|
||||||
password: ${{ github.token }}
|
|
||||||
- name: Build and push cloudserver image
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
context: .
|
|
||||||
provenance: false
|
|
||||||
tags: |
|
|
||||||
ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
labels: |
|
|
||||||
git.repository=${{ github.repository }}
|
|
||||||
git.commit-sha=${{ github.sha }}
|
|
||||||
cache-from: type=gha,scope=cloudserver
|
|
||||||
cache-to: type=gha,mode=max,scope=cloudserver
|
|
||||||
- name: Build and push pykmip image
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
context: .github/pykmip
|
|
||||||
tags: |
|
|
||||||
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
|
||||||
labels: |
|
|
||||||
git.repository=${{ github.repository }}
|
|
||||||
git.commit-sha=${{ github.sha }}
|
|
||||||
cache-from: type=gha,scope=pykmip
|
|
||||||
cache-to: type=gha,mode=max,scope=pykmip
|
|
||||||
- name: Build and push MongoDB
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
context: .github/docker/mongodb
|
|
||||||
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
cache-from: type=gha,scope=mongodb
|
|
||||||
cache-to: type=gha,mode=max,scope=mongodb
|
|
||||||
|
|
||||||
multiple-backend:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
env:
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
S3BACKEND: mem
|
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
|
||||||
S3DATA: multiple
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Login to Registry
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.repository_owner }}
|
|
||||||
password: ${{ github.token }}
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose --profile sproxyd up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run multiple backend test
|
|
||||||
run: |-
|
|
||||||
set -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
bash wait_for_local_port.bash 81 40
|
|
||||||
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
|
||||||
env:
|
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
mongo-v0-ft-tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
env:
|
|
||||||
S3BACKEND: mem
|
|
||||||
MPU_TESTING: "yes"
|
|
||||||
S3METADATA: mongodb
|
|
||||||
S3KMS: file
|
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
|
||||||
DEFAULT_BUCKET_KEY_FORMAT: v0
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose --profile mongo up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run functional tests
|
|
||||||
run: |-
|
|
||||||
set -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
|
||||||
env:
|
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
mongo-v1-ft-tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
env:
|
|
||||||
S3BACKEND: mem
|
|
||||||
MPU_TESTING: "yes"
|
|
||||||
S3METADATA: mongodb
|
|
||||||
S3KMS: file
|
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
|
||||||
DEFAULT_BUCKET_KEY_FORMAT: v1
|
|
||||||
METADATA_MAX_CACHED_BUCKETS: 1
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose --profile mongo up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run functional tests
|
|
||||||
run: |-
|
|
||||||
set -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
|
||||||
yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
|
|
||||||
env:
|
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
file-ft-tests:
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- job-name: file-ft-tests
|
|
||||||
name: ${{ matrix.job-name }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
env:
|
|
||||||
S3BACKEND: file
|
|
||||||
S3VAULT: mem
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
MPU_TESTING: "yes"
|
|
||||||
JOB_NAME: ${{ matrix.job-name }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Setup matrix job artifacts directory
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -exu
|
|
||||||
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run file ft tests
|
|
||||||
run: |-
|
|
||||||
set -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
utapi-v2-tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
env:
|
|
||||||
ENABLE_UTAPI_V2: t
|
|
||||||
S3BACKEND: mem
|
|
||||||
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run file utapi v2 tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
quota-tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
inflights:
|
|
||||||
- name: "With Inflights"
|
|
||||||
value: "true"
|
|
||||||
- name: "Without Inflights"
|
|
||||||
value: "false"
|
|
||||||
env:
|
|
||||||
S3METADATA: mongodb
|
|
||||||
S3BACKEND: mem
|
|
||||||
S3QUOTA: scuba
|
|
||||||
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
|
|
||||||
SCUBA_HOST: localhost
|
|
||||||
SCUBA_PORT: 8100
|
|
||||||
SCUBA_HEALTHCHECK_FREQUENCY: 100
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose --profile mongo up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run quota tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
kmip-ft-tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
env:
|
|
||||||
S3BACKEND: file
|
|
||||||
S3VAULT: mem
|
|
||||||
MPU_TESTING: "yes"
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Copy KMIP certs
|
|
||||||
run: cp -r ./certs /tmp/ssl-kmip
|
|
||||||
working-directory: .github/pykmip
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose --profile pykmip up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run file KMIP tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
bash wait_for_local_port.bash 5696 40
|
|
||||||
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
ceph-backend-test:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
env:
|
|
||||||
S3BACKEND: mem
|
|
||||||
S3DATA: multiple
|
|
||||||
S3KMS: file
|
|
||||||
CI_CEPH: 'true'
|
|
||||||
MPU_TESTING: "yes"
|
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Login to GitHub Registry
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.repository_owner }}
|
|
||||||
password: ${{ github.token }}
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- uses: ruby/setup-ruby@v1
|
|
||||||
with:
|
|
||||||
ruby-version: '2.5.9'
|
|
||||||
- name: Install Ruby dependencies
|
|
||||||
run: |
|
|
||||||
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
|
|
||||||
- name: Install Java dependencies
|
|
||||||
run: |
|
|
||||||
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose --profile ceph up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
env:
|
|
||||||
S3METADATA: mongodb
|
|
||||||
- name: Run Ceph multiple backend tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
bash .github/ceph/wait_for_ceph.sh
|
|
||||||
bash wait_for_local_port.bash 27018 40
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
|
|
||||||
env:
|
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
|
||||||
S3METADATA: mem
|
|
||||||
- name: Run Java tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
|
|
||||||
working-directory: tests/functional/jaws
|
|
||||||
- name: Run Ruby tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
|
|
||||||
working-directory: tests/functional/fog
|
|
||||||
- name: Run Javascript AWS SDK tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
|
|
||||||
yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
|
|
||||||
env:
|
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
|
|
||||||
S3BACKEND: file
|
|
||||||
S3VAULT: mem
|
|
||||||
S3METADATA: mongodb
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
81
Dockerfile
81
Dockerfile
|
@ -1,38 +1,5 @@
|
||||||
ARG NODE_VERSION=16.20-bullseye-slim
|
FROM node:16.13.2-slim
|
||||||
|
MAINTAINER Giorgio Regni <gr@scality.com>
|
||||||
FROM node:${NODE_VERSION} as builder
|
|
||||||
|
|
||||||
WORKDIR /usr/src/app
|
|
||||||
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends \
|
|
||||||
build-essential \
|
|
||||||
ca-certificates \
|
|
||||||
curl \
|
|
||||||
git \
|
|
||||||
gnupg2 \
|
|
||||||
jq \
|
|
||||||
python3 \
|
|
||||||
ssh \
|
|
||||||
wget \
|
|
||||||
libffi-dev \
|
|
||||||
zlib1g-dev \
|
|
||||||
&& apt-get clean \
|
|
||||||
&& mkdir -p /root/ssh \
|
|
||||||
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
|
|
||||||
|
|
||||||
ENV PYTHON=python3
|
|
||||||
COPY package.json yarn.lock /usr/src/app/
|
|
||||||
RUN npm install typescript -g
|
|
||||||
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
FROM node:${NODE_VERSION}
|
|
||||||
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install -y --no-install-recommends \
|
|
||||||
jq \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
ENV NO_PROXY localhost,127.0.0.1
|
ENV NO_PROXY localhost,127.0.0.1
|
||||||
ENV no_proxy localhost,127.0.0.1
|
ENV no_proxy localhost,127.0.0.1
|
||||||
|
@ -40,21 +7,47 @@ ENV no_proxy localhost,127.0.0.1
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
EXPOSE 8002
|
EXPOSE 8002
|
||||||
|
|
||||||
RUN apt-get update && \
|
COPY ./package.json /usr/src/app/
|
||||||
apt-get install -y --no-install-recommends \
|
COPY ./yarn.lock /usr/src/app/
|
||||||
jq \
|
|
||||||
tini \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /usr/src/app
|
WORKDIR /usr/src/app
|
||||||
|
|
||||||
# Keep the .git directory in order to properly report version
|
RUN apt-get update \
|
||||||
COPY . /usr/src/app
|
&& apt-get install -y \
|
||||||
COPY --from=builder /usr/src/app/node_modules ./node_modules/
|
curl \
|
||||||
|
gnupg2
|
||||||
|
|
||||||
|
RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
|
||||||
|
&& echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
ca-certificates \
|
||||||
|
git \
|
||||||
|
jq \
|
||||||
|
python3 \
|
||||||
|
ssh \
|
||||||
|
yarn \
|
||||||
|
wget \
|
||||||
|
libffi-dev \
|
||||||
|
zlib1g-dev \
|
||||||
|
&& mkdir -p /root/ssh \
|
||||||
|
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
|
||||||
|
|
||||||
|
ENV PYTHON=python3
|
||||||
|
RUN yarn cache clean \
|
||||||
|
&& yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
|
||||||
|
&& apt-get autoremove --purge -y python git build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& yarn cache clean \
|
||||||
|
&& rm -rf ~/.node-gyp \
|
||||||
|
&& rm -rf /tmp/yarn-*
|
||||||
|
|
||||||
|
COPY . /usr/src/app
|
||||||
|
|
||||||
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
|
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
|
||||||
|
|
||||||
ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
|
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
|
||||||
|
|
||||||
CMD [ "yarn", "start" ]
|
CMD [ "yarn", "start" ]
|
||||||
|
|
|
@ -7,7 +7,7 @@ COPY . /usr/src/app
|
||||||
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y jq python git build-essential --no-install-recommends \
|
&& apt-get install -y jq python git build-essential --no-install-recommends \
|
||||||
&& yarn install --production \
|
&& yarn install --production --network-concurrency 1 \
|
||||||
&& apt-get autoremove --purge -y python git build-essential \
|
&& apt-get autoremove --purge -y python git build-essential \
|
||||||
&& rm -rf /var/lib/apt/lists/* \
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
&& yarn cache clean \
|
&& yarn cache clean \
|
||||||
|
|
175
README.md
175
README.md
|
@ -1,7 +1,10 @@
|
||||||
# Zenko CloudServer with Vitastor Backend
|
# Zenko CloudServer
|
||||||
|
|
||||||
![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
|
![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
|
||||||
|
|
||||||
|
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/zenko/cloudserver)
|
||||||
|
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
|
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
|
||||||
|
@ -11,71 +14,137 @@ Scality’s Open Source Multi-Cloud Data Controller.
|
||||||
CloudServer provides a single AWS S3 API interface to access multiple
|
CloudServer provides a single AWS S3 API interface to access multiple
|
||||||
backend data storage both on-premise or public in the cloud.
|
backend data storage both on-premise or public in the cloud.
|
||||||
|
|
||||||
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
|
CloudServer is useful for Developers, either to run as part of a
|
||||||
backend support.
|
continous integration test environment to emulate the AWS S3 service locally
|
||||||
|
or as an abstraction layer to develop object storage enabled
|
||||||
|
application on the go.
|
||||||
|
|
||||||
## Quick Start with Vitastor
|
## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
|
||||||
|
|
||||||
Vitastor Backend is in experimental status, however you can already try to
|
## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
|
||||||
run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
|
|
||||||
it works too 😊.
|
|
||||||
|
|
||||||
Installation instructions:
|
## Docker
|
||||||
|
|
||||||
### Install Vitastor
|
[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/zenko/cloudserver/)
|
||||||
|
|
||||||
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
|
## Contributing
|
||||||
|
|
||||||
### Install Zenko with Vitastor Backend
|
In order to contribute, please follow the
|
||||||
|
[Contributing Guidelines](
|
||||||
|
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
|
||||||
|
|
||||||
- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
|
## Installation
|
||||||
- Install dependencies: `npm install --omit dev` or just `npm install`
|
|
||||||
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
|
|
||||||
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
|
|
||||||
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
|
|
||||||
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
|
|
||||||
|
|
||||||
### Install and Configure MongoDB
|
### Dependencies
|
||||||
|
|
||||||
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
|
Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
|
||||||
|
. Up-to-date versions can be found at
|
||||||
|
[Nodesource](https://github.com/nodesource/distributions).
|
||||||
|
|
||||||
### Setup Zenko
|
### Clone source code
|
||||||
|
|
||||||
- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
|
```shell
|
||||||
- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
|
git clone https://github.com/scality/S3.git
|
||||||
- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
|
|
||||||
- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
|
|
||||||
- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
|
|
||||||
and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
|
|
||||||
access keys, but it's not published, so let's use a file for now.
|
|
||||||
- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
|
|
||||||
You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
|
|
||||||
in this file.
|
|
||||||
|
|
||||||
Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
|
|
||||||
instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
|
|
||||||
|
|
||||||
### Start Zenko
|
|
||||||
|
|
||||||
Start the S3 server with: `node index.js`
|
|
||||||
|
|
||||||
If you use default settings, Zenko CloudServer starts on port 8000.
|
|
||||||
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
|
|
||||||
|
|
||||||
Now you can access your S3 with `s3cmd` or `geesefs`:
|
|
||||||
|
|
||||||
```
|
|
||||||
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
|
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
### Install js dependencies
|
||||||
AWS_ACCESS_KEY_ID=accessKey1 \
|
|
||||||
AWS_SECRET_ACCESS_KEY=verySecretKey1 \
|
Go to the ./S3 folder,
|
||||||
geesefs --endpoint http://localhost:8000 testbucket mountdir
|
|
||||||
|
```shell
|
||||||
|
yarn install --frozen-lockfile
|
||||||
```
|
```
|
||||||
|
|
||||||
# Author & License
|
If you get an error regarding installation of the diskUsage module,
|
||||||
|
please install g++.
|
||||||
|
|
||||||
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
If you get an error regarding level-down bindings, try clearing your yarn cache:
|
||||||
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
|
|
||||||
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way)
|
```shell
|
||||||
|
yarn cache clean
|
||||||
|
```
|
||||||
|
|
||||||
|
## Run it with a file backend
|
||||||
|
|
||||||
|
```shell
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
|
||||||
|
9991 are also open locally for internal transfer of metadata and data,
|
||||||
|
respectively.
|
||||||
|
|
||||||
|
The default access key is accessKey1 with
|
||||||
|
a secret key of verySecretKey1.
|
||||||
|
|
||||||
|
By default the metadata files will be saved in the
|
||||||
|
localMetadata directory and the data files will be saved
|
||||||
|
in the localData directory within the ./S3 directory on your
|
||||||
|
machine. These directories have been pre-created within the
|
||||||
|
repository. If you would like to save the data or metadata in
|
||||||
|
different locations of your choice, you must specify them with absolute paths.
|
||||||
|
So, when starting the server:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
mkdir -m 700 $(pwd)/myFavoriteDataPath
|
||||||
|
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
|
||||||
|
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
|
||||||
|
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
## Run it with multiple data backends
|
||||||
|
|
||||||
|
```shell
|
||||||
|
export S3DATA='multiple'
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer on port 8000.
|
||||||
|
The default access key is accessKey1 with
|
||||||
|
a secret key of verySecretKey1.
|
||||||
|
|
||||||
|
With multiple backends, you have the ability to
|
||||||
|
choose where each object will be saved by setting
|
||||||
|
the following header with a locationConstraint on
|
||||||
|
a PUT request:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
|
||||||
|
```
|
||||||
|
|
||||||
|
If no header is sent with a PUT object request, the
|
||||||
|
location constraint of the bucket will determine
|
||||||
|
where the data is saved. If the bucket has no location
|
||||||
|
constraint, the endpoint of the PUT request will be
|
||||||
|
used to determine location.
|
||||||
|
|
||||||
|
See the Configuration section in our documentation
|
||||||
|
[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
|
||||||
|
to learn how to set location constraints.
|
||||||
|
|
||||||
|
## Run it with an in-memory backend
|
||||||
|
|
||||||
|
```shell
|
||||||
|
yarn run mem_backend
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer on port 8000.
|
||||||
|
The default access key is accessKey1 with
|
||||||
|
a secret key of verySecretKey1.
|
||||||
|
|
||||||
|
## Run it with Vault user management
|
||||||
|
|
||||||
|
Note: Vault is proprietary and must be accessed separately.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
export S3VAULT=vault
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer using Vault for user management.
|
||||||
|
|
||||||
|
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
|
||||||
|
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
|
||||||
|
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
|
||||||
|
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
'use strict'; // eslint-disable-line strict
|
||||||
|
|
||||||
|
const {
|
||||||
|
startWSManagementClient,
|
||||||
|
startPushConnectionHealthCheckServer,
|
||||||
|
} = require('../lib/management/push');
|
||||||
|
|
||||||
|
const logger = require('../lib/utilities/logger');
|
||||||
|
|
||||||
|
const {
|
||||||
|
PUSH_ENDPOINT: pushEndpoint,
|
||||||
|
INSTANCE_ID: instanceId,
|
||||||
|
MANAGEMENT_TOKEN: managementToken,
|
||||||
|
} = process.env;
|
||||||
|
|
||||||
|
if (!pushEndpoint) {
|
||||||
|
logger.error('missing push endpoint env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!instanceId) {
|
||||||
|
logger.error('missing instance id env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!managementToken) {
|
||||||
|
logger.error('missing management token env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
startPushConnectionHealthCheckServer(err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('could not start healthcheck server', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
const url = `${pushEndpoint}/${instanceId}/ws?metrics=1`;
|
||||||
|
startWSManagementClient(url, managementToken, err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('connection failed, exiting', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
logger.info('no more connection, exiting');
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
});
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
'use strict'; // eslint-disable-line strict
|
||||||
|
|
||||||
|
const {
|
||||||
|
startWSManagementClient,
|
||||||
|
startPushConnectionHealthCheckServer,
|
||||||
|
} = require('../lib/management/push');
|
||||||
|
|
||||||
|
const logger = require('../lib/utilities/logger');
|
||||||
|
|
||||||
|
const {
|
||||||
|
PUSH_ENDPOINT: pushEndpoint,
|
||||||
|
INSTANCE_ID: instanceId,
|
||||||
|
MANAGEMENT_TOKEN: managementToken,
|
||||||
|
} = process.env;
|
||||||
|
|
||||||
|
if (!pushEndpoint) {
|
||||||
|
logger.error('missing push endpoint env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!instanceId) {
|
||||||
|
logger.error('missing instance id env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!managementToken) {
|
||||||
|
logger.error('missing management token env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
startPushConnectionHealthCheckServer(err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('could not start healthcheck server', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
const url = `${pushEndpoint}/${instanceId}/ws?proxy=1`;
|
||||||
|
startWSManagementClient(url, managementToken, err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('connection failed, exiting', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
logger.info('no more connection, exiting');
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
});
|
|
@ -4,7 +4,6 @@
|
||||||
"metricsPort": 8002,
|
"metricsPort": 8002,
|
||||||
"metricsListenOn": [],
|
"metricsListenOn": [],
|
||||||
"replicationGroupId": "RG001",
|
"replicationGroupId": "RG001",
|
||||||
"workers": 4,
|
|
||||||
"restEndpoints": {
|
"restEndpoints": {
|
||||||
"localhost": "us-east-1",
|
"localhost": "us-east-1",
|
||||||
"127.0.0.1": "us-east-1",
|
"127.0.0.1": "us-east-1",
|
||||||
|
@ -102,14 +101,6 @@
|
||||||
"readPreference": "primary",
|
"readPreference": "primary",
|
||||||
"database": "metadata"
|
"database": "metadata"
|
||||||
},
|
},
|
||||||
"authdata": "authdata.json",
|
|
||||||
"backends": {
|
|
||||||
"auth": "file",
|
|
||||||
"data": "file",
|
|
||||||
"metadata": "mongodb",
|
|
||||||
"kms": "file",
|
|
||||||
"quota": "none"
|
|
||||||
},
|
|
||||||
"externalBackends": {
|
"externalBackends": {
|
||||||
"aws_s3": {
|
"aws_s3": {
|
||||||
"httpAgent": {
|
"httpAgent": {
|
|
@ -1,71 +0,0 @@
|
||||||
{
|
|
||||||
"port": 8000,
|
|
||||||
"listenOn": [],
|
|
||||||
"metricsPort": 8002,
|
|
||||||
"metricsListenOn": [],
|
|
||||||
"replicationGroupId": "RG001",
|
|
||||||
"restEndpoints": {
|
|
||||||
"localhost": "STANDARD",
|
|
||||||
"127.0.0.1": "STANDARD",
|
|
||||||
"yourhostname.ru": "STANDARD"
|
|
||||||
},
|
|
||||||
"websiteEndpoints": [
|
|
||||||
"static.yourhostname.ru"
|
|
||||||
],
|
|
||||||
"replicationEndpoints": [ {
|
|
||||||
"site": "zenko",
|
|
||||||
"servers": ["127.0.0.1:8000"],
|
|
||||||
"default": true
|
|
||||||
} ],
|
|
||||||
"log": {
|
|
||||||
"logLevel": "info",
|
|
||||||
"dumpLevel": "error"
|
|
||||||
},
|
|
||||||
"healthChecks": {
|
|
||||||
"allowFrom": ["127.0.0.1/8", "::1"]
|
|
||||||
},
|
|
||||||
"backends": {
|
|
||||||
"metadata": "mongodb"
|
|
||||||
},
|
|
||||||
"mongodb": {
|
|
||||||
"replicaSetHosts": "127.0.0.1:27017",
|
|
||||||
"writeConcern": "majority",
|
|
||||||
"replicaSet": "rs0",
|
|
||||||
"readPreference": "primary",
|
|
||||||
"database": "s3",
|
|
||||||
"authCredentials": {
|
|
||||||
"username": "s3",
|
|
||||||
"password": ""
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"externalBackends": {
|
|
||||||
"aws_s3": {
|
|
||||||
"httpAgent": {
|
|
||||||
"keepAlive": false,
|
|
||||||
"keepAliveMsecs": 1000,
|
|
||||||
"maxFreeSockets": 256,
|
|
||||||
"maxSockets": null
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"gcp": {
|
|
||||||
"httpAgent": {
|
|
||||||
"keepAlive": true,
|
|
||||||
"keepAliveMsecs": 1000,
|
|
||||||
"maxFreeSockets": 256,
|
|
||||||
"maxSockets": null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"requests": {
|
|
||||||
"viaProxy": false,
|
|
||||||
"trustedProxyCIDRs": [],
|
|
||||||
"extractClientIPFromHeader": ""
|
|
||||||
},
|
|
||||||
"bucketNotificationDestinations": [
|
|
||||||
{
|
|
||||||
"resource": "target1",
|
|
||||||
"type": "dummy",
|
|
||||||
"host": "localhost:6000"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
53
constants.js
53
constants.js
|
@ -104,6 +104,7 @@ const constants = {
|
||||||
'policyStatus',
|
'policyStatus',
|
||||||
'publicAccessBlock',
|
'publicAccessBlock',
|
||||||
'requestPayment',
|
'requestPayment',
|
||||||
|
'restore',
|
||||||
'torrent',
|
'torrent',
|
||||||
],
|
],
|
||||||
|
|
||||||
|
@ -116,8 +117,7 @@ const constants = {
|
||||||
],
|
],
|
||||||
|
|
||||||
// user metadata header to set object locationConstraint
|
// user metadata header to set object locationConstraint
|
||||||
objectLocationConstraintHeader: 'x-amz-storage-class',
|
objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
|
||||||
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
|
|
||||||
legacyLocations: ['sproxyd', 'legacy'],
|
legacyLocations: ['sproxyd', 'legacy'],
|
||||||
// declare here all existing service accounts and their properties
|
// declare here all existing service accounts and their properties
|
||||||
// (if any, otherwise an empty object)
|
// (if any, otherwise an empty object)
|
||||||
|
@ -130,7 +130,7 @@ const constants = {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
/* eslint-disable camelcase */
|
/* eslint-disable camelcase */
|
||||||
externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
|
externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true },
|
||||||
// some of the available data backends (if called directly rather
|
// some of the available data backends (if called directly rather
|
||||||
// than through the multiple backend gateway) need a key provided
|
// than through the multiple backend gateway) need a key provided
|
||||||
// as a string as first parameter of the get/delete methods.
|
// as a string as first parameter of the get/delete methods.
|
||||||
|
@ -176,8 +176,6 @@ const constants = {
|
||||||
'objectDeleteTagging',
|
'objectDeleteTagging',
|
||||||
'objectGetTagging',
|
'objectGetTagging',
|
||||||
'objectPutTagging',
|
'objectPutTagging',
|
||||||
'objectPutLegalHold',
|
|
||||||
'objectPutRetention',
|
|
||||||
],
|
],
|
||||||
// response header to be sent when there are invalid
|
// response header to be sent when there are invalid
|
||||||
// user metadata in the object's metadata
|
// user metadata in the object's metadata
|
||||||
|
@ -185,6 +183,7 @@ const constants = {
|
||||||
// Bucket specific queries supported by AWS that we do not currently support
|
// Bucket specific queries supported by AWS that we do not currently support
|
||||||
// these queries may or may not be supported at object level
|
// these queries may or may not be supported at object level
|
||||||
unsupportedBucketQueries: [
|
unsupportedBucketQueries: [
|
||||||
|
'tagging',
|
||||||
],
|
],
|
||||||
suppressedUtapiEventFields: [
|
suppressedUtapiEventFields: [
|
||||||
'object',
|
'object',
|
||||||
|
@ -198,51 +197,7 @@ const constants = {
|
||||||
'user',
|
'user',
|
||||||
'bucket',
|
'bucket',
|
||||||
],
|
],
|
||||||
arrayOfAllowed: [
|
|
||||||
'objectPutTagging',
|
|
||||||
'objectPutLegalHold',
|
|
||||||
'objectPutRetention',
|
|
||||||
],
|
|
||||||
allowedUtapiEventFilterStates: ['allow', 'deny'],
|
allowedUtapiEventFilterStates: ['allow', 'deny'],
|
||||||
allowedRestoreObjectRequestTierValues: ['Standard'],
|
|
||||||
lifecycleListing: {
|
|
||||||
CURRENT_TYPE: 'current',
|
|
||||||
NON_CURRENT_TYPE: 'noncurrent',
|
|
||||||
ORPHAN_DM_TYPE: 'orphan',
|
|
||||||
},
|
|
||||||
multiObjectDeleteConcurrency: 50,
|
|
||||||
maxScannedLifecycleListingEntries: 10000,
|
|
||||||
overheadField: [
|
|
||||||
'content-length',
|
|
||||||
'owner-id',
|
|
||||||
'versionId',
|
|
||||||
'isNull',
|
|
||||||
'isDeleteMarker',
|
|
||||||
],
|
|
||||||
unsupportedSignatureChecksums: new Set([
|
|
||||||
'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
|
|
||||||
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
|
|
||||||
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
|
|
||||||
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
|
|
||||||
]),
|
|
||||||
supportedSignatureChecksums: new Set([
|
|
||||||
'UNSIGNED-PAYLOAD',
|
|
||||||
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
|
|
||||||
]),
|
|
||||||
ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
|
|
||||||
ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
|
|
||||||
// The AWS assumed Role resource type
|
|
||||||
assumedRoleArnResourceType: 'assumed-role',
|
|
||||||
// Session name of the backbeat lifecycle assumed role session.
|
|
||||||
backbeatLifecycleSessionName: 'backbeat-lifecycle',
|
|
||||||
actionsToConsiderAsObjectPut: [
|
|
||||||
'initiateMultipartUpload',
|
|
||||||
'objectPutPart',
|
|
||||||
'completeMultipartUpload',
|
|
||||||
],
|
|
||||||
// if requester is not bucket owner, bucket policy actions should be denied with
|
|
||||||
// MethodNotAllowed error
|
|
||||||
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = constants;
|
module.exports = constants;
|
||||||
|
|
|
@ -199,10 +199,6 @@ if [[ -n "$BUCKET_DENY_FILTER" ]]; then
|
||||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
|
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$TESTING_MODE" ]]; then
|
|
||||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $JQ_FILTERS_CONFIG != "." ]]; then
|
if [[ $JQ_FILTERS_CONFIG != "." ]]; then
|
||||||
jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
|
jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
|
||||||
mv config.json.tmp config.json
|
mv config.json.tmp config.json
|
||||||
|
|
|
@ -2,12 +2,11 @@
|
||||||
|
|
||||||
## Docker Image Generation
|
## Docker Image Generation
|
||||||
|
|
||||||
Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
|
Docker images are hosted on [registry.scality.com](registry.scality.com).
|
||||||
CloudServer has a few images there:
|
CloudServer has two namespaces there:
|
||||||
|
|
||||||
* Cloudserver container image: ghcr.io/scality/cloudserver
|
* Production Namespace: registry.scality.com/cloudserver
|
||||||
* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
|
* Dev Namespace: registry.scality.com/cloudserver-dev
|
||||||
* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
|
|
||||||
|
|
||||||
With every CI build, the CI will push images, tagging the
|
With every CI build, the CI will push images, tagging the
|
||||||
content with the developer branch's short SHA-1 commit hash.
|
content with the developer branch's short SHA-1 commit hash.
|
||||||
|
@ -19,55 +18,63 @@ Tagged versions of cloudserver will be stored in the production namespace.
|
||||||
## How to Pull Docker Images
|
## How to Pull Docker Images
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker pull ghcr.io/scality/cloudserver:<commit hash>
|
docker pull registry.scality.com/cloudserver-dev/cloudserver:<commit hash>
|
||||||
docker pull ghcr.io/scality/cloudserver:<tag>
|
docker pull registry.scality.com/cloudserver/cloudserver:<tag>
|
||||||
```
|
```
|
||||||
|
|
||||||
## Release Process
|
## Release Process
|
||||||
|
|
||||||
To release a production image:
|
To release a production image:
|
||||||
|
|
||||||
* Create a PR to bump the package version
|
* Checkout the relevant branch. In this example,
|
||||||
Update Cloudserver's `package.json` by bumping it to the relevant next
|
we are working on development/8.3, and we want to release version `8.3.0`.
|
||||||
version in a new PR. Per example if the last released version was
|
|
||||||
`8.4.7`, the next version would be `8.4.8`.
|
```sh
|
||||||
|
git checkout development/8.3
|
||||||
|
```
|
||||||
|
|
||||||
|
* Tag the branch with the release version. In this example, `8.3.0`
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git tag -a 8.3.0
|
||||||
|
# The message should be 'v<version>'
|
||||||
|
v8.3.0
|
||||||
|
```
|
||||||
|
|
||||||
|
* Push the tags to GitHub.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git push --tags
|
||||||
|
```
|
||||||
|
|
||||||
|
* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
|
||||||
|
|
||||||
|
* Branch Name: The one used for the tag earlier. In this example 'development/8.3'
|
||||||
|
* Override Stage: 'release'
|
||||||
|
* Extra properties:
|
||||||
|
* name: `'tag'`, value: `[release version]`, in this example`'8.3.0'`
|
||||||
|
|
||||||
|
* Once the docker image is present on [registry.scality.com](registry.scality.com),
|
||||||
|
update CloudServers' `package.json`
|
||||||
|
by bumping it to the relevant next version in a new PR.
|
||||||
|
In this case, `8.3.1` .
|
||||||
|
|
||||||
```js
|
```js
|
||||||
{
|
{
|
||||||
"name": "cloudserver",
|
"name": "@zenko/cloudserver",
|
||||||
"version": "8.4.8", <--- Here
|
"version": "8.3.1", <--- Here
|
||||||
[...]
|
[...]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
* Review & merge the PR
|
* Finally, once your PR has been reviewed, release the release version on Jira,
|
||||||
|
set up the next version, and approve your PR.
|
||||||
* Create the release on GitHub
|
|
||||||
|
|
||||||
* Go the Release tab (https://github.com/scality/cloudserver/releases);
|
|
||||||
* Click on the `Draft new release button`;
|
|
||||||
* In the `tag` field, type the name of the release (`8.4.8`), and confirm
|
|
||||||
to create the tag on publish;
|
|
||||||
* Click on `Generate release notes` button to fill the fields;
|
|
||||||
* Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
|
|
||||||
* Click to `Publish the release` to create the GitHub release and git tag
|
|
||||||
|
|
||||||
Notes:
|
|
||||||
* the Git tag will be created automatically.
|
|
||||||
* this should be done as soon as the PR is merged, so that the tag
|
|
||||||
is put on the "version bump" commit.
|
|
||||||
|
|
||||||
* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
|
|
||||||
|
|
||||||
* Branch Name: The one used for the tag earlier. In this example `development/8.4`
|
|
||||||
* Override Stage: 'release'
|
|
||||||
* Extra properties:
|
|
||||||
* name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
|
|
||||||
|
|
||||||
* Release the release version on Jira
|
|
||||||
|
|
||||||
* Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
|
* Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
|
||||||
* Create a next version
|
* Create a new version if necessary
|
||||||
* Name: `[next version]`, in this example `8.4.9`
|
* Name: `[next version]`, in this example `8.3.1`
|
||||||
* Click `...` and select `Release` on the recently released version (`8.4.8`)
|
* Start Date: `[date of the release]`
|
||||||
* Fill in the field to move incomplete version to the next one
|
* Click `...` and select `Release` on the release version
|
||||||
|
* Return to the release ticket,
|
||||||
|
change the fix version of the ticket to the new version
|
||||||
|
* Return to your PR and type `/approve`
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
script_full_path=$(readlink -f "$0")
|
||||||
|
file_dir=$(dirname "$script_full_path")/..
|
||||||
|
|
||||||
|
PACKAGE_VERSION=$(cat $file_dir/package.json \
|
||||||
|
| grep version \
|
||||||
|
| head -1 \
|
||||||
|
| awk -F: '{ print $2 }' \
|
||||||
|
| sed 's/[",]//g' \
|
||||||
|
| tr -d '[[:space:]]')
|
||||||
|
|
||||||
|
echo $PACKAGE_VERSION
|
|
@ -0,0 +1,605 @@
|
||||||
|
---
|
||||||
|
version: 0.2
|
||||||
|
|
||||||
|
branches:
|
||||||
|
feature/*, documentation/*, improvement/*, bugfix/*, w/*, q/*, hotfix/*, dependabot/*, user/*:
|
||||||
|
stage: pre-merge
|
||||||
|
|
||||||
|
models:
|
||||||
|
- env: &global-env
|
||||||
|
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
|
||||||
|
%(secret:azure_storage_access_key)s
|
||||||
|
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
|
||||||
|
%(secret:azure_storage_account_name)s
|
||||||
|
azurebackend_AZURE_STORAGE_ENDPOINT: >-
|
||||||
|
%(secret:azure_storage_endpoint)s
|
||||||
|
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
|
||||||
|
%(secret:azure_storage_access_key_2)s
|
||||||
|
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
|
||||||
|
%(secret:azure_storage_account_name_2)s
|
||||||
|
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
|
||||||
|
%(secret:azure_storage_endpoint_2)s
|
||||||
|
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
|
||||||
|
%(secret:azure_storage_access_key)s
|
||||||
|
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
|
||||||
|
%(secret:azure_storage_account_name)s
|
||||||
|
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
|
||||||
|
%(secret:azure_storage_endpoint)s
|
||||||
|
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
|
||||||
|
%(secret:azure_storage_access_key)s
|
||||||
|
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
|
||||||
|
%(secret:azure_storage_account_name)s
|
||||||
|
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
|
||||||
|
%(secret:azure_storage_endpoint)s
|
||||||
|
azuretest_AZURE_BLOB_ENDPOINT: "%(secret:azure_storage_endpoint)s"
|
||||||
|
b2backend_B2_ACCOUNT_ID: "%(secret:b2backend_b2_account_id)s"
|
||||||
|
b2backend_B2_STORAGE_ACCESS_KEY: >-
|
||||||
|
%(secret:b2backend_b2_storage_access_key)s
|
||||||
|
GOOGLE_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
|
||||||
|
GOOGLE_SERVICE_KEY: "%(secret:gcp_service_key)s"
|
||||||
|
AWS_S3_BACKEND_ACCESS_KEY: "%(secret:aws_s3_backend_access_key)s"
|
||||||
|
AWS_S3_BACKEND_SECRET_KEY: "%(secret:aws_s3_backend_secret_key)s"
|
||||||
|
AWS_S3_BACKEND_ACCESS_KEY_2: "%(secret:aws_s3_backend_access_key_2)s"
|
||||||
|
AWS_S3_BACKEND_SECRET_KEY_2: "%(secret:aws_s3_backend_secret_key_2)s"
|
||||||
|
AWS_GCP_BACKEND_ACCESS_KEY: "%(secret:aws_gcp_backend_access_key)s"
|
||||||
|
AWS_GCP_BACKEND_SECRET_KEY: "%(secret:aws_gcp_backend_secret_key)s"
|
||||||
|
AWS_GCP_BACKEND_ACCESS_KEY_2: "%(secret:aws_gcp_backend_access_key_2)s"
|
||||||
|
AWS_GCP_BACKEND_SECRET_KEY_2: "%(secret:aws_gcp_backend_secret_key_2)s"
|
||||||
|
b2backend_B2_STORAGE_ENDPOINT: "%(secret:b2backend_b2_storage_endpoint)s"
|
||||||
|
gcpbackend2_GCP_SERVICE_EMAIL: "%(secret:gcp2_service_email)s"
|
||||||
|
gcpbackend2_GCP_SERVICE_KEY: "%(secret:gcp2_service_key)s"
|
||||||
|
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
||||||
|
gcpbackend_GCP_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
|
||||||
|
gcpbackend_GCP_SERVICE_KEY: "%(secret:gcp_service_key)s"
|
||||||
|
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
|
||||||
|
%(secret:gcpbackendmismatch_gcp_service_email)s
|
||||||
|
gcpbackendmismatch_GCP_SERVICE_KEY: >-
|
||||||
|
%(secret:gcpbackendmismatch_gcp_service_key)s
|
||||||
|
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
||||||
|
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
||||||
|
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
||||||
|
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
|
||||||
|
- env: &mongo-vars
|
||||||
|
S3BACKEND: "mem"
|
||||||
|
MPU_TESTING: "yes"
|
||||||
|
S3METADATA: mongodb
|
||||||
|
S3KMS: "file"
|
||||||
|
- env: &multiple-backend-vars
|
||||||
|
S3BACKEND: "mem"
|
||||||
|
S3DATA: "multiple"
|
||||||
|
MPU_TESTING: "yes"
|
||||||
|
S3KMS: "file"
|
||||||
|
- env: &file-mem-mpu
|
||||||
|
S3BACKEND: "file"
|
||||||
|
S3VAULT: "mem"
|
||||||
|
MPU_TESTING: "yes"
|
||||||
|
- env: &oras
|
||||||
|
REGISTRY: 'registry.scality.com'
|
||||||
|
PROJECT: '%(prop:git_slug)s'
|
||||||
|
LAYERS: >-
|
||||||
|
dashboard.json:application/grafana-dashboard+json
|
||||||
|
alerts.yaml:application/prometheus-alerts+yaml
|
||||||
|
- Git: &clone
|
||||||
|
name: Pull repo
|
||||||
|
repourl: '%(prop:git_reference)s'
|
||||||
|
shallow: true
|
||||||
|
retryFetch: true
|
||||||
|
haltOnFailure: true
|
||||||
|
- ShellCommand: &credentials
|
||||||
|
name: Setup Credentials
|
||||||
|
command: bash eve/workers/build/credentials.bash
|
||||||
|
haltOnFailure: true
|
||||||
|
env: *global-env
|
||||||
|
- ShellCommand: &yarn-install
|
||||||
|
name: install modules
|
||||||
|
command: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
|
||||||
|
haltOnFailure: true
|
||||||
|
- ShellCommand: &check-s3-action-logs
|
||||||
|
name: Check s3 action logs
|
||||||
|
command: |
|
||||||
|
LOGS=`cat /artifacts/s3.log | grep 'No actionLog'`
|
||||||
|
test `echo -n ${LOGS} | wc -l` -eq 0 || (echo $LOGS && false)
|
||||||
|
- Upload: &upload-artifacts
|
||||||
|
source: /artifacts
|
||||||
|
urls:
|
||||||
|
- "*"
|
||||||
|
- ShellCommand: &follow-s3-log
|
||||||
|
logfiles:
|
||||||
|
s3:
|
||||||
|
filename: /artifacts/s3.log
|
||||||
|
follow: true
|
||||||
|
- ShellCommand: &follow-s3-ceph-logs
|
||||||
|
logfiles:
|
||||||
|
ceph:
|
||||||
|
filename: /artifacts/ceph.log
|
||||||
|
follow: true
|
||||||
|
s3:
|
||||||
|
filename: /artifacts/s3.log
|
||||||
|
follow: true
|
||||||
|
- ShellCommand: &add-hostname
|
||||||
|
name: add hostname
|
||||||
|
command: |
|
||||||
|
echo "127.0.0.1 testrequestbucket.localhost" >> /etc/hosts
|
||||||
|
echo \
|
||||||
|
"127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" \
|
||||||
|
>> /etc/hosts
|
||||||
|
haltOnFailure: true
|
||||||
|
- ShellCommand: &setup-junit-upload
|
||||||
|
name: preparing junit files for upload
|
||||||
|
command: |
|
||||||
|
mkdir -p artifacts/junit
|
||||||
|
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
|
||||||
|
alwaysRun: true
|
||||||
|
- Upload: &upload-junits
|
||||||
|
source: artifacts
|
||||||
|
urls:
|
||||||
|
- "*"
|
||||||
|
alwaysRun: true
|
||||||
|
- env: &docker_env
|
||||||
|
DEVELOPMENT_DOCKER_IMAGE_NAME: >-
|
||||||
|
registry.scality.com/%(prop:git_slug)s-dev/%(prop:git_slug)s
|
||||||
|
PRODUCTION_DOCKER_IMAGE_NAME: >-
|
||||||
|
registry.scality.com/%(prop:git_slug)s/%(prop:git_slug)s
|
||||||
|
- ShellCommand: &docker_login
|
||||||
|
name: Login to docker registry
|
||||||
|
command: >
|
||||||
|
docker login
|
||||||
|
-u "${HARBOR_LOGIN}"
|
||||||
|
-p "${HARBOR_PASSWORD}"
|
||||||
|
registry.scality.com
|
||||||
|
usePTY: true
|
||||||
|
env:
|
||||||
|
HARBOR_LOGIN: '%(secret:harbor_login)s'
|
||||||
|
HARBOR_PASSWORD: '%(secret:harbor_password)s'
|
||||||
|
- ShellCommand: &wait_docker_daemon
|
||||||
|
name: Wait for Docker daemon to be ready
|
||||||
|
command: |
|
||||||
|
bash -c '
|
||||||
|
for i in {1..150}
|
||||||
|
do
|
||||||
|
docker info &> /dev/null && exit
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
echo "Could not reach Docker daemon from buildbot worker" >&2
|
||||||
|
exit 1'
|
||||||
|
haltOnFailure: true
|
||||||
|
|
||||||
|
|
||||||
|
stages:
|
||||||
|
pre-merge:
|
||||||
|
worker:
|
||||||
|
type: local
|
||||||
|
steps:
|
||||||
|
- TriggerStages:
|
||||||
|
name: Launch all workers
|
||||||
|
stage_names:
|
||||||
|
- docker-build
|
||||||
|
- linting-coverage
|
||||||
|
- file-ft-tests
|
||||||
|
- multiple-backend-test
|
||||||
|
- mongo-v0-ft-tests
|
||||||
|
- mongo-v1-ft-tests
|
||||||
|
- ceph-backend-tests
|
||||||
|
- kmip-ft-tests
|
||||||
|
- utapi-v2-tests
|
||||||
|
waitForFinish: true
|
||||||
|
haltOnFailure: true
|
||||||
|
|
||||||
|
linting-coverage:
|
||||||
|
worker:
|
||||||
|
type: docker
|
||||||
|
path: eve/workers/build
|
||||||
|
volumes: &default_volumes
|
||||||
|
- '/home/eve/workspace'
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand: *add-hostname
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand:
|
||||||
|
name: Unit Coverage mandatory file
|
||||||
|
command: |
|
||||||
|
set -ex
|
||||||
|
test -f .git/HEAD
|
||||||
|
- ShellCommand:
|
||||||
|
name: Linting
|
||||||
|
command: |
|
||||||
|
set -ex
|
||||||
|
yarn run --silent lint -- --max-warnings 0
|
||||||
|
yarn run --silent lint_md
|
||||||
|
flake8 $(git ls-files "*.py")
|
||||||
|
yamllint -c yamllint.yml $(git ls-files "*.yml")
|
||||||
|
- ShellCommand:
|
||||||
|
name: Unit Coverage
|
||||||
|
command: |
|
||||||
|
set -ex
|
||||||
|
unset HTTP_PROXY HTTPS_PROXY NO_PROXY
|
||||||
|
unset http_proxy https_proxy no_proxy
|
||||||
|
mkdir -p $CIRCLE_TEST_REPORTS/unit
|
||||||
|
yarn test
|
||||||
|
yarn run test_versionid_base62
|
||||||
|
yarn run test_legacy_location
|
||||||
|
env: &shared-vars
|
||||||
|
<<: *global-env
|
||||||
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
|
CIRCLE_TEST_REPORTS: /tmp
|
||||||
|
CIRCLE_ARTIFACTS: /tmp
|
||||||
|
CI_REPORTS: /tmp
|
||||||
|
- ShellCommand:
|
||||||
|
name: Unit Coverage logs
|
||||||
|
command: find /tmp/unit -exec cat {} \;
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
multiple-backend-test:
|
||||||
|
worker:
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/pod.yaml
|
||||||
|
images:
|
||||||
|
aggressor: eve/workers/build
|
||||||
|
s3: "."
|
||||||
|
vars:
|
||||||
|
aggressorMem: "2560Mi"
|
||||||
|
s3Mem: "2560Mi"
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
<<: *global-env
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
bash -c "
|
||||||
|
source /root/.aws/exports &> /dev/null
|
||||||
|
set -ex
|
||||||
|
bash wait_for_local_port.bash 8000 40
|
||||||
|
yarn run multiple_backend_test
|
||||||
|
yarn run ft_awssdk_external_backends"
|
||||||
|
<<: *follow-s3-log
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
<<: *global-env
|
||||||
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
|
- ShellCommand:
|
||||||
|
command: mvn test
|
||||||
|
workdir: build/tests/functional/jaws
|
||||||
|
<<: *follow-s3-log
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
- ShellCommand:
|
||||||
|
command: rspec tests.rb
|
||||||
|
workdir: build/tests/functional/fog
|
||||||
|
<<: *follow-s3-log
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
- ShellCommand: *check-s3-action-logs
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-artifacts
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
ceph-backend-tests:
|
||||||
|
worker:
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/pod.yaml
|
||||||
|
images:
|
||||||
|
aggressor: eve/workers/build
|
||||||
|
s3: "."
|
||||||
|
ceph: eve/workers/ceph
|
||||||
|
vars:
|
||||||
|
aggressorMem: "2500Mi"
|
||||||
|
s3Mem: "2560Mi"
|
||||||
|
redis: enabled
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
<<: *global-env
|
||||||
|
S3METADATA: mongodb
|
||||||
|
CI_CEPH: "true"
|
||||||
|
MPU_TESTING: "yes"
|
||||||
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
bash -c "
|
||||||
|
source /root/.aws/exports &> /dev/null
|
||||||
|
set -ex
|
||||||
|
bash eve/workers/ceph/wait_for_ceph.sh
|
||||||
|
bash wait_for_local_port.bash 27018 40
|
||||||
|
bash wait_for_local_port.bash 8000 40
|
||||||
|
yarn run multiple_backend_test"
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
<<: *global-env
|
||||||
|
S3METADATA: mem
|
||||||
|
<<: *follow-s3-ceph-logs
|
||||||
|
- ShellCommand:
|
||||||
|
command: mvn test
|
||||||
|
workdir: build/tests/functional/jaws
|
||||||
|
<<: *follow-s3-ceph-logs
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
- ShellCommand:
|
||||||
|
command: rspec tests.rb
|
||||||
|
workdir: build/tests/functional/fog
|
||||||
|
<<: *follow-s3-ceph-logs
|
||||||
|
env:
|
||||||
|
<<: *multiple-backend-vars
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
yarn run ft_awssdk &&
|
||||||
|
yarn run ft_s3cmd
|
||||||
|
env:
|
||||||
|
<<: *file-mem-mpu
|
||||||
|
<<: *global-env
|
||||||
|
S3METADATA: mongodb
|
||||||
|
S3_LOCATION_FILE: "/kube_pod-prod-cloudserver-backend-0/\
|
||||||
|
build/tests/locationConfig/locationConfigCeph.json"
|
||||||
|
<<: *follow-s3-ceph-logs
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-artifacts
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
mongo-v0-ft-tests:
|
||||||
|
worker:
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/pod.yaml
|
||||||
|
images:
|
||||||
|
aggressor: eve/workers/build
|
||||||
|
s3: "."
|
||||||
|
vars:
|
||||||
|
aggressorMem: "2Gi"
|
||||||
|
s3Mem: "1664Mi"
|
||||||
|
redis: enabled
|
||||||
|
env:
|
||||||
|
<<: *mongo-vars
|
||||||
|
<<: *global-env
|
||||||
|
DEFAULT_BUCKET_KEY_FORMAT: "v0"
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
set -ex
|
||||||
|
bash wait_for_local_port.bash 8000 40
|
||||||
|
yarn run ft_test
|
||||||
|
<<: *follow-s3-log
|
||||||
|
env:
|
||||||
|
<<: *mongo-vars
|
||||||
|
<<: *global-env
|
||||||
|
DEFAULT_BUCKET_KEY_FORMAT: "v0"
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-artifacts
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
mongo-v1-ft-tests:
|
||||||
|
worker:
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/pod.yaml
|
||||||
|
images:
|
||||||
|
aggressor: eve/workers/build
|
||||||
|
s3: "."
|
||||||
|
vars:
|
||||||
|
aggressorMem: "2Gi"
|
||||||
|
s3Mem: "1664Mi"
|
||||||
|
redis: enabled
|
||||||
|
env:
|
||||||
|
<<: *mongo-vars
|
||||||
|
<<: *global-env
|
||||||
|
DEFAULT_BUCKET_KEY_FORMAT: "v1"
|
||||||
|
METADATA_MAX_CACHED_BUCKETS: "1"
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
set -ex
|
||||||
|
bash wait_for_local_port.bash 8000 40
|
||||||
|
yarn run ft_test
|
||||||
|
yarn run ft_mixed_bucket_format_version
|
||||||
|
<<: *follow-s3-log
|
||||||
|
env:
|
||||||
|
<<: *mongo-vars
|
||||||
|
<<: *global-env
|
||||||
|
DEFAULT_BUCKET_KEY_FORMAT: "v1"
|
||||||
|
METADATA_MAX_CACHED_BUCKETS: "1"
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-artifacts
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
file-ft-tests:
|
||||||
|
worker:
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/pod.yaml
|
||||||
|
images:
|
||||||
|
aggressor: eve/workers/build
|
||||||
|
s3: "."
|
||||||
|
vars:
|
||||||
|
aggressorMem: "3Gi"
|
||||||
|
s3Mem: "2560Mi"
|
||||||
|
redis: enabled
|
||||||
|
env:
|
||||||
|
<<: *file-mem-mpu
|
||||||
|
<<: *global-env
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
set -ex
|
||||||
|
bash wait_for_local_port.bash 8000 40
|
||||||
|
yarn run ft_test
|
||||||
|
<<: *follow-s3-log
|
||||||
|
env:
|
||||||
|
<<: *file-mem-mpu
|
||||||
|
<<: *global-env
|
||||||
|
- ShellCommand: *check-s3-action-logs
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-artifacts
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
kmip-ft-tests:
|
||||||
|
worker:
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/pod.yaml
|
||||||
|
images:
|
||||||
|
aggressor: eve/workers/build
|
||||||
|
s3: "."
|
||||||
|
pykmip: eve/workers/pykmip
|
||||||
|
vars:
|
||||||
|
aggressorMem: "2Gi"
|
||||||
|
s3Mem: "1664Mi"
|
||||||
|
redis: enabled
|
||||||
|
pykmip: enabled
|
||||||
|
env:
|
||||||
|
<<: *file-mem-mpu
|
||||||
|
<<: *global-env
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
set -ex
|
||||||
|
bash wait_for_local_port.bash 8000 40
|
||||||
|
bash wait_for_local_port.bash 5696 40
|
||||||
|
yarn run ft_kmip
|
||||||
|
logfiles:
|
||||||
|
pykmip:
|
||||||
|
filename: /artifacts/pykmip.log
|
||||||
|
follow: true
|
||||||
|
s3:
|
||||||
|
filename: /artifacts/s3.log
|
||||||
|
follow: true
|
||||||
|
env:
|
||||||
|
<<: *file-mem-mpu
|
||||||
|
<<: *global-env
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-artifacts
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
utapi-v2-tests:
|
||||||
|
worker:
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/pod.yaml
|
||||||
|
images:
|
||||||
|
aggressor: eve/workers/build
|
||||||
|
s3: "."
|
||||||
|
vars:
|
||||||
|
aggressorMem: "2Gi"
|
||||||
|
s3Mem: "2Gi"
|
||||||
|
env:
|
||||||
|
ENABLE_UTAPI_V2: t
|
||||||
|
S3BACKEND: mem
|
||||||
|
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *credentials
|
||||||
|
- ShellCommand: *yarn-install
|
||||||
|
- ShellCommand:
|
||||||
|
command: |
|
||||||
|
bash -c "
|
||||||
|
source /root/.aws/exports &> /dev/null
|
||||||
|
set -ex
|
||||||
|
bash wait_for_local_port.bash 8000 40
|
||||||
|
yarn run test_utapi_v2"
|
||||||
|
<<: *follow-s3-log
|
||||||
|
env:
|
||||||
|
ENABLE_UTAPI_V2: t
|
||||||
|
S3BACKEND: mem
|
||||||
|
- ShellCommand: *check-s3-action-logs
|
||||||
|
- ShellCommand: *setup-junit-upload
|
||||||
|
- Upload: *upload-artifacts
|
||||||
|
- Upload: *upload-junits
|
||||||
|
|
||||||
|
# The docker-build stage ensures that your images are built on every commit
|
||||||
|
# and also hosted on the registry to help you pull it up and
|
||||||
|
# test it in a real environment if needed.
|
||||||
|
# It also allows us to pull and rename it when performing a release.
|
||||||
|
docker-build:
|
||||||
|
worker: &docker_worker
|
||||||
|
type: kube_pod
|
||||||
|
path: eve/workers/docker/pod.yaml
|
||||||
|
images:
|
||||||
|
worker: eve/workers/docker
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *wait_docker_daemon
|
||||||
|
- ShellCommand: *docker_login
|
||||||
|
- ShellCommand:
|
||||||
|
name: docker build
|
||||||
|
command: >-
|
||||||
|
docker build .
|
||||||
|
--tag=${DEVELOPMENT_DOCKER_IMAGE_NAME}:%(prop:commit_short_revision)s
|
||||||
|
env: *docker_env
|
||||||
|
haltOnFailure: true
|
||||||
|
- ShellCommand:
|
||||||
|
name: push docker image into the development namespace
|
||||||
|
command: docker push ${DEVELOPMENT_DOCKER_IMAGE_NAME}
|
||||||
|
haltOnFailure: true
|
||||||
|
env: *docker_env
|
||||||
|
- ShellCommand: &oras_login
|
||||||
|
name: Oras login
|
||||||
|
command:
|
||||||
|
oras login --username "${HARBOR_LOGIN}" --password "${HARBOR_PASSWORD}" ${REGISTRY}
|
||||||
|
env:
|
||||||
|
<<: *oras
|
||||||
|
HARBOR_LOGIN: '%(secret:harbor_login)s'
|
||||||
|
HARBOR_PASSWORD: '%(secret:harbor_password)s'
|
||||||
|
- ShellCommand:
|
||||||
|
name: push dashboards to the development namespace
|
||||||
|
command: |
|
||||||
|
for revision in %(prop:commit_short_revision)s latest ; do
|
||||||
|
oras push ${REGISTRY}/${PROJECT}-dev/${PROJECT}-dashboards:$revision ${LAYERS}
|
||||||
|
done
|
||||||
|
env: *oras
|
||||||
|
workdir: build/monitoring/
|
||||||
|
|
||||||
|
|
||||||
|
# This stage can be used to release your Docker image.
|
||||||
|
# To use this stage:
|
||||||
|
# 1. Tag the repository
|
||||||
|
# 2. Force a build using:
|
||||||
|
# * A branch that ideally matches the tag
|
||||||
|
# * The release stage
|
||||||
|
# * An extra property with the name tag and its value being the actual tag
|
||||||
|
release:
|
||||||
|
worker:
|
||||||
|
type: local
|
||||||
|
steps:
|
||||||
|
- TriggerStages:
|
||||||
|
stage_names:
|
||||||
|
- docker-release
|
||||||
|
haltOnFailure: true
|
||||||
|
docker-release:
|
||||||
|
worker: *docker_worker
|
||||||
|
steps:
|
||||||
|
- Git: *clone
|
||||||
|
- ShellCommand: *wait_docker_daemon
|
||||||
|
- ShellCommand: *docker_login
|
||||||
|
- ShellCommand:
|
||||||
|
name: Checkout tag
|
||||||
|
command: git checkout refs/tags/%(prop:tag)s
|
||||||
|
haltOnFailure: true
|
||||||
|
- ShellCommand:
|
||||||
|
name: docker build
|
||||||
|
command: >-
|
||||||
|
docker build .
|
||||||
|
--tag=${PRODUCTION_DOCKER_IMAGE_NAME}:%(prop:tag)s
|
||||||
|
env: *docker_env
|
||||||
|
- ShellCommand:
|
||||||
|
name: publish docker image to Scality Production OCI registry
|
||||||
|
command: docker push ${PRODUCTION_DOCKER_IMAGE_NAME}:%(prop:tag)s
|
||||||
|
env: *docker_env
|
||||||
|
- ShellCommand: *oras_login
|
||||||
|
- ShellCommand:
|
||||||
|
name: push dashboards to the production namespace
|
||||||
|
command: |
|
||||||
|
oras push ${REGISTRY}/${PROJECT}/${PROJECT}-dashboards:%(prop:tag)s ${LAYERS}
|
||||||
|
env: *oras
|
||||||
|
workdir: build/monitoring/
|
|
@ -0,0 +1,62 @@
|
||||||
|
FROM buildpack-deps:bionic-curl
|
||||||
|
|
||||||
|
#
|
||||||
|
# Install packages needed by the buildchain
|
||||||
|
#
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
|
||||||
|
RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
|
||||||
|
&& echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
|
||||||
|
&& apt-get update \
|
||||||
|
&& apt-get install -y yarn \
|
||||||
|
&& cat /tmp/*packages.list | xargs apt-get install -y \
|
||||||
|
&& update-ca-certificates \
|
||||||
|
&& git clone https://github.com/tj/n.git \
|
||||||
|
&& make -C ./n \
|
||||||
|
&& n 16.13.2 \
|
||||||
|
&& pip install pip==9.0.1 \
|
||||||
|
&& rm -rf ./n \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& rm -f /tmp/packages.list
|
||||||
|
|
||||||
|
#
|
||||||
|
# Add user eve
|
||||||
|
#
|
||||||
|
|
||||||
|
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
|
||||||
|
&& adduser eve sudo \
|
||||||
|
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
|
||||||
|
#
|
||||||
|
# Install Dependencies
|
||||||
|
#
|
||||||
|
|
||||||
|
# Install RVM and gems
|
||||||
|
ENV RUBY_VERSION="2.5.0"
|
||||||
|
RUN gem update --system
|
||||||
|
|
||||||
|
RUN gpg2 --keyserver hkp://pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB \
|
||||||
|
&& curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
|
||||||
|
&& usermod -a -G rvm eve
|
||||||
|
|
||||||
|
COPY ./gems.list /tmp/
|
||||||
|
|
||||||
|
RUN /bin/bash -l -c "\
|
||||||
|
source /usr/local/rvm/scripts/rvm \
|
||||||
|
&& cat /tmp/gems.list | xargs gem install \
|
||||||
|
&& rm /tmp/gems.list"
|
||||||
|
|
||||||
|
# Install Pip packages
|
||||||
|
COPY ./pip_packages.list /tmp/
|
||||||
|
RUN cat /tmp/pip_packages.list | xargs pip install \
|
||||||
|
&& rm -f /tmp/pip_packages.list \
|
||||||
|
&& mkdir /home/eve/.aws \
|
||||||
|
&& chown eve /home/eve/.aws
|
||||||
|
|
||||||
|
#
|
||||||
|
# Run buildbot-worker on startup
|
||||||
|
#
|
||||||
|
|
||||||
|
ARG BUILDBOT_VERSION
|
||||||
|
RUN pip install buildbot-worker==$BUILDBOT_VERSION
|
||||||
|
|
||||||
|
CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS && buildbot-worker start --nodaemon"]
|
|
@ -0,0 +1,13 @@
|
||||||
|
ca-certificates
|
||||||
|
git
|
||||||
|
gnupg
|
||||||
|
libffi-dev
|
||||||
|
libssl-dev
|
||||||
|
python-pip
|
||||||
|
python2.7
|
||||||
|
python2.7-dev
|
||||||
|
software-properties-common
|
||||||
|
sudo
|
||||||
|
tcl
|
||||||
|
wget
|
||||||
|
procps
|
|
@ -2,9 +2,9 @@
|
||||||
set -x #echo on
|
set -x #echo on
|
||||||
set -e #exit at the first error
|
set -e #exit at the first error
|
||||||
|
|
||||||
mkdir -p $HOME/.aws
|
mkdir -p ~/.aws
|
||||||
|
|
||||||
cat >>$HOME/.aws/credentials <<EOF
|
cat >>/root/.aws/credentials <<EOF
|
||||||
[default]
|
[default]
|
||||||
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
|
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
|
||||||
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
|
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
|
|
@ -0,0 +1,5 @@
|
||||||
|
nokogiri:1.12.5
|
||||||
|
fog-aws:1.3.0
|
||||||
|
json
|
||||||
|
mime-types:3.1
|
||||||
|
rspec:3.5
|
|
@ -0,0 +1,3 @@
|
||||||
|
flake8
|
||||||
|
s3cmd==1.6.1
|
||||||
|
yamllint
|
|
@ -0,0 +1,15 @@
|
||||||
|
build-essential
|
||||||
|
ca-certificates
|
||||||
|
curl
|
||||||
|
default-jdk
|
||||||
|
gnupg2
|
||||||
|
libdigest-hmac-perl
|
||||||
|
lsof
|
||||||
|
maven
|
||||||
|
netcat
|
||||||
|
redis-server
|
||||||
|
yarn
|
||||||
|
zlib1g-dev
|
||||||
|
jq
|
||||||
|
openssl
|
||||||
|
ruby-full
|
|
@ -0,0 +1,33 @@
|
||||||
|
FROM centos:7
|
||||||
|
|
||||||
|
ARG BUILDBOT_VERSION=0.9.12
|
||||||
|
|
||||||
|
VOLUME /home/eve/workspace
|
||||||
|
|
||||||
|
WORKDIR /home/eve/workspace
|
||||||
|
|
||||||
|
RUN yum install -y epel-release \
|
||||||
|
&& yum-config-manager \
|
||||||
|
--add-repo \
|
||||||
|
https://download.docker.com/linux/centos/docker-ce.repo \
|
||||||
|
&& yum install -y \
|
||||||
|
python-devel \
|
||||||
|
python-pip \
|
||||||
|
python36 \
|
||||||
|
python36-devel \
|
||||||
|
python36-pip \
|
||||||
|
git \
|
||||||
|
docker-ce-cli-18.09.6 \
|
||||||
|
which \
|
||||||
|
&& adduser -u 1042 --home /home/eve eve --groups docker \
|
||||||
|
&& chown -R eve:eve /home/eve \
|
||||||
|
&& pip3 install buildbot-worker==${BUILDBOT_VERSION}
|
||||||
|
|
||||||
|
|
||||||
|
ARG ORAS_VERSION=0.12.0
|
||||||
|
RUN curl -LO https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz && \
|
||||||
|
mkdir -p oras-install/ && \
|
||||||
|
tar -zxf oras_${ORAS_VERSION}_*.tar.gz -C /usr/local/bin oras && \
|
||||||
|
rm -rf oras_${ORAS_VERSION}_*.tar.gz oras-install/
|
||||||
|
|
||||||
|
CMD buildbot-worker create-worker . ${BUILDMASTER}:${BUILDMASTER_PORT} ${WORKERNAME} ${WORKERPASS} && buildbot-worker start --nodaemon
|
|
@ -0,0 +1,43 @@
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: worker
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: build-worker
|
||||||
|
image: "{{ images.worker }}"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: "250m"
|
||||||
|
memory: 2Gi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 2Gi
|
||||||
|
env:
|
||||||
|
- name: DOCKER_HOST
|
||||||
|
value: localhost:2375
|
||||||
|
volumeMounts:
|
||||||
|
- name: worker-workspace
|
||||||
|
mountPath: /home/eve/workspace
|
||||||
|
- name: dind-daemon
|
||||||
|
image: docker:18.09.6-dind
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: "500m"
|
||||||
|
memory: 2Gi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 2Gi
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- name: docker-storage
|
||||||
|
mountPath: /var/lib/docker
|
||||||
|
- name: worker-workspace
|
||||||
|
mountPath: /home/eve/workspace
|
||||||
|
volumes:
|
||||||
|
- name: docker-storage
|
||||||
|
emptyDir: {}
|
||||||
|
- name: worker-workspace
|
||||||
|
emptyDir: {}
|
|
@ -0,0 +1,233 @@
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: "proxy-ci-test-pod"
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
terminationGracePeriodSeconds: 10
|
||||||
|
hostAliases:
|
||||||
|
- ip: "127.0.0.1"
|
||||||
|
hostnames:
|
||||||
|
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com"
|
||||||
|
- "testrequestbucket.localhost"
|
||||||
|
- "pykmip.local"
|
||||||
|
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
|
||||||
|
initContainers:
|
||||||
|
- name: kmip-certs-installer
|
||||||
|
image: {{ images.pykmip }}
|
||||||
|
command: [ 'sh', '-c', 'cp /ssl/* /ssl-kmip/']
|
||||||
|
volumeMounts:
|
||||||
|
- name: kmip-certs
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /ssl-kmip
|
||||||
|
{%- endif %}
|
||||||
|
containers:
|
||||||
|
{% if vars.env.S3METADATA is defined and vars.env.S3METADATA == "mongodb" -%}
|
||||||
|
- name: mongo
|
||||||
|
image: scality/ci-mongo:3.6.8
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
{%- endif %}
|
||||||
|
- name: aggressor
|
||||||
|
image: {{ images.aggressor }}
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: "1"
|
||||||
|
memory: {{ vars.aggressorMem }}
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: {{ vars.aggressorMem }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: creds
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /root/.aws
|
||||||
|
- name: artifacts
|
||||||
|
readOnly: true
|
||||||
|
mountPath: /artifacts
|
||||||
|
command:
|
||||||
|
- bash
|
||||||
|
- -lc
|
||||||
|
- |
|
||||||
|
buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS
|
||||||
|
buildbot-worker start --nodaemon
|
||||||
|
env:
|
||||||
|
- name: CI
|
||||||
|
value: "true"
|
||||||
|
- name: ENABLE_LOCAL_CACHE
|
||||||
|
value: "true"
|
||||||
|
- name: REPORT_TOKEN
|
||||||
|
value: "report-token-1"
|
||||||
|
- name: REMOTE_MANAGEMENT_DISABLE
|
||||||
|
value: "1"
|
||||||
|
{% for key, value in vars.env.items() %}
|
||||||
|
- name: {{ key }}
|
||||||
|
value: "{{ value }}"
|
||||||
|
{% endfor %}
|
||||||
|
- name: s3
|
||||||
|
image: {{ images.s3 }}
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: "1750m"
|
||||||
|
memory: {{ vars.s3Mem }}
|
||||||
|
limits:
|
||||||
|
cpu: "1750m"
|
||||||
|
memory: {{ vars.s3Mem }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: creds
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /root/.aws
|
||||||
|
- name: certs
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /tmp
|
||||||
|
- name: artifacts
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /artifacts
|
||||||
|
- name: kmip-certs
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /ssl-kmip
|
||||||
|
command:
|
||||||
|
- bash
|
||||||
|
- -ec
|
||||||
|
- |
|
||||||
|
sleep 10 # wait for mongo
|
||||||
|
/usr/src/app/docker-entrypoint.sh yarn start | tee -a /artifacts/s3.log
|
||||||
|
env:
|
||||||
|
{% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" and vars.env.CI_CEPH is not defined -%}
|
||||||
|
- name: S3_LOCATION_FILE
|
||||||
|
value: "/usr/src/app/tests/locationConfig/locationConfigTests.json"
|
||||||
|
{%- endif %}
|
||||||
|
{% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" and vars.env.CI_CEPH is defined and vars.env.CI_CEPH == "true" -%}
|
||||||
|
- name: S3_LOCATION_FILE
|
||||||
|
value: "/usr/src/app/tests/locationConfig/locationConfigCeph.json"
|
||||||
|
{%- endif %}
|
||||||
|
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
|
||||||
|
- name: S3KMS
|
||||||
|
value: kmip
|
||||||
|
- name: S3KMIP_PORT
|
||||||
|
value: "5696"
|
||||||
|
- name: S3KMIP_HOSTS
|
||||||
|
value: "pykmip.local"
|
||||||
|
- name: S3KMIP_COMPOUND_CREATE
|
||||||
|
value: "false"
|
||||||
|
- name: S3KMIP_BUCKET_ATTRIBUTE_NAME
|
||||||
|
value: ''
|
||||||
|
- name: S3KMIP_PIPELINE_DEPTH
|
||||||
|
value: "8"
|
||||||
|
- name: S3KMIP_KEY
|
||||||
|
value: /ssl-kmip/kmip-client-key.pem
|
||||||
|
- name: S3KMIP_CERT
|
||||||
|
value: /ssl-kmip/kmip-client-cert.pem
|
||||||
|
- name: S3KMIP_CA
|
||||||
|
value: /ssl-kmip/kmip-ca.pem
|
||||||
|
{%- endif %}
|
||||||
|
- name: CI
|
||||||
|
value: "true"
|
||||||
|
- name: ENABLE_LOCAL_CACHE
|
||||||
|
value: "true"
|
||||||
|
- name: MONGODB_HOSTS
|
||||||
|
value: "localhost:27018"
|
||||||
|
- name: MONGODB_RS
|
||||||
|
value: "rs0"
|
||||||
|
- name: REDIS_HOST
|
||||||
|
value: "localhost"
|
||||||
|
- name: REDIS_PORT
|
||||||
|
value: "6379"
|
||||||
|
- name: REPORT_TOKEN
|
||||||
|
value: "report-token-1"
|
||||||
|
- name: REMOTE_MANAGEMENT_DISABLE
|
||||||
|
value: "1"
|
||||||
|
- name: HEALTHCHECKS_ALLOWFROM
|
||||||
|
value: "0.0.0.0/0"
|
||||||
|
{% for key, value in vars.env.items() %}
|
||||||
|
- name: {{ key }}
|
||||||
|
value: "{{ value }}"
|
||||||
|
{% endfor %}
|
||||||
|
{% if vars.redis is defined and vars.redis == "enabled" -%}
|
||||||
|
- name: redis
|
||||||
|
image: redis:alpine
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 128Mi
|
||||||
|
{%- endif %}
|
||||||
|
{% if vars.env.CI_PROXY is defined and vars.env.CI_PROXY == "true" -%}
|
||||||
|
- name: squid
|
||||||
|
image: scality/ci-squid
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 128Mi
|
||||||
|
volumeMounts:
|
||||||
|
- name: certs
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /ssl
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -exc
|
||||||
|
- |
|
||||||
|
mkdir -p /ssl
|
||||||
|
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
|
||||||
|
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
|
||||||
|
-keyout /ssl/myca.pem -out /ssl/myca.pem
|
||||||
|
cp /ssl/myca.pem /ssl/CA.pem
|
||||||
|
squid -f /etc/squid/squid.conf -N -z
|
||||||
|
squid -f /etc/squid/squid.conf -NYCd 1
|
||||||
|
{%- endif %}
|
||||||
|
{% if vars.env.CI_CEPH is defined and vars.env.CI_CEPH == "true" -%}
|
||||||
|
- name: ceph
|
||||||
|
image: {{ images.ceph }}
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1536Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1536Mi
|
||||||
|
volumeMounts:
|
||||||
|
- name: artifacts
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /artifacts
|
||||||
|
{%- endif %}
|
||||||
|
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
|
||||||
|
- name: pykmip
|
||||||
|
image: {{ images.pykmip }}
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
volumeMounts:
|
||||||
|
- name: artifacts
|
||||||
|
readOnly: false
|
||||||
|
mountPath: /artifacts
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
{%- endif %}
|
||||||
|
volumes:
|
||||||
|
- name: creds
|
||||||
|
emptyDir: {}
|
||||||
|
- name: certs
|
||||||
|
emptyDir: {}
|
||||||
|
- name: artifacts
|
||||||
|
emptyDir: {}
|
||||||
|
- name: kmip-certs
|
||||||
|
emptyDir: {}
|
|
@ -48,7 +48,7 @@ signed_headers = 'host;x-amz-content-sha256;x-amz-date'
|
||||||
canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
|
canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
|
||||||
.format(method, canonical_uri, canonical_querystring, canonical_headers,
|
.format(method, canonical_uri, canonical_querystring, canonical_headers,
|
||||||
signed_headers, payload_hash)
|
signed_headers, payload_hash)
|
||||||
print(canonical_request)
|
print canonical_request
|
||||||
|
|
||||||
credential_scope = '{0}/{1}/{2}/aws4_request' \
|
credential_scope = '{0}/{1}/{2}/aws4_request' \
|
||||||
.format(date_stamp, region, service)
|
.format(date_stamp, region, service)
|
||||||
|
@ -76,4 +76,4 @@ headers = {
|
||||||
endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
|
endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
|
||||||
|
|
||||||
r = requests.get(endpoint, headers=headers)
|
r = requests.get(endpoint, headers=headers)
|
||||||
print(r.text)
|
print (r.text)
|
||||||
|
|
|
@ -1,28 +0,0 @@
|
||||||
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
|
|
||||||
|
|
||||||
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
|
|
||||||
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
|
|
||||||
|
|
||||||
COPY . ${HOME_DIR}/s3
|
|
||||||
RUN chown -R ${USER} ${HOME_DIR}
|
|
||||||
RUN pip3 install redis===3.5.3 requests==2.27.1 && \
|
|
||||||
apt-get install -y git-lfs
|
|
||||||
|
|
||||||
USER ${USER}
|
|
||||||
WORKDIR ${HOME_DIR}/s3
|
|
||||||
RUN rm -f ~/.gitconfig && \
|
|
||||||
git config --global --add safe.directory . && \
|
|
||||||
git lfs install && \
|
|
||||||
GIT_LFS_SKIP_SMUDGE=1 && \
|
|
||||||
yarn global add typescript && \
|
|
||||||
yarn install --frozen-lockfile --production --network-concurrency 1 && \
|
|
||||||
yarn cache clean --all && \
|
|
||||||
yarn global remove typescript
|
|
||||||
|
|
||||||
# run symlinking separately to avoid yarn installation errors
|
|
||||||
# we might have to check if the symlinking is really needed!
|
|
||||||
RUN ln -sf /scality-kms node_modules
|
|
||||||
|
|
||||||
EXPOSE 8000
|
|
||||||
|
|
||||||
CMD bash -c "source ${CONF_DIR}/env && export && supervisord -c ${CONF_DIR}/supervisord.conf"
|
|
7
index.js
7
index.js
|
@ -1,10 +1,3 @@
|
||||||
'use strict'; // eslint-disable-line strict
|
'use strict'; // eslint-disable-line strict
|
||||||
|
|
||||||
require('werelogs').stderrUtils.catchAndTimestampStderr(
|
|
||||||
undefined,
|
|
||||||
// Do not exit as workers have their own listener that will exit
|
|
||||||
// But primary don't have another listener
|
|
||||||
require('cluster').isPrimary ? 1 : null,
|
|
||||||
);
|
|
||||||
|
|
||||||
require('./lib/server.js')();
|
require('./lib/server.js')();
|
||||||
|
|
661
lib/Config.js
661
lib/Config.js
|
@ -8,18 +8,18 @@ const crypto = require('crypto');
|
||||||
const { v4: uuidv4 } = require('uuid');
|
const { v4: uuidv4 } = require('uuid');
|
||||||
const cronParser = require('cron-parser');
|
const cronParser = require('cron-parser');
|
||||||
const joi = require('@hapi/joi');
|
const joi = require('@hapi/joi');
|
||||||
const { s3routes, auth: arsenalAuth, s3middleware } = require('arsenal');
|
|
||||||
const { isValidBucketName } = s3routes.routesUtils;
|
const { isValidBucketName } = require('arsenal').s3routes.routesUtils;
|
||||||
const validateAuthConfig = arsenalAuth.inMemory.validateAuthConfig;
|
const validateAuthConfig = require('arsenal').auth.inMemory.validateAuthConfig;
|
||||||
const { buildAuthDataAccount } = require('./auth/in_memory/builder');
|
const { buildAuthDataAccount } = require('./auth/in_memory/builder');
|
||||||
const validExternalBackends = require('../constants').externalBackends;
|
const validExternalBackends = require('../constants').externalBackends;
|
||||||
const { azureAccountNameRegex, base64Regex,
|
const { azureAccountNameRegex, base64Regex,
|
||||||
allowedUtapiEventFilterFields, allowedUtapiEventFilterStates,
|
allowedUtapiEventFilterFields, allowedUtapiEventFilterStates,
|
||||||
} = require('../constants');
|
} = require('../constants');
|
||||||
const { utapiVersion } = require('utapi');
|
const { utapiVersion } = require('utapi');
|
||||||
const { scaleMsPerDay } = s3middleware.objectUtils;
|
const { versioning } = require('arsenal');
|
||||||
|
|
||||||
const constants = require('../constants');
|
const versionIdUtils = versioning.VersionID;
|
||||||
|
|
||||||
// config paths
|
// config paths
|
||||||
const configSearchPaths = [
|
const configSearchPaths = [
|
||||||
|
@ -107,47 +107,6 @@ function parseSproxydConfig(configSproxyd) {
|
||||||
return joi.attempt(configSproxyd, joiSchema, 'bad config');
|
return joi.attempt(configSproxyd, joiSchema, 'bad config');
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseRedisConfig(redisConfig) {
|
|
||||||
const joiSchema = joi.object({
|
|
||||||
password: joi.string().allow(''),
|
|
||||||
host: joi.string(),
|
|
||||||
port: joi.number(),
|
|
||||||
retry: joi.object({
|
|
||||||
connectBackoff: joi.object({
|
|
||||||
min: joi.number().required(),
|
|
||||||
max: joi.number().required(),
|
|
||||||
jitter: joi.number().required(),
|
|
||||||
factor: joi.number().required(),
|
|
||||||
deadline: joi.number().required(),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
// sentinel config
|
|
||||||
sentinels: joi.alternatives().try(
|
|
||||||
joi.string()
|
|
||||||
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
|
|
||||||
.custom(hosts => hosts.split(',').map(item => {
|
|
||||||
const [host, port] = item.split(':');
|
|
||||||
return { host, port: Number.parseInt(port, 10) };
|
|
||||||
})),
|
|
||||||
joi.array().items(
|
|
||||||
joi.object({
|
|
||||||
host: joi.string().required(),
|
|
||||||
port: joi.number().required(),
|
|
||||||
})
|
|
||||||
).min(1),
|
|
||||||
),
|
|
||||||
name: joi.string(),
|
|
||||||
sentinelPassword: joi.string().allow(''),
|
|
||||||
})
|
|
||||||
.and('host', 'port')
|
|
||||||
.and('sentinels', 'name')
|
|
||||||
.xor('host', 'sentinels')
|
|
||||||
.without('sentinels', ['host', 'port'])
|
|
||||||
.without('host', ['sentinels', 'sentinelPassword']);
|
|
||||||
|
|
||||||
return joi.attempt(redisConfig, joiSchema, 'bad config');
|
|
||||||
}
|
|
||||||
|
|
||||||
function restEndpointsAssert(restEndpoints, locationConstraints) {
|
function restEndpointsAssert(restEndpoints, locationConstraints) {
|
||||||
assert(typeof restEndpoints === 'object',
|
assert(typeof restEndpoints === 'object',
|
||||||
'bad config: restEndpoints must be an object of endpoints');
|
'bad config: restEndpoints must be an object of endpoints');
|
||||||
|
@ -178,71 +137,26 @@ function gcpLocationConstraintAssert(location, locationObj) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function azureGetStorageAccountName(location, locationDetails) {
|
function azureLocationConstraintAssert(location, locationObj) {
|
||||||
const { azureStorageAccountName } = locationDetails;
|
const {
|
||||||
|
azureStorageEndpoint,
|
||||||
|
azureStorageAccountName,
|
||||||
|
azureStorageAccessKey,
|
||||||
|
azureContainerName,
|
||||||
|
} = locationObj.details;
|
||||||
|
const storageEndpointFromEnv =
|
||||||
|
process.env[`${location}_AZURE_STORAGE_ENDPOINT`];
|
||||||
const storageAccountNameFromEnv =
|
const storageAccountNameFromEnv =
|
||||||
process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`];
|
process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`];
|
||||||
return storageAccountNameFromEnv || azureStorageAccountName;
|
const storageAccessKeyFromEnv =
|
||||||
}
|
process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`];
|
||||||
|
|
||||||
function azureGetLocationCredentials(location, locationDetails) {
|
|
||||||
const storageAccessKey =
|
|
||||||
process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`] ||
|
|
||||||
locationDetails.azureStorageAccessKey;
|
|
||||||
const sasToken =
|
|
||||||
process.env[`${location}_AZURE_SAS_TOKEN`] ||
|
|
||||||
locationDetails.sasToken;
|
|
||||||
const clientKey =
|
|
||||||
process.env[`${location}_AZURE_CLIENT_KEY`] ||
|
|
||||||
locationDetails.clientKey;
|
|
||||||
|
|
||||||
const authMethod =
|
|
||||||
process.env[`${location}_AZURE_AUTH_METHOD`] ||
|
|
||||||
locationDetails.authMethod ||
|
|
||||||
(storageAccessKey && 'shared-key') ||
|
|
||||||
(sasToken && 'shared-access-signature') ||
|
|
||||||
(clientKey && 'client-secret') ||
|
|
||||||
'shared-key';
|
|
||||||
|
|
||||||
switch (authMethod) {
|
|
||||||
case 'shared-key':
|
|
||||||
default:
|
|
||||||
return {
|
|
||||||
authMethod,
|
|
||||||
storageAccountName:
|
|
||||||
azureGetStorageAccountName(location, locationDetails),
|
|
||||||
storageAccessKey,
|
|
||||||
};
|
|
||||||
|
|
||||||
case 'shared-access-signature':
|
|
||||||
return {
|
|
||||||
authMethod,
|
|
||||||
sasToken,
|
|
||||||
};
|
|
||||||
|
|
||||||
case 'client-secret':
|
|
||||||
return {
|
|
||||||
authMethod,
|
|
||||||
tenantId:
|
|
||||||
process.env[`${location}_AZURE_TENANT_ID`] ||
|
|
||||||
locationDetails.tenantId,
|
|
||||||
clientId:
|
|
||||||
process.env[`${location}_AZURE_CLIENT_ID`] ||
|
|
||||||
locationDetails.clientId,
|
|
||||||
clientKey,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function azureLocationConstraintAssert(location, locationObj) {
|
|
||||||
const locationParams = {
|
const locationParams = {
|
||||||
...azureGetLocationCredentials(location, locationObj.details),
|
azureStorageEndpoint: storageEndpointFromEnv || azureStorageEndpoint,
|
||||||
azureStorageEndpoint:
|
azureStorageAccountName:
|
||||||
process.env[`${location}_AZURE_STORAGE_ENDPOINT`] ||
|
storageAccountNameFromEnv || azureStorageAccountName,
|
||||||
locationObj.details.azureStorageEndpoint,
|
azureStorageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
|
||||||
azureContainerName: locationObj.details.azureContainerName,
|
azureContainerName,
|
||||||
};
|
};
|
||||||
|
|
||||||
Object.keys(locationParams).forEach(param => {
|
Object.keys(locationParams).forEach(param => {
|
||||||
const value = locationParams[param];
|
const value = locationParams[param];
|
||||||
assert.notEqual(value, undefined,
|
assert.notEqual(value, undefined,
|
||||||
|
@ -252,16 +166,13 @@ function azureLocationConstraintAssert(location, locationObj) {
|
||||||
`bad location constraint: "${location}" ${param} ` +
|
`bad location constraint: "${location}" ${param} ` +
|
||||||
`"${value}" must be a string`);
|
`"${value}" must be a string`);
|
||||||
});
|
});
|
||||||
|
assert(azureAccountNameRegex.test(locationParams.azureStorageAccountName),
|
||||||
if (locationParams.authMethod === 'shared-key') {
|
`bad location constraint: "${location}" azureStorageAccountName ` +
|
||||||
assert(azureAccountNameRegex.test(locationParams.storageAccountName),
|
`"${locationParams.storageAccountName}" is an invalid value`);
|
||||||
`bad location constraint: "${location}" azureStorageAccountName ` +
|
assert(base64Regex.test(locationParams.azureStorageAccessKey),
|
||||||
`"${locationParams.storageAccountName}" is an invalid value`);
|
`bad location constraint: "${location}" ` +
|
||||||
assert(base64Regex.test(locationParams.storageAccessKey),
|
'azureStorageAccessKey is not a valid base64 string');
|
||||||
`bad location constraint: "${location}" ` +
|
assert(isValidBucketName(azureContainerName, []),
|
||||||
'azureStorageAccessKey is not a valid base64 string');
|
|
||||||
}
|
|
||||||
assert(isValidBucketName(locationParams.azureContainerName, []),
|
|
||||||
`bad location constraint: "${location}" ` +
|
`bad location constraint: "${location}" ` +
|
||||||
'azureContainerName is an invalid container name');
|
'azureContainerName is an invalid container name');
|
||||||
}
|
}
|
||||||
|
@ -280,104 +191,10 @@ function hdClientLocationConstraintAssert(configHd) {
|
||||||
return hdclientFields;
|
return hdclientFields;
|
||||||
}
|
}
|
||||||
|
|
||||||
function azureArchiveLocationConstraintAssert(locationObj) {
|
|
||||||
const checkedFields = [
|
|
||||||
'azureContainerName',
|
|
||||||
'azureStorageEndpoint',
|
|
||||||
];
|
|
||||||
if (Object.keys(locationObj.details).length === 0 ||
|
|
||||||
!checkedFields.every(field => field in locationObj.details)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const {
|
|
||||||
azureContainerName,
|
|
||||||
azureStorageEndpoint,
|
|
||||||
} = locationObj.details;
|
|
||||||
const stringFields = [
|
|
||||||
azureContainerName,
|
|
||||||
azureStorageEndpoint,
|
|
||||||
];
|
|
||||||
stringFields.forEach(field => {
|
|
||||||
assert(typeof field === 'string',
|
|
||||||
`bad config: ${field} must be a string`);
|
|
||||||
});
|
|
||||||
|
|
||||||
let hasAuthMethod = false;
|
|
||||||
if (locationObj.details.sasToken !== undefined) {
|
|
||||||
assert(typeof locationObj.details.sasToken === 'string',
|
|
||||||
`bad config: ${locationObj.details.sasToken} must be a string`);
|
|
||||||
hasAuthMethod = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (locationObj.details.azureStorageAccountName !== undefined &&
|
|
||||||
locationObj.details.azureStorageAccessKey !== undefined) {
|
|
||||||
assert(typeof locationObj.details.azureStorageAccountName === 'string',
|
|
||||||
`bad config: ${locationObj.details.azureStorageAccountName} must be a string`);
|
|
||||||
assert(typeof locationObj.details.azureStorageAccessKey === 'string',
|
|
||||||
`bad config: ${locationObj.details.azureStorageAccessKey} must be a string`);
|
|
||||||
assert(!hasAuthMethod, 'Multiple authentication methods are not allowed');
|
|
||||||
hasAuthMethod = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (locationObj.details.tenantId !== undefined &&
|
|
||||||
locationObj.details.clientId !== undefined &&
|
|
||||||
locationObj.details.clientKey !== undefined) {
|
|
||||||
assert(typeof locationObj.details.tenantId === 'string',
|
|
||||||
`bad config: ${locationObj.details.tenantId} must be a string`);
|
|
||||||
assert(typeof locationObj.details.clientId === 'string',
|
|
||||||
`bad config: ${locationObj.details.clientId} must be a string`);
|
|
||||||
assert(typeof locationObj.details.clientKey === 'string',
|
|
||||||
`bad config: ${locationObj.details.clientKey} must be a string`);
|
|
||||||
assert(!hasAuthMethod, 'Multiple authentication methods are not allowed');
|
|
||||||
hasAuthMethod = true;
|
|
||||||
}
|
|
||||||
assert(hasAuthMethod, 'Missing authentication method');
|
|
||||||
}
|
|
||||||
|
|
||||||
function dmfLocationConstraintAssert(locationObj) {
|
|
||||||
const checkedFields = [
|
|
||||||
'endpoint',
|
|
||||||
'username',
|
|
||||||
'password',
|
|
||||||
'repoId',
|
|
||||||
'nsId',
|
|
||||||
];
|
|
||||||
if (Object.keys(locationObj.details).length === 0 ||
|
|
||||||
!checkedFields.every(field => field in locationObj.details)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const {
|
|
||||||
endpoint,
|
|
||||||
username,
|
|
||||||
password,
|
|
||||||
repoId,
|
|
||||||
nsId,
|
|
||||||
} = locationObj.details;
|
|
||||||
const stringFields = [
|
|
||||||
endpoint,
|
|
||||||
username,
|
|
||||||
password,
|
|
||||||
nsId,
|
|
||||||
];
|
|
||||||
stringFields.forEach(field => {
|
|
||||||
assert(typeof field === 'string',
|
|
||||||
`bad config: ${field} must be a string`);
|
|
||||||
});
|
|
||||||
assert.strictEqual(Array.isArray(repoId), true);
|
|
||||||
repoId.forEach(rId => {
|
|
||||||
assert(typeof rId === 'string',
|
|
||||||
`bad config: ${rId} must be a string`);
|
|
||||||
});
|
|
||||||
assert(repoId.every(
|
|
||||||
r => typeof r === 'string'),
|
|
||||||
'bad config: each repoId must be a string',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function locationConstraintAssert(locationConstraints) {
|
function locationConstraintAssert(locationConstraints) {
|
||||||
const supportedBackends =
|
const supportedBackends =
|
||||||
['mem', 'file', 'scality',
|
['mem', 'file', 'scality',
|
||||||
'mongodb', 'dmf', 'azure_archive', 'vitastor'].concat(Object.keys(validExternalBackends));
|
'mongodb'].concat(Object.keys(validExternalBackends));
|
||||||
assert(typeof locationConstraints === 'object',
|
assert(typeof locationConstraints === 'object',
|
||||||
'bad config: locationConstraints must be an object');
|
'bad config: locationConstraints must be an object');
|
||||||
Object.keys(locationConstraints).forEach(l => {
|
Object.keys(locationConstraints).forEach(l => {
|
||||||
|
@ -485,12 +302,6 @@ function locationConstraintAssert(locationConstraints) {
|
||||||
if (locationConstraints[l].type === 'gcp') {
|
if (locationConstraints[l].type === 'gcp') {
|
||||||
gcpLocationConstraintAssert(l, locationConstraints[l]);
|
gcpLocationConstraintAssert(l, locationConstraints[l]);
|
||||||
}
|
}
|
||||||
if (locationConstraints[l].type === 'dmf') {
|
|
||||||
dmfLocationConstraintAssert(locationConstraints[l]);
|
|
||||||
}
|
|
||||||
if (locationConstraints[l].type === 'azure_archive') {
|
|
||||||
azureArchiveLocationConstraintAssert(locationConstraints[l]);
|
|
||||||
}
|
|
||||||
if (locationConstraints[l].type === 'pfs') {
|
if (locationConstraints[l].type === 'pfs') {
|
||||||
assert(typeof details.pfsDaemonEndpoint === 'object',
|
assert(typeof details.pfsDaemonEndpoint === 'object',
|
||||||
'bad config: pfsDaemonEndpoint is mandatory and must be an object');
|
'bad config: pfsDaemonEndpoint is mandatory and must be an object');
|
||||||
|
@ -502,33 +313,26 @@ function locationConstraintAssert(locationConstraints) {
|
||||||
locationConstraints[l].details.connector.hdclient);
|
locationConstraints[l].details.connector.hdclient);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
assert(Object.keys(locationConstraints)
|
||||||
|
.includes('us-east-1'), 'bad locationConfig: must ' +
|
||||||
|
'include us-east-1 as a locationConstraint');
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseUtapiReindex(config) {
|
function parseUtapiReindex({ enabled, schedule, sentinel, bucketd }) {
|
||||||
const {
|
|
||||||
enabled,
|
|
||||||
schedule,
|
|
||||||
redis,
|
|
||||||
bucketd,
|
|
||||||
onlyCountLatestWhenObjectLocked,
|
|
||||||
} = config;
|
|
||||||
assert(typeof enabled === 'boolean',
|
assert(typeof enabled === 'boolean',
|
||||||
'bad config: utapi.reindex.enabled must be a boolean');
|
'bad config: utapi.reindex.enabled must be a boolean');
|
||||||
|
assert(typeof sentinel === 'object',
|
||||||
const parsedRedis = parseRedisConfig(redis);
|
'bad config: utapi.reindex.sentinel must be an object');
|
||||||
assert(Array.isArray(parsedRedis.sentinels),
|
assert(typeof sentinel.port === 'number',
|
||||||
'bad config: utapi reindex redis config requires a list of sentinels');
|
'bad config: utapi.reindex.sentinel.port must be a number');
|
||||||
|
assert(typeof sentinel.name === 'string',
|
||||||
|
'bad config: utapi.reindex.sentinel.name must be a string');
|
||||||
assert(typeof bucketd === 'object',
|
assert(typeof bucketd === 'object',
|
||||||
'bad config: utapi.reindex.bucketd must be an object');
|
'bad config: utapi.reindex.bucketd must be an object');
|
||||||
assert(typeof bucketd.port === 'number',
|
assert(typeof bucketd.port === 'number',
|
||||||
'bad config: utapi.reindex.bucketd.port must be a number');
|
'bad config: utapi.reindex.bucketd.port must be a number');
|
||||||
assert(typeof schedule === 'string',
|
assert(typeof schedule === 'string',
|
||||||
'bad config: utapi.reindex.schedule must be a string');
|
'bad config: utapi.reindex.schedule must be a string');
|
||||||
if (onlyCountLatestWhenObjectLocked !== undefined) {
|
|
||||||
assert(typeof onlyCountLatestWhenObjectLocked === 'boolean',
|
|
||||||
'bad config: utapi.reindex.onlyCountLatestWhenObjectLocked must be a boolean');
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
cronParser.parseExpression(schedule);
|
cronParser.parseExpression(schedule);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
@ -536,13 +340,6 @@ function parseUtapiReindex(config) {
|
||||||
'bad config: utapi.reindex.schedule must be a valid ' +
|
'bad config: utapi.reindex.schedule must be a valid ' +
|
||||||
`cron schedule. ${e.message}.`);
|
`cron schedule. ${e.message}.`);
|
||||||
}
|
}
|
||||||
return {
|
|
||||||
enabled,
|
|
||||||
schedule,
|
|
||||||
redis: parsedRedis,
|
|
||||||
bucketd,
|
|
||||||
onlyCountLatestWhenObjectLocked,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function requestsConfigAssert(requestsConfig) {
|
function requestsConfigAssert(requestsConfig) {
|
||||||
|
@ -630,6 +427,7 @@ class Config extends EventEmitter {
|
||||||
// Read config automatically
|
// Read config automatically
|
||||||
this._getLocationConfig();
|
this._getLocationConfig();
|
||||||
this._getConfig();
|
this._getConfig();
|
||||||
|
this._configureBackends();
|
||||||
}
|
}
|
||||||
|
|
||||||
_getLocationConfig() {
|
_getLocationConfig() {
|
||||||
|
@ -762,6 +560,12 @@ class Config extends EventEmitter {
|
||||||
const { replicationEndpoints } = config;
|
const { replicationEndpoints } = config;
|
||||||
assert(replicationEndpoints instanceof Array, 'bad config: ' +
|
assert(replicationEndpoints instanceof Array, 'bad config: ' +
|
||||||
'`replicationEndpoints` property must be an array');
|
'`replicationEndpoints` property must be an array');
|
||||||
|
// if (replicationEndpoints.length > 1) {
|
||||||
|
// const hasDefault = replicationEndpoints.some(
|
||||||
|
// replicationEndpoint => replicationEndpoint.default);
|
||||||
|
// assert(hasDefault, 'bad config: `replicationEndpoints` must ' +
|
||||||
|
// 'contain a default endpoint');
|
||||||
|
// }
|
||||||
replicationEndpoints.forEach(replicationEndpoint => {
|
replicationEndpoints.forEach(replicationEndpoint => {
|
||||||
assert.strictEqual(typeof replicationEndpoint, 'object',
|
assert.strictEqual(typeof replicationEndpoint, 'object',
|
||||||
'bad config: `replicationEndpoints` property must be an ' +
|
'bad config: `replicationEndpoints` property must be an ' +
|
||||||
|
@ -779,7 +583,7 @@ class Config extends EventEmitter {
|
||||||
assert(validExternalBackends[type], 'bad config: `type` ' +
|
assert(validExternalBackends[type], 'bad config: `type` ' +
|
||||||
'property of `replicationEndpoints` object must be ' +
|
'property of `replicationEndpoints` object must be ' +
|
||||||
'a valid external backend (one of: "' +
|
'a valid external backend (one of: "' +
|
||||||
`${Object.keys(validExternalBackends).join('", "')}")`);
|
`${Object.keys(validExternalBackends).join('", "')})`);
|
||||||
} else {
|
} else {
|
||||||
assert.notStrictEqual(servers, undefined, 'bad config: ' +
|
assert.notStrictEqual(servers, undefined, 'bad config: ' +
|
||||||
'each object of `replicationEndpoints` array that is ' +
|
'each object of `replicationEndpoints` array that is ' +
|
||||||
|
@ -841,11 +645,11 @@ class Config extends EventEmitter {
|
||||||
this.websiteEndpoints = config.websiteEndpoints;
|
this.websiteEndpoints = config.websiteEndpoints;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.workers = false;
|
this.clusters = false;
|
||||||
if (config.workers !== undefined) {
|
if (config.clusters !== undefined) {
|
||||||
assert(Number.isInteger(config.workers) && config.workers > 0,
|
assert(Number.isInteger(config.clusters) && config.clusters > 0,
|
||||||
'bad config: workers must be a positive integer');
|
'bad config: clusters must be a positive integer');
|
||||||
this.workers = config.workers;
|
this.clusters = config.clusters;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.usEastBehavior !== undefined) {
|
if (config.usEastBehavior !== undefined) {
|
||||||
|
@ -1083,7 +887,8 @@ class Config extends EventEmitter {
|
||||||
assert(typeof config.localCache.port === 'number',
|
assert(typeof config.localCache.port === 'number',
|
||||||
'config: bad port for localCache. port must be a number');
|
'config: bad port for localCache. port must be a number');
|
||||||
if (config.localCache.password !== undefined) {
|
if (config.localCache.password !== undefined) {
|
||||||
assert(typeof config.localCache.password === 'string',
|
assert(
|
||||||
|
this._verifyRedisPassword(config.localCache.password),
|
||||||
'config: vad password for localCache. password must' +
|
'config: vad password for localCache. password must' +
|
||||||
' be a string');
|
' be a string');
|
||||||
}
|
}
|
||||||
|
@ -1109,46 +914,56 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.redis) {
|
if (config.redis) {
|
||||||
this.redis = parseRedisConfig(config.redis);
|
if (config.redis.sentinels) {
|
||||||
}
|
this.redis = { sentinels: [], name: null };
|
||||||
if (config.scuba) {
|
|
||||||
this.scuba = {};
|
assert(typeof config.redis.name === 'string',
|
||||||
if (config.scuba.host) {
|
'bad config: redis sentinel name must be a string');
|
||||||
assert(typeof config.scuba.host === 'string',
|
this.redis.name = config.redis.name;
|
||||||
'bad config: scuba host must be a string');
|
assert(Array.isArray(config.redis.sentinels) ||
|
||||||
this.scuba.host = config.scuba.host;
|
typeof config.redis.sentinels === 'string',
|
||||||
|
'bad config: redis sentinels must be an array or string');
|
||||||
|
|
||||||
|
if (typeof config.redis.sentinels === 'string') {
|
||||||
|
config.redis.sentinels.split(',').forEach(item => {
|
||||||
|
const [host, port] = item.split(':');
|
||||||
|
this.redis.sentinels.push({ host,
|
||||||
|
port: Number.parseInt(port, 10) });
|
||||||
|
});
|
||||||
|
} else if (Array.isArray(config.redis.sentinels)) {
|
||||||
|
config.redis.sentinels.forEach(item => {
|
||||||
|
const { host, port } = item;
|
||||||
|
assert(typeof host === 'string',
|
||||||
|
'bad config: redis sentinel host must be a string');
|
||||||
|
assert(typeof port === 'number',
|
||||||
|
'bad config: redis sentinel port must be a number');
|
||||||
|
this.redis.sentinels.push({ host, port });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.redis.sentinelPassword !== undefined) {
|
||||||
|
assert(
|
||||||
|
this._verifyRedisPassword(config.redis.sentinelPassword));
|
||||||
|
this.redis.sentinelPassword = config.redis.sentinelPassword;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// check for standalone configuration
|
||||||
|
this.redis = {};
|
||||||
|
assert(typeof config.redis.host === 'string',
|
||||||
|
'bad config: redis.host must be a string');
|
||||||
|
assert(typeof config.redis.port === 'number',
|
||||||
|
'bad config: redis.port must be a number');
|
||||||
|
this.redis.host = config.redis.host;
|
||||||
|
this.redis.port = config.redis.port;
|
||||||
}
|
}
|
||||||
if (config.scuba.port) {
|
if (config.redis.password !== undefined) {
|
||||||
assert(Number.isInteger(config.scuba.port)
|
assert(
|
||||||
&& config.scuba.port > 0,
|
this._verifyRedisPassword(config.redis.password),
|
||||||
'bad config: scuba port must be a positive integer');
|
'bad config: invalid password for redis. password must ' +
|
||||||
this.scuba.port = config.scuba.port;
|
'be a string');
|
||||||
|
this.redis.password = config.redis.password;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (process.env.SCUBA_HOST && process.env.SCUBA_PORT) {
|
|
||||||
assert(typeof process.env.SCUBA_HOST === 'string',
|
|
||||||
'bad config: scuba host must be a string');
|
|
||||||
assert(Number.isInteger(Number(process.env.SCUBA_PORT))
|
|
||||||
&& Number(process.env.SCUBA_PORT) > 0,
|
|
||||||
'bad config: scuba port must be a positive integer');
|
|
||||||
this.scuba = {
|
|
||||||
host: process.env.SCUBA_HOST,
|
|
||||||
port: Number(process.env.SCUBA_PORT),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (this.scuba) {
|
|
||||||
this.quotaEnabled = true;
|
|
||||||
}
|
|
||||||
const maxStaleness = Number(process.env.QUOTA_MAX_STALENESS_MS) ||
|
|
||||||
config.quota?.maxStatenessMS ||
|
|
||||||
24 * 60 * 60 * 1000;
|
|
||||||
assert(Number.isInteger(maxStaleness), 'bad config: maxStalenessMS must be an integer');
|
|
||||||
const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' ||
|
|
||||||
config.quota?.enableInflights || false;
|
|
||||||
this.quota = {
|
|
||||||
maxStaleness,
|
|
||||||
enableInflights,
|
|
||||||
};
|
|
||||||
if (config.utapi) {
|
if (config.utapi) {
|
||||||
this.utapi = { component: 's3' };
|
this.utapi = { component: 's3' };
|
||||||
if (config.utapi.host) {
|
if (config.utapi.host) {
|
||||||
|
@ -1177,8 +992,50 @@ class Config extends EventEmitter {
|
||||||
assert(config.redis, 'missing required property of utapi ' +
|
assert(config.redis, 'missing required property of utapi ' +
|
||||||
'configuration: redis');
|
'configuration: redis');
|
||||||
if (config.utapi.redis) {
|
if (config.utapi.redis) {
|
||||||
this.utapi.redis = parseRedisConfig(config.utapi.redis);
|
if (config.utapi.redis.sentinels) {
|
||||||
if (this.utapi.redis.retry === undefined) {
|
this.utapi.redis = { sentinels: [], name: null };
|
||||||
|
|
||||||
|
assert(typeof config.utapi.redis.name === 'string',
|
||||||
|
'bad config: redis sentinel name must be a string');
|
||||||
|
this.utapi.redis.name = config.utapi.redis.name;
|
||||||
|
|
||||||
|
assert(Array.isArray(config.utapi.redis.sentinels),
|
||||||
|
'bad config: redis sentinels must be an array');
|
||||||
|
config.utapi.redis.sentinels.forEach(item => {
|
||||||
|
const { host, port } = item;
|
||||||
|
assert(typeof host === 'string',
|
||||||
|
'bad config: redis sentinel host must be a string');
|
||||||
|
assert(typeof port === 'number',
|
||||||
|
'bad config: redis sentinel port must be a number');
|
||||||
|
this.utapi.redis.sentinels.push({ host, port });
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// check for standalone configuration
|
||||||
|
this.utapi.redis = {};
|
||||||
|
assert(typeof config.utapi.redis.host === 'string',
|
||||||
|
'bad config: redis.host must be a string');
|
||||||
|
assert(typeof config.utapi.redis.port === 'number',
|
||||||
|
'bad config: redis.port must be a number');
|
||||||
|
this.utapi.redis.host = config.utapi.redis.host;
|
||||||
|
this.utapi.redis.port = config.utapi.redis.port;
|
||||||
|
}
|
||||||
|
if (config.utapi.redis.retry !== undefined) {
|
||||||
|
if (config.utapi.redis.retry.connectBackoff !== undefined) {
|
||||||
|
const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
|
||||||
|
assert.strictEqual(typeof min, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: min must be a number');
|
||||||
|
assert.strictEqual(typeof max, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: max must be a number');
|
||||||
|
assert.strictEqual(typeof jitter, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: jitter must be a number');
|
||||||
|
assert.strictEqual(typeof factor, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: factor must be a number');
|
||||||
|
assert.strictEqual(typeof deadline, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: deadline must be a number');
|
||||||
|
}
|
||||||
|
|
||||||
|
this.utapi.redis.retry = config.utapi.redis.retry;
|
||||||
|
} else {
|
||||||
this.utapi.redis.retry = {
|
this.utapi.redis.retry = {
|
||||||
connectBackoff: {
|
connectBackoff: {
|
||||||
min: 10,
|
min: 10,
|
||||||
|
@ -1189,6 +1046,22 @@ class Config extends EventEmitter {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (config.utapi.redis.password !== undefined) {
|
||||||
|
assert(
|
||||||
|
this._verifyRedisPassword(config.utapi.redis.password),
|
||||||
|
'config: invalid password for utapi redis. password' +
|
||||||
|
' must be a string');
|
||||||
|
this.utapi.redis.password = config.utapi.redis.password;
|
||||||
|
}
|
||||||
|
if (config.utapi.redis.sentinelPassword !== undefined) {
|
||||||
|
assert(
|
||||||
|
this._verifyRedisPassword(
|
||||||
|
config.utapi.redis.sentinelPassword),
|
||||||
|
'config: invalid password for utapi redis. password' +
|
||||||
|
' must be a string');
|
||||||
|
this.utapi.redis.sentinelPassword =
|
||||||
|
config.utapi.redis.sentinelPassword;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (config.utapi.metrics) {
|
if (config.utapi.metrics) {
|
||||||
this.utapi.metrics = config.utapi.metrics;
|
this.utapi.metrics = config.utapi.metrics;
|
||||||
|
@ -1258,7 +1131,8 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.utapi && config.utapi.reindex) {
|
if (config.utapi && config.utapi.reindex) {
|
||||||
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
|
parseUtapiReindex(config.utapi.reindex);
|
||||||
|
this.utapi.reindex = config.utapi.reindex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1303,8 +1177,6 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.authdata = config.authdata || 'authdata.json';
|
|
||||||
|
|
||||||
this.kms = {};
|
this.kms = {};
|
||||||
if (config.kms) {
|
if (config.kms) {
|
||||||
assert(typeof config.kms.userName === 'string');
|
assert(typeof config.kms.userName === 'string');
|
||||||
|
@ -1524,6 +1396,25 @@ class Config extends EventEmitter {
|
||||||
this.outboundProxy.certs = certObj.certs;
|
this.outboundProxy.certs = certObj.certs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.managementAgent = {};
|
||||||
|
this.managementAgent.port = 8010;
|
||||||
|
this.managementAgent.host = 'localhost';
|
||||||
|
if (config.managementAgent !== undefined) {
|
||||||
|
if (config.managementAgent.port !== undefined) {
|
||||||
|
assert(Number.isInteger(config.managementAgent.port)
|
||||||
|
&& config.managementAgent.port > 0,
|
||||||
|
'bad config: managementAgent port must be a positive ' +
|
||||||
|
'integer');
|
||||||
|
this.managementAgent.port = config.managementAgent.port;
|
||||||
|
}
|
||||||
|
if (config.managementAgent.host !== undefined) {
|
||||||
|
assert.strictEqual(typeof config.managementAgent.host, 'string',
|
||||||
|
'bad config: management agent host must ' +
|
||||||
|
'be a string');
|
||||||
|
this.managementAgent.host = config.managementAgent.host;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Ephemeral token to protect the reporting endpoint:
|
// Ephemeral token to protect the reporting endpoint:
|
||||||
// try inherited from parent first, then hardcoded in conf file,
|
// try inherited from parent first, then hardcoded in conf file,
|
||||||
// then create a fresh one as last resort.
|
// then create a fresh one as last resort.
|
||||||
|
@ -1576,120 +1467,64 @@ class Config extends EventEmitter {
|
||||||
requestsConfigAssert(config.requests);
|
requestsConfigAssert(config.requests);
|
||||||
this.requests = config.requests;
|
this.requests = config.requests;
|
||||||
}
|
}
|
||||||
// CLDSRV-378: on 8.x branches, null version compatibility
|
if (process.env.VERSION_ID_ENCODING_TYPE !== undefined) {
|
||||||
// mode is enforced because null keys are not supported by the
|
// override config
|
||||||
// MongoDB backend.
|
config.versionIdEncodingType = process.env.VERSION_ID_ENCODING_TYPE;
|
||||||
this.nullVersionCompatMode = true;
|
}
|
||||||
|
if (config.versionIdEncodingType) {
|
||||||
|
if (config.versionIdEncodingType === 'hex') {
|
||||||
|
this.versionIdEncodingType = versionIdUtils.ENC_TYPE_HEX;
|
||||||
|
} else if (config.versionIdEncodingType === 'base62') {
|
||||||
|
this.versionIdEncodingType = versionIdUtils.ENC_TYPE_BASE62;
|
||||||
|
} else {
|
||||||
|
throw new Error(`Invalid versionIdEncodingType: ${config.versionIdEncodingType}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.versionIdEncodingType = versionIdUtils.ENC_TYPE_HEX;
|
||||||
|
}
|
||||||
if (config.bucketNotificationDestinations) {
|
if (config.bucketNotificationDestinations) {
|
||||||
this.bucketNotificationDestinations = bucketNotifAssert(config.bucketNotificationDestinations);
|
this.bucketNotificationDestinations = bucketNotifAssert(config.bucketNotificationDestinations);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.lifecycleRoleName = config.lifecycleRoleName || null;
|
|
||||||
|
|
||||||
// Version of the configuration we're running under
|
// Version of the configuration we're running under
|
||||||
this.overlayVersion = config.overlayVersion || 0;
|
this.overlayVersion = config.overlayVersion || 0;
|
||||||
|
|
||||||
this._setTimeOptions();
|
|
||||||
this.multiObjectDeleteConcurrency = constants.multiObjectDeleteConcurrency;
|
|
||||||
const extractedNumber = Number.parseInt(config.multiObjectDeleteConcurrency, 10);
|
|
||||||
if (!isNaN(extractedNumber) && extractedNumber > 0 && extractedNumber < 1000) {
|
|
||||||
this.multiObjectDeleteConcurrency = extractedNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.multiObjectDeleteEnableOptimizations = true;
|
|
||||||
if (config.multiObjectDeleteEnableOptimizations === false) {
|
|
||||||
this.multiObjectDeleteEnableOptimizations = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.testingMode = config.testingMode || false;
|
|
||||||
|
|
||||||
this.maxScannedLifecycleListingEntries = constants.maxScannedLifecycleListingEntries;
|
|
||||||
if (config.maxScannedLifecycleListingEntries !== undefined) {
|
|
||||||
// maxScannedLifecycleListingEntries > 2 is required as a minimum because we must
|
|
||||||
// scan at least three entries to determine version eligibility.
|
|
||||||
// Two entries representing the master key and the following one representing the non-current version.
|
|
||||||
assert(Number.isInteger(config.maxScannedLifecycleListingEntries) &&
|
|
||||||
config.maxScannedLifecycleListingEntries > 2,
|
|
||||||
'bad config: maxScannedLifecycleListingEntries must be greater than 2');
|
|
||||||
this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
|
|
||||||
}
|
|
||||||
|
|
||||||
this._configureBackends(config);
|
|
||||||
}
|
|
||||||
|
|
||||||
_setTimeOptions() {
|
|
||||||
// NOTE: EXPIRE_ONE_DAY_EARLIER and TRANSITION_ONE_DAY_EARLIER are deprecated in favor of
|
|
||||||
// TIME_PROGRESSION_FACTOR which decreases the weight attributed to a day in order to among other things
|
|
||||||
// expedite the lifecycle of objects.
|
|
||||||
|
|
||||||
// moves lifecycle expiration deadlines 1 day earlier, mostly for testing
|
|
||||||
const expireOneDayEarlier = process.env.EXPIRE_ONE_DAY_EARLIER === 'true';
|
|
||||||
// moves lifecycle transition deadlines 1 day earlier, mostly for testing
|
|
||||||
const transitionOneDayEarlier = process.env.TRANSITION_ONE_DAY_EARLIER === 'true';
|
|
||||||
// decreases the weight attributed to a day in order to expedite the lifecycle of objects.
|
|
||||||
const timeProgressionFactor = Number.parseInt(process.env.TIME_PROGRESSION_FACTOR, 10) || 1;
|
|
||||||
|
|
||||||
const isIncompatible = (expireOneDayEarlier || transitionOneDayEarlier) && (timeProgressionFactor > 1);
|
|
||||||
assert(!isIncompatible, 'The environment variables "EXPIRE_ONE_DAY_EARLIER" or ' +
|
|
||||||
'"TRANSITION_ONE_DAY_EARLIER" are not compatible with the "TIME_PROGRESSION_FACTOR" variable.');
|
|
||||||
|
|
||||||
// The scaledMsPerDay value is initially set to the number of milliseconds per day
|
|
||||||
// (24 * 60 * 60 * 1000) as the default value.
|
|
||||||
// However, during testing, if the timeProgressionFactor is defined and greater than 1,
|
|
||||||
// the scaledMsPerDay value is decreased. This adjustment allows for simulating actions occurring
|
|
||||||
// earlier in time.
|
|
||||||
const scaledMsPerDay = scaleMsPerDay(timeProgressionFactor);
|
|
||||||
|
|
||||||
this.timeOptions = {
|
|
||||||
expireOneDayEarlier,
|
|
||||||
transitionOneDayEarlier,
|
|
||||||
timeProgressionFactor,
|
|
||||||
scaledMsPerDay,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
getTimeOptions() {
|
|
||||||
return this.timeOptions;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_getAuthData() {
|
_getAuthData() {
|
||||||
return JSON.parse(fs.readFileSync(findConfigFile(process.env.S3AUTH_CONFIG || this.authdata), { encoding: 'utf-8' }));
|
return require(findConfigFile(process.env.S3AUTH_CONFIG || 'authdata.json'));
|
||||||
}
|
}
|
||||||
|
|
||||||
_configureBackends(config) {
|
_configureBackends() {
|
||||||
const backends = config.backends || {};
|
|
||||||
/**
|
/**
|
||||||
* Configure the backends for Authentication, Data and Metadata.
|
* Configure the backends for Authentication, Data and Metadata.
|
||||||
*/
|
*/
|
||||||
let auth = backends.auth || 'mem';
|
let auth = 'mem';
|
||||||
let data = backends.data || 'multiple';
|
let data = 'multiple';
|
||||||
let metadata = backends.metadata || 'file';
|
let metadata = 'file';
|
||||||
let kms = backends.kms || 'file';
|
let kms = 'file';
|
||||||
let quota = backends.quota || 'none';
|
|
||||||
if (process.env.S3BACKEND) {
|
if (process.env.S3BACKEND) {
|
||||||
const validBackends = ['mem', 'file', 'scality', 'cdmi'];
|
const validBackends = ['mem', 'file', 'scality', 'cdmi'];
|
||||||
assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
|
assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
|
||||||
'bad environment variable: S3BACKEND environment variable ' +
|
'bad environment variable: S3BACKEND environment variable ' +
|
||||||
'should be one of mem/file/scality/cdmi'
|
'should be one of mem/file/scality/cdmi'
|
||||||
);
|
);
|
||||||
auth = process.env.S3BACKEND == 'scality' ? 'scality' : 'mem';
|
auth = process.env.S3BACKEND;
|
||||||
data = process.env.S3BACKEND;
|
data = process.env.S3BACKEND;
|
||||||
metadata = process.env.S3BACKEND;
|
metadata = process.env.S3BACKEND;
|
||||||
kms = process.env.S3BACKEND;
|
kms = process.env.S3BACKEND;
|
||||||
}
|
}
|
||||||
if (process.env.S3VAULT) {
|
if (process.env.S3VAULT) {
|
||||||
auth = process.env.S3VAULT;
|
auth = process.env.S3VAULT;
|
||||||
auth = (auth === 'file' || auth === 'mem' || auth === 'cdmi' ? 'mem' : auth);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
|
if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
|
||||||
// Auth only checks for 'mem' since mem === file
|
// Auth only checks for 'mem' since mem === file
|
||||||
|
auth = 'mem';
|
||||||
let authData;
|
let authData;
|
||||||
if (process.env.SCALITY_ACCESS_KEY_ID &&
|
if (process.env.SCALITY_ACCESS_KEY_ID &&
|
||||||
process.env.SCALITY_SECRET_ACCESS_KEY) {
|
process.env.SCALITY_SECRET_ACCESS_KEY) {
|
||||||
authData = buildAuthDataAccount(
|
authData = buildAuthDataAccount(
|
||||||
process.env.SCALITY_ACCESS_KEY_ID,
|
process.env.SCALITY_ACCESS_KEY_ID,
|
||||||
process.env.SCALITY_SECRET_ACCESS_KEY);
|
process.env.SCALITY_SECRET_ACCESS_KEY);
|
||||||
} else {
|
} else {
|
||||||
authData = this._getAuthData();
|
authData = this._getAuthData();
|
||||||
}
|
}
|
||||||
|
@ -1697,7 +1532,7 @@ class Config extends EventEmitter {
|
||||||
throw new Error('bad config: invalid auth config file.');
|
throw new Error('bad config: invalid auth config file.');
|
||||||
}
|
}
|
||||||
this.authData = authData;
|
this.authData = authData;
|
||||||
} else if (auth === 'multiple') {
|
} else if (auth === 'multiple') {
|
||||||
const authData = this._getAuthData();
|
const authData = this._getAuthData();
|
||||||
if (validateAuthConfig(authData)) {
|
if (validateAuthConfig(authData)) {
|
||||||
throw new Error('bad config: invalid auth config file.');
|
throw new Error('bad config: invalid auth config file.');
|
||||||
|
@ -1712,9 +1547,9 @@ class Config extends EventEmitter {
|
||||||
'should be one of mem/file/scality/multiple'
|
'should be one of mem/file/scality/multiple'
|
||||||
);
|
);
|
||||||
data = process.env.S3DATA;
|
data = process.env.S3DATA;
|
||||||
if (data === 'scality' || data === 'multiple') {
|
}
|
||||||
data = 'multiple';
|
if (data === 'scality' || data === 'multiple') {
|
||||||
}
|
data = 'multiple';
|
||||||
}
|
}
|
||||||
assert(this.locationConstraints !== undefined &&
|
assert(this.locationConstraints !== undefined &&
|
||||||
this.restEndpoints !== undefined,
|
this.restEndpoints !== undefined,
|
||||||
|
@ -1727,18 +1562,18 @@ class Config extends EventEmitter {
|
||||||
if (process.env.S3KMS) {
|
if (process.env.S3KMS) {
|
||||||
kms = process.env.S3KMS;
|
kms = process.env.S3KMS;
|
||||||
}
|
}
|
||||||
if (process.env.S3QUOTA) {
|
|
||||||
quota = process.env.S3QUOTA;
|
|
||||||
}
|
|
||||||
this.backends = {
|
this.backends = {
|
||||||
auth,
|
auth,
|
||||||
data,
|
data,
|
||||||
metadata,
|
metadata,
|
||||||
kms,
|
kms,
|
||||||
quota,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_verifyRedisPassword(password) {
|
||||||
|
return typeof password === 'string';
|
||||||
|
}
|
||||||
|
|
||||||
setAuthDataAccounts(accounts) {
|
setAuthDataAccounts(accounts) {
|
||||||
this.authData.accounts = accounts;
|
this.authData.accounts = accounts;
|
||||||
this.emit('authdata-update');
|
this.emit('authdata-update');
|
||||||
|
@ -1785,8 +1620,9 @@ class Config extends EventEmitter {
|
||||||
|
|
||||||
getAzureEndpoint(locationConstraint) {
|
getAzureEndpoint(locationConstraint) {
|
||||||
let azureStorageEndpoint =
|
let azureStorageEndpoint =
|
||||||
process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] ||
|
process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] ||
|
||||||
this.locationConstraints[locationConstraint].details.azureStorageEndpoint;
|
this.locationConstraints[locationConstraint]
|
||||||
|
.details.azureStorageEndpoint;
|
||||||
if (!azureStorageEndpoint.endsWith('/')) {
|
if (!azureStorageEndpoint.endsWith('/')) {
|
||||||
// append the trailing slash
|
// append the trailing slash
|
||||||
azureStorageEndpoint = `${azureStorageEndpoint}/`;
|
azureStorageEndpoint = `${azureStorageEndpoint}/`;
|
||||||
|
@ -1795,40 +1631,23 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
getAzureStorageAccountName(locationConstraint) {
|
getAzureStorageAccountName(locationConstraint) {
|
||||||
const accountName = azureGetStorageAccountName(
|
const { azureStorageAccountName } =
|
||||||
locationConstraint,
|
this.locationConstraints[locationConstraint].details;
|
||||||
this.locationConstraints[locationConstraint].details
|
const storageAccountNameFromEnv =
|
||||||
);
|
process.env[`${locationConstraint}_AZURE_STORAGE_ACCOUNT_NAME`];
|
||||||
if (accountName) {
|
return storageAccountNameFromEnv || azureStorageAccountName;
|
||||||
return accountName;
|
|
||||||
}
|
|
||||||
|
|
||||||
// For SAS & ServicePrincipal, retrieve the accountName from the endpoint
|
|
||||||
const endpoint = this.getAzureEndpoint(locationConstraint);
|
|
||||||
const url = new URL(endpoint);
|
|
||||||
const fragments = url.hostname.split('.', 3);
|
|
||||||
if (fragments.length === 3 && fragments[1] === 'blob') {
|
|
||||||
return fragments[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
// We should always reach here, though it may not be the case for "mock" servers,
|
|
||||||
// where the accoutName is in the path
|
|
||||||
const path = url.pathname.replace(/^\//, '').replace(/\/$/, '');
|
|
||||||
if (path) {
|
|
||||||
return path;
|
|
||||||
}
|
|
||||||
|
|
||||||
// We should not reach here; if that happens, use the endpoint itself, which
|
|
||||||
// should be close-enough since this function is used for detecting when two
|
|
||||||
// locations actually point to the same account
|
|
||||||
return endpoint;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
getAzureStorageCredentials(locationConstraint) {
|
getAzureStorageCredentials(locationConstraint) {
|
||||||
return azureGetLocationCredentials(
|
const { azureStorageAccessKey } =
|
||||||
locationConstraint,
|
this.locationConstraints[locationConstraint].details;
|
||||||
this.locationConstraints[locationConstraint].details
|
const storageAccessKeyFromEnv =
|
||||||
);
|
process.env[`${locationConstraint}_AZURE_STORAGE_ACCESS_KEY`];
|
||||||
|
return {
|
||||||
|
storageAccountName:
|
||||||
|
this.getAzureStorageAccountName(locationConstraint),
|
||||||
|
storageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
getPfsDaemonEndpoint(locationConstraint) {
|
getPfsDaemonEndpoint(locationConstraint) {
|
||||||
|
@ -1861,25 +1680,13 @@ class Config extends EventEmitter {
|
||||||
.update(instanceId)
|
.update(instanceId)
|
||||||
.digest('hex');
|
.digest('hex');
|
||||||
}
|
}
|
||||||
|
|
||||||
isQuotaEnabled() {
|
|
||||||
return !!this.quotaEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
isQuotaInflightEnabled() {
|
|
||||||
return this.quota.enableInflights;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
parseSproxydConfig,
|
parseSproxydConfig,
|
||||||
parseRedisConfig,
|
|
||||||
locationConstraintAssert,
|
locationConstraintAssert,
|
||||||
ConfigObject: Config,
|
ConfigObject: Config,
|
||||||
config: new Config(),
|
config: new Config(),
|
||||||
requestsConfigAssert,
|
requestsConfigAssert,
|
||||||
bucketNotifAssert,
|
bucketNotifAssert,
|
||||||
azureGetStorageAccountName,
|
|
||||||
azureGetLocationCredentials,
|
|
||||||
azureArchiveLocationConstraintAssert,
|
|
||||||
};
|
};
|
||||||
|
|
235
lib/api/api.js
235
lib/api/api.js
|
@ -7,7 +7,6 @@ const bucketDeleteEncryption = require('./bucketDeleteEncryption');
|
||||||
const bucketDeleteWebsite = require('./bucketDeleteWebsite');
|
const bucketDeleteWebsite = require('./bucketDeleteWebsite');
|
||||||
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
|
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
|
||||||
const bucketDeletePolicy = require('./bucketDeletePolicy');
|
const bucketDeletePolicy = require('./bucketDeletePolicy');
|
||||||
const bucketDeleteQuota = require('./bucketDeleteQuota');
|
|
||||||
const { bucketGet } = require('./bucketGet');
|
const { bucketGet } = require('./bucketGet');
|
||||||
const bucketGetACL = require('./bucketGetACL');
|
const bucketGetACL = require('./bucketGetACL');
|
||||||
const bucketGetCors = require('./bucketGetCors');
|
const bucketGetCors = require('./bucketGetCors');
|
||||||
|
@ -18,16 +17,12 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
|
||||||
const bucketGetNotification = require('./bucketGetNotification');
|
const bucketGetNotification = require('./bucketGetNotification');
|
||||||
const bucketGetObjectLock = require('./bucketGetObjectLock');
|
const bucketGetObjectLock = require('./bucketGetObjectLock');
|
||||||
const bucketGetPolicy = require('./bucketGetPolicy');
|
const bucketGetPolicy = require('./bucketGetPolicy');
|
||||||
const bucketGetQuota = require('./bucketGetQuota');
|
|
||||||
const bucketGetEncryption = require('./bucketGetEncryption');
|
const bucketGetEncryption = require('./bucketGetEncryption');
|
||||||
const bucketHead = require('./bucketHead');
|
const bucketHead = require('./bucketHead');
|
||||||
const { bucketPut } = require('./bucketPut');
|
const { bucketPut } = require('./bucketPut');
|
||||||
const bucketPutACL = require('./bucketPutACL');
|
const bucketPutACL = require('./bucketPutACL');
|
||||||
const bucketPutCors = require('./bucketPutCors');
|
const bucketPutCors = require('./bucketPutCors');
|
||||||
const bucketPutVersioning = require('./bucketPutVersioning');
|
const bucketPutVersioning = require('./bucketPutVersioning');
|
||||||
const bucketPutTagging = require('./bucketPutTagging');
|
|
||||||
const bucketDeleteTagging = require('./bucketDeleteTagging');
|
|
||||||
const bucketGetTagging = require('./bucketGetTagging');
|
|
||||||
const bucketPutWebsite = require('./bucketPutWebsite');
|
const bucketPutWebsite = require('./bucketPutWebsite');
|
||||||
const bucketPutReplication = require('./bucketPutReplication');
|
const bucketPutReplication = require('./bucketPutReplication');
|
||||||
const bucketPutLifecycle = require('./bucketPutLifecycle');
|
const bucketPutLifecycle = require('./bucketPutLifecycle');
|
||||||
|
@ -35,7 +30,6 @@ const bucketPutNotification = require('./bucketPutNotification');
|
||||||
const bucketPutEncryption = require('./bucketPutEncryption');
|
const bucketPutEncryption = require('./bucketPutEncryption');
|
||||||
const bucketPutPolicy = require('./bucketPutPolicy');
|
const bucketPutPolicy = require('./bucketPutPolicy');
|
||||||
const bucketPutObjectLock = require('./bucketPutObjectLock');
|
const bucketPutObjectLock = require('./bucketPutObjectLock');
|
||||||
const bucketUpdateQuota = require('./bucketUpdateQuota');
|
|
||||||
const bucketGetReplication = require('./bucketGetReplication');
|
const bucketGetReplication = require('./bucketGetReplication');
|
||||||
const bucketDeleteReplication = require('./bucketDeleteReplication');
|
const bucketDeleteReplication = require('./bucketDeleteReplication');
|
||||||
const corsPreflight = require('./corsPreflight');
|
const corsPreflight = require('./corsPreflight');
|
||||||
|
@ -47,7 +41,7 @@ const metadataSearch = require('./metadataSearch');
|
||||||
const { multiObjectDelete } = require('./multiObjectDelete');
|
const { multiObjectDelete } = require('./multiObjectDelete');
|
||||||
const multipartDelete = require('./multipartDelete');
|
const multipartDelete = require('./multipartDelete');
|
||||||
const objectCopy = require('./objectCopy');
|
const objectCopy = require('./objectCopy');
|
||||||
const { objectDelete } = require('./objectDelete');
|
const objectDelete = require('./objectDelete');
|
||||||
const objectDeleteTagging = require('./objectDeleteTagging');
|
const objectDeleteTagging = require('./objectDeleteTagging');
|
||||||
const objectGet = require('./objectGet');
|
const objectGet = require('./objectGet');
|
||||||
const objectGetACL = require('./objectGetACL');
|
const objectGetACL = require('./objectGetACL');
|
||||||
|
@ -62,12 +56,12 @@ const objectPutTagging = require('./objectPutTagging');
|
||||||
const objectPutPart = require('./objectPutPart');
|
const objectPutPart = require('./objectPutPart');
|
||||||
const objectPutCopyPart = require('./objectPutCopyPart');
|
const objectPutCopyPart = require('./objectPutCopyPart');
|
||||||
const objectPutRetention = require('./objectPutRetention');
|
const objectPutRetention = require('./objectPutRetention');
|
||||||
const objectRestore = require('./objectRestore');
|
|
||||||
const prepareRequestContexts
|
const prepareRequestContexts
|
||||||
= require('./apiUtils/authorization/prepareRequestContexts');
|
= require('./apiUtils/authorization/prepareRequestContexts');
|
||||||
const serviceGet = require('./serviceGet');
|
const serviceGet = require('./serviceGet');
|
||||||
const vault = require('../auth/vault');
|
const vault = require('../auth/vault');
|
||||||
const website = require('./website');
|
const websiteGet = require('./websiteGet');
|
||||||
|
const websiteHead = require('./websiteHead');
|
||||||
const writeContinue = require('../utilities/writeContinue');
|
const writeContinue = require('../utilities/writeContinue');
|
||||||
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
|
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
|
||||||
const parseCopySource = require('./apiUtils/object/parseCopySource');
|
const parseCopySource = require('./apiUtils/object/parseCopySource');
|
||||||
|
@ -85,10 +79,6 @@ const api = {
|
||||||
// Attach the apiMethod method to the request, so it can used by monitoring in the server
|
// Attach the apiMethod method to the request, so it can used by monitoring in the server
|
||||||
// eslint-disable-next-line no-param-reassign
|
// eslint-disable-next-line no-param-reassign
|
||||||
request.apiMethod = apiMethod;
|
request.apiMethod = apiMethod;
|
||||||
// Array of end of API callbacks, used to perform some logic
|
|
||||||
// at the end of an API.
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
request.finalizerHooks = [];
|
|
||||||
|
|
||||||
const actionLog = monitoringMap[apiMethod];
|
const actionLog = monitoringMap[apiMethod];
|
||||||
if (!actionLog &&
|
if (!actionLog &&
|
||||||
|
@ -123,7 +113,6 @@ const api = {
|
||||||
// no need to check auth on website or cors preflight requests
|
// no need to check auth on website or cors preflight requests
|
||||||
if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
|
if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
|
||||||
apiMethod === 'corsPreflight') {
|
apiMethod === 'corsPreflight') {
|
||||||
request.actionImplicitDenies = false;
|
|
||||||
return this[apiMethod](request, log, callback);
|
return this[apiMethod](request, log, callback);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,25 +135,15 @@ const api = {
|
||||||
|
|
||||||
const requestContexts = prepareRequestContexts(apiMethod, request,
|
const requestContexts = prepareRequestContexts(apiMethod, request,
|
||||||
sourceBucket, sourceObject, sourceVersionId);
|
sourceBucket, sourceObject, sourceVersionId);
|
||||||
// Extract all the _apiMethods and store them in an array
|
|
||||||
const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : [];
|
|
||||||
// Attach the names to the current request
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
request.apiMethods = apiMethods;
|
|
||||||
|
|
||||||
function checkAuthResults(authResults) {
|
function checkAuthResults(authResults) {
|
||||||
let returnTagCount = true;
|
let returnTagCount = true;
|
||||||
const isImplicitDeny = {};
|
|
||||||
let isOnlyImplicitDeny = true;
|
|
||||||
if (apiMethod === 'objectGet') {
|
if (apiMethod === 'objectGet') {
|
||||||
// first item checks s3:GetObject(Version) action
|
// first item checks s3:GetObject(Version) action
|
||||||
if (!authResults[0].isAllowed && !authResults[0].isImplicit) {
|
if (!authResults[0].isAllowed) {
|
||||||
log.trace('get object authorization denial from Vault');
|
log.trace('get object authorization denial from Vault');
|
||||||
return errors.AccessDenied;
|
return errors.AccessDenied;
|
||||||
}
|
}
|
||||||
// TODO add support for returnTagCount in the bucket policy
|
|
||||||
// checks
|
|
||||||
isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
|
|
||||||
// second item checks s3:GetObject(Version)Tagging action
|
// second item checks s3:GetObject(Version)Tagging action
|
||||||
if (!authResults[1].isAllowed) {
|
if (!authResults[1].isAllowed) {
|
||||||
log.trace('get tagging authorization denial ' +
|
log.trace('get tagging authorization denial ' +
|
||||||
|
@ -173,138 +152,113 @@ const api = {
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for (let i = 0; i < authResults.length; i++) {
|
for (let i = 0; i < authResults.length; i++) {
|
||||||
isImplicitDeny[authResults[i].action] = true;
|
if (!authResults[i].isAllowed) {
|
||||||
if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
|
|
||||||
// Any explicit deny rejects the current API call
|
|
||||||
log.trace('authorization denial from Vault');
|
log.trace('authorization denial from Vault');
|
||||||
return errors.AccessDenied;
|
return errors.AccessDenied;
|
||||||
}
|
}
|
||||||
if (authResults[i].isAllowed) {
|
|
||||||
// If the action is allowed, the result is not implicit
|
|
||||||
// Deny.
|
|
||||||
isImplicitDeny[authResults[i].action] = false;
|
|
||||||
isOnlyImplicitDeny = false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// These two APIs cannot use ACLs or Bucket Policies, hence, any
|
return returnTagCount;
|
||||||
// implicit deny from vault must be treated as an explicit deny.
|
|
||||||
if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
|
|
||||||
return errors.AccessDenied;
|
|
||||||
}
|
|
||||||
return { returnTagCount, isImplicitDeny };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
next => auth.server.doAuth(
|
next => auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) =>
|
||||||
request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
|
next(err, userInfo, authorizationResults, streamingV4Params), 's3', requestContexts),
|
||||||
if (err) {
|
(userInfo, authorizationResults, streamingV4Params, next) => {
|
||||||
// VaultClient returns standard errors, but the route requires
|
if (authorizationResults) {
|
||||||
// Arsenal errors
|
const checkedResults = checkAuthResults(authorizationResults);
|
||||||
const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
|
if (checkedResults instanceof Error) {
|
||||||
log.trace('authentication error', { error: err });
|
return next(checkedResults);
|
||||||
return next(arsenalError);
|
|
||||||
}
|
}
|
||||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
returnTagCount = checkedResults;
|
||||||
}, 's3', requestContexts),
|
|
||||||
(userInfo, authorizationResults, streamingV4Params, infos, next) => {
|
|
||||||
const authNames = { accountName: userInfo.getAccountDisplayName() };
|
|
||||||
if (userInfo.isRequesterAnIAMUser()) {
|
|
||||||
authNames.userName = userInfo.getIAMdisplayName();
|
|
||||||
}
|
}
|
||||||
if (isRequesterASessionUser(userInfo)) {
|
return tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log,
|
||||||
authNames.sessionName = userInfo.getShortid().split(':')[1];
|
(err, tagAuthResults, updatedContexts) =>
|
||||||
}
|
next(err, tagAuthResults, authorizationResults, userInfo, streamingV4Params, updatedContexts));
|
||||||
log.addDefaultFields(authNames);
|
|
||||||
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
|
||||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
|
||||||
}
|
|
||||||
// issue 100 Continue to the client
|
|
||||||
writeContinue(request, response);
|
|
||||||
const MAX_POST_LENGTH = request.method === 'POST' ?
|
|
||||||
1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
|
|
||||||
const post = [];
|
|
||||||
let postLength = 0;
|
|
||||||
request.on('data', chunk => {
|
|
||||||
postLength += chunk.length;
|
|
||||||
// Sanity check on post length
|
|
||||||
if (postLength <= MAX_POST_LENGTH) {
|
|
||||||
post.push(chunk);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
request.on('error', err => {
|
|
||||||
log.trace('error receiving request', {
|
|
||||||
error: err,
|
|
||||||
});
|
|
||||||
return next(errors.InternalError);
|
|
||||||
});
|
|
||||||
|
|
||||||
request.on('end', () => {
|
|
||||||
if (postLength > MAX_POST_LENGTH) {
|
|
||||||
log.error('body length is too long for request type',
|
|
||||||
{ postLength });
|
|
||||||
return next(errors.InvalidRequest);
|
|
||||||
}
|
|
||||||
// Convert array of post buffers into one string
|
|
||||||
request.post = Buffer.concat(post, postLength).toString();
|
|
||||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
|
||||||
});
|
|
||||||
return undefined;
|
|
||||||
},
|
},
|
||||||
// Tag condition keys require information from CloudServer for evaluation
|
], (err, tagAuthResults, authorizationResults, userInfo, streamingV4Params, updatedContexts) => {
|
||||||
(userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
|
|
||||||
authorizationResults,
|
|
||||||
request,
|
|
||||||
requestContexts,
|
|
||||||
apiMethod,
|
|
||||||
log,
|
|
||||||
(err, authResultsWithTags) => {
|
|
||||||
if (err) {
|
|
||||||
log.trace('tag authentication error', { error: err });
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
|
|
||||||
},
|
|
||||||
),
|
|
||||||
], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
|
|
||||||
if (err) {
|
if (err) {
|
||||||
|
log.trace('authentication error', { error: err });
|
||||||
return callback(err);
|
return callback(err);
|
||||||
}
|
}
|
||||||
request.accountQuotas = infos?.accountQuota;
|
const authNames = { accountName: userInfo.getAccountDisplayName() };
|
||||||
if (authorizationResults) {
|
if (userInfo.isRequesterAnIAMUser()) {
|
||||||
const checkedResults = checkAuthResults(authorizationResults);
|
authNames.userName = userInfo.getIAMdisplayName();
|
||||||
|
}
|
||||||
|
if (isRequesterASessionUser(userInfo)) {
|
||||||
|
authNames.sessionName = userInfo.getShortid().split(':')[1];
|
||||||
|
}
|
||||||
|
log.addDefaultFields(authNames);
|
||||||
|
if (tagAuthResults) {
|
||||||
|
const checkedResults = checkAuthResults(tagAuthResults);
|
||||||
if (checkedResults instanceof Error) {
|
if (checkedResults instanceof Error) {
|
||||||
return callback(checkedResults);
|
return callback(checkedResults);
|
||||||
}
|
}
|
||||||
returnTagCount = checkedResults.returnTagCount;
|
returnTagCount = checkedResults;
|
||||||
request.actionImplicitDenies = checkedResults.isImplicitDeny;
|
|
||||||
} else {
|
|
||||||
// create an object of keys apiMethods with all values to false:
|
|
||||||
// for backward compatibility, all apiMethods are allowed by default
|
|
||||||
// thus it is explicitly allowed, so implicit deny is false
|
|
||||||
request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
|
|
||||||
acc[curr] = false;
|
|
||||||
return acc;
|
|
||||||
}, {});
|
|
||||||
}
|
}
|
||||||
const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
|
|
||||||
(hook, done) => hook(err, done),
|
|
||||||
() => callback(err, ...results));
|
|
||||||
|
|
||||||
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
||||||
request._response = response;
|
request._response = response;
|
||||||
return this[apiMethod](userInfo, request, streamingV4Params,
|
return this[apiMethod](userInfo, request, streamingV4Params,
|
||||||
log, methodCallback, authorizationResults);
|
log, callback, authorizationResults);
|
||||||
}
|
}
|
||||||
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
|
// issue 100 Continue to the client
|
||||||
return this[apiMethod](userInfo, request, sourceBucket,
|
writeContinue(request, response);
|
||||||
sourceObject, sourceVersionId, log, methodCallback);
|
const MAX_POST_LENGTH = request.method.toUpperCase() === 'POST' ?
|
||||||
}
|
1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
|
||||||
if (apiMethod === 'objectGet') {
|
const post = [];
|
||||||
return this[apiMethod](userInfo, request, returnTagCount, log, callback);
|
let postLength = 0;
|
||||||
}
|
request.on('data', chunk => {
|
||||||
return this[apiMethod](userInfo, request, log, methodCallback);
|
postLength += chunk.length;
|
||||||
|
// Sanity check on post length
|
||||||
|
if (postLength <= MAX_POST_LENGTH) {
|
||||||
|
post.push(chunk);
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
});
|
||||||
|
|
||||||
|
request.on('error', err => {
|
||||||
|
log.trace('error receiving request', {
|
||||||
|
error: err,
|
||||||
|
});
|
||||||
|
return callback(errors.InternalError);
|
||||||
|
});
|
||||||
|
|
||||||
|
request.on('end', () => {
|
||||||
|
if (postLength > MAX_POST_LENGTH) {
|
||||||
|
log.error('body length is too long for request type',
|
||||||
|
{ postLength });
|
||||||
|
return callback(errors.InvalidRequest);
|
||||||
|
}
|
||||||
|
// Convert array of post buffers into one string
|
||||||
|
request.post = Buffer.concat(post, postLength).toString();
|
||||||
|
|
||||||
|
// IAM policy -Tag condition keys require information from CloudServer for evaluation
|
||||||
|
return tagConditionKeyAuth(authorizationResults, request, (updatedContexts || requestContexts),
|
||||||
|
apiMethod, log, (err, tagAuthResults) => {
|
||||||
|
if (err) {
|
||||||
|
log.trace('tag authentication error', { error: err });
|
||||||
|
return callback(err);
|
||||||
|
}
|
||||||
|
if (tagAuthResults) {
|
||||||
|
const checkedResults = checkAuthResults(tagAuthResults);
|
||||||
|
if (checkedResults instanceof Error) {
|
||||||
|
return callback(checkedResults);
|
||||||
|
}
|
||||||
|
returnTagCount = checkedResults;
|
||||||
|
}
|
||||||
|
if (apiMethod === 'objectCopy' ||
|
||||||
|
apiMethod === 'objectPutCopyPart') {
|
||||||
|
return this[apiMethod](userInfo, request, sourceBucket,
|
||||||
|
sourceObject, sourceVersionId, log, callback);
|
||||||
|
}
|
||||||
|
if (apiMethod === 'objectGet') {
|
||||||
|
return this[apiMethod](userInfo, request,
|
||||||
|
returnTagCount, log, callback);
|
||||||
|
}
|
||||||
|
return this[apiMethod](userInfo, request, log, callback);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return undefined;
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
bucketDelete,
|
bucketDelete,
|
||||||
|
@ -324,21 +278,15 @@ const api = {
|
||||||
bucketPutACL,
|
bucketPutACL,
|
||||||
bucketPutCors,
|
bucketPutCors,
|
||||||
bucketPutVersioning,
|
bucketPutVersioning,
|
||||||
bucketPutTagging,
|
|
||||||
bucketDeleteTagging,
|
|
||||||
bucketGetTagging,
|
|
||||||
bucketPutWebsite,
|
bucketPutWebsite,
|
||||||
bucketPutReplication,
|
bucketPutReplication,
|
||||||
bucketGetReplication,
|
bucketGetReplication,
|
||||||
bucketDeleteReplication,
|
bucketDeleteReplication,
|
||||||
bucketDeleteQuota,
|
|
||||||
bucketPutLifecycle,
|
bucketPutLifecycle,
|
||||||
bucketUpdateQuota,
|
|
||||||
bucketGetLifecycle,
|
bucketGetLifecycle,
|
||||||
bucketDeleteLifecycle,
|
bucketDeleteLifecycle,
|
||||||
bucketPutPolicy,
|
bucketPutPolicy,
|
||||||
bucketGetPolicy,
|
bucketGetPolicy,
|
||||||
bucketGetQuota,
|
|
||||||
bucketDeletePolicy,
|
bucketDeletePolicy,
|
||||||
bucketPutObjectLock,
|
bucketPutObjectLock,
|
||||||
bucketPutNotification,
|
bucketPutNotification,
|
||||||
|
@ -368,10 +316,9 @@ const api = {
|
||||||
objectPutPart,
|
objectPutPart,
|
||||||
objectPutCopyPart,
|
objectPutCopyPart,
|
||||||
objectPutRetention,
|
objectPutRetention,
|
||||||
objectRestore,
|
|
||||||
serviceGet,
|
serviceGet,
|
||||||
websiteGet: website,
|
websiteGet,
|
||||||
websiteHead: website,
|
websiteHead,
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = api;
|
module.exports = api;
|
||||||
|
|
|
@ -1,23 +1,11 @@
|
||||||
const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
|
const { evaluators, actionMaps, RequestContext } = require('arsenal').policies;
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const { parseCIDR, isValid } = require('ipaddr.js');
|
|
||||||
const constants = require('../../../../constants');
|
const constants = require('../../../../constants');
|
||||||
const { config } = require('../../../Config');
|
|
||||||
|
|
||||||
const {
|
const { allAuthedUsersId, bucketOwnerActions, logId, publicId } = constants;
|
||||||
allAuthedUsersId,
|
|
||||||
bucketOwnerActions,
|
|
||||||
logId,
|
|
||||||
publicId,
|
|
||||||
arrayOfAllowed,
|
|
||||||
assumedRoleArnResourceType,
|
|
||||||
backbeatLifecycleSessionName,
|
|
||||||
actionsToConsiderAsObjectPut,
|
|
||||||
} = constants;
|
|
||||||
|
|
||||||
// whitelist buckets to allow public read on objects
|
// whitelist buckets to allow public read on objects
|
||||||
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
|
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ?
|
||||||
? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
|
process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
|
||||||
|
|
||||||
function getServiceAccountProperties(canonicalID) {
|
function getServiceAccountProperties(canonicalID) {
|
||||||
const canonicalIDArray = canonicalID.split('/');
|
const canonicalIDArray = canonicalID.split('/');
|
||||||
|
@ -38,41 +26,13 @@ function isRequesterNonAccountUser(authInfo) {
|
||||||
return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
|
return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function checkBucketAcls(bucket, requestType, canonicalID) {
|
||||||
* Checks the access control for a given bucket based on the request type and user's canonical ID.
|
|
||||||
*
|
|
||||||
* @param {Bucket} bucket - The bucket to check access control for.
|
|
||||||
* @param {string} requestType - The list of s3 actions to check within the API call.
|
|
||||||
* @param {string} canonicalID - The canonical ID of the user making the request.
|
|
||||||
* @param {string} mainApiCall - The main API call (first item of the requestType).
|
|
||||||
*
|
|
||||||
* @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
|
|
||||||
*/
|
|
||||||
|
|
||||||
function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
|
||||||
// Same logic applies on the Versioned APIs, so let's simplify it.
|
|
||||||
let requestTypeParsed = requestType.endsWith('Version') ?
|
|
||||||
requestType.slice(0, 'Version'.length * -1) : requestType;
|
|
||||||
requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
|
|
||||||
'objectPut' : requestTypeParsed;
|
|
||||||
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
|
|
||||||
'objectPut' : mainApiCall;
|
|
||||||
if (bucket.getOwner() === canonicalID) {
|
if (bucket.getOwner() === canonicalID) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (parsedMainApiCall === 'objectGet') {
|
|
||||||
if (requestTypeParsed === 'objectGetTagging') {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (parsedMainApiCall === 'objectPut') {
|
|
||||||
if (arrayOfAllowed.includes(requestTypeParsed)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const bucketAcl = bucket.getAcl();
|
const bucketAcl = bucket.getAcl();
|
||||||
if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') {
|
if (requestType === 'bucketGet' || requestType === 'bucketHead') {
|
||||||
if (bucketAcl.Canned === 'public-read'
|
if (bucketAcl.Canned === 'public-read'
|
||||||
|| bucketAcl.Canned === 'public-read-write'
|
|| bucketAcl.Canned === 'public-read-write'
|
||||||
|| (bucketAcl.Canned === 'authenticated-read'
|
|| (bucketAcl.Canned === 'authenticated-read'
|
||||||
|
@ -90,7 +50,7 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (requestTypeParsed === 'bucketGetACL') {
|
if (requestType === 'bucketGetACL') {
|
||||||
if ((bucketAcl.Canned === 'log-delivery-write'
|
if ((bucketAcl.Canned === 'log-delivery-write'
|
||||||
&& canonicalID === logId)
|
&& canonicalID === logId)
|
||||||
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|
||||||
|
@ -106,7 +66,7 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'bucketPutACL') {
|
if (requestType === 'bucketPutACL') {
|
||||||
if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|
if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|
||||||
|| bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
|
|| bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
|
||||||
return true;
|
return true;
|
||||||
|
@ -120,7 +80,11 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') {
|
if (requestType === 'bucketDelete' && bucket.getOwner() === canonicalID) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (requestType === 'objectDelete' || requestType === 'objectPut') {
|
||||||
if (bucketAcl.Canned === 'public-read-write'
|
if (bucketAcl.Canned === 'public-read-write'
|
||||||
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|
||||||
|| bucketAcl.WRITE.indexOf(canonicalID) > -1) {
|
|| bucketAcl.WRITE.indexOf(canonicalID) > -1) {
|
||||||
|
@ -140,39 +104,25 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
||||||
// objectPutACL, objectGetACL, objectHead or objectGet, the bucket
|
// objectPutACL, objectGetACL, objectHead or objectGet, the bucket
|
||||||
// authorization check should just return true so can move on to check
|
// authorization check should just return true so can move on to check
|
||||||
// rights at the object level.
|
// rights at the object level.
|
||||||
return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
|
return (requestType === 'objectPutACL' || requestType === 'objectGetACL' ||
|
||||||
|| requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
|
requestType === 'objectGet' || requestType === 'objectHead');
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
|
function checkObjectAcls(bucket, objectMD, requestType, canonicalID) {
|
||||||
isUserUnauthenticated, mainApiCall) {
|
|
||||||
const bucketOwner = bucket.getOwner();
|
const bucketOwner = bucket.getOwner();
|
||||||
const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
|
|
||||||
'objectPut' : requestType;
|
|
||||||
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
|
|
||||||
'objectPut' : mainApiCall;
|
|
||||||
// acls don't distinguish between users and accounts, so both should be allowed
|
// acls don't distinguish between users and accounts, so both should be allowed
|
||||||
if (bucketOwnerActions.includes(requestTypeParsed)
|
if (bucketOwnerActions.includes(requestType)
|
||||||
&& (bucketOwner === canonicalID)) {
|
&& (bucketOwner === canonicalID)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (objectMD['owner-id'] === canonicalID) {
|
if (objectMD['owner-id'] === canonicalID) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Backward compatibility
|
|
||||||
if (parsedMainApiCall === 'objectGet') {
|
|
||||||
if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
|
|
||||||
&& requestTypeParsed === 'objectGetTagging') {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!objectMD.acl) {
|
if (!objectMD.acl) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') {
|
if (requestType === 'objectGet' || requestType === 'objectHead') {
|
||||||
if (objectMD.acl.Canned === 'public-read'
|
if (objectMD.acl.Canned === 'public-read'
|
||||||
|| objectMD.acl.Canned === 'public-read-write'
|
|| objectMD.acl.Canned === 'public-read-write'
|
||||||
|| (objectMD.acl.Canned === 'authenticated-read'
|
|| (objectMD.acl.Canned === 'authenticated-read'
|
||||||
|
@ -198,11 +148,11 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
|
|
||||||
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
||||||
// bucket has canned ACL public-read-write
|
// bucket has canned ACL public-read-write
|
||||||
if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') {
|
if (requestType === 'objectPut' || requestType === 'objectDelete') {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'objectPutACL') {
|
if (requestType === 'objectPutACL') {
|
||||||
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
||||||
&& bucketOwner === canonicalID)
|
&& bucketOwner === canonicalID)
|
||||||
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
||||||
|
@ -218,7 +168,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'objectGetACL') {
|
if (requestType === 'objectGetACL') {
|
||||||
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
||||||
&& bucketOwner === canonicalID)
|
&& bucketOwner === canonicalID)
|
||||||
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
||||||
|
@ -237,9 +187,9 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
// allow public reads on buckets that are whitelisted for anonymous reads
|
// allow public reads on buckets that are whitelisted for anonymous reads
|
||||||
// TODO: remove this after bucket policies are implemented
|
// TODO: remove this after bucket policies are implemented
|
||||||
const bucketAcl = bucket.getAcl();
|
const bucketAcl = bucket.getAcl();
|
||||||
const allowPublicReads = publicReadBuckets.includes(bucket.getName())
|
const allowPublicReads = publicReadBuckets.includes(bucket.getName()) &&
|
||||||
&& bucketAcl.Canned === 'public-read'
|
bucketAcl.Canned === 'public-read' &&
|
||||||
&& (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
|
(requestType === 'objectGet' || requestType === 'objectHead');
|
||||||
if (allowPublicReads) {
|
if (allowPublicReads) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -266,20 +216,6 @@ function _checkBucketPolicyResources(request, resource, log) {
|
||||||
return evaluators.isResourceApplicable(requestContext, resource, log);
|
return evaluators.isResourceApplicable(requestContext, resource, log);
|
||||||
}
|
}
|
||||||
|
|
||||||
function _checkBucketPolicyConditions(request, conditions, log) {
|
|
||||||
const ip = request ? requestUtils.getClientIp(request, config) : undefined;
|
|
||||||
if (!conditions) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
// build request context from the request!
|
|
||||||
const requestContext = new RequestContext(request.headers, request.query,
|
|
||||||
request.bucketName, request.objectKey, ip,
|
|
||||||
request.connection.encrypted, request.resourceType, 's3', null, null,
|
|
||||||
null, null, null, null, null, null, null, null, null,
|
|
||||||
request.objectLockRetentionDays);
|
|
||||||
return evaluators.meetConditions(requestContext, conditions, log);
|
|
||||||
}
|
|
||||||
|
|
||||||
function _getAccountId(arn) {
|
function _getAccountId(arn) {
|
||||||
// account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
|
// account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
|
||||||
return arn.substr(13, 12);
|
return arn.substr(13, 12);
|
||||||
|
@ -324,11 +260,11 @@ function _checkPrincipals(canonicalID, arn, principal) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) {
|
function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request) {
|
||||||
let permission = 'defaultDeny';
|
let permission = 'defaultDeny';
|
||||||
// if requester is user within bucket owner account, actions should be
|
// if requester is user within bucket owner account, actions should be
|
||||||
// allowed unless explicitly denied (assumes allowed by IAM policy)
|
// allowed unless explicitly denied (assumes allowed by IAM policy)
|
||||||
if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) {
|
if (bucketOwner === canonicalID) {
|
||||||
permission = 'allow';
|
permission = 'allow';
|
||||||
}
|
}
|
||||||
let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
|
let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
|
||||||
|
@ -337,13 +273,12 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l
|
||||||
const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
|
const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
|
||||||
const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
|
const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
|
||||||
const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
|
const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
|
||||||
const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
|
|
||||||
|
|
||||||
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') {
|
if (principalMatch && actionMatch && resourceMatch && s.Effect === 'Deny') {
|
||||||
// explicit deny trumps any allows, so return immediately
|
// explicit deny trumps any allows, so return immediately
|
||||||
return 'explicitDeny';
|
return 'explicitDeny';
|
||||||
}
|
}
|
||||||
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') {
|
if (principalMatch && actionMatch && resourceMatch && s.Effect === 'Allow') {
|
||||||
permission = 'allow';
|
permission = 'allow';
|
||||||
}
|
}
|
||||||
copiedStatement = copiedStatement.splice(1);
|
copiedStatement = copiedStatement.splice(1);
|
||||||
|
@ -351,141 +286,80 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l
|
||||||
return permission;
|
return permission;
|
||||||
}
|
}
|
||||||
|
|
||||||
function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log,
|
function isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request) {
|
||||||
request, aclPermission, results, actionImplicitDenies) {
|
// Check to see if user is authorized to perform a
|
||||||
const bucketPolicy = bucket.getBucketPolicy();
|
// particular action on bucket based on ACLs.
|
||||||
let processedResult = results[requestType];
|
// TODO: Add IAM checks
|
||||||
if (!bucketPolicy) {
|
let requesterIsNotUser = true;
|
||||||
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
|
let arn = null;
|
||||||
} else {
|
if (authInfo) {
|
||||||
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
|
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
|
||||||
bucketOwner, log, request, actionImplicitDenies);
|
arn = authInfo.getArn();
|
||||||
|
|
||||||
if (bucketPolicyPermission === 'explicitDeny') {
|
|
||||||
processedResult = false;
|
|
||||||
} else if (bucketPolicyPermission === 'allow') {
|
|
||||||
processedResult = true;
|
|
||||||
} else {
|
|
||||||
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return processedResult;
|
// if the bucket owner is an account, users should not have default access
|
||||||
|
if (((bucket.getOwner() === canonicalID) && requesterIsNotUser)
|
||||||
|
|| isServiceAccount(canonicalID)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const aclPermission = checkBucketAcls(bucket, requestType, canonicalID);
|
||||||
|
const bucketPolicy = bucket.getBucketPolicy();
|
||||||
|
if (!bucketPolicy) {
|
||||||
|
return aclPermission;
|
||||||
|
}
|
||||||
|
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
|
||||||
|
canonicalID, arn, bucket.getOwner(), log, request);
|
||||||
|
if (bucketPolicyPermission === 'explicitDeny') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (aclPermission || (bucketPolicyPermission === 'allow'));
|
||||||
}
|
}
|
||||||
|
|
||||||
function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
|
function isObjAuthorized(bucket, objectMD, requestType, canonicalID, authInfo, log, request) {
|
||||||
actionImplicitDeniesInput = {}, isWebsite = false) {
|
const bucketOwner = bucket.getOwner();
|
||||||
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
|
if (!objectMD) {
|
||||||
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
|
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
||||||
const mainApiCall = requestTypes[0];
|
// bucket has canned ACL public-read-write
|
||||||
const results = {};
|
if (requestType === 'objectPut' || requestType === 'objectDelete') {
|
||||||
return requestTypes.every(_requestType => {
|
return true;
|
||||||
// By default, all missing actions are defined as allowed from IAM, to be
|
|
||||||
// backward compatible
|
|
||||||
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
|
|
||||||
// Check to see if user is authorized to perform a
|
|
||||||
// particular action on bucket based on ACLs.
|
|
||||||
// TODO: Add IAM checks
|
|
||||||
let requesterIsNotUser = true;
|
|
||||||
let arn = null;
|
|
||||||
if (authInfo) {
|
|
||||||
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
|
|
||||||
arn = authInfo.getArn();
|
|
||||||
}
|
}
|
||||||
// if the bucket owner is an account, users should not have default access
|
// check bucket has read access
|
||||||
if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
|
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
|
||||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
return isBucketAuthorized(bucket, 'bucketGet', canonicalID, authInfo, log, request);
|
||||||
return results[_requestType];
|
}
|
||||||
}
|
let requesterIsNotUser = true;
|
||||||
const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
|
let arn = null;
|
||||||
// In case of error bucket access is checked with bucketGet
|
if (authInfo) {
|
||||||
// For website, bucket policy only uses objectGet and ignores bucketGet
|
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
|
arn = authInfo.getArn();
|
||||||
// bucketGet should be used to check acl but switched to objectGet for bucket policy
|
}
|
||||||
if (isWebsite && _requestType === 'bucketGet') {
|
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser) {
|
||||||
// eslint-disable-next-line no-param-reassign
|
return true;
|
||||||
_requestType = 'objectGet';
|
}
|
||||||
actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
|
|
||||||
}
|
|
||||||
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
|
|
||||||
request, aclPermission, results, actionImplicitDenies);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {},
|
if (isServiceAccount(canonicalID)) {
|
||||||
log, request) {
|
return true;
|
||||||
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
|
}
|
||||||
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
|
// account is authorized if:
|
||||||
const results = {};
|
// - requesttype is included in bucketOwnerActions and
|
||||||
return requestTypes.every(_requestType => {
|
// - account is the bucket owner
|
||||||
// By default, all missing actions are defined as allowed from IAM, to be
|
// - requester is account, not user
|
||||||
// backward compatible
|
if (bucketOwnerActions.includes(requestType)
|
||||||
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
|
|
||||||
let arn = null;
|
|
||||||
if (authInfo) {
|
|
||||||
arn = authInfo.getArn();
|
|
||||||
}
|
|
||||||
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
|
|
||||||
request, true, results, actionImplicitDenies);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
|
|
||||||
actionImplicitDeniesInput = {}, isWebsite = false) {
|
|
||||||
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
|
|
||||||
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
|
|
||||||
const results = {};
|
|
||||||
const mainApiCall = requestTypes[0];
|
|
||||||
return requestTypes.every(_requestType => {
|
|
||||||
// By default, all missing actions are defined as allowed from IAM, to be
|
|
||||||
// backward compatible
|
|
||||||
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
|
|
||||||
const parsedMethodName = _requestType.endsWith('Version')
|
|
||||||
? _requestType.slice(0, -7) : _requestType;
|
|
||||||
const bucketOwner = bucket.getOwner();
|
|
||||||
if (!objectMD) {
|
|
||||||
// check bucket has read access
|
|
||||||
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
|
|
||||||
let permission = 'bucketGet';
|
|
||||||
if (actionsToConsiderAsObjectPut.includes(_requestType)) {
|
|
||||||
permission = 'objectPut';
|
|
||||||
}
|
|
||||||
results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
|
|
||||||
actionImplicitDenies, isWebsite);
|
|
||||||
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
|
||||||
// bucket has canned ACL public-read-write
|
|
||||||
if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
|
|
||||||
&& results[_requestType] === false) {
|
|
||||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
|
||||||
}
|
|
||||||
return results[_requestType];
|
|
||||||
}
|
|
||||||
let requesterIsNotUser = true;
|
|
||||||
let arn = null;
|
|
||||||
let isUserUnauthenticated = false;
|
|
||||||
if (authInfo) {
|
|
||||||
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
|
|
||||||
arn = authInfo.getArn();
|
|
||||||
isUserUnauthenticated = arn === undefined;
|
|
||||||
}
|
|
||||||
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
|
|
||||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
|
||||||
return results[_requestType];
|
|
||||||
}
|
|
||||||
// account is authorized if:
|
|
||||||
// - requesttype is included in bucketOwnerActions and
|
|
||||||
// - account is the bucket owner
|
|
||||||
// - requester is account, not user
|
|
||||||
if (bucketOwnerActions.includes(parsedMethodName)
|
|
||||||
&& (bucketOwner === canonicalID)
|
&& (bucketOwner === canonicalID)
|
||||||
&& requesterIsNotUser) {
|
&& requesterIsNotUser) {
|
||||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
return true;
|
||||||
return results[_requestType];
|
}
|
||||||
}
|
const aclPermission = checkObjectAcls(bucket, objectMD, requestType,
|
||||||
const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName,
|
canonicalID);
|
||||||
canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall);
|
const bucketPolicy = bucket.getBucketPolicy();
|
||||||
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner,
|
if (!bucketPolicy) {
|
||||||
log, request, aclPermission, results, actionImplicitDenies);
|
return aclPermission;
|
||||||
});
|
}
|
||||||
|
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
|
||||||
|
canonicalID, arn, bucket.getOwner(), log, request);
|
||||||
|
if (bucketPolicyPermission === 'explicitDeny') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (aclPermission || (bucketPolicyPermission === 'allow'));
|
||||||
}
|
}
|
||||||
|
|
||||||
function _checkResource(resource, bucketArn) {
|
function _checkResource(resource, bucketArn) {
|
||||||
|
@ -514,117 +388,6 @@ function validatePolicyResource(bucketName, policy) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkIp(value) {
|
|
||||||
const errString = 'Invalid IP address in Conditions';
|
|
||||||
|
|
||||||
const values = Array.isArray(value) ? value : [value];
|
|
||||||
|
|
||||||
for (let i = 0; i < values.length; i++) {
|
|
||||||
// these preliminary checks are validating the provided
|
|
||||||
// ip address against ipaddr.js, the library we use when
|
|
||||||
// evaluating IP condition keys. It ensures compatibility,
|
|
||||||
// but additional checks are required to enforce the right
|
|
||||||
// notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
|
|
||||||
// we would accept different ip formats, which is not
|
|
||||||
// standard in an AWS use case.
|
|
||||||
try {
|
|
||||||
try {
|
|
||||||
parseCIDR(values[i]);
|
|
||||||
} catch (err) {
|
|
||||||
isValid(values[i]);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
return errString;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply the existing IP validation logic to each element
|
|
||||||
const validateIpRegex = ip => {
|
|
||||||
if (constants.ipv4Regex.test(ip)) {
|
|
||||||
return ip.split('.').every(part => parseInt(part, 10) <= 255);
|
|
||||||
}
|
|
||||||
if (constants.ipv6Regex.test(ip)) {
|
|
||||||
return ip.split(':').every(part => part.length <= 4);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
if (validateIpRegex(values[i]) !== true) {
|
|
||||||
return errString;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the function hasn't returned by now, all elements are valid
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// This function checks all bucket policy conditions if the values provided
|
|
||||||
// are valid for the condition type. If not it returns a relevant Malformed policy error string
|
|
||||||
function validatePolicyConditions(policy) {
|
|
||||||
const validConditions = [
|
|
||||||
{ conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
|
|
||||||
{ conditionKey: 's3:object-lock-remaining-retention-days' },
|
|
||||||
];
|
|
||||||
// keys where value type does not seem to be checked by AWS:
|
|
||||||
// - s3:object-lock-remaining-retention-days
|
|
||||||
|
|
||||||
if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// there can be multiple statements in the policy, each with a Condition enclosure
|
|
||||||
for (let i = 0; i < policy.Statement.length; i++) {
|
|
||||||
const s = policy.Statement[i];
|
|
||||||
if (s.Condition) {
|
|
||||||
const conditionOperators = Object.keys(s.Condition);
|
|
||||||
// there can be multiple condition operations in the Condition enclosure
|
|
||||||
// eslint-disable-next-line no-restricted-syntax
|
|
||||||
for (const conditionOperator of conditionOperators) {
|
|
||||||
const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
|
|
||||||
const conditionValue = s.Condition[conditionOperator][conditionKey];
|
|
||||||
const validCondition = validConditions.find(validCondition =>
|
|
||||||
validCondition.conditionKey === conditionKey
|
|
||||||
);
|
|
||||||
// AWS returns does not return an error if the condition starts with 'aws:'
|
|
||||||
// so we reproduce this behaviour
|
|
||||||
if (!validCondition && !conditionKey.startsWith('aws:')) {
|
|
||||||
return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
|
|
||||||
}
|
|
||||||
if (validCondition && validCondition.conditionValueTypeChecker) {
|
|
||||||
const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
|
|
||||||
if (conditionValueTypeError) {
|
|
||||||
return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
|
|
||||||
* @param {string} arn - Amazon resource name - example:
|
|
||||||
* arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
|
|
||||||
* @return {boolean} true if Lifecycle assumed role session arn, false if not.
|
|
||||||
*/
|
|
||||||
function isLifecycleSession(arn) {
|
|
||||||
if (!arn) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const arnSplits = arn.split(':');
|
|
||||||
const service = arnSplits[2];
|
|
||||||
|
|
||||||
const resourceNames = arnSplits[arnSplits.length - 1].split('/');
|
|
||||||
|
|
||||||
const resourceType = resourceNames[0];
|
|
||||||
const sessionName = resourceNames[resourceNames.length - 1];
|
|
||||||
|
|
||||||
return (service === 'sts'
|
|
||||||
&& resourceType === assumedRoleArnResourceType
|
|
||||||
&& sessionName === backbeatLifecycleSessionName);
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
isBucketAuthorized,
|
isBucketAuthorized,
|
||||||
isObjAuthorized,
|
isObjAuthorized,
|
||||||
|
@ -635,7 +398,4 @@ module.exports = {
|
||||||
checkBucketAcls,
|
checkBucketAcls,
|
||||||
checkObjectAcls,
|
checkObjectAcls,
|
||||||
validatePolicyResource,
|
validatePolicyResource,
|
||||||
validatePolicyConditions,
|
|
||||||
isLifecycleSession,
|
|
||||||
evaluateBucketPolicyWithIAM,
|
|
||||||
};
|
};
|
||||||
|
|
|
@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||||
apiMethod, 's3');
|
apiMethod, 's3');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (apiMethod === 'bucketPut') {
|
if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -65,17 +65,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||||
|
|
||||||
const requestContexts = [];
|
const requestContexts = [];
|
||||||
|
|
||||||
if (apiMethod === 'multiObjectDelete') {
|
if (apiMethodAfterVersionCheck === 'objectCopy'
|
||||||
// MultiObjectDelete does not require any authorization when evaluating
|
|
||||||
// the API. Instead, we authorize each object passed.
|
|
||||||
// But in order to get any relevant information from the authorization service
|
|
||||||
// for example, the account quota, we must send a request context object
|
|
||||||
// with no `specificResource`. We expect the result to be an implicit deny.
|
|
||||||
// In the API, we then ignore these authorization results, and we can use
|
|
||||||
// any information returned, e.g., the quota.
|
|
||||||
const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
|
|
||||||
requestContexts.push(requestContextMultiObjectDelete);
|
|
||||||
} else if (apiMethodAfterVersionCheck === 'objectCopy'
|
|
||||||
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') {
|
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') {
|
||||||
const objectGetAction = sourceVersionId ? 'objectGetVersion' :
|
const objectGetAction = sourceVersionId ? 'objectGetVersion' :
|
||||||
'objectGet';
|
'objectGet';
|
||||||
|
@ -107,126 +97,27 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||||
const objectGetTaggingAction = (request.query &&
|
const objectGetTaggingAction = (request.query &&
|
||||||
request.query.versionId) ? 'objectGetTaggingVersion' :
|
request.query.versionId) ? 'objectGetTaggingVersion' :
|
||||||
'objectGetTagging';
|
'objectGetTagging';
|
||||||
if (request.headers['x-amz-version-id']) {
|
|
||||||
const objectGetVersionAction = 'objectGetVersion';
|
|
||||||
const getVersionResourceVersion =
|
|
||||||
generateRequestContext(objectGetVersionAction);
|
|
||||||
requestContexts.push(getVersionResourceVersion);
|
|
||||||
}
|
|
||||||
const getRequestContext =
|
const getRequestContext =
|
||||||
generateRequestContext(apiMethodAfterVersionCheck);
|
generateRequestContext(apiMethodAfterVersionCheck);
|
||||||
const getTaggingRequestContext =
|
const getTaggingRequestContext =
|
||||||
generateRequestContext(objectGetTaggingAction);
|
generateRequestContext(objectGetTaggingAction);
|
||||||
requestContexts.push(getRequestContext, getTaggingRequestContext);
|
requestContexts.push(getRequestContext, getTaggingRequestContext);
|
||||||
} else if (apiMethodAfterVersionCheck === 'objectGetTagging') {
|
|
||||||
const objectGetTaggingAction = 'objectGetTagging';
|
|
||||||
const getTaggingResourceVersion =
|
|
||||||
generateRequestContext(objectGetTaggingAction);
|
|
||||||
requestContexts.push(getTaggingResourceVersion);
|
|
||||||
if (request.headers['x-amz-version-id']) {
|
|
||||||
const objectGetTaggingVersionAction = 'objectGetTaggingVersion';
|
|
||||||
const getTaggingVersionResourceVersion =
|
|
||||||
generateRequestContext(objectGetTaggingVersionAction);
|
|
||||||
requestContexts.push(getTaggingVersionResourceVersion);
|
|
||||||
}
|
|
||||||
} else if (apiMethodAfterVersionCheck === 'objectHead') {
|
|
||||||
const objectHeadAction = 'objectHead';
|
|
||||||
const headObjectAction =
|
|
||||||
generateRequestContext(objectHeadAction);
|
|
||||||
requestContexts.push(headObjectAction);
|
|
||||||
if (request.headers['x-amz-version-id']) {
|
|
||||||
const objectHeadVersionAction = 'objectGetVersion';
|
|
||||||
const headObjectVersion =
|
|
||||||
generateRequestContext(objectHeadVersionAction);
|
|
||||||
requestContexts.push(headObjectVersion);
|
|
||||||
}
|
|
||||||
} else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
|
|
||||||
const putObjectTaggingRequestContext =
|
|
||||||
generateRequestContext('objectPutTagging');
|
|
||||||
requestContexts.push(putObjectTaggingRequestContext);
|
|
||||||
if (request.headers['x-amz-version-id']) {
|
|
||||||
const putObjectVersionRequestContext =
|
|
||||||
generateRequestContext('objectPutTaggingVersion');
|
|
||||||
requestContexts.push(putObjectVersionRequestContext);
|
|
||||||
}
|
|
||||||
} else if (apiMethodAfterVersionCheck === 'objectPutCopyPart') {
|
|
||||||
const putObjectRequestContext =
|
|
||||||
generateRequestContext('objectPut');
|
|
||||||
requestContexts.push(putObjectRequestContext);
|
|
||||||
const getObjectRequestContext =
|
|
||||||
generateRequestContext('objectGet');
|
|
||||||
requestContexts.push(getObjectRequestContext);
|
|
||||||
} else if (apiMethodAfterVersionCheck === 'objectPut') {
|
} else if (apiMethodAfterVersionCheck === 'objectPut') {
|
||||||
// if put object with version
|
const putRequestContext =
|
||||||
if (request.headers['x-scal-s3-version-id'] ||
|
generateRequestContext(apiMethodAfterVersionCheck);
|
||||||
request.headers['x-scal-s3-version-id'] === '') {
|
requestContexts.push(putRequestContext);
|
||||||
const putVersionRequestContext =
|
// if put object (versioning) with tag set
|
||||||
generateRequestContext('objectPutVersion');
|
if (request.headers['x-amz-tagging']) {
|
||||||
requestContexts.push(putVersionRequestContext);
|
const putTaggingRequestContext =
|
||||||
} else {
|
generateRequestContext('objectPutTagging');
|
||||||
const putRequestContext =
|
requestContexts.push(putTaggingRequestContext);
|
||||||
generateRequestContext(apiMethodAfterVersionCheck);
|
|
||||||
requestContexts.push(putRequestContext);
|
|
||||||
// if put object (versioning) with tag set
|
|
||||||
if (request.headers['x-amz-tagging']) {
|
|
||||||
const putTaggingRequestContext =
|
|
||||||
generateRequestContext('objectPutTagging');
|
|
||||||
requestContexts.push(putTaggingRequestContext);
|
|
||||||
}
|
|
||||||
if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
|
|
||||||
const putLegalHoldStatusAction =
|
|
||||||
generateRequestContext('objectPutLegalHold');
|
|
||||||
requestContexts.push(putLegalHoldStatusAction);
|
|
||||||
}
|
|
||||||
// if put object (versioning) with ACL
|
|
||||||
if (isHeaderAcl(request.headers)) {
|
|
||||||
const putAclRequestContext =
|
|
||||||
generateRequestContext('objectPutACL');
|
|
||||||
requestContexts.push(putAclRequestContext);
|
|
||||||
}
|
|
||||||
if (request.headers['x-amz-object-lock-mode']) {
|
|
||||||
const putObjectLockRequestContext =
|
|
||||||
generateRequestContext('objectPutRetention');
|
|
||||||
requestContexts.push(putObjectLockRequestContext);
|
|
||||||
}
|
|
||||||
if (request.headers['x-amz-version-id']) {
|
|
||||||
const putObjectVersionRequestContext =
|
|
||||||
generateRequestContext('objectPutTaggingVersion');
|
|
||||||
requestContexts.push(putObjectVersionRequestContext);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
|
|
||||||
apiMethodAfterVersionCheck === 'objectPutPart' ||
|
|
||||||
apiMethodAfterVersionCheck === 'completeMultipartUpload'
|
|
||||||
) {
|
|
||||||
if (request.headers['x-scal-s3-version-id'] ||
|
|
||||||
request.headers['x-scal-s3-version-id'] === '') {
|
|
||||||
const putVersionRequestContext =
|
|
||||||
generateRequestContext('objectPutVersion');
|
|
||||||
requestContexts.push(putVersionRequestContext);
|
|
||||||
} else {
|
|
||||||
const putRequestContext =
|
|
||||||
generateRequestContext(apiMethodAfterVersionCheck);
|
|
||||||
requestContexts.push(putRequestContext);
|
|
||||||
}
|
|
||||||
|
|
||||||
// if put object (versioning) with ACL
|
// if put object (versioning) with ACL
|
||||||
if (isHeaderAcl(request.headers)) {
|
if (isHeaderAcl(request.headers)) {
|
||||||
const putAclRequestContext =
|
const putAclRequestContext =
|
||||||
generateRequestContext('objectPutACL');
|
generateRequestContext('objectPutACL');
|
||||||
requestContexts.push(putAclRequestContext);
|
requestContexts.push(putAclRequestContext);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (request.headers['x-amz-object-lock-mode']) {
|
|
||||||
const putObjectLockRequestContext =
|
|
||||||
generateRequestContext('objectPutRetention');
|
|
||||||
requestContexts.push(putObjectLockRequestContext);
|
|
||||||
}
|
|
||||||
if (request.headers['x-amz-version-id']) {
|
|
||||||
const putObjectVersionRequestContext =
|
|
||||||
generateRequestContext('objectPutTaggingVersion');
|
|
||||||
requestContexts.push(putObjectVersionRequestContext);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
const requestContext =
|
const requestContext =
|
||||||
generateRequestContext(apiMethodAfterVersionCheck);
|
generateRequestContext(apiMethodAfterVersionCheck);
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
|
const assert = require('assert');
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
|
|
||||||
const { auth, s3middleware } = require('arsenal');
|
const { auth, s3middleware } = require('arsenal');
|
||||||
const metadata = require('../../../metadata/wrapper');
|
const metadata = require('../../../metadata/wrapper');
|
||||||
const { decodeVersionId } = require('../object/versioning');
|
const { decodeVersionId } = require('../object/versioning');
|
||||||
|
@ -12,66 +12,63 @@ function makeTagQuery(tags) {
|
||||||
.join('&');
|
.join('&');
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) {
|
function updateRequestContexts(request, requestContexts, apiMethod, log, cb) {
|
||||||
async.waterfall([
|
requestContexts.forEach(rc => {
|
||||||
next => {
|
rc.setNeedTagEval(true);
|
||||||
if (request.headers['x-amz-tagging']) {
|
|
||||||
return next(null, request.headers['x-amz-tagging']);
|
async.series([
|
||||||
}
|
next => {
|
||||||
if (request.post && apiMethod === 'objectPutTagging') {
|
if (request.headers['x-amz-tagging']) {
|
||||||
return parseTagXml(request.post, log, (err, tags) => {
|
rc.setRequestObjTags(request.headers['x-amz-tagging']);
|
||||||
|
process.nextTick(() => next());
|
||||||
|
} else if (request.post && apiMethod === 'objectPutTagging') {
|
||||||
|
parseTagXml(request.post, log, (err, tags) => {
|
||||||
|
if (err) {
|
||||||
|
log.trace('error parsing request tags');
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
rc.setRequestObjTags(makeTagQuery(tags));
|
||||||
|
return next();
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
process.nextTick(() => next());
|
||||||
|
}
|
||||||
|
},
|
||||||
|
next => {
|
||||||
|
const objectKey = request.objectKey;
|
||||||
|
const bucketName = request.bucketName;
|
||||||
|
const decodedVidResult = decodeVersionId(request.query);
|
||||||
|
if (decodedVidResult instanceof Error) {
|
||||||
|
log.trace('invalid versionId query', {
|
||||||
|
versionId: request.query.versionId,
|
||||||
|
error: decodedVidResult,
|
||||||
|
});
|
||||||
|
return process.nextTick(() => next(decodedVidResult));
|
||||||
|
}
|
||||||
|
const reqVersionId = decodedVidResult;
|
||||||
|
return metadata.getObjectMD(bucketName, objectKey, { versionId: reqVersionId }, log,
|
||||||
|
(err, objMD) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log.trace('error parsing request tags');
|
if (err.is.NoSuchKey) {
|
||||||
return next(err);
|
return next();
|
||||||
}
|
|
||||||
return next(null, makeTagQuery(tags));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return next(null, null);
|
|
||||||
},
|
|
||||||
(requestTagsQuery, next) => {
|
|
||||||
const objectKey = request.objectKey;
|
|
||||||
const bucketName = request.bucketName;
|
|
||||||
const decodedVidResult = decodeVersionId(request.query);
|
|
||||||
if (decodedVidResult instanceof Error) {
|
|
||||||
log.trace('invalid versionId query', {
|
|
||||||
versionId: request.query.versionId,
|
|
||||||
error: decodedVidResult,
|
|
||||||
});
|
|
||||||
return next(decodedVidResult);
|
|
||||||
}
|
|
||||||
const reqVersionId = decodedVidResult;
|
|
||||||
return metadata.getObjectMD(
|
|
||||||
bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => {
|
|
||||||
if (err) {
|
|
||||||
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
|
|
||||||
if (err.NoSuchKey) {
|
|
||||||
return next(null, requestTagsQuery, null);
|
|
||||||
}
|
}
|
||||||
log.trace('error getting request object tags');
|
log.trace('error getting request object tags');
|
||||||
return next(err);
|
return next(err);
|
||||||
}
|
}
|
||||||
const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags);
|
const existingTags = objMD.tags;
|
||||||
return next(null, requestTagsQuery, existingTagsQuery);
|
if (existingTags) {
|
||||||
|
rc.setExistingObjTag(makeTagQuery(existingTags));
|
||||||
|
}
|
||||||
|
return next();
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
], (err, requestTagsQuery, existingTagsQuery) => {
|
], err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log.trace('error processing tag condition key evaluation');
|
log.trace('error processing tag condition key evaluation');
|
||||||
return cb(err);
|
return cb(err);
|
||||||
}
|
|
||||||
// FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
|
|
||||||
// eslint-disable-next-line no-restricted-syntax
|
|
||||||
for (const rc of requestContexts) {
|
|
||||||
rc.setNeedTagEval(true);
|
|
||||||
if (requestTagsQuery) {
|
|
||||||
rc.setRequestObjTags(requestTagsQuery);
|
|
||||||
}
|
}
|
||||||
if (existingTagsQuery) {
|
return cb(null, requestContexts);
|
||||||
rc.setExistingObjTag(existingTagsQuery);
|
});
|
||||||
}
|
|
||||||
}
|
|
||||||
return cb();
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -80,20 +77,23 @@ function tagConditionKeyAuth(authorizationResults, request, requestContexts, api
|
||||||
return cb();
|
return cb();
|
||||||
}
|
}
|
||||||
if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
|
if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
|
||||||
return cb(null, authorizationResults);
|
return cb();
|
||||||
}
|
}
|
||||||
|
|
||||||
return updateRequestContextsWithTags(request, requestContexts, apiMethod, log, err => {
|
return updateRequestContexts(request, requestContexts, apiMethod, log, (err, updatedContexts) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return cb(err);
|
return cb(err);
|
||||||
}
|
}
|
||||||
|
if (assert.deepStrictEqual(requestContexts, updatedContexts)) {
|
||||||
|
return cb();
|
||||||
|
}
|
||||||
return auth.server.doAuth(request, log,
|
return auth.server.doAuth(request, log,
|
||||||
(err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts);
|
(err, userInfo, tagAuthResults) => cb(err, tagAuthResults), 's3', updatedContexts);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
tagConditionKeyAuth,
|
tagConditionKeyAuth,
|
||||||
updateRequestContextsWithTags,
|
updateRequestContexts,
|
||||||
makeTagQuery,
|
makeTagQuery,
|
||||||
};
|
};
|
||||||
|
|
|
@ -41,7 +41,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
|
||||||
usersBucket : oldUsersBucket;
|
usersBucket : oldUsersBucket;
|
||||||
return metadata.putObjectMD(usersBucketBeingCalled, key,
|
return metadata.putObjectMD(usersBucketBeingCalled, key,
|
||||||
omVal, {}, log, err => {
|
omVal, {}, log, err => {
|
||||||
if (err?.is?.NoSuchBucket) {
|
if (err?.is.NoSuchBucket) {
|
||||||
// There must be no usersBucket so createBucket
|
// There must be no usersBucket so createBucket
|
||||||
// one using the new format
|
// one using the new format
|
||||||
log.trace('users bucket does not exist, ' +
|
log.trace('users bucket does not exist, ' +
|
||||||
|
@ -61,8 +61,8 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
|
||||||
// from getting a BucketAlreadyExists
|
// from getting a BucketAlreadyExists
|
||||||
// error with respect
|
// error with respect
|
||||||
// to the usersBucket.
|
// to the usersBucket.
|
||||||
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
|
if (err &&
|
||||||
if (err && !err.BucketAlreadyExists) {
|
!err.is.BucketAlreadyExists) {
|
||||||
log.error('error from metadata', {
|
log.error('error from metadata', {
|
||||||
error: err,
|
error: err,
|
||||||
});
|
});
|
||||||
|
@ -223,8 +223,7 @@ function createBucket(authInfo, bucketName, headers,
|
||||||
},
|
},
|
||||||
getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
|
getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
|
||||||
metadata.getBucket(bucketName, log, (err, data) => {
|
metadata.getBucket(bucketName, log, (err, data) => {
|
||||||
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
|
if (err?.is.NoSuchBucket) {
|
||||||
if (err && err.NoSuchBucket) {
|
|
||||||
return callback(null, 'NoBucketYet');
|
return callback(null, 'NoBucketYet');
|
||||||
}
|
}
|
||||||
if (err) {
|
if (err) {
|
||||||
|
|
|
@ -16,15 +16,14 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
|
||||||
`${mpuBucketPrefix}${destinationBucketName}`;
|
`${mpuBucketPrefix}${destinationBucketName}`;
|
||||||
return metadata.deleteBucket(mpuBucketName, log, err => {
|
return metadata.deleteBucket(mpuBucketName, log, err => {
|
||||||
// If the mpu bucket does not exist, just move on
|
// If the mpu bucket does not exist, just move on
|
||||||
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
|
if (err?.is.NoSuchBucket) {
|
||||||
if (err && err.NoSuchBucket) {
|
|
||||||
return cb();
|
return cb();
|
||||||
}
|
}
|
||||||
return cb(err);
|
return cb(err);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) {
|
function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, log, cb) {
|
||||||
async.mapLimit(mpus, 1, (mpu, next) => {
|
async.mapLimit(mpus, 1, (mpu, next) => {
|
||||||
const splitterChar = mpu.key.includes(oldSplitter) ?
|
const splitterChar = mpu.key.includes(oldSplitter) ?
|
||||||
oldSplitter : splitter;
|
oldSplitter : splitter;
|
||||||
|
@ -40,7 +39,7 @@ function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log,
|
||||||
byteLength: partSizeSum,
|
byteLength: partSizeSum,
|
||||||
});
|
});
|
||||||
next(err);
|
next(err);
|
||||||
}, request);
|
});
|
||||||
}, cb);
|
}, cb);
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
|
@ -49,13 +48,11 @@ function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log,
|
||||||
* @param {object} bucketMD - bucket attributes/metadata
|
* @param {object} bucketMD - bucket attributes/metadata
|
||||||
* @param {string} bucketName - bucket in which objectMetadata is stored
|
* @param {string} bucketName - bucket in which objectMetadata is stored
|
||||||
* @param {string} canonicalID - account canonicalID of requester
|
* @param {string} canonicalID - account canonicalID of requester
|
||||||
* @param {object} request - request object given by router
|
|
||||||
* including normalized headers
|
|
||||||
* @param {object} log - Werelogs logger
|
* @param {object} log - Werelogs logger
|
||||||
* @param {function} cb - callback from async.waterfall in bucketDelete
|
* @param {function} cb - callback from async.waterfall in bucketDelete
|
||||||
* @return {undefined}
|
* @return {undefined}
|
||||||
*/
|
*/
|
||||||
function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, cb) {
|
function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, log, cb) {
|
||||||
log.trace('deleting bucket from metadata');
|
log.trace('deleting bucket from metadata');
|
||||||
assert.strictEqual(typeof bucketName, 'string');
|
assert.strictEqual(typeof bucketName, 'string');
|
||||||
assert.strictEqual(typeof canonicalID, 'string');
|
assert.strictEqual(typeof canonicalID, 'string');
|
||||||
|
@ -102,7 +99,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
|
||||||
}
|
}
|
||||||
if (objectsListRes.Contents.length) {
|
if (objectsListRes.Contents.length) {
|
||||||
return _deleteOngoingMPUs(authInfo, bucketName,
|
return _deleteOngoingMPUs(authInfo, bucketName,
|
||||||
bucketMD, objectsListRes.Contents, request, log, err => {
|
bucketMD, objectsListRes.Contents, log, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return next(err);
|
return next(err);
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,9 +30,6 @@ function bucketShield(bucket, requestType) {
|
||||||
// Otherwise return an error to the client
|
// Otherwise return an error to the client
|
||||||
if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
|
if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
|
||||||
(requestType !== 'objectPut' &&
|
(requestType !== 'objectPut' &&
|
||||||
requestType !== 'initiateMultipartUpload' &&
|
|
||||||
requestType !== 'objectPutPart' &&
|
|
||||||
requestType !== 'completeMultipartUpload' &&
|
|
||||||
requestType !== 'bucketPutACL' &&
|
requestType !== 'bucketPutACL' &&
|
||||||
requestType !== 'bucketDelete')) {
|
requestType !== 'bucketDelete')) {
|
||||||
return true;
|
return true;
|
||||||
|
|
|
@ -20,8 +20,7 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
|
||||||
oldSplitter, bucketName);
|
oldSplitter, bucketName);
|
||||||
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
|
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
|
||||||
{}, log, error => {
|
{}, log, error => {
|
||||||
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
|
if (error && !error.is.NoSuchKey) {
|
||||||
if (error && !error.NoSuchKey) {
|
|
||||||
log.error('from metadata while deleting user bucket',
|
log.error('from metadata while deleting user bucket',
|
||||||
{ error });
|
{ error });
|
||||||
return cb(error);
|
return cb(error);
|
||||||
|
|
|
@ -3,7 +3,7 @@ const async = require('async');
|
||||||
const constants = require('../../../../constants');
|
const constants = require('../../../../constants');
|
||||||
const { data } = require('../../../data/wrapper');
|
const { data } = require('../../../data/wrapper');
|
||||||
const locationConstraintCheck = require('../object/locationConstraintCheck');
|
const locationConstraintCheck = require('../object/locationConstraintCheck');
|
||||||
const { standardMetadataValidateBucketAndObj } =
|
const { metadataValidateBucketAndObj } =
|
||||||
require('../../../metadata/metadataUtils');
|
require('../../../metadata/metadataUtils');
|
||||||
const services = require('../../../services');
|
const services = require('../../../services');
|
||||||
|
|
||||||
|
@ -14,7 +14,7 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
uploadId,
|
uploadId,
|
||||||
preciseRequestType: request.apiMethods || 'multipartDelete',
|
preciseRequestType: 'multipartDelete',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
// For validating the request at the destinationBucket level
|
// For validating the request at the destinationBucket level
|
||||||
|
@ -22,11 +22,10 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
|
||||||
// but the requestType is the more general 'objectDelete'
|
// but the requestType is the more general 'objectDelete'
|
||||||
const metadataValParams = Object.assign({}, metadataValMPUparams);
|
const metadataValParams = Object.assign({}, metadataValMPUparams);
|
||||||
metadataValParams.requestType = 'objectPut';
|
metadataValParams.requestType = 'objectPut';
|
||||||
const authzIdentityResult = request ? request.actionImplicitDenies : false;
|
|
||||||
|
|
||||||
async.waterfall([
|
async.waterfall([
|
||||||
function checkDestBucketVal(next) {
|
function checkDestBucketVal(next) {
|
||||||
standardMetadataValidateBucketAndObj(metadataValParams, authzIdentityResult, log,
|
metadataValidateBucketAndObj(metadataValParams, log,
|
||||||
(err, destinationBucket) => {
|
(err, destinationBucket) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return next(err, destinationBucket);
|
return next(err, destinationBucket);
|
||||||
|
@ -57,14 +56,9 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
|
||||||
function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket,
|
function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket,
|
||||||
next) {
|
next) {
|
||||||
const location = mpuOverviewObj.controllingLocationConstraint;
|
const location = mpuOverviewObj.controllingLocationConstraint;
|
||||||
const originalIdentityAuthzResults = request.actionImplicitDenies;
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
delete request.actionImplicitDenies;
|
|
||||||
return data.abortMPU(objectKey, uploadId, location, bucketName,
|
return data.abortMPU(objectKey, uploadId, location, bucketName,
|
||||||
request, destBucket, locationConstraintCheck, log,
|
request, destBucket, locationConstraintCheck, log,
|
||||||
(err, skipDataDelete) => {
|
(err, skipDataDelete) => {
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
request.actionImplicitDenies = originalIdentityAuthzResults;
|
|
||||||
if (err) {
|
if (err) {
|
||||||
return next(err, destBucket);
|
return next(err, destBucket);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,247 +0,0 @@
|
||||||
/*
|
|
||||||
* Code based on Yutaka Oishi (Fujifilm) contributions
|
|
||||||
* Date: 11 Sep 2020
|
|
||||||
*/
|
|
||||||
const { ObjectMDArchive } = require('arsenal').models;
|
|
||||||
const errors = require('arsenal').errors;
|
|
||||||
const { config } = require('../../../Config');
|
|
||||||
const { locationConstraints } = config;
|
|
||||||
|
|
||||||
const { scaledMsPerDay } = config.getTimeOptions();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get response header "x-amz-restore"
|
|
||||||
* Be called by objectHead.js
|
|
||||||
* @param {object} objMD - object's metadata
|
|
||||||
* @returns {string|undefined} x-amz-restore
|
|
||||||
*/
|
|
||||||
function getAmzRestoreResHeader(objMD) {
|
|
||||||
if (objMD.archive &&
|
|
||||||
objMD.archive.restoreRequestedAt &&
|
|
||||||
!objMD.archive.restoreCompletedAt) {
|
|
||||||
// Avoid race condition by relying on the `archive` MD of the object
|
|
||||||
// and return the right header after a RESTORE request.
|
|
||||||
// eslint-disable-next-line
|
|
||||||
return `ongoing-request="true"`;
|
|
||||||
}
|
|
||||||
if (objMD['x-amz-restore']) {
|
|
||||||
if (objMD['x-amz-restore']['expiry-date']) {
|
|
||||||
const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString();
|
|
||||||
// eslint-disable-next-line
|
|
||||||
return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if restore can be done.
|
|
||||||
*
|
|
||||||
* @param {ObjectMD} objectMD - object metadata
|
|
||||||
* @param {object} log - werelogs logger
|
|
||||||
* @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
|
|
||||||
*/
|
|
||||||
function _validateStartRestore(objectMD, log) {
|
|
||||||
if (objectMD.archive?.restoreCompletedAt) {
|
|
||||||
if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
|
|
||||||
// return InvalidObjectState error if the restored object is expired
|
|
||||||
// but restore info md of this object has not yet been cleared
|
|
||||||
log.debug('The restored object already expired.',
|
|
||||||
{
|
|
||||||
archive: objectMD.archive,
|
|
||||||
method: '_validateStartRestore',
|
|
||||||
});
|
|
||||||
return errors.InvalidObjectState;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If object is already restored, no further check is needed
|
|
||||||
// Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have
|
|
||||||
// been reset.
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
|
|
||||||
if (!isLocationCold) {
|
|
||||||
// return InvalidObjectState error if the object is not in cold storage,
|
|
||||||
// not in cold storage means either location cold flag not exists or cold flag is explicit false
|
|
||||||
log.debug('The bucket of the object is not in a cold storage location.',
|
|
||||||
{
|
|
||||||
isLocationCold,
|
|
||||||
method: '_validateStartRestore',
|
|
||||||
});
|
|
||||||
return errors.InvalidObjectState;
|
|
||||||
}
|
|
||||||
if (objectMD.archive?.restoreRequestedAt) {
|
|
||||||
// return RestoreAlreadyInProgress error if the object is currently being restored
|
|
||||||
// check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
|
|
||||||
log.debug('The object is currently being restored.',
|
|
||||||
{
|
|
||||||
archive: objectMD.archive,
|
|
||||||
method: '_validateStartRestore',
|
|
||||||
});
|
|
||||||
return errors.RestoreAlreadyInProgress;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if "put version id" is allowed
|
|
||||||
*
|
|
||||||
* @param {ObjectMD} objMD - object metadata
|
|
||||||
* @param {string} versionId - object's version id
|
|
||||||
* @param {object} log - werelogs logger
|
|
||||||
* @return {ArsenalError|undefined} - undefined if "put version id" is allowed
|
|
||||||
*/
|
|
||||||
function validatePutVersionId(objMD, versionId, log) {
|
|
||||||
if (!objMD) {
|
|
||||||
const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
|
|
||||||
log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
|
|
||||||
return err;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (objMD.isDeleteMarker) {
|
|
||||||
log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
|
|
||||||
return errors.MethodNotAllowed;
|
|
||||||
}
|
|
||||||
|
|
||||||
const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
|
|
||||||
if (!isLocationCold) {
|
|
||||||
log.error('The object data is not stored in a cold storage location.',
|
|
||||||
{
|
|
||||||
isLocationCold,
|
|
||||||
dataStoreName: objMD.dataStoreName,
|
|
||||||
method: 'validatePutVersionId',
|
|
||||||
});
|
|
||||||
return errors.InvalidObjectState;
|
|
||||||
}
|
|
||||||
|
|
||||||
// make sure object archive restoration is in progress
|
|
||||||
// NOTE: we do not use putObjectVersion to update the restoration period.
|
|
||||||
if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
|
|
||||||
|| objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
|
|
||||||
log.error('object archive restoration is not in progress',
|
|
||||||
{ method: 'validatePutVersionId', versionId });
|
|
||||||
return errors.InvalidObjectState;
|
|
||||||
}
|
|
||||||
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if the object is already restored, and update the expiration date accordingly:
|
|
||||||
* > After restoring an archived object, you can update the restoration period by reissuing the
|
|
||||||
* > request with a new period. Amazon S3 updates the restoration period relative to the current
|
|
||||||
* > time.
|
|
||||||
*
|
|
||||||
* @param {ObjectMD} objectMD - object metadata
|
|
||||||
* @param {object} log - werelogs logger
|
|
||||||
* @return {boolean} - true if the object is already restored
|
|
||||||
*/
|
|
||||||
function _updateObjectExpirationDate(objectMD, log) {
|
|
||||||
// Check if restoreCompletedAt field exists
|
|
||||||
// Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
|
|
||||||
// checked earlier in the process, so checking again here would create weird states
|
|
||||||
const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
|
|
||||||
log.debug('The restore status of the object.', {
|
|
||||||
isObjectAlreadyRestored,
|
|
||||||
method: 'isObjectAlreadyRestored'
|
|
||||||
});
|
|
||||||
if (isObjectAlreadyRestored) {
|
|
||||||
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
|
|
||||||
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
|
|
||||||
|
|
||||||
/* eslint-disable no-param-reassign */
|
|
||||||
objectMD.archive.restoreWillExpireAt = expiryDate;
|
|
||||||
objectMD['x-amz-restore'] = {
|
|
||||||
'ongoing-request': false,
|
|
||||||
'expiry-date': expiryDate,
|
|
||||||
};
|
|
||||||
/* eslint-enable no-param-reassign */
|
|
||||||
}
|
|
||||||
return isObjectAlreadyRestored;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* update restore expiration date.
|
|
||||||
*
|
|
||||||
* @param {ObjectMD} objectMD - objectMD instance
|
|
||||||
* @param {object} restoreParam - restore param
|
|
||||||
* @param {object} log - werelogs logger
|
|
||||||
* @return {ArsenalError|undefined} internal error if object MD is not valid
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
function _updateRestoreInfo(objectMD, restoreParam, log) {
|
|
||||||
if (!objectMD.archive) {
|
|
||||||
log.debug('objectMD.archive doesn\'t exits', {
|
|
||||||
objectMD,
|
|
||||||
method: '_updateRestoreInfo'
|
|
||||||
});
|
|
||||||
return errors.InternalError.customizeDescription('Archive metadata is missing.');
|
|
||||||
}
|
|
||||||
/* eslint-disable no-param-reassign */
|
|
||||||
objectMD.archive.restoreRequestedAt = new Date();
|
|
||||||
objectMD.archive.restoreRequestedDays = restoreParam.days;
|
|
||||||
objectMD.originOp = 's3:ObjectRestore:Post';
|
|
||||||
/* eslint-enable no-param-reassign */
|
|
||||||
if (!ObjectMDArchive.isValid(objectMD.archive)) {
|
|
||||||
log.debug('archive is not valid', {
|
|
||||||
archive: objectMD.archive,
|
|
||||||
method: '_updateRestoreInfo'
|
|
||||||
});
|
|
||||||
return errors.InternalError.customizeDescription('Invalid archive metadata.');
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* start to restore object.
|
|
||||||
* If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
|
|
||||||
* calculate restore expiry-date and add it to objectMD.
|
|
||||||
* Be called by objectRestore.js
|
|
||||||
*
|
|
||||||
* @param {ObjectMD} objectMD - objectMd instance
|
|
||||||
* @param {object} restoreParam - bucket name
|
|
||||||
* @param {object} log - werelogs logger
|
|
||||||
* @param {function} cb - bucket name
|
|
||||||
* @return {undefined}
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
function startRestore(objectMD, restoreParam, log, cb) {
|
|
||||||
log.info('Validating if restore can be done or not.');
|
|
||||||
const checkResultError = _validateStartRestore(objectMD, log);
|
|
||||||
if (checkResultError) {
|
|
||||||
return cb(checkResultError);
|
|
||||||
}
|
|
||||||
log.info('Updating restore information.');
|
|
||||||
const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
|
|
||||||
if (updateResultError) {
|
|
||||||
return cb(updateResultError);
|
|
||||||
}
|
|
||||||
const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
|
|
||||||
return cb(null, isObjectAlreadyRestored);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* checks if object data is available or if it's in cold storage
|
|
||||||
* @param {ObjectMD} objMD Object metadata
|
|
||||||
* @returns {ArsenalError|null} error if object data is not available
|
|
||||||
*/
|
|
||||||
function verifyColdObjectAvailable(objMD) {
|
|
||||||
// return error when object is cold
|
|
||||||
if (objMD.archive &&
|
|
||||||
// Object is in cold backend
|
|
||||||
(!objMD.archive.restoreRequestedAt ||
|
|
||||||
// Object is being restored
|
|
||||||
(objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
|
|
||||||
const err = errors.InvalidObjectState
|
|
||||||
.customizeDescription('The operation is not valid for the object\'s storage class');
|
|
||||||
return err;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
startRestore,
|
|
||||||
getAmzRestoreResHeader,
|
|
||||||
validatePutVersionId,
|
|
||||||
verifyColdObjectAvailable,
|
|
||||||
};
|
|
|
@ -5,9 +5,10 @@ const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
|
||||||
const constants = require('../../../../constants');
|
const constants = require('../../../../constants');
|
||||||
const { data } = require('../../../data/wrapper');
|
const { data } = require('../../../data/wrapper');
|
||||||
const services = require('../../../services');
|
const services = require('../../../services');
|
||||||
|
const logger = require('../../../utilities/logger');
|
||||||
const { dataStore } = require('./storeObject');
|
const { dataStore } = require('./storeObject');
|
||||||
const locationConstraintCheck = require('./locationConstraintCheck');
|
const locationConstraintCheck = require('./locationConstraintCheck');
|
||||||
const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
|
const { versioningPreprocessing } = require('./versioning');
|
||||||
const removeAWSChunked = require('./removeAWSChunked');
|
const removeAWSChunked = require('./removeAWSChunked');
|
||||||
const getReplicationInfo = require('./getReplicationInfo');
|
const getReplicationInfo = require('./getReplicationInfo');
|
||||||
const { config } = require('../../../Config');
|
const { config } = require('../../../Config');
|
||||||
|
@ -20,7 +21,7 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' +
|
||||||
'a versioned object to a location-constraint of type Azure or GCP.';
|
'a versioned object to a location-constraint of type Azure or GCP.';
|
||||||
|
|
||||||
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
||||||
metadataStoreParams, dataToDelete, log, requestMethod, callback) {
|
metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
|
||||||
services.metadataStoreObject(bucketName, dataGetInfo,
|
services.metadataStoreObject(bucketName, dataGetInfo,
|
||||||
cipherBundle, metadataStoreParams, (err, result) => {
|
cipherBundle, metadataStoreParams, (err, result) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
|
@ -30,7 +31,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
||||||
const newDataStoreName = Array.isArray(dataGetInfo) ?
|
const newDataStoreName = Array.isArray(dataGetInfo) ?
|
||||||
dataGetInfo[0].dataStoreName : null;
|
dataGetInfo[0].dataStoreName : null;
|
||||||
return data.batchDelete(dataToDelete, requestMethod,
|
return data.batchDelete(dataToDelete, requestMethod,
|
||||||
newDataStoreName, log, err => callback(err, result));
|
newDataStoreName, deleteLog, err => callback(err, result));
|
||||||
}
|
}
|
||||||
return callback(null, result);
|
return callback(null, result);
|
||||||
});
|
});
|
||||||
|
@ -50,9 +51,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
||||||
* @param {(object|null)} streamingV4Params - if v4 auth, object containing
|
* @param {(object|null)} streamingV4Params - if v4 auth, object containing
|
||||||
* accessKey, signatureFromRequest, region, scopeDate, timestamp, and
|
* accessKey, signatureFromRequest, region, scopeDate, timestamp, and
|
||||||
* credentialScope (to be used for streaming v4 auth if applicable)
|
* credentialScope (to be used for streaming v4 auth if applicable)
|
||||||
* @param {(object|null)} overheadField - fields to be included in metadata overhead
|
|
||||||
* @param {RequestLogger} log - logger instance
|
* @param {RequestLogger} log - logger instance
|
||||||
* @param {string} originOp - Origin operation
|
|
||||||
* @param {function} callback - callback function
|
* @param {function} callback - callback function
|
||||||
* @return {undefined} and call callback with (err, result) -
|
* @return {undefined} and call callback with (err, result) -
|
||||||
* result.contentMD5 - content md5 of new object or version
|
* result.contentMD5 - content md5 of new object or version
|
||||||
|
@ -60,10 +59,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
||||||
*/
|
*/
|
||||||
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
|
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
|
||||||
overheadField, log, originOp, callback) {
|
log, callback) {
|
||||||
const putVersionId = request.headers['x-scal-s3-version-id'];
|
|
||||||
const isPutVersion = putVersionId || putVersionId === '';
|
|
||||||
|
|
||||||
const size = isDeleteMarker ? 0 : request.parsedContentLength;
|
const size = isDeleteMarker ? 0 : request.parsedContentLength;
|
||||||
// although the request method may actually be 'DELETE' if creating a
|
// although the request method may actually be 'DELETE' if creating a
|
||||||
// delete marker, for our purposes we consider this to be a 'PUT'
|
// delete marker, for our purposes we consider this to be a 'PUT'
|
||||||
|
@ -116,7 +112,6 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
isDeleteMarker,
|
isDeleteMarker,
|
||||||
replicationInfo: getReplicationInfo(
|
replicationInfo: getReplicationInfo(
|
||||||
objectKey, bucketMD, false, size, null, null, authInfo),
|
objectKey, bucketMD, false, size, null, null, authInfo),
|
||||||
overheadField,
|
|
||||||
log,
|
log,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -134,6 +129,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (!isDeleteMarker) {
|
if (!isDeleteMarker) {
|
||||||
metadataStoreParams.contentType = request.headers['content-type'];
|
metadataStoreParams.contentType = request.headers['content-type'];
|
||||||
metadataStoreParams.cacheControl = request.headers['cache-control'];
|
metadataStoreParams.cacheControl = request.headers['cache-control'];
|
||||||
|
@ -143,7 +139,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
removeAWSChunked(request.headers['content-encoding']);
|
removeAWSChunked(request.headers['content-encoding']);
|
||||||
metadataStoreParams.expires = request.headers.expires;
|
metadataStoreParams.expires = request.headers.expires;
|
||||||
metadataStoreParams.tagging = request.headers['x-amz-tagging'];
|
metadataStoreParams.tagging = request.headers['x-amz-tagging'];
|
||||||
metadataStoreParams.originOp = originOp;
|
metadataStoreParams.originOp = 's3:ObjectCreated:Put';
|
||||||
const defaultObjectLockConfiguration
|
const defaultObjectLockConfiguration
|
||||||
= bucketMD.getObjectLockConfiguration();
|
= bucketMD.getObjectLockConfiguration();
|
||||||
if (defaultObjectLockConfiguration) {
|
if (defaultObjectLockConfiguration) {
|
||||||
|
@ -158,7 +154,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
// eslint-disable-next-line no-param-reassign
|
// eslint-disable-next-line no-param-reassign
|
||||||
request.headers[constants.objectLocationConstraintHeader] =
|
request.headers[constants.objectLocationConstraintHeader] =
|
||||||
objMD[constants.objectLocationConstraintHeader];
|
objMD[constants.objectLocationConstraintHeader];
|
||||||
metadataStoreParams.originOp = originOp;
|
metadataStoreParams.originOp = 's3:ObjectRemoved:DeleteMarkerCreated';
|
||||||
}
|
}
|
||||||
|
|
||||||
const backendInfoObj =
|
const backendInfoObj =
|
||||||
|
@ -189,17 +185,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (objMD && objMD.uploadId) {
|
|
||||||
metadataStoreParams.oldReplayId = objMD.uploadId;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* eslint-disable camelcase */
|
/* eslint-disable camelcase */
|
||||||
const dontSkipBackend = externalBackends;
|
const dontSkipBackend = externalBackends;
|
||||||
/* eslint-enable camelcase */
|
/* eslint-enable camelcase */
|
||||||
|
|
||||||
|
const requestLogger =
|
||||||
|
logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
|
||||||
const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
|
const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
|
||||||
const mdOnlySize = request.headers['x-amz-meta-size'];
|
const mdOnlySize = request.headers['x-amz-meta-size'];
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
function storeData(next) {
|
function storeData(next) {
|
||||||
if (size === 0) {
|
if (size === 0) {
|
||||||
|
@ -264,11 +257,6 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
return next(null, dataGetInfoArr);
|
return next(null, dataGetInfoArr);
|
||||||
},
|
},
|
||||||
function getVersioningInfo(infoArr, next) {
|
function getVersioningInfo(infoArr, next) {
|
||||||
// if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
|
|
||||||
if (isPutVersion) {
|
|
||||||
const options = overwritingVersioning(objMD, metadataStoreParams);
|
|
||||||
return process.nextTick(() => next(null, options, infoArr));
|
|
||||||
}
|
|
||||||
return versioningPreprocessing(bucketName, bucketMD,
|
return versioningPreprocessing(bucketName, bucketMD,
|
||||||
metadataStoreParams.objectKey, objMD, log, (err, options) => {
|
metadataStoreParams.objectKey, objMD, log, (err, options) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
|
@ -288,13 +276,11 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
metadataStoreParams.versionId = options.versionId;
|
metadataStoreParams.versionId = options.versionId;
|
||||||
metadataStoreParams.versioning = options.versioning;
|
metadataStoreParams.versioning = options.versioning;
|
||||||
metadataStoreParams.isNull = options.isNull;
|
metadataStoreParams.isNull = options.isNull;
|
||||||
metadataStoreParams.deleteNullKey = options.deleteNullKey;
|
metadataStoreParams.nullVersionId = options.nullVersionId;
|
||||||
if (options.extraMD) {
|
metadataStoreParams.nullUploadId = options.nullUploadId;
|
||||||
Object.assign(metadataStoreParams, options.extraMD);
|
|
||||||
}
|
|
||||||
return _storeInMDandDeleteData(bucketName, infoArr,
|
return _storeInMDandDeleteData(bucketName, infoArr,
|
||||||
cipherBundle, metadataStoreParams,
|
cipherBundle, metadataStoreParams,
|
||||||
options.dataToDelete, log, requestMethod, next);
|
options.dataToDelete, requestLogger, requestMethod, next);
|
||||||
},
|
},
|
||||||
], callback);
|
], callback);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,18 +0,0 @@
|
||||||
/**
|
|
||||||
* _bucketRequiresOplogUpdate - DELETE an object from a bucket
|
|
||||||
* @param {BucketInfo} bucket - bucket object
|
|
||||||
* @return {boolean} whether objects require oplog updates on deletion, or not
|
|
||||||
*/
|
|
||||||
function _bucketRequiresOplogUpdate(bucket) {
|
|
||||||
// Default behavior is to require an oplog update
|
|
||||||
if (!bucket || !bucket.getLifecycleConfiguration || !bucket.getNotificationConfiguration) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
// If the bucket has lifecycle configuration or notification configuration
|
|
||||||
// set, we also require an oplog update
|
|
||||||
return bucket.getLifecycleConfiguration() || bucket.getNotificationConfiguration();
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
_bucketRequiresOplogUpdate,
|
|
||||||
};
|
|
|
@ -4,25 +4,23 @@ const {
|
||||||
LifecycleDateTime,
|
LifecycleDateTime,
|
||||||
LifecycleUtils,
|
LifecycleUtils,
|
||||||
} = require('arsenal').s3middleware.lifecycleHelpers;
|
} = require('arsenal').s3middleware.lifecycleHelpers;
|
||||||
const { config } = require('../../../Config');
|
|
||||||
|
|
||||||
const {
|
// moves lifecycle transition deadlines 1 day earlier, mostly for testing
|
||||||
expireOneDayEarlier,
|
const transitionOneDayEarlier = process.env.TRANSITION_ONE_DAY_EARLIER === 'true';
|
||||||
transitionOneDayEarlier,
|
// moves lifecycle expiration deadlines 1 day earlier, mostly for testing
|
||||||
timeProgressionFactor,
|
const expireOneDayEarlier = process.env.EXPIRE_ONE_DAY_EARLIER === 'true';
|
||||||
scaledMsPerDay,
|
|
||||||
} = config.getTimeOptions();
|
|
||||||
|
|
||||||
const lifecycleDateTime = new LifecycleDateTime({
|
const lifecycleDateTime = new LifecycleDateTime({
|
||||||
transitionOneDayEarlier,
|
transitionOneDayEarlier,
|
||||||
expireOneDayEarlier,
|
expireOneDayEarlier,
|
||||||
timeProgressionFactor,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor);
|
const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime);
|
||||||
|
|
||||||
|
const oneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
|
||||||
|
|
||||||
function calculateDate(objDate, expDays, datetime) {
|
function calculateDate(objDate, expDays, datetime) {
|
||||||
return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay));
|
return new Date(datetime.getTimestamp(objDate) + expDays * oneDay);
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatExpirationHeader(date, id) {
|
function formatExpirationHeader(date, id) {
|
||||||
|
@ -39,10 +37,8 @@ const AMZ_ABORT_ID_HEADER = 'x-amz-abort-rule-id';
|
||||||
|
|
||||||
function _generateExpHeadersObjects(rules, params, datetime) {
|
function _generateExpHeadersObjects(rules, params, datetime) {
|
||||||
const tags = {
|
const tags = {
|
||||||
TagSet: params.tags
|
TagSet: Object.keys(params.tags)
|
||||||
? Object.keys(params.tags)
|
.map(key => ({ Key: key, Value: params.tags[key] })),
|
||||||
.map(key => ({ Key: key, Value: params.tags[key] }))
|
|
||||||
: [],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const objectInfo = { Key: params.key };
|
const objectInfo = { Key: params.key };
|
||||||
|
|
|
@ -1,190 +0,0 @@
|
||||||
const { versioning } = require('arsenal');
|
|
||||||
const versionIdUtils = versioning.VersionID;
|
|
||||||
|
|
||||||
const { lifecycleListing } = require('../../../../constants');
|
|
||||||
const { CURRENT_TYPE, NON_CURRENT_TYPE, ORPHAN_DM_TYPE } = lifecycleListing;
|
|
||||||
|
|
||||||
function _makeTags(tags) {
|
|
||||||
const res = [];
|
|
||||||
Object.entries(tags).forEach(([key, value]) =>
|
|
||||||
res.push(
|
|
||||||
{
|
|
||||||
Key: key,
|
|
||||||
Value: value,
|
|
||||||
}
|
|
||||||
));
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
function processCurrents(bucketName, listParams, isBucketVersioned, list) {
|
|
||||||
const data = {
|
|
||||||
Name: bucketName,
|
|
||||||
Prefix: listParams.prefix,
|
|
||||||
MaxKeys: listParams.maxKeys,
|
|
||||||
MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
|
|
||||||
IsTruncated: !!list.IsTruncated,
|
|
||||||
Marker: listParams.marker,
|
|
||||||
BeforeDate: listParams.beforeDate,
|
|
||||||
NextMarker: list.NextMarker,
|
|
||||||
Contents: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
list.Contents.forEach(item => {
|
|
||||||
const v = item.value;
|
|
||||||
|
|
||||||
const content = {
|
|
||||||
Key: item.key,
|
|
||||||
LastModified: v.LastModified,
|
|
||||||
ETag: `"${v.ETag}"`,
|
|
||||||
Size: v.Size,
|
|
||||||
Owner: {
|
|
||||||
ID: v.Owner.ID,
|
|
||||||
DisplayName: v.Owner.DisplayName,
|
|
||||||
},
|
|
||||||
StorageClass: v.StorageClass,
|
|
||||||
TagSet: _makeTags(v.tags),
|
|
||||||
IsLatest: true, // for compatibility with AWS ListObjectVersions.
|
|
||||||
DataStoreName: v.dataStoreName,
|
|
||||||
ListType: CURRENT_TYPE,
|
|
||||||
};
|
|
||||||
|
|
||||||
// NOTE: The current versions listed to be lifecycle should include version id
|
|
||||||
// if the bucket is versioned.
|
|
||||||
if (isBucketVersioned) {
|
|
||||||
const versionId = (v.IsNull || v.VersionId === undefined) ?
|
|
||||||
'null' : versionIdUtils.encode(v.VersionId);
|
|
||||||
content.VersionId = versionId;
|
|
||||||
}
|
|
||||||
|
|
||||||
data.Contents.push(content);
|
|
||||||
});
|
|
||||||
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
|
|
||||||
function _encodeVersionId(vid) {
|
|
||||||
let versionId = vid;
|
|
||||||
if (versionId && versionId !== 'null') {
|
|
||||||
versionId = versionIdUtils.encode(versionId);
|
|
||||||
}
|
|
||||||
return versionId;
|
|
||||||
}
|
|
||||||
|
|
||||||
function processNonCurrents(bucketName, listParams, list) {
|
|
||||||
const nextVersionIdMarker = _encodeVersionId(list.NextVersionIdMarker);
|
|
||||||
const versionIdMarker = _encodeVersionId(listParams.versionIdMarker);
|
|
||||||
|
|
||||||
const data = {
|
|
||||||
Name: bucketName,
|
|
||||||
Prefix: listParams.prefix,
|
|
||||||
MaxKeys: listParams.maxKeys,
|
|
||||||
MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
|
|
||||||
IsTruncated: !!list.IsTruncated,
|
|
||||||
KeyMarker: listParams.keyMarker,
|
|
||||||
VersionIdMarker: versionIdMarker,
|
|
||||||
BeforeDate: listParams.beforeDate,
|
|
||||||
NextKeyMarker: list.NextKeyMarker,
|
|
||||||
NextVersionIdMarker: nextVersionIdMarker,
|
|
||||||
Contents: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
list.Contents.forEach(item => {
|
|
||||||
const v = item.value;
|
|
||||||
const versionId = (v.IsNull || v.VersionId === undefined) ?
|
|
||||||
'null' : versionIdUtils.encode(v.VersionId);
|
|
||||||
|
|
||||||
const content = {
|
|
||||||
Key: item.key,
|
|
||||||
LastModified: v.LastModified,
|
|
||||||
ETag: `"${v.ETag}"`,
|
|
||||||
Size: v.Size,
|
|
||||||
Owner: {
|
|
||||||
ID: v.Owner.ID,
|
|
||||||
DisplayName: v.Owner.DisplayName,
|
|
||||||
},
|
|
||||||
StorageClass: v.StorageClass,
|
|
||||||
TagSet: _makeTags(v.tags),
|
|
||||||
staleDate: v.staleDate, // lowerCamelCase to be compatible with existing lifecycle.
|
|
||||||
VersionId: versionId,
|
|
||||||
DataStoreName: v.dataStoreName,
|
|
||||||
ListType: NON_CURRENT_TYPE,
|
|
||||||
};
|
|
||||||
|
|
||||||
data.Contents.push(content);
|
|
||||||
});
|
|
||||||
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
|
|
||||||
function processOrphans(bucketName, listParams, list) {
|
|
||||||
const data = {
|
|
||||||
Name: bucketName,
|
|
||||||
Prefix: listParams.prefix,
|
|
||||||
MaxKeys: listParams.maxKeys,
|
|
||||||
MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
|
|
||||||
IsTruncated: !!list.IsTruncated,
|
|
||||||
Marker: listParams.marker,
|
|
||||||
BeforeDate: listParams.beforeDate,
|
|
||||||
NextMarker: list.NextMarker,
|
|
||||||
Contents: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
list.Contents.forEach(item => {
|
|
||||||
const v = item.value;
|
|
||||||
const versionId = (v.IsNull || v.VersionId === undefined) ?
|
|
||||||
'null' : versionIdUtils.encode(v.VersionId);
|
|
||||||
data.Contents.push({
|
|
||||||
Key: item.key,
|
|
||||||
LastModified: v.LastModified,
|
|
||||||
Owner: {
|
|
||||||
ID: v.Owner.ID,
|
|
||||||
DisplayName: v.Owner.DisplayName,
|
|
||||||
},
|
|
||||||
VersionId: versionId,
|
|
||||||
IsLatest: true, // for compatibility with AWS ListObjectVersions.
|
|
||||||
ListType: ORPHAN_DM_TYPE,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getLocationConstraintErrorMessage(locationName) {
|
|
||||||
return 'value of the location you are attempting to set ' +
|
|
||||||
`- ${locationName} - is not listed in the locationConstraint config`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* validateMaxScannedEntries - Validates and returns the maximum scanned entries value.
|
|
||||||
*
|
|
||||||
* @param {object} params - Query parameters
|
|
||||||
* @param {object} config - CloudServer configuration
|
|
||||||
* @param {number} min - Minimum number of entries to be scanned
|
|
||||||
* @returns {Object} - An object indicating the validation result:
|
|
||||||
* - isValid (boolean): Whether the validation is successful.
|
|
||||||
* - maxScannedLifecycleListingEntries (number): The validated maximum scanned entries value if isValid is true.
|
|
||||||
*/
|
|
||||||
function validateMaxScannedEntries(params, config, min) {
|
|
||||||
let maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
|
|
||||||
|
|
||||||
if (params['max-scanned-lifecycle-listing-entries']) {
|
|
||||||
const maxEntriesParams = Number.parseInt(params['max-scanned-lifecycle-listing-entries'], 10);
|
|
||||||
|
|
||||||
if (Number.isNaN(maxEntriesParams) || maxEntriesParams < min ||
|
|
||||||
maxEntriesParams > maxScannedLifecycleListingEntries) {
|
|
||||||
return { isValid: false };
|
|
||||||
}
|
|
||||||
|
|
||||||
maxScannedLifecycleListingEntries = maxEntriesParams;
|
|
||||||
}
|
|
||||||
|
|
||||||
return { isValid: true, maxScannedLifecycleListingEntries };
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
processCurrents,
|
|
||||||
processNonCurrents,
|
|
||||||
processOrphans,
|
|
||||||
getLocationConstraintErrorMessage,
|
|
||||||
validateMaxScannedEntries,
|
|
||||||
};
|
|
|
@ -1,11 +1,5 @@
|
||||||
const { errors, auth, policies } = require('arsenal');
|
const { errors } = require('arsenal');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
const { config } = require('../../../Config');
|
|
||||||
const vault = require('../../../auth/vault');
|
|
||||||
const { evaluateBucketPolicyWithIAM } = require('../authorization/permissionChecks');
|
|
||||||
|
|
||||||
const { scaledMsPerDay } = config.getTimeOptions();
|
|
||||||
/**
|
/**
|
||||||
* Calculates retain until date for the locked object version
|
* Calculates retain until date for the locked object version
|
||||||
* @param {object} retention - includes days or years retention period
|
* @param {object} retention - includes days or years retention period
|
||||||
|
@ -21,9 +15,8 @@ function calculateRetainUntilDate(retention) {
|
||||||
const date = moment();
|
const date = moment();
|
||||||
// Calculate the number of days to retain the lock on the object
|
// Calculate the number of days to retain the lock on the object
|
||||||
const retainUntilDays = days || years * 365;
|
const retainUntilDays = days || years * 365;
|
||||||
const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay;
|
|
||||||
const retainUntilDate
|
const retainUntilDate
|
||||||
= date.add(retainUntilDaysInMs, 'ms');
|
= date.add(retainUntilDays, 'days');
|
||||||
return retainUntilDate.toISOString();
|
return retainUntilDate.toISOString();
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
|
@ -50,7 +43,7 @@ function validateHeaders(bucket, headers, log) {
|
||||||
!(objectLockMode && objectLockDate)) {
|
!(objectLockMode && objectLockDate)) {
|
||||||
return errors.InvalidArgument.customizeDescription(
|
return errors.InvalidArgument.customizeDescription(
|
||||||
'x-amz-object-lock-retain-until-date and ' +
|
'x-amz-object-lock-retain-until-date and ' +
|
||||||
'x-amz-object-lock-mode must both be supplied',
|
'x-amz-object-lock-mode must both be supplied'
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']);
|
const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']);
|
||||||
|
@ -133,216 +126,73 @@ function setObjectLockInformation(headers, md, defaultRetention) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper class for check object lock state checks
|
* isObjectLocked - checks whether object is locked or not
|
||||||
|
* @param {obect} bucket - bucket metadata
|
||||||
|
* @param {object} objectMD - object metadata
|
||||||
|
* @param {array} headers - request headers
|
||||||
|
* @return {boolean} - indicates whether object is locked or not
|
||||||
*/
|
*/
|
||||||
class ObjectLockInfo {
|
function isObjectLocked(bucket, objectMD, headers) {
|
||||||
/**
|
if (bucket.isObjectLockEnabled()) {
|
||||||
*
|
const objectLegalHold = objectMD.legalHold;
|
||||||
* @param {object} retentionInfo - The object lock retention policy
|
if (objectLegalHold) {
|
||||||
* @param {"GOVERNANCE" | "COMPLIANCE" | null} retentionInfo.mode - Retention policy mode.
|
|
||||||
* @param {string} retentionInfo.date - Expiration date of retention policy. A string in ISO-8601 format
|
|
||||||
* @param {bool} retentionInfo.legalHold - Whether a legal hold is enable for the object
|
|
||||||
*/
|
|
||||||
constructor(retentionInfo) {
|
|
||||||
this.mode = retentionInfo.mode || null;
|
|
||||||
this.date = retentionInfo.date || null;
|
|
||||||
this.legalHold = retentionInfo.legalHold || false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ObjectLockInfo.isLocked
|
|
||||||
* @returns {bool} - Whether the retention policy is active and protecting the object
|
|
||||||
*/
|
|
||||||
isLocked() {
|
|
||||||
if (this.legalHold) {
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
const retentionMode = objectMD.retentionMode;
|
||||||
if (!this.mode || !this.date) {
|
const retentionDate = objectMD.retentionDate;
|
||||||
|
if (!retentionMode || !retentionDate) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if (retentionMode === 'GOVERNANCE' &&
|
||||||
return !this.isExpired();
|
headers['x-amz-bypass-governance-retention']) {
|
||||||
}
|
return false;
|
||||||
|
}
|
||||||
/**
|
const objectDate = moment(retentionDate);
|
||||||
* ObjectLockInfo.isGovernanceMode
|
|
||||||
* @returns {bool} - true if retention mode is GOVERNANCE
|
|
||||||
*/
|
|
||||||
isGovernanceMode() {
|
|
||||||
return this.mode === 'GOVERNANCE';
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ObjectLockInfo.isComplianceMode
|
|
||||||
* @returns {bool} - True if retention mode is COMPLIANCE
|
|
||||||
*/
|
|
||||||
isComplianceMode() {
|
|
||||||
return this.mode === 'COMPLIANCE';
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ObjectLockInfo.isExpired
|
|
||||||
* @returns {bool} - True if the retention policy has expired
|
|
||||||
*/
|
|
||||||
isExpired() {
|
|
||||||
const now = moment();
|
const now = moment();
|
||||||
return this.date === null || now.isSameOrAfter(this.date);
|
// indicates retain until date has expired
|
||||||
}
|
if (now.isSameOrAfter(objectDate)) {
|
||||||
|
return false;
|
||||||
/**
|
|
||||||
* ObjectLockInfo.isExtended
|
|
||||||
* @param {string} timestamp - Timestamp in ISO-8601 format
|
|
||||||
* @returns {bool} - True if the given timestamp is after the policy expiration date or if no expiration date is set
|
|
||||||
*/
|
|
||||||
isExtended(timestamp) {
|
|
||||||
return timestamp !== undefined && (this.date === null || moment(timestamp).isSameOrAfter(this.date));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ObjectLockInfo.canModifyObject
|
|
||||||
* @param {bool} hasGovernanceBypass - Whether to bypass governance retention policies
|
|
||||||
* @returns {bool} - True if the retention policy allows the objects data to be modified (overwritten/deleted)
|
|
||||||
*/
|
|
||||||
canModifyObject(hasGovernanceBypass) {
|
|
||||||
// can modify object if object is not locked
|
|
||||||
// cannot modify object in any cases if legal hold is enabled
|
|
||||||
// if no legal hold, can only modify object if bypassing governance when locked
|
|
||||||
if (!this.isLocked()) {
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
return !this.legalHold && this.isGovernanceMode() && !!hasGovernanceBypass;
|
return true;
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ObjectLockInfo.canModifyPolicy
|
|
||||||
* @param {object} policyChanges - Proposed changes to the retention policy
|
|
||||||
* @param {"GOVERNANCE" | "COMPLIANCE" | undefined} policyChanges.mode - Retention policy mode.
|
|
||||||
* @param {string} policyChanges.date - Expiration date of retention policy. A string in ISO-8601 format
|
|
||||||
* @param {bool} hasGovernanceBypass - Whether to bypass governance retention policies
|
|
||||||
* @returns {bool} - True if the changes are allowed to be applied to the retention policy
|
|
||||||
*/
|
|
||||||
canModifyPolicy(policyChanges, hasGovernanceBypass) {
|
|
||||||
// If an object does not have a retention policy or it is expired then all changes are allowed
|
|
||||||
if (!this.isLocked()) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// The only allowed change in compliance mode is extending the retention period
|
|
||||||
if (this.isComplianceMode()) {
|
|
||||||
if (policyChanges.mode === 'COMPLIANCE' && this.isExtended(policyChanges.date)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.isGovernanceMode()) {
|
|
||||||
// Extensions are always allowed in governance mode
|
|
||||||
if (policyChanges.mode === 'GOVERNANCE' && this.isExtended(policyChanges.date)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// All other changes in governance mode require a bypass
|
|
||||||
if (hasGovernanceBypass) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function validateObjectLockUpdate(objectMD, retentionInfo, bypassGovernance) {
|
||||||
*
|
const { retentionMode: existingMode, retentionDate: existingDateISO } = objectMD;
|
||||||
* @param {object} headers - s3 request headers
|
if (!existingMode) {
|
||||||
* @returns {bool} - True if the headers is present and === "true"
|
return null;
|
||||||
*/
|
}
|
||||||
function hasGovernanceBypassHeader(headers) {
|
|
||||||
const bypassHeader = headers['x-amz-bypass-governance-retention'] || '';
|
|
||||||
return bypassHeader.toLowerCase() === 'true';
|
|
||||||
}
|
|
||||||
|
|
||||||
|
const existingDate = new Date(existingDateISO);
|
||||||
|
const isExpired = existingDate < Date.now();
|
||||||
|
const isExtended = new Date(retentionInfo.date) > existingDate;
|
||||||
|
|
||||||
/**
|
if (existingMode === 'GOVERNANCE' && !isExpired && !bypassGovernance) {
|
||||||
* checkUserGovernanceBypass
|
if (retentionInfo.mode === 'GOVERNANCE' && isExtended) {
|
||||||
*
|
return null;
|
||||||
* Checks for the presence of the s3:BypassGovernanceRetention permission for a given user
|
}
|
||||||
*
|
return errors.AccessDenied;
|
||||||
* @param {object} request - Incoming s3 request
|
}
|
||||||
* @param {object} authInfo - s3 authentication info
|
|
||||||
* @param {object} bucketMD - bucket metadata
|
|
||||||
* @param {string} objectKey - object key
|
|
||||||
* @param {object} log - Werelogs logger
|
|
||||||
* @param {function} cb - callback returns errors.AccessDenied if the authorization fails
|
|
||||||
* @returns {undefined} -
|
|
||||||
*/
|
|
||||||
function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log, cb) {
|
|
||||||
log.trace(
|
|
||||||
'object in GOVERNANCE mode and is user, checking for attached policies',
|
|
||||||
{ method: 'checkUserPolicyGovernanceBypass' },
|
|
||||||
);
|
|
||||||
|
|
||||||
const authParams = auth.server.extractParams(request, log, 's3', request.query);
|
if (existingMode === 'COMPLIANCE') {
|
||||||
const ip = policies.requestUtils.getClientIp(request, config);
|
if (retentionInfo.mode === 'GOVERNANCE' && !isExpired) {
|
||||||
const requestContextParams = {
|
return errors.AccessDenied;
|
||||||
constantParams: {
|
}
|
||||||
headers: request.headers,
|
|
||||||
query: request.query,
|
|
||||||
generalResource: bucketMD.getName(),
|
|
||||||
specificResource: { key: objectKey },
|
|
||||||
requesterIp: ip,
|
|
||||||
sslEnabled: request.connection.encrypted,
|
|
||||||
apiMethod: 'bypassGovernanceRetention',
|
|
||||||
awsService: 's3',
|
|
||||||
locationConstraint: bucketMD.getLocationConstraint(),
|
|
||||||
requesterInfo: authInfo,
|
|
||||||
signatureVersion: authParams.params.data.signatureVersion,
|
|
||||||
authType: authParams.params.data.authType,
|
|
||||||
signatureAge: authParams.params.data.signatureAge,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
return vault.checkPolicies(requestContextParams,
|
|
||||||
authInfo.getArn(), log, (err, authorizationResults) => {
|
|
||||||
if (err) {
|
|
||||||
return cb(err);
|
|
||||||
}
|
|
||||||
const explicitDenyExists = authorizationResults.some(
|
|
||||||
authzResult => authzResult.isAllowed === false && !authzResult.isImplicit);
|
|
||||||
if (explicitDenyExists) {
|
|
||||||
log.trace('authorization check failed for user',
|
|
||||||
{
|
|
||||||
'method': 'checkUserPolicyGovernanceBypass',
|
|
||||||
's3:BypassGovernanceRetention': false,
|
|
||||||
});
|
|
||||||
return cb(errors.AccessDenied);
|
|
||||||
}
|
|
||||||
// Convert authorization results into an easier to handle format
|
|
||||||
const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => {
|
|
||||||
const apiMethod = authorizationResults[idx].action;
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
acc[apiMethod] = curr.isImplicit;
|
|
||||||
return acc;
|
|
||||||
}, {});
|
|
||||||
|
|
||||||
// Evaluate against the bucket policies
|
if (!isExtended) {
|
||||||
const areAllActionsAllowed = evaluateBucketPolicyWithIAM(
|
return errors.AccessDenied;
|
||||||
bucketMD,
|
}
|
||||||
Object.keys(actionImplicitDenies),
|
}
|
||||||
authInfo.getCanonicalID(),
|
|
||||||
authInfo,
|
|
||||||
actionImplicitDenies,
|
|
||||||
log,
|
|
||||||
request);
|
|
||||||
|
|
||||||
return cb(areAllActionsAllowed === true ? null : errors.AccessDenied);
|
return null;
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
calculateRetainUntilDate,
|
calculateRetainUntilDate,
|
||||||
compareObjectLockInformation,
|
compareObjectLockInformation,
|
||||||
setObjectLockInformation,
|
setObjectLockInformation,
|
||||||
|
isObjectLocked,
|
||||||
validateHeaders,
|
validateHeaders,
|
||||||
hasGovernanceBypassHeader,
|
validateObjectLockUpdate,
|
||||||
checkUserGovernanceBypass,
|
|
||||||
ObjectLockInfo,
|
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,172 +0,0 @@
|
||||||
const async = require('async');
|
|
||||||
const { errors, s3middleware } = require('arsenal');
|
|
||||||
|
|
||||||
const { allowedRestoreObjectRequestTierValues } = require('../../../../constants');
|
|
||||||
const coldStorage = require('./coldStorage');
|
|
||||||
const monitoring = require('../../../utilities/monitoringHandler');
|
|
||||||
const { pushMetric } = require('../../../utapi/utilities');
|
|
||||||
const { decodeVersionId } = require('./versioning');
|
|
||||||
const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
|
|
||||||
const { parseRestoreRequestXml } = s3middleware.objectRestore;
|
|
||||||
const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if tier is supported
|
|
||||||
* @param {object} restoreInfo - restore information
|
|
||||||
* @returns {ArsenalError|undefined} return NotImplemented error if tier not support
|
|
||||||
*/
|
|
||||||
function checkTierSupported(restoreInfo) {
|
|
||||||
if (!allowedRestoreObjectRequestTierValues.includes(restoreInfo.tier)) {
|
|
||||||
return errors.NotImplemented;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* POST Object restore process
|
|
||||||
*
|
|
||||||
* @param {MetadataWrapper} metadata - metadata wrapper
|
|
||||||
* @param {object} mdUtils - utility object to treat metadata
|
|
||||||
* @param {AuthInfo} userInfo - Instance of AuthInfo class with requester's info
|
|
||||||
* @param {IncomingMessage} request - request info
|
|
||||||
* @param {object} log - Werelogs logger
|
|
||||||
* @param {function} callback callback function
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
|
|
||||||
const METHOD = 'objectRestore';
|
|
||||||
|
|
||||||
const { bucketName, objectKey } = request;
|
|
||||||
|
|
||||||
log.debug('processing request', { method: METHOD });
|
|
||||||
|
|
||||||
const decodedVidResult = decodeVersionId(request.query);
|
|
||||||
if (decodedVidResult instanceof Error) {
|
|
||||||
log.trace('invalid versionId query',
|
|
||||||
{
|
|
||||||
method: METHOD,
|
|
||||||
versionId: request.query.versionId,
|
|
||||||
error: decodedVidResult,
|
|
||||||
});
|
|
||||||
return process.nextTick(() => callback(decodedVidResult));
|
|
||||||
}
|
|
||||||
|
|
||||||
let isObjectRestored = false;
|
|
||||||
|
|
||||||
const mdValueParams = {
|
|
||||||
authInfo: userInfo,
|
|
||||||
bucketName,
|
|
||||||
objectKey,
|
|
||||||
versionId: decodedVidResult,
|
|
||||||
requestType: request.apiMethods || 'restoreObject',
|
|
||||||
/**
|
|
||||||
* Restoring an object might not cause any impact on
|
|
||||||
* the storage, if the object is already restored: in
|
|
||||||
* this case, the duration is extended. We disable the
|
|
||||||
* quota evaluation and trigger it manually.
|
|
||||||
*/
|
|
||||||
checkQuota: false,
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
|
|
||||||
return async.waterfall([
|
|
||||||
// get metadata of bucket and object
|
|
||||||
function validateBucketAndObject(next) {
|
|
||||||
return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies,
|
|
||||||
log, (err, bucketMD, objectMD) => {
|
|
||||||
if (err) {
|
|
||||||
log.trace('request authorization failed', { method: METHOD, error: err });
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
// Call back error if object metadata could not be obtained
|
|
||||||
if (!objectMD) {
|
|
||||||
const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey;
|
|
||||||
log.trace('error no object metadata found', { method: METHOD, error: err });
|
|
||||||
return next(err, bucketMD);
|
|
||||||
}
|
|
||||||
// If object metadata is delete marker,
|
|
||||||
// call back NoSuchKey or MethodNotAllowed depending on specifying versionId
|
|
||||||
if (objectMD.isDeleteMarker) {
|
|
||||||
let err = errors.NoSuchKey;
|
|
||||||
if (decodedVidResult) {
|
|
||||||
err = errors.MethodNotAllowed;
|
|
||||||
}
|
|
||||||
log.trace('version is a delete marker', { method: METHOD, error: err });
|
|
||||||
return next(err, bucketMD, objectMD);
|
|
||||||
}
|
|
||||||
log.info('it acquired the object metadata.', {
|
|
||||||
'method': METHOD,
|
|
||||||
});
|
|
||||||
return next(null, bucketMD, objectMD);
|
|
||||||
});
|
|
||||||
},
|
|
||||||
|
|
||||||
// generate restore param obj from xml of request body and check tier validity
|
|
||||||
function parseRequestXmlAndCheckTier(bucketMD, objectMD, next) {
|
|
||||||
log.trace('parsing object restore information');
|
|
||||||
return parseRestoreRequestXml(request.post, log, (err, restoreInfo) => {
|
|
||||||
if (err) {
|
|
||||||
return next(err, bucketMD, objectMD, restoreInfo);
|
|
||||||
}
|
|
||||||
log.info('it parsed xml of the request body.', { method: METHOD, value: restoreInfo });
|
|
||||||
const checkTierResult = checkTierSupported(restoreInfo);
|
|
||||||
if (checkTierResult instanceof Error) {
|
|
||||||
return next(checkTierResult);
|
|
||||||
}
|
|
||||||
return next(null, bucketMD, objectMD, restoreInfo);
|
|
||||||
});
|
|
||||||
},
|
|
||||||
// start restore process
|
|
||||||
function startRestore(bucketMD, objectMD, restoreInfo, next) {
|
|
||||||
return coldStorage.startRestore(objectMD, restoreInfo, log,
|
|
||||||
(err, _isObjectRestored) => {
|
|
||||||
isObjectRestored = _isObjectRestored;
|
|
||||||
return next(err, bucketMD, objectMD);
|
|
||||||
});
|
|
||||||
},
|
|
||||||
function evaluateQuotas(bucketMD, objectMD, next) {
|
|
||||||
if (isObjectRestored) {
|
|
||||||
return next(null, bucketMD, objectMD);
|
|
||||||
}
|
|
||||||
const actions = Array.isArray(mdValueParams.requestType) ?
|
|
||||||
mdValueParams.requestType : [mdValueParams.requestType];
|
|
||||||
const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
|
|
||||||
return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
|
|
||||||
false, log, err => next(err, bucketMD, objectMD));
|
|
||||||
},
|
|
||||||
function updateObjectMD(bucketMD, objectMD, next) {
|
|
||||||
const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
|
|
||||||
metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
|
|
||||||
log, err => next(err, bucketMD, objectMD));
|
|
||||||
},
|
|
||||||
],
|
|
||||||
(err, bucketMD) => {
|
|
||||||
// generate CORS response header
|
|
||||||
const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD);
|
|
||||||
if (err) {
|
|
||||||
log.trace('error processing request',
|
|
||||||
{
|
|
||||||
method: METHOD,
|
|
||||||
error: err,
|
|
||||||
});
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'POST', bucketName, err.code, 'restoreObject');
|
|
||||||
return callback(err, err.code, responseHeaders);
|
|
||||||
}
|
|
||||||
pushMetric('restoreObject', log, {
|
|
||||||
userInfo,
|
|
||||||
bucket: bucketName,
|
|
||||||
});
|
|
||||||
if (isObjectRestored) {
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'POST', bucketName, '200', 'restoreObject');
|
|
||||||
return callback(null, 200, responseHeaders);
|
|
||||||
}
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'POST', bucketName, '202', 'restoreObject');
|
|
||||||
return callback(null, 202, responseHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
module.exports = objectRestore;
|
|
|
@ -1,32 +0,0 @@
|
||||||
const { errors } = require('arsenal');
|
|
||||||
|
|
||||||
const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants');
|
|
||||||
|
|
||||||
function validateChecksumHeaders(headers) {
|
|
||||||
// If the x-amz-trailer header is present the request is using one of the
|
|
||||||
// trailing checksum algorithms, which are not supported.
|
|
||||||
if (headers['x-amz-trailer'] !== undefined) {
|
|
||||||
return errors.BadRequest.customizeDescription('trailing checksum is not supported');
|
|
||||||
}
|
|
||||||
|
|
||||||
const signatureChecksum = headers['x-amz-content-sha256'];
|
|
||||||
if (signatureChecksum === undefined) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (supportedSignatureChecksums.has(signatureChecksum)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the value is not one of the possible checksum algorithms
|
|
||||||
// the only other valid value is the actual sha256 checksum of the payload.
|
|
||||||
// Do a simple sanity check of the length to guard against future algos.
|
|
||||||
// If the value is an unknown algo, then it will fail checksum validation.
|
|
||||||
if (!unsupportedSignatureChecksums.has(signatureChecksum) && signatureChecksum.length === 64) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return errors.BadRequest.customizeDescription('unsupported checksum algorithm');
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = validateChecksumHeaders;
|
|
|
@ -4,40 +4,13 @@ const async = require('async');
|
||||||
const metadata = require('../../../metadata/wrapper');
|
const metadata = require('../../../metadata/wrapper');
|
||||||
const { config } = require('../../../Config');
|
const { config } = require('../../../Config');
|
||||||
|
|
||||||
const { scaledMsPerDay } = config.getTimeOptions();
|
|
||||||
|
|
||||||
const versionIdUtils = versioning.VersionID;
|
const versionIdUtils = versioning.VersionID;
|
||||||
// Use Arsenal function to generate a version ID used internally by metadata
|
// Use Arsenal function to generate a version ID used internally by metadata
|
||||||
// for null versions that are created before bucket versioning is configured
|
// for null versions that are created before bucket versioning is configured
|
||||||
const nonVersionedObjId =
|
const nonVersionedObjId =
|
||||||
versionIdUtils.getInfVid(config.replicationGroupId);
|
versionIdUtils.getInfVid(config.replicationGroupId);
|
||||||
|
|
||||||
/** decodeVID - decode the version id
|
/** decodedVidResult - decode the version id from a query object
|
||||||
* @param {string} versionId - version ID
|
|
||||||
* @return {(Error|string|undefined)} - return Invalid Argument if decryption
|
|
||||||
* fails due to improper format, otherwise undefined or the decoded version id
|
|
||||||
*/
|
|
||||||
function decodeVID(versionId) {
|
|
||||||
if (versionId === 'null') {
|
|
||||||
return versionId;
|
|
||||||
}
|
|
||||||
|
|
||||||
let decoded;
|
|
||||||
const invalidErr = errors.InvalidArgument.customizeDescription('Invalid version id specified');
|
|
||||||
try {
|
|
||||||
decoded = versionIdUtils.decode(versionId);
|
|
||||||
} catch (err) {
|
|
||||||
return invalidErr;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (decoded instanceof Error) {
|
|
||||||
return invalidErr;
|
|
||||||
}
|
|
||||||
|
|
||||||
return decoded;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** decodeVersionId - decode the version id from a query object
|
|
||||||
* @param {object} [reqQuery] - request query object
|
* @param {object} [reqQuery] - request query object
|
||||||
* @param {string} [reqQuery.versionId] - version ID sent in request query
|
* @param {string} [reqQuery.versionId] - version ID sent in request query
|
||||||
* @return {(Error|string|undefined)} - return Invalid Argument if decryption
|
* @return {(Error|string|undefined)} - return Invalid Argument if decryption
|
||||||
|
@ -47,7 +20,16 @@ function decodeVersionId(reqQuery) {
|
||||||
if (!reqQuery || !reqQuery.versionId) {
|
if (!reqQuery || !reqQuery.versionId) {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
return decodeVID(reqQuery.versionId);
|
let versionId = reqQuery.versionId;
|
||||||
|
if (versionId === 'null') {
|
||||||
|
return versionId;
|
||||||
|
}
|
||||||
|
versionId = versionIdUtils.decode(versionId);
|
||||||
|
if (versionId instanceof Error) {
|
||||||
|
return errors.InvalidArgument
|
||||||
|
.customizeDescription('Invalid version id specified');
|
||||||
|
}
|
||||||
|
return versionId;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** getVersionIdResHeader - return encrypted version ID if appropriate
|
/** getVersionIdResHeader - return encrypted version ID if appropriate
|
||||||
|
@ -58,10 +40,11 @@ function decodeVersionId(reqQuery) {
|
||||||
*/
|
*/
|
||||||
function getVersionIdResHeader(verCfg, objectMD) {
|
function getVersionIdResHeader(verCfg, objectMD) {
|
||||||
if (verCfg) {
|
if (verCfg) {
|
||||||
if (objectMD.isNull || !objectMD.versionId) {
|
if (objectMD.isNull || (objectMD && !objectMD.versionId)) {
|
||||||
return 'null';
|
return 'null';
|
||||||
}
|
}
|
||||||
return versionIdUtils.encode(objectMD.versionId);
|
return versionIdUtils.encode(objectMD.versionId,
|
||||||
|
config.versionIdEncodingType);
|
||||||
}
|
}
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
@ -79,34 +62,17 @@ function checkQueryVersionId(query) {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb) {
|
function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
|
||||||
// In compatibility mode, create null versioned keys instead of null keys
|
metadata.putObjectMD(bucketName, objKey, objMD, options, log, err => {
|
||||||
let versionId;
|
|
||||||
let nullVersionMD;
|
|
||||||
if (config.nullVersionCompatMode) {
|
|
||||||
versionId = nullVersionId;
|
|
||||||
nullVersionMD = Object.assign({}, objMD, {
|
|
||||||
versionId: nullVersionId,
|
|
||||||
isNull: true,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
versionId = 'null';
|
|
||||||
nullVersionMD = Object.assign({}, objMD, {
|
|
||||||
versionId: nullVersionId,
|
|
||||||
isNull: true,
|
|
||||||
isNull2: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => {
|
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error from metadata storing null version as new version',
|
log.debug('error from metadata storing null version as new version',
|
||||||
{ error: err });
|
{ error: err });
|
||||||
}
|
}
|
||||||
cb(err);
|
cb(err, options);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** check existence and get location of null version data for deletion
|
/** get location of null version data for deletion
|
||||||
* @param {string} bucketName - name of bucket
|
* @param {string} bucketName - name of bucket
|
||||||
* @param {string} objKey - name of object key
|
* @param {string} objKey - name of object key
|
||||||
* @param {object} options - metadata options for getting object MD
|
* @param {object} options - metadata options for getting object MD
|
||||||
|
@ -117,56 +83,50 @@ function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb)
|
||||||
* @param {function} cb - callback
|
* @param {function} cb - callback
|
||||||
* @return {undefined} - and call callback with (err, dataToDelete)
|
* @return {undefined} - and call callback with (err, dataToDelete)
|
||||||
*/
|
*/
|
||||||
function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) {
|
function _getNullVersionsToDelete(bucketName, objKey, options, mst, log, cb) {
|
||||||
const nullOptions = {};
|
|
||||||
if (!options.deleteData) {
|
|
||||||
return process.nextTick(cb, null, nullOptions);
|
|
||||||
}
|
|
||||||
if (options.versionId === mst.versionId) {
|
if (options.versionId === mst.versionId) {
|
||||||
// no need to get another key as the master is the target
|
// no need to get delete location, we already have the master's metadata
|
||||||
nullOptions.dataToDelete = mst.objLocation;
|
const dataToDelete = mst.objLocation;
|
||||||
return process.nextTick(cb, null, nullOptions);
|
return process.nextTick(cb, null, dataToDelete);
|
||||||
}
|
|
||||||
if (options.versionId === 'null') {
|
|
||||||
// deletion of the null key will be done by the main metadata
|
|
||||||
// PUT via this option
|
|
||||||
nullOptions.deleteNullKey = true;
|
|
||||||
}
|
}
|
||||||
return metadata.getObjectMD(bucketName, objKey, options, log,
|
return metadata.getObjectMD(bucketName, objKey, options, log,
|
||||||
(err, versionMD) => {
|
(err, versionMD) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
// the null key may not exist, hence it's a normal
|
log.debug('err from metadata getting specified version', {
|
||||||
// situation to have a NoSuchKey error, in which case
|
error: err,
|
||||||
// there is nothing to delete
|
method: '_getNullVersionsToDelete',
|
||||||
if (err.is.NoSuchKey) {
|
});
|
||||||
log.debug('null version does not exist', {
|
|
||||||
method: '_prepareNullVersionDeletion',
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
log.warn('could not get null version metadata', {
|
|
||||||
error: err,
|
|
||||||
method: '_prepareNullVersionDeletion',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return cb(err);
|
return cb(err);
|
||||||
}
|
}
|
||||||
if (versionMD.location) {
|
if (!versionMD.location) {
|
||||||
const dataToDelete = Array.isArray(versionMD.location) ?
|
return cb();
|
||||||
versionMD.location : [versionMD.location];
|
|
||||||
nullOptions.dataToDelete = dataToDelete;
|
|
||||||
}
|
}
|
||||||
return cb(null, nullOptions);
|
const dataToDelete = Array.isArray(versionMD.location) ?
|
||||||
|
versionMD.location : [versionMD.location];
|
||||||
|
return cb(null, dataToDelete);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
|
function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
|
||||||
return metadata.deleteObjectMD(bucketName, objKey, options, log, err => {
|
return _getNullVersionsToDelete(bucketName, objKey, options, mst, log,
|
||||||
if (err) {
|
(err, nullDataToDelete) => {
|
||||||
log.warn('metadata error deleting null versioned key',
|
if (err) {
|
||||||
{ bucketName, objKey, error: err, method: '_deleteNullVersionMD' });
|
log.warn('could not find null version metadata', {
|
||||||
}
|
error: err,
|
||||||
return cb(err);
|
method: '_deleteNullVersionMD',
|
||||||
});
|
});
|
||||||
|
return cb(err);
|
||||||
|
}
|
||||||
|
return metadata.deleteObjectMD(bucketName, objKey, options, log,
|
||||||
|
err => {
|
||||||
|
if (err) {
|
||||||
|
log.warn('metadata error deleting null version',
|
||||||
|
{ error: err, method: '_deleteNullVersionMD' });
|
||||||
|
return cb(err);
|
||||||
|
}
|
||||||
|
return cb(null, nullDataToDelete);
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -176,103 +136,73 @@ function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
|
||||||
* @param {object} mst - state of master version, as returned by
|
* @param {object} mst - state of master version, as returned by
|
||||||
* getMasterState()
|
* getMasterState()
|
||||||
* @param {string} vstat - bucket versioning status: 'Enabled' or 'Suspended'
|
* @param {string} vstat - bucket versioning status: 'Enabled' or 'Suspended'
|
||||||
* @param {boolean} nullVersionCompatMode - if true, behaves in null
|
|
||||||
* version compatibility mode and return appropriate values: this mode
|
|
||||||
* does not attempt to create null keys but create null versioned keys
|
|
||||||
* instead
|
|
||||||
*
|
*
|
||||||
* @return {object} result object with the following attributes:
|
* @return {object} result object with the following attributes:
|
||||||
* - {object} options: versioning-related options to pass to the
|
* - {object} options: versioning-related options to pass to the
|
||||||
services.metadataStoreObject() call
|
services.metadataStoreObject() call
|
||||||
* - {object} [options.extraMD]: extra attributes to set in object metadata
|
* - {object} [storeOptions]: options for metadata to create a new
|
||||||
* - {string} [nullVersionId]: null version key to create, if needed
|
null version key, if needed
|
||||||
* - {object} [delOptions]: options for metadata to delete the null
|
* - {object} [delOptions]: options for metadata to delete the null
|
||||||
version key, if needed
|
version key, if needed
|
||||||
*/
|
*/
|
||||||
function processVersioningState(mst, vstat, nullVersionCompatMode) {
|
function processVersioningState(mst, vstat) {
|
||||||
const versioningSuspended = (vstat === 'Suspended');
|
const options = {};
|
||||||
const masterIsNull = mst.exists && (mst.isNull || !mst.versionId);
|
const storeOptions = {};
|
||||||
|
const delOptions = {};
|
||||||
if (versioningSuspended) {
|
// object does not exist or is not versioned (before versioning)
|
||||||
// versioning is suspended: overwrite the existing null version
|
if (mst.versionId === undefined || mst.isNull) {
|
||||||
const options = { versionId: '', isNull: true };
|
// versioning is suspended, overwrite existing master version
|
||||||
if (masterIsNull) {
|
if (vstat === 'Suspended') {
|
||||||
// if the null version exists, clean it up prior to put
|
options.versionId = '';
|
||||||
if (mst.objLocation) {
|
options.isNull = true;
|
||||||
options.dataToDelete = mst.objLocation;
|
options.dataToDelete = mst.objLocation;
|
||||||
}
|
// if null version exists, clean it up prior to put
|
||||||
// backward-compat: a null version key may exist even with
|
if (mst.isNull) {
|
||||||
// a null master (due to S3C-7526), if so, delete it (its
|
delOptions.versionId = mst.versionId;
|
||||||
// data will be deleted as part of the master cleanup, so
|
if (mst.uploadId) {
|
||||||
// no "deleteData" param is needed)
|
delOptions.replayId = mst.uploadId;
|
||||||
//
|
}
|
||||||
// "isNull2" attribute is set in master metadata when
|
|
||||||
// null keys are used, which is used as an optimization to
|
|
||||||
// avoid having to check the versioned key since there can
|
|
||||||
// be no more versioned key to clean up
|
|
||||||
if (mst.isNull && mst.versionId && !mst.isNull2) {
|
|
||||||
const delOptions = { versionId: mst.versionId };
|
|
||||||
return { options, delOptions };
|
return { options, delOptions };
|
||||||
}
|
}
|
||||||
return { options };
|
return { options };
|
||||||
}
|
}
|
||||||
if (mst.nullVersionId) {
|
// versioning is enabled, create a new version
|
||||||
// backward-compat: delete the null versioned key and data
|
options.versioning = true;
|
||||||
const delOptions = { versionId: mst.nullVersionId, deleteData: true };
|
if (mst.exists) {
|
||||||
if (mst.nullUploadId) {
|
// store master version in a new key
|
||||||
delOptions.replayId = mst.nullUploadId;
|
const versionId = mst.isNull ? mst.versionId : nonVersionedObjId;
|
||||||
|
storeOptions.versionId = versionId;
|
||||||
|
storeOptions.isNull = true;
|
||||||
|
options.nullVersionId = versionId;
|
||||||
|
// non-versioned (non-null) MPU objects don't have a
|
||||||
|
// replay ID, so don't reference their uploadId
|
||||||
|
if (mst.isNull && mst.uploadId) {
|
||||||
|
options.nullUploadId = mst.uploadId;
|
||||||
}
|
}
|
||||||
return { options, delOptions };
|
return { options, storeOptions };
|
||||||
|
}
|
||||||
|
return { options };
|
||||||
|
}
|
||||||
|
// master is versioned and is not a null version
|
||||||
|
const nullVersionId = mst.nullVersionId;
|
||||||
|
if (vstat === 'Suspended') {
|
||||||
|
// versioning is suspended, overwrite the existing master version
|
||||||
|
options.versionId = '';
|
||||||
|
options.isNull = true;
|
||||||
|
if (nullVersionId === undefined) {
|
||||||
|
return { options };
|
||||||
|
}
|
||||||
|
delOptions.versionId = nullVersionId;
|
||||||
|
if (mst.nullUploadId) {
|
||||||
|
delOptions.replayId = mst.nullUploadId;
|
||||||
}
|
}
|
||||||
// clean up the eventual null key's location data prior to put
|
|
||||||
|
|
||||||
// NOTE: due to metadata v1 internal format, we cannot guess
|
|
||||||
// from the master key whether there is an associated null
|
|
||||||
// key, because the master key may be removed whenever the
|
|
||||||
// latest version becomes a delete marker. Hence we need to
|
|
||||||
// pessimistically try to get the null key metadata and delete
|
|
||||||
// it if it exists.
|
|
||||||
const delOptions = { versionId: 'null', deleteData: true };
|
|
||||||
return { options, delOptions };
|
return { options, delOptions };
|
||||||
}
|
}
|
||||||
|
// versioning is enabled, put the new version
|
||||||
// versioning is enabled: create a new version
|
options.versioning = true;
|
||||||
const options = { versioning: true };
|
options.nullVersionId = nullVersionId;
|
||||||
if (masterIsNull) {
|
if (mst.nullUploadId) {
|
||||||
// if master is a null version or a non-versioned key,
|
options.nullUploadId = mst.nullUploadId;
|
||||||
// copy it to a new null key
|
|
||||||
const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
|
|
||||||
if (nullVersionCompatMode) {
|
|
||||||
options.extraMD = {
|
|
||||||
nullVersionId,
|
|
||||||
};
|
|
||||||
if (mst.uploadId) {
|
|
||||||
options.extraMD.nullUploadId = mst.uploadId;
|
|
||||||
}
|
|
||||||
return { options, nullVersionId };
|
|
||||||
}
|
|
||||||
if (mst.isNull && !mst.isNull2) {
|
|
||||||
// if master null version was put with an older
|
|
||||||
// Cloudserver (or in compat mode), there is a
|
|
||||||
// possibility that it also has a null versioned key
|
|
||||||
// associated, so we need to delete it as we write the
|
|
||||||
// null key
|
|
||||||
const delOptions = {
|
|
||||||
versionId: nullVersionId,
|
|
||||||
};
|
|
||||||
return { options, nullVersionId, delOptions };
|
|
||||||
}
|
|
||||||
return { options, nullVersionId };
|
|
||||||
}
|
|
||||||
// backward-compat: keep a reference to the existing null
|
|
||||||
// versioned key
|
|
||||||
if (mst.nullVersionId) {
|
|
||||||
options.extraMD = {
|
|
||||||
nullVersionId: mst.nullVersionId,
|
|
||||||
};
|
|
||||||
if (mst.nullUploadId) {
|
|
||||||
options.extraMD.nullUploadId = mst.nullUploadId;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return { options };
|
return { options };
|
||||||
}
|
}
|
||||||
|
@ -299,7 +229,6 @@ function getMasterState(objMD) {
|
||||||
versionId: objMD.versionId,
|
versionId: objMD.versionId,
|
||||||
uploadId: objMD.uploadId,
|
uploadId: objMD.uploadId,
|
||||||
isNull: objMD.isNull,
|
isNull: objMD.isNull,
|
||||||
isNull2: objMD.isNull2,
|
|
||||||
nullVersionId: objMD.nullVersionId,
|
nullVersionId: objMD.nullVersionId,
|
||||||
nullUploadId: objMD.nullUploadId,
|
nullUploadId: objMD.nullUploadId,
|
||||||
};
|
};
|
||||||
|
@ -323,6 +252,9 @@ function getMasterState(objMD) {
|
||||||
* ('' overwrites the master version)
|
* ('' overwrites the master version)
|
||||||
* options.versioning - (true/undefined) metadata instruction to create new ver
|
* options.versioning - (true/undefined) metadata instruction to create new ver
|
||||||
* options.isNull - (true/undefined) whether new version is null or not
|
* options.isNull - (true/undefined) whether new version is null or not
|
||||||
|
* options.nullVersionId - if storing a null version in version history, the
|
||||||
|
* version id of the null version
|
||||||
|
* options.deleteNullVersionData - whether to delete the data of the null ver
|
||||||
*/
|
*/
|
||||||
function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
|
function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
|
||||||
log, callback) {
|
log, callback) {
|
||||||
|
@ -334,102 +266,42 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
|
||||||
return process.nextTick(callback, null, options);
|
return process.nextTick(callback, null, options);
|
||||||
}
|
}
|
||||||
// bucket is versioning configured
|
// bucket is versioning configured
|
||||||
const { options, nullVersionId, delOptions } =
|
const { options, storeOptions, delOptions } =
|
||||||
processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode);
|
processVersioningState(mst, vCfg.Status);
|
||||||
return async.series([
|
return async.series([
|
||||||
function storeNullVersionMD(next) {
|
function storeVersion(next) {
|
||||||
if (!nullVersionId) {
|
if (!storeOptions) {
|
||||||
return process.nextTick(next);
|
return process.nextTick(next);
|
||||||
}
|
}
|
||||||
return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
|
const versionMD = Object.assign({}, objMD, storeOptions);
|
||||||
|
const params = { versionId: storeOptions.versionId };
|
||||||
|
return _storeNullVersionMD(bucketName, objectKey, versionMD,
|
||||||
|
params, log, next);
|
||||||
},
|
},
|
||||||
function prepareNullVersionDeletion(next) {
|
function deleteNullVersion(next) {
|
||||||
if (!delOptions) {
|
if (!delOptions) {
|
||||||
return process.nextTick(next);
|
return process.nextTick(next);
|
||||||
}
|
}
|
||||||
return _prepareNullVersionDeletion(
|
return _deleteNullVersionMD(bucketName, objectKey, delOptions, mst,
|
||||||
bucketName, objectKey, delOptions, mst, log,
|
log, (err, nullDataToDelete) => {
|
||||||
(err, nullOptions) => {
|
|
||||||
if (err) {
|
if (err) {
|
||||||
return next(err);
|
log.warn('unexpected error deleting null version md', {
|
||||||
|
error: err,
|
||||||
|
method: 'versioningPreprocessing',
|
||||||
|
});
|
||||||
|
// it's possible there was a concurrent request to
|
||||||
|
// delete the null version, so proceed with putting a
|
||||||
|
// new version
|
||||||
|
if (err.is.NoSuchKey) {
|
||||||
|
return next(null, options);
|
||||||
|
}
|
||||||
|
return next(errors.InternalError);
|
||||||
}
|
}
|
||||||
Object.assign(options, nullOptions);
|
Object.assign(options, { dataToDelete: nullDataToDelete });
|
||||||
return next();
|
return next();
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
function deleteNullVersionMD(next) {
|
], err => callback(err, options));
|
||||||
if (delOptions &&
|
|
||||||
delOptions.versionId &&
|
|
||||||
delOptions.versionId !== 'null') {
|
|
||||||
// backward-compat: delete old null versioned key
|
|
||||||
return _deleteNullVersionMD(
|
|
||||||
bucketName, objectKey, { versionId: delOptions.versionId }, log, next);
|
|
||||||
}
|
|
||||||
return process.nextTick(next);
|
|
||||||
},
|
|
||||||
], err => {
|
|
||||||
// it's possible there was a prior request that deleted the
|
|
||||||
// null version, so proceed with putting a new version
|
|
||||||
if (err && err.is.NoSuchKey) {
|
|
||||||
return callback(null, options);
|
|
||||||
}
|
|
||||||
return callback(err, options);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Return options to pass to Metadata layer for version-specific
|
|
||||||
* operations with the given requested version ID
|
|
||||||
*
|
|
||||||
* @param {object} objectMD - object metadata
|
|
||||||
* @param {boolean} nullVersionCompatMode - if true, behaves in null
|
|
||||||
* version compatibility mode
|
|
||||||
* @return {object} options object with params:
|
|
||||||
* {string} [options.versionId] - specific versionId to update
|
|
||||||
* {boolean} [options.isNull=true|false|undefined] - if set, tells the
|
|
||||||
* Metadata backend if we're updating or deleting a new-style null
|
|
||||||
* version (stored in master or null key), or not a null version.
|
|
||||||
*/
|
|
||||||
function getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode) {
|
|
||||||
// Use the internal versionId if it is a "real" null version (not
|
|
||||||
// non-versioned)
|
|
||||||
//
|
|
||||||
// If the target object is non-versioned: do not specify a
|
|
||||||
// "versionId" attribute nor "isNull"
|
|
||||||
//
|
|
||||||
// If the target version is a null version, i.e. has the "isNull"
|
|
||||||
// attribute:
|
|
||||||
//
|
|
||||||
// - send the "isNull=true" param to Metadata if the version is
|
|
||||||
// already a null key put by a non-compat mode Cloudserver, to
|
|
||||||
// let Metadata know that the null key is to be updated or
|
|
||||||
// deleted. This is the case if the "isNull2" metadata attribute
|
|
||||||
// exists
|
|
||||||
//
|
|
||||||
// - otherwise, do not send the "isNull" parameter to hint
|
|
||||||
// Metadata that it is a legacy null version
|
|
||||||
//
|
|
||||||
// If the target version is not a null version and is versioned:
|
|
||||||
//
|
|
||||||
// - send the "isNull=false" param to Metadata in non-compat
|
|
||||||
// mode (mandatory for v1 format)
|
|
||||||
//
|
|
||||||
// - otherwise, do not send the "isNull" parameter to hint
|
|
||||||
// Metadata that an existing null version may not be stored in a
|
|
||||||
// null key
|
|
||||||
//
|
|
||||||
//
|
|
||||||
if (objectMD.versionId === undefined) {
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
const options = { versionId: objectMD.versionId };
|
|
||||||
if (objectMD.isNull) {
|
|
||||||
if (objectMD.isNull2) {
|
|
||||||
options.isNull = true;
|
|
||||||
}
|
|
||||||
} else if (!nullVersionCompatMode) {
|
|
||||||
options.isNull = false;
|
|
||||||
}
|
|
||||||
return options;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** preprocessingVersioningDelete - return versioning information for S3 to
|
/** preprocessingVersioningDelete - return versioning information for S3 to
|
||||||
|
@ -438,113 +310,64 @@ function getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode) {
|
||||||
* @param {object} bucketMD - bucket metadata
|
* @param {object} bucketMD - bucket metadata
|
||||||
* @param {object} objectMD - obj metadata
|
* @param {object} objectMD - obj metadata
|
||||||
* @param {string} [reqVersionId] - specific version ID sent as part of request
|
* @param {string} [reqVersionId] - specific version ID sent as part of request
|
||||||
* @param {boolean} nullVersionCompatMode - if true, behaves in null version compatibility mode
|
* @param {RequestLogger} log - logger instance
|
||||||
* @return {object} options object with params:
|
* @param {function} callback - callback
|
||||||
* {boolean} [options.deleteData=true|undefined] - whether to delete data (if undefined
|
* @return {undefined} and call callback with params (err, options):
|
||||||
|
* options.deleteData - (true/undefined) whether to delete data (if undefined
|
||||||
* means creating a delete marker instead)
|
* means creating a delete marker instead)
|
||||||
* {string} [options.versionId] - specific versionId to delete
|
* options.versionId - specific versionId to delete
|
||||||
* {boolean} [options.isNull=true|false|undefined] - if set, tells the
|
|
||||||
* Metadata backend if we're deleting a new-style null version (stored
|
|
||||||
* in master or null key), or not a null version.
|
|
||||||
*/
|
*/
|
||||||
function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersionId, nullVersionCompatMode) {
|
function preprocessingVersioningDelete(bucketName, bucketMD, objectMD,
|
||||||
let options = {};
|
reqVersionId, log, callback) {
|
||||||
if (bucketMD.getVersioningConfiguration() && reqVersionId) {
|
const options = {};
|
||||||
options = getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode);
|
// bucket is not versioning enabled
|
||||||
}
|
if (!bucketMD.getVersioningConfiguration()) {
|
||||||
if (!bucketMD.getVersioningConfiguration() || reqVersionId) {
|
|
||||||
// delete data if bucket is non-versioned or the request
|
|
||||||
// deletes a specific version
|
|
||||||
options.deleteData = true;
|
options.deleteData = true;
|
||||||
|
return callback(null, options);
|
||||||
}
|
}
|
||||||
return options;
|
// bucket is versioning enabled
|
||||||
}
|
if (reqVersionId && reqVersionId !== 'null') {
|
||||||
|
// deleting a specific version
|
||||||
/**
|
options.deleteData = true;
|
||||||
* Keep metadatas when the object is restored from cold storage
|
options.versionId = reqVersionId;
|
||||||
* but remove the specific ones we don't want to keep
|
if (objectMD.uploadId) {
|
||||||
* @param {object} objMD - obj metadata
|
options.replayId = objectMD.uploadId;
|
||||||
* @param {object} metadataStoreParams - custom built object containing resource details.
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function restoreMetadata(objMD, metadataStoreParams) {
|
|
||||||
/* eslint-disable no-param-reassign */
|
|
||||||
const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt'];
|
|
||||||
// We need to keep user metadata and tags
|
|
||||||
Object.keys(objMD).forEach(key => {
|
|
||||||
if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) {
|
|
||||||
metadataStoreParams.metaHeaders[key] = objMD[key];
|
|
||||||
}
|
}
|
||||||
});
|
return callback(null, options);
|
||||||
|
}
|
||||||
if (objMD['x-amz-website-redirect-location']) {
|
if (reqVersionId) {
|
||||||
if (!metadataStoreParams.headers) {
|
// deleting the 'null' version if it exists
|
||||||
metadataStoreParams.headers = {};
|
if (objectMD.versionId === undefined) {
|
||||||
|
// object is not versioned, deleting it
|
||||||
|
options.deleteData = true;
|
||||||
|
// non-versioned (non-null) MPU objects don't have a
|
||||||
|
// replay ID, so don't reference their uploadId
|
||||||
|
return callback(null, options);
|
||||||
}
|
}
|
||||||
metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location'];
|
if (objectMD.isNull) {
|
||||||
|
// master is the null version
|
||||||
|
options.deleteData = true;
|
||||||
|
options.versionId = objectMD.versionId;
|
||||||
|
if (objectMD.uploadId) {
|
||||||
|
options.replayId = objectMD.uploadId;
|
||||||
|
}
|
||||||
|
return callback(null, options);
|
||||||
|
}
|
||||||
|
if (objectMD.nullVersionId) {
|
||||||
|
// null version exists, deleting it
|
||||||
|
options.deleteData = true;
|
||||||
|
options.versionId = objectMD.nullVersionId;
|
||||||
|
if (objectMD.nullUploadId) {
|
||||||
|
options.replayId = objectMD.nullUploadId;
|
||||||
|
}
|
||||||
|
return callback(null, options);
|
||||||
|
}
|
||||||
|
// null version does not exist, no deletion
|
||||||
|
// TODO check AWS behaviour for no deletion (seems having no error)
|
||||||
|
return callback(errors.NoSuchKey);
|
||||||
}
|
}
|
||||||
|
// not deleting any specific version, making a delete marker instead
|
||||||
if (objMD.replicationInfo) {
|
return callback(null, options);
|
||||||
metadataStoreParams.replicationInfo = objMD.replicationInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (objMD.legalHold) {
|
|
||||||
metadataStoreParams.legalHold = objMD.legalHold;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (objMD.acl) {
|
|
||||||
metadataStoreParams.acl = objMD.acl;
|
|
||||||
}
|
|
||||||
|
|
||||||
metadataStoreParams.creationTime = objMD['creation-time'];
|
|
||||||
metadataStoreParams.lastModifiedDate = objMD['last-modified'];
|
|
||||||
metadataStoreParams.taggingCopy = objMD.tags;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** overwritingVersioning - return versioning information for S3 to handle
|
|
||||||
* storing version metadata with a specific version id.
|
|
||||||
* @param {object} objMD - obj metadata
|
|
||||||
* @param {object} metadataStoreParams - custom built object containing resource details.
|
|
||||||
* @return {object} options
|
|
||||||
* options.versionId - specific versionId to overwrite in metadata
|
|
||||||
* options.isNull - (true/undefined) whether new version is null or not
|
|
||||||
* options.nullVersionId - if storing a null version in version history, the
|
|
||||||
* version id of the null version
|
|
||||||
*/
|
|
||||||
function overwritingVersioning(objMD, metadataStoreParams) {
|
|
||||||
metadataStoreParams.updateMicroVersionId = true;
|
|
||||||
metadataStoreParams.amzStorageClass = objMD['x-amz-storage-class'];
|
|
||||||
|
|
||||||
// set correct originOp
|
|
||||||
metadataStoreParams.originOp = 's3:ObjectRestore:Completed';
|
|
||||||
|
|
||||||
// update restore
|
|
||||||
const days = objMD.archive?.restoreRequestedDays;
|
|
||||||
const now = Date.now();
|
|
||||||
metadataStoreParams.archive = {
|
|
||||||
archiveInfo: objMD.archive?.archiveInfo,
|
|
||||||
restoreRequestedAt: objMD.archive?.restoreRequestedAt,
|
|
||||||
restoreRequestedDays: objMD.archive?.restoreRequestedDays,
|
|
||||||
restoreCompletedAt: new Date(now),
|
|
||||||
restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
|
|
||||||
};
|
|
||||||
|
|
||||||
/* eslint-enable no-param-reassign */
|
|
||||||
|
|
||||||
const versionId = objMD.versionId || undefined;
|
|
||||||
const options = {
|
|
||||||
versionId,
|
|
||||||
isNull: objMD.isNull,
|
|
||||||
};
|
|
||||||
if (objMD.nullVersionId) {
|
|
||||||
options.extraMD = {
|
|
||||||
nullVersionId: objMD.nullVersionId,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
restoreMetadata(objMD, metadataStoreParams);
|
|
||||||
|
|
||||||
return options;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
@ -554,8 +377,5 @@ module.exports = {
|
||||||
processVersioningState,
|
processVersioningState,
|
||||||
getMasterState,
|
getMasterState,
|
||||||
versioningPreprocessing,
|
versioningPreprocessing,
|
||||||
getVersionSpecificMetadataOptions,
|
|
||||||
preprocessingVersioningDelete,
|
preprocessingVersioningDelete,
|
||||||
overwritingVersioning,
|
|
||||||
decodeVID,
|
|
||||||
};
|
};
|
||||||
|
|
|
@ -101,33 +101,8 @@ function validateWebsiteHeader(header) {
|
||||||
header.startsWith('http://') || header.startsWith('https://'));
|
header.startsWith('http://') || header.startsWith('https://'));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* appendWebsiteIndexDocument - append index to objectKey if necessary
|
|
||||||
* @param {object} request - normalized request object
|
|
||||||
* @param {string} indexDocumentSuffix - index document from website config
|
|
||||||
* @param {boolean} force - flag to force append index
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function appendWebsiteIndexDocument(request, indexDocumentSuffix, force = false) {
|
|
||||||
const reqObjectKey = request.objectKey ? request.objectKey : '';
|
|
||||||
/* eslint-disable no-param-reassign */
|
|
||||||
|
|
||||||
// find index document if "directory" sent in request
|
|
||||||
if (reqObjectKey.endsWith('/')) {
|
|
||||||
request.objectKey += indexDocumentSuffix;
|
|
||||||
// find index document if no key provided
|
|
||||||
} else if (reqObjectKey === '') {
|
|
||||||
request.objectKey = indexDocumentSuffix;
|
|
||||||
// force for redirect 302 on folder without trailing / that has an index
|
|
||||||
} else if (force) {
|
|
||||||
request.objectKey += `/${indexDocumentSuffix}`;
|
|
||||||
}
|
|
||||||
/* eslint-enable no-param-reassign */
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
findRoutingRule,
|
findRoutingRule,
|
||||||
extractRedirectInfo,
|
extractRedirectInfo,
|
||||||
validateWebsiteHeader,
|
validateWebsiteHeader,
|
||||||
appendWebsiteIndexDocument,
|
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,314 +0,0 @@
|
||||||
const async = require('async');
|
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const monitoring = require('../../../utilities/monitoringHandler');
|
|
||||||
const {
|
|
||||||
actionNeedQuotaCheckCopy,
|
|
||||||
actionNeedQuotaCheck,
|
|
||||||
actionWithDataDeletion,
|
|
||||||
} = require('arsenal').policies;
|
|
||||||
const { config } = require('../../../Config');
|
|
||||||
const QuotaService = require('../../../quotas/quotas');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Process the bytes to write based on the request and object metadata
|
|
||||||
* @param {string} apiMethod - api method
|
|
||||||
* @param {BucketInfo} bucket - bucket info
|
|
||||||
* @param {string} versionId - version id of the object
|
|
||||||
* @param {number} contentLength - content length of the object
|
|
||||||
* @param {object} objMD - object metadata
|
|
||||||
* @param {object} destObjMD - destination object metadata
|
|
||||||
* @return {number} processed content length
|
|
||||||
*/
|
|
||||||
function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
|
|
||||||
let bytes = contentLength;
|
|
||||||
if (apiMethod === 'objectRestore') {
|
|
||||||
// object is being restored
|
|
||||||
bytes = Number.parseInt(objMD['content-length'], 10);
|
|
||||||
} else if (!bytes && objMD?.['content-length']) {
|
|
||||||
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
|
|
||||||
if (!destObjMD || bucket.isVersioningEnabled()) {
|
|
||||||
// object is being copied
|
|
||||||
bytes = Number.parseInt(objMD['content-length'], 10);
|
|
||||||
} else if (!bucket.isVersioningEnabled()) {
|
|
||||||
// object is being copied and replaces the target
|
|
||||||
bytes = Number.parseInt(objMD['content-length'], 10) -
|
|
||||||
Number.parseInt(destObjMD['content-length'], 10);
|
|
||||||
}
|
|
||||||
} else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
|
|
||||||
// object is being deleted
|
|
||||||
bytes = -Number.parseInt(objMD['content-length'], 10);
|
|
||||||
}
|
|
||||||
} else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
|
|
||||||
// object is being replaced: store the diff, if the bucket is not versioned
|
|
||||||
bytes = bytes - Number.parseInt(objMD['content-length'], 10);
|
|
||||||
}
|
|
||||||
return bytes || 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if a metric is stale based on the provided parameters.
|
|
||||||
*
|
|
||||||
* @param {Object} metric - The metric object to check.
|
|
||||||
* @param {string} resourceType - The type of the resource.
|
|
||||||
* @param {string} resourceName - The name of the resource.
|
|
||||||
* @param {string} action - The action being performed.
|
|
||||||
* @param {number} inflight - The number of inflight requests.
|
|
||||||
* @param {Object} log - The logger object.
|
|
||||||
* @returns {boolean} Returns true if the metric is stale, false otherwise.
|
|
||||||
*/
|
|
||||||
function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
|
|
||||||
if (metric.date && Date.now() - new Date(metric.date).getTime() >
|
|
||||||
QuotaService.maxStaleness) {
|
|
||||||
log.warn('Stale metrics from the quota service, allowing the request', {
|
|
||||||
resourceType,
|
|
||||||
resourceName,
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
});
|
|
||||||
monitoring.requestWithQuotaMetricsUnavailable.inc();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Evaluates quotas for a bucket and an account and update inflight count.
|
|
||||||
*
|
|
||||||
* @param {number} bucketQuota - The quota limit for the bucket.
|
|
||||||
* @param {number} accountQuota - The quota limit for the account.
|
|
||||||
* @param {object} bucket - The bucket object.
|
|
||||||
* @param {object} account - The account object.
|
|
||||||
* @param {number} inflight - The number of inflight requests.
|
|
||||||
* @param {number} inflightForCheck - The number of inflight requests for checking quotas.
|
|
||||||
* @param {string} action - The action being performed.
|
|
||||||
* @param {object} log - The logger object.
|
|
||||||
* @param {function} callback - The callback function to be called when evaluation is complete.
|
|
||||||
* @returns {object} - The result of the evaluation.
|
|
||||||
*/
|
|
||||||
function _evaluateQuotas(
|
|
||||||
bucketQuota,
|
|
||||||
accountQuota,
|
|
||||||
bucket,
|
|
||||||
account,
|
|
||||||
inflight,
|
|
||||||
inflightForCheck,
|
|
||||||
action,
|
|
||||||
log,
|
|
||||||
callback,
|
|
||||||
) {
|
|
||||||
let bucketQuotaExceeded = false;
|
|
||||||
let accountQuotaExceeded = false;
|
|
||||||
const creationDate = new Date(bucket.getCreationDate()).getTime();
|
|
||||||
return async.parallel({
|
|
||||||
bucketQuota: parallelDone => {
|
|
||||||
if (bucketQuota > 0) {
|
|
||||||
return QuotaService.getUtilizationMetrics('bucket',
|
|
||||||
`${bucket.getName()}_${creationDate}`, null, {
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
}, (err, bucketMetrics) => {
|
|
||||||
if (err || inflight < 0) {
|
|
||||||
return parallelDone(err);
|
|
||||||
}
|
|
||||||
if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
|
|
||||||
bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
|
|
||||||
log.debug('Bucket quota exceeded', {
|
|
||||||
bucket: bucket.getName(),
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
quota: bucketQuota,
|
|
||||||
bytesTotal: bucketMetrics.bytesTotal,
|
|
||||||
});
|
|
||||||
bucketQuotaExceeded = true;
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
},
|
|
||||||
accountQuota: parallelDone => {
|
|
||||||
if (accountQuota > 0 && account?.account) {
|
|
||||||
return QuotaService.getUtilizationMetrics('account',
|
|
||||||
account.account, null, {
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
}, (err, accountMetrics) => {
|
|
||||||
if (err || inflight < 0) {
|
|
||||||
return parallelDone(err);
|
|
||||||
}
|
|
||||||
if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
|
|
||||||
accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
|
|
||||||
log.debug('Account quota exceeded', {
|
|
||||||
accountId: account.account,
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
quota: accountQuota,
|
|
||||||
bytesTotal: accountMetrics.bytesTotal,
|
|
||||||
});
|
|
||||||
accountQuotaExceeded = true;
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
},
|
|
||||||
}, err => {
|
|
||||||
if (err) {
|
|
||||||
log.warn('Error evaluating quotas', {
|
|
||||||
error: err.name,
|
|
||||||
description: err.message,
|
|
||||||
isInflightDeletion: inflight < 0,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Monitors the duration of quota evaluation for a specific API method.
|
|
||||||
*
|
|
||||||
* @param {string} apiMethod - The name of the API method being monitored.
|
|
||||||
* @param {string} type - The type of quota being evaluated.
|
|
||||||
* @param {string} code - The code associated with the quota being evaluated.
|
|
||||||
* @param {number} duration - The duration of the quota evaluation in nanoseconds.
|
|
||||||
* @returns {undefined} - Returns nothing.
|
|
||||||
*/
|
|
||||||
function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
|
|
||||||
monitoring.quotaEvaluationDuration.labels({
|
|
||||||
action: apiMethod,
|
|
||||||
type,
|
|
||||||
code,
|
|
||||||
}).observe(duration / 1e9);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param {Request} request - request object
|
|
||||||
* @param {BucketInfo} bucket - bucket object
|
|
||||||
* @param {Account} account - account object
|
|
||||||
* @param {array} apiNames - action names: operations to authorize
|
|
||||||
* @param {string} apiMethod - the main API call
|
|
||||||
* @param {number} inflight - inflight bytes
|
|
||||||
* @param {boolean} isStorageReserved - Flag to check if the current quota, minus
|
|
||||||
* the incoming bytes, are under the limit.
|
|
||||||
* @param {Logger} log - logger
|
|
||||||
* @param {function} callback - callback function
|
|
||||||
* @returns {boolean} - true if the quota is valid, false otherwise
|
|
||||||
*/
|
|
||||||
function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
|
|
||||||
if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
|
|
||||||
return callback(null);
|
|
||||||
}
|
|
||||||
let type;
|
|
||||||
let bucketQuotaExceeded = false;
|
|
||||||
let accountQuotaExceeded = false;
|
|
||||||
let quotaEvaluationDuration;
|
|
||||||
const requestStartTime = process.hrtime.bigint();
|
|
||||||
const bucketQuota = bucket.getQuota();
|
|
||||||
const accountQuota = account?.quota || 0;
|
|
||||||
const shouldSendInflights = config.isQuotaInflightEnabled();
|
|
||||||
|
|
||||||
if (bucketQuota && accountQuota) {
|
|
||||||
type = 'bucket+account';
|
|
||||||
} else if (bucketQuota) {
|
|
||||||
type = 'bucket';
|
|
||||||
} else {
|
|
||||||
type = 'account';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (actionWithDataDeletion[apiMethod]) {
|
|
||||||
type = 'delete';
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
|
|
||||||
if (bucketQuota > 0 || accountQuota > 0) {
|
|
||||||
log.warn('quota is set for a bucket, but the quota service is disabled', {
|
|
||||||
bucketName: bucket.getName(),
|
|
||||||
});
|
|
||||||
monitoring.requestWithQuotaMetricsUnavailable.inc();
|
|
||||||
}
|
|
||||||
return callback(null);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isStorageReserved) {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
inflight = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
return async.forEach(apiNames, (apiName, done) => {
|
|
||||||
// Object copy operations first check the target object,
|
|
||||||
// meaning the source object, containing the current bytes,
|
|
||||||
// is checked second. This logic handles these APIs calls by
|
|
||||||
// ensuring the bytes are positives (i.e., not an object
|
|
||||||
// replacement).
|
|
||||||
if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
inflight = Math.abs(inflight);
|
|
||||||
} else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
|
|
||||||
return done();
|
|
||||||
}
|
|
||||||
// When inflights are disabled, the sum of the current utilization metrics
|
|
||||||
// and the current bytes are compared with the quota. The current bytes
|
|
||||||
// are not sent to the utilization service. When inflights are enabled,
|
|
||||||
// the sum of the current utilization metrics only are compared with the
|
|
||||||
// quota. They include the current inflight bytes sent in the request.
|
|
||||||
let _inflights = shouldSendInflights ? inflight : undefined;
|
|
||||||
const inflightForCheck = shouldSendInflights ? 0 : inflight;
|
|
||||||
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
|
|
||||||
inflightForCheck, apiName, log,
|
|
||||||
(err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
|
|
||||||
if (err) {
|
|
||||||
return done(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
bucketQuotaExceeded = _bucketQuotaExceeded;
|
|
||||||
accountQuotaExceeded = _accountQuotaExceeded;
|
|
||||||
|
|
||||||
// Inflights are inverted: in case of cleanup, we just re-issue
|
|
||||||
// the same API call.
|
|
||||||
if (_inflights) {
|
|
||||||
_inflights = -_inflights;
|
|
||||||
}
|
|
||||||
|
|
||||||
request.finalizerHooks.push((errorFromAPI, _done) => {
|
|
||||||
const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
|
|
||||||
const quotaCleanUpStartTime = process.hrtime.bigint();
|
|
||||||
// Quotas are cleaned only in case of error in the API
|
|
||||||
async.waterfall([
|
|
||||||
cb => {
|
|
||||||
if (errorFromAPI) {
|
|
||||||
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
|
|
||||||
null, apiName, log, cb);
|
|
||||||
}
|
|
||||||
return cb();
|
|
||||||
},
|
|
||||||
], () => {
|
|
||||||
monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
|
|
||||||
Number(process.hrtime.bigint() - quotaCleanUpStartTime));
|
|
||||||
return _done();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
return done();
|
|
||||||
});
|
|
||||||
}, err => {
|
|
||||||
quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
|
|
||||||
if (err) {
|
|
||||||
log.warn('Error getting metrics from the quota service, allowing the request', {
|
|
||||||
error: err.name,
|
|
||||||
description: err.message,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (!actionWithDataDeletion[apiMethod] &&
|
|
||||||
(bucketQuotaExceeded || accountQuotaExceeded)) {
|
|
||||||
return callback(errors.QuotaExceeded);
|
|
||||||
}
|
|
||||||
return callback();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
processBytesToWrite,
|
|
||||||
isMetricStale,
|
|
||||||
validateQuotas,
|
|
||||||
};
|
|
|
@ -1,117 +0,0 @@
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const constants = require('../../../constants');
|
|
||||||
const services = require('../../services');
|
|
||||||
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
|
|
||||||
const { pushMetric } = require('../../utapi/utilities');
|
|
||||||
const monitoring = require('../../utilities/monitoringHandler');
|
|
||||||
const { getLocationConstraintErrorMessage, processCurrents,
|
|
||||||
validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
|
|
||||||
const { config } = require('../../Config');
|
|
||||||
|
|
||||||
function handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, list, isBucketVersioned, log, callback) {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
listParams.maxKeys = requestMaxKeys;
|
|
||||||
const res = processCurrents(bucketName, listParams, isBucketVersioned, list);
|
|
||||||
|
|
||||||
pushMetric('listLifecycleCurrents', log, { authInfo, bucket: bucketName });
|
|
||||||
monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleCurrents');
|
|
||||||
return callback(null, res);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* listLifecycleCurrents - Return list of current versions/masters in bucket
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with
|
|
||||||
* requester's info
|
|
||||||
* @param {array} locationConstraints - array of location contraint
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {function} log - Werelogs request logger
|
|
||||||
* @param {function} callback - callback to respond to http request
|
|
||||||
* with either error code or xml response body
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function listLifecycleCurrents(authInfo, locationConstraints, request, log, callback) {
|
|
||||||
const params = request.query;
|
|
||||||
const bucketName = request.bucketName;
|
|
||||||
|
|
||||||
log.debug('processing request', { method: 'listLifecycleCurrents' });
|
|
||||||
const requestMaxKeys = params['max-keys'] ?
|
|
||||||
Number.parseInt(params['max-keys'], 10) : 1000;
|
|
||||||
if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, 400, 'listLifecycleCurrents');
|
|
||||||
return callback(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
|
|
||||||
|
|
||||||
const minEntriesToBeScanned = 1;
|
|
||||||
const { isValid, maxScannedLifecycleListingEntries } =
|
|
||||||
validateMaxScannedEntries(params, config, minEntriesToBeScanned);
|
|
||||||
if (!isValid) {
|
|
||||||
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
|
|
||||||
return callback(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
|
|
||||||
const excludedDataStoreName = params['excluded-data-store-name'];
|
|
||||||
if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
|
|
||||||
const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
|
|
||||||
log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
|
|
||||||
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
|
|
||||||
|
|
||||||
return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
|
|
||||||
}
|
|
||||||
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: 'listLifecycleCurrents',
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
const listParams = {
|
|
||||||
listingType: 'DelimiterCurrent',
|
|
||||||
maxKeys: actualMaxKeys,
|
|
||||||
prefix: params.prefix,
|
|
||||||
beforeDate: params['before-date'],
|
|
||||||
marker: params.marker,
|
|
||||||
excludedDataStoreName,
|
|
||||||
maxScannedLifecycleListingEntries,
|
|
||||||
};
|
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'listLifecycleCurrents');
|
|
||||||
return callback(err, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
const vcfg = bucket.getVersioningConfiguration();
|
|
||||||
const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
|
|
||||||
|
|
||||||
if (!requestMaxKeys) {
|
|
||||||
const emptyList = {
|
|
||||||
Contents: [],
|
|
||||||
IsTruncated: false,
|
|
||||||
};
|
|
||||||
return handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, emptyList, isBucketVersioned, log, callback);
|
|
||||||
}
|
|
||||||
|
|
||||||
return services.getLifecycleListing(bucketName, listParams, log,
|
|
||||||
(err, list) => {
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'listLifecycleCurrents');
|
|
||||||
return callback(err, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
return handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, list, isBucketVersioned, log, callback);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
listLifecycleCurrents,
|
|
||||||
};
|
|
|
@ -1,127 +0,0 @@
|
||||||
const { errors, versioning } = require('arsenal');
|
|
||||||
const constants = require('../../../constants');
|
|
||||||
const services = require('../../services');
|
|
||||||
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
|
|
||||||
const { pushMetric } = require('../../utapi/utilities');
|
|
||||||
const versionIdUtils = versioning.VersionID;
|
|
||||||
const monitoring = require('../../utilities/monitoringHandler');
|
|
||||||
const { getLocationConstraintErrorMessage, processNonCurrents,
|
|
||||||
validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
|
|
||||||
const { config } = require('../../Config');
|
|
||||||
|
|
||||||
function handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, list, log, callback) {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
listParams.maxKeys = requestMaxKeys;
|
|
||||||
const res = processNonCurrents(bucketName, listParams, list);
|
|
||||||
|
|
||||||
pushMetric('listLifecycleNonCurrents', log, { authInfo, bucket: bucketName });
|
|
||||||
monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleNonCurrents');
|
|
||||||
return callback(null, res);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* listLifecycleNonCurrents - Return list of non-current versions in bucket
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with
|
|
||||||
* requester's info
|
|
||||||
* @param {array} locationConstraints - array of location contraint
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {function} log - Werelogs request logger
|
|
||||||
* @param {function} callback - callback to respond to http request
|
|
||||||
* with either error code or xml response body
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function listLifecycleNonCurrents(authInfo, locationConstraints, request, log, callback) {
|
|
||||||
const params = request.query;
|
|
||||||
const bucketName = request.bucketName;
|
|
||||||
|
|
||||||
log.debug('processing request', { method: 'listLifecycleNonCurrents' });
|
|
||||||
const requestMaxKeys = params['max-keys'] ?
|
|
||||||
Number.parseInt(params['max-keys'], 10) : 1000;
|
|
||||||
if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, 400, 'listLifecycleNonCurrents');
|
|
||||||
return callback(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
|
|
||||||
|
|
||||||
// 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
|
|
||||||
// Two entries representing the master key and the following one representing the non-current version.
|
|
||||||
const minEntriesToBeScanned = 3;
|
|
||||||
const { isValid, maxScannedLifecycleListingEntries } =
|
|
||||||
validateMaxScannedEntries(params, config, minEntriesToBeScanned);
|
|
||||||
if (!isValid) {
|
|
||||||
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleNonCurrents');
|
|
||||||
return callback(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
|
|
||||||
const excludedDataStoreName = params['excluded-data-store-name'];
|
|
||||||
if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
|
|
||||||
const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
|
|
||||||
log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
|
|
||||||
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
|
|
||||||
|
|
||||||
return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
|
|
||||||
}
|
|
||||||
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: 'listLifecycleNonCurrents',
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
const listParams = {
|
|
||||||
listingType: 'DelimiterNonCurrent',
|
|
||||||
maxKeys: actualMaxKeys,
|
|
||||||
prefix: params.prefix,
|
|
||||||
beforeDate: params['before-date'],
|
|
||||||
keyMarker: params['key-marker'],
|
|
||||||
excludedDataStoreName,
|
|
||||||
maxScannedLifecycleListingEntries,
|
|
||||||
};
|
|
||||||
|
|
||||||
listParams.versionIdMarker = params['version-id-marker'] ?
|
|
||||||
versionIdUtils.decode(params['version-id-marker']) : undefined;
|
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'listLifecycleNonCurrents');
|
|
||||||
return callback(err, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
const vcfg = bucket.getVersioningConfiguration();
|
|
||||||
const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
|
|
||||||
if (!isBucketVersioned) {
|
|
||||||
log.debug('bucket is not versioned');
|
|
||||||
return callback(errors.InvalidRequest.customizeDescription(
|
|
||||||
'bucket is not versioned'), null);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!requestMaxKeys) {
|
|
||||||
const emptyList = {
|
|
||||||
Contents: [],
|
|
||||||
IsTruncated: false,
|
|
||||||
};
|
|
||||||
return handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, emptyList, log, callback);
|
|
||||||
}
|
|
||||||
|
|
||||||
return services.getLifecycleListing(bucketName, listParams, log,
|
|
||||||
(err, list) => {
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'listLifecycleNonCurrents');
|
|
||||||
return callback(err, null);
|
|
||||||
}
|
|
||||||
return handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, list, log, callback);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
listLifecycleNonCurrents,
|
|
||||||
};
|
|
|
@ -1,112 +0,0 @@
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const constants = require('../../../constants');
|
|
||||||
const services = require('../../services');
|
|
||||||
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
|
|
||||||
const { pushMetric } = require('../../utapi/utilities');
|
|
||||||
const monitoring = require('../../utilities/monitoringHandler');
|
|
||||||
const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
|
|
||||||
const { config } = require('../../Config');
|
|
||||||
|
|
||||||
function handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, list, log, callback) {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
listParams.maxKeys = requestMaxKeys;
|
|
||||||
const res = processOrphans(bucketName, listParams, list);
|
|
||||||
|
|
||||||
pushMetric('listLifecycleOrphanDeleteMarkers', log, { authInfo, bucket: bucketName });
|
|
||||||
monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleOrphanDeleteMarkers');
|
|
||||||
return callback(null, res);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* listLifecycleOrphanDeleteMarkers - Return list of expired object delete marker in bucket
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with
|
|
||||||
* requester's info
|
|
||||||
* @param {array} locationConstraints - array of location contraint
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {function} log - Werelogs request logger
|
|
||||||
* @param {function} callback - callback to respond to http request
|
|
||||||
* with either error code or xml response body
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request, log, callback) {
|
|
||||||
const params = request.query;
|
|
||||||
const bucketName = request.bucketName;
|
|
||||||
|
|
||||||
log.debug('processing request', { method: 'listLifecycleOrphanDeleteMarkers' });
|
|
||||||
const requestMaxKeys = params['max-keys'] ?
|
|
||||||
Number.parseInt(params['max-keys'], 10) : 1000;
|
|
||||||
if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
|
|
||||||
return callback(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
|
|
||||||
|
|
||||||
// 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
|
|
||||||
// Two entries representing the master key and the following one representing the non-current version.
|
|
||||||
const minEntriesToBeScanned = 3;
|
|
||||||
const { isValid, maxScannedLifecycleListingEntries } =
|
|
||||||
validateMaxScannedEntries(params, config, minEntriesToBeScanned);
|
|
||||||
if (!isValid) {
|
|
||||||
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
|
|
||||||
return callback(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: 'listLifecycleOrphanDeleteMarkers',
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
const listParams = {
|
|
||||||
listingType: 'DelimiterOrphanDeleteMarker',
|
|
||||||
maxKeys: actualMaxKeys,
|
|
||||||
prefix: params.prefix,
|
|
||||||
beforeDate: params['before-date'],
|
|
||||||
marker: params.marker,
|
|
||||||
maxScannedLifecycleListingEntries,
|
|
||||||
};
|
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
|
|
||||||
return callback(err, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
const vcfg = bucket.getVersioningConfiguration();
|
|
||||||
const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
|
|
||||||
if (!isBucketVersioned) {
|
|
||||||
log.debug('bucket is not versioned or suspended');
|
|
||||||
return callback(errors.InvalidRequest.customizeDescription(
|
|
||||||
'bucket is not versioned'), null);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!requestMaxKeys) {
|
|
||||||
const emptyList = {
|
|
||||||
Contents: [],
|
|
||||||
IsTruncated: false,
|
|
||||||
};
|
|
||||||
return handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, emptyList, log, callback);
|
|
||||||
}
|
|
||||||
|
|
||||||
return services.getLifecycleListing(bucketName, listParams, log,
|
|
||||||
(err, list) => {
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', { error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
|
|
||||||
return callback(err, null);
|
|
||||||
}
|
|
||||||
return handleResult(listParams, requestMaxKeys, authInfo,
|
|
||||||
bucketName, list, log, callback);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
listLifecycleOrphanDeleteMarkers,
|
|
||||||
};
|
|
|
@ -2,7 +2,7 @@ const { errors } = require('arsenal');
|
||||||
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const deleteBucket = require('./apiUtils/bucket/bucketDeletion');
|
const deleteBucket = require('./apiUtils/bucket/bucketDeletion');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
|
|
||||||
|
@ -34,7 +34,7 @@ function bucketDelete(authInfo, request, log, cb) {
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
return metadataValidateBucket(metadataValParams, log,
|
||||||
(err, bucketMD) => {
|
(err, bucketMD) => {
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucketMD);
|
request.method, bucketMD);
|
||||||
|
@ -48,7 +48,7 @@ function bucketDelete(authInfo, request, log, cb) {
|
||||||
log.trace('passed checks',
|
log.trace('passed checks',
|
||||||
{ method: 'metadataValidateBucket' });
|
{ method: 'metadataValidateBucket' });
|
||||||
return deleteBucket(authInfo, bucketMD, bucketName,
|
return deleteBucket(authInfo, bucketMD, bucketName,
|
||||||
authInfo.getCanonicalID(), request, log, err => {
|
authInfo.getCanonicalID(), log, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
monitoring.promMetrics(
|
monitoring.promMetrics(
|
||||||
'DELETE', bucketName, err.code, 'deleteBucket');
|
'DELETE', bucketName, err.code, 'deleteBucket');
|
||||||
|
|
|
@ -38,8 +38,7 @@ function bucketDeleteCors(authInfo, request, log, callback) {
|
||||||
}
|
}
|
||||||
log.trace('found bucket in metadata');
|
log.trace('found bucket in metadata');
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request)) {
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketDeleteCors',
|
method: 'bucketDeleteCors',
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
|
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const { checkExpectedBucketOwner } = require('./apiUtils/authorization/bucketOwner');
|
const { checkExpectedBucketOwner } = require('./apiUtils/authorization/bucketOwner');
|
||||||
|
@ -21,12 +21,12 @@ function bucketDeleteEncryption(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeleteEncryption',
|
requestType: 'bucketDeleteEncryption',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next),
|
next => metadataValidateBucket(metadataValParams, log, next),
|
||||||
(bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)),
|
(bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)),
|
||||||
(bucket, next) => {
|
(bucket, next) => {
|
||||||
const sseConfig = bucket.getServerSideEncryption();
|
const sseConfig = bucket.getServerSideEncryption();
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
|
@ -18,10 +18,10 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeleteLifecycle',
|
requestType: 'bucketDeleteLifecycle',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error processing request', {
|
log.debug('error processing request', {
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -16,10 +16,10 @@ function bucketDeletePolicy(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeletePolicy',
|
requestType: 'bucketDeletePolicy',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error processing request', {
|
log.debug('error processing request', {
|
||||||
|
|
|
@ -1,58 +0,0 @@
|
||||||
const { waterfall } = require('async');
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
|
||||||
const metadata = require('../metadata/wrapper');
|
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
|
||||||
|
|
||||||
const requestType = 'bucketDeleteQuota';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bucket Update Quota - Update bucket quota
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {object} log - Werelogs logger
|
|
||||||
* @param {function} callback - callback to server
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function bucketDeleteQuota(authInfo, request, log, callback) {
|
|
||||||
log.debug('processing request', { method: 'bucketDeleteQuota' });
|
|
||||||
|
|
||||||
const { bucketName } = request;
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: request.apiMethods || requestType,
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
return waterfall([
|
|
||||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
|
||||||
(err, bucket) => next(err, bucket)),
|
|
||||||
(bucket, next) => {
|
|
||||||
bucket.setQuota(0);
|
|
||||||
metadata.updateBucket(bucket.getName(), bucket, log, err =>
|
|
||||||
next(err, bucket));
|
|
||||||
},
|
|
||||||
], (err, bucket) => {
|
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
|
||||||
request.method, bucket);
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', {
|
|
||||||
error: err,
|
|
||||||
method: 'bucketDeleteQuota'
|
|
||||||
});
|
|
||||||
monitoring.promMetrics('DELETE', bucketName, err.code,
|
|
||||||
'bucketDeleteQuota');
|
|
||||||
return callback(err, err.code, corsHeaders);
|
|
||||||
}
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'DELETE', bucketName, '204', 'bucketDeleteQuota');
|
|
||||||
pushMetric('bucketDeleteQuota', log, {
|
|
||||||
authInfo,
|
|
||||||
bucket: bucketName,
|
|
||||||
});
|
|
||||||
return callback(null, 204, corsHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = bucketDeleteQuota;
|
|
|
@ -1,5 +1,5 @@
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
|
@ -18,10 +18,10 @@ function bucketDeleteReplication(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeleteReplication',
|
requestType: 'bucketDeleteReplication',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error processing request', {
|
log.debug('error processing request', {
|
||||||
|
|
|
@ -1,62 +0,0 @@
|
||||||
const { waterfall } = require('async');
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
|
||||||
const metadata = require('../metadata/wrapper');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bucket Delete Tagging - Delete a bucket's Tagging
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {object} log - Werelogs logger
|
|
||||||
* @param {function} callback - callback to server
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function bucketDeleteTagging(authInfo, request, log, callback) {
|
|
||||||
const bucketName = request.bucketName;
|
|
||||||
log.debug('processing request', { method: 'bucketDeleteTagging', bucketName });
|
|
||||||
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: request.apiMethods || 'bucketDeleteTagging',
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
|
|
||||||
let bucket = null;
|
|
||||||
return waterfall([
|
|
||||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
|
||||||
(err, b) => {
|
|
||||||
if (err) {
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
bucket = b;
|
|
||||||
bucket.setTags([]);
|
|
||||||
return next();
|
|
||||||
}),
|
|
||||||
next => metadata.updateBucket(bucket.getName(), bucket, log, next),
|
|
||||||
], err => {
|
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
|
||||||
request.method, bucket);
|
|
||||||
if (err) {
|
|
||||||
log.error('error processing request', {
|
|
||||||
error: err,
|
|
||||||
method: 'deleteBucketTagging',
|
|
||||||
bucketName
|
|
||||||
});
|
|
||||||
monitoring.promMetrics('DELETE', bucketName, err.code,
|
|
||||||
'deleteBucketTagging');
|
|
||||||
return callback(err, corsHeaders);
|
|
||||||
}
|
|
||||||
pushMetric('deleteBucketTagging', log, {
|
|
||||||
authInfo,
|
|
||||||
bucket: bucketName,
|
|
||||||
});
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'DELETE', bucketName, '200', 'deleteBucketTagging');
|
|
||||||
return callback(err, corsHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = bucketDeleteTagging;
|
|
|
@ -30,8 +30,7 @@ function bucketDeleteWebsite(authInfo, request, log, callback) {
|
||||||
}
|
}
|
||||||
log.trace('found bucket in metadata');
|
log.trace('found bucket in metadata');
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request)) {
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketDeleteWebsite',
|
method: 'bucketDeleteWebsite',
|
||||||
|
|
|
@ -2,7 +2,7 @@ const querystring = require('querystring');
|
||||||
const { errors, versioning, s3middleware } = require('arsenal');
|
const { errors, versioning, s3middleware } = require('arsenal');
|
||||||
const constants = require('../../constants');
|
const constants = require('../../constants');
|
||||||
const services = require('../services');
|
const services = require('../services');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const escapeForXml = s3middleware.escapeForXml;
|
const escapeForXml = s3middleware.escapeForXml;
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
|
@ -10,6 +10,7 @@ const versionIdUtils = versioning.VersionID;
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
const { generateToken, decryptToken }
|
const { generateToken, decryptToken }
|
||||||
= require('../api/apiUtils/object/continueToken');
|
= require('../api/apiUtils/object/continueToken');
|
||||||
|
const { config } = require('../Config');
|
||||||
|
|
||||||
// do not url encode the continuation tokens
|
// do not url encode the continuation tokens
|
||||||
const skipUrlEncoding = new Set([
|
const skipUrlEncoding = new Set([
|
||||||
|
@ -104,7 +105,7 @@ const skipUrlEncoding = new Set([
|
||||||
*/
|
*/
|
||||||
/* eslint-enable max-len */
|
/* eslint-enable max-len */
|
||||||
|
|
||||||
function processVersions(bucketName, listParams, list) {
|
function processVersions(bucketName, listParams, list, encType) {
|
||||||
const xml = [];
|
const xml = [];
|
||||||
xml.push(
|
xml.push(
|
||||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||||
|
@ -129,7 +130,7 @@ function processVersions(bucketName, listParams, list) {
|
||||||
xmlParams.forEach(p => {
|
xmlParams.forEach(p => {
|
||||||
if (p.value) {
|
if (p.value) {
|
||||||
const val = p.tag !== 'NextVersionIdMarker' || p.value === 'null' ?
|
const val = p.tag !== 'NextVersionIdMarker' || p.value === 'null' ?
|
||||||
p.value : versionIdUtils.encode(p.value);
|
p.value : versionIdUtils.encode(p.value, encType);
|
||||||
xml.push(`<${p.tag}>${escapeXmlFn(val)}</${p.tag}>`);
|
xml.push(`<${p.tag}>${escapeXmlFn(val)}</${p.tag}>`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -145,7 +146,7 @@ function processVersions(bucketName, listParams, list) {
|
||||||
`<Key>${objectKey}</Key>`,
|
`<Key>${objectKey}</Key>`,
|
||||||
'<VersionId>',
|
'<VersionId>',
|
||||||
(v.IsNull || v.VersionId === undefined) ?
|
(v.IsNull || v.VersionId === undefined) ?
|
||||||
'null' : versionIdUtils.encode(v.VersionId),
|
'null' : versionIdUtils.encode(v.VersionId, encType),
|
||||||
'</VersionId>',
|
'</VersionId>',
|
||||||
`<IsLatest>${isLatest}</IsLatest>`,
|
`<IsLatest>${isLatest}</IsLatest>`,
|
||||||
`<LastModified>${v.LastModified}</LastModified>`,
|
`<LastModified>${v.LastModified}</LastModified>`,
|
||||||
|
@ -182,7 +183,6 @@ function processMasterVersions(bucketName, listParams, list) {
|
||||||
{ tag: 'EncodingType', value: listParams.encoding },
|
{ tag: 'EncodingType', value: listParams.encoding },
|
||||||
{ tag: 'IsTruncated', value: isTruncated },
|
{ tag: 'IsTruncated', value: isTruncated },
|
||||||
];
|
];
|
||||||
|
|
||||||
if (listParams.v2) {
|
if (listParams.v2) {
|
||||||
xmlParams.push(
|
xmlParams.push(
|
||||||
{ tag: 'StartAfter', value: listParams.startAfter || '' });
|
{ tag: 'StartAfter', value: listParams.startAfter || '' });
|
||||||
|
@ -210,13 +210,12 @@ function processMasterVersions(bucketName, listParams, list) {
|
||||||
xmlParams.forEach(p => {
|
xmlParams.forEach(p => {
|
||||||
if (p.value && skipUrlEncoding.has(p.tag)) {
|
if (p.value && skipUrlEncoding.has(p.tag)) {
|
||||||
xml.push(`<${p.tag}>${p.value}</${p.tag}>`);
|
xml.push(`<${p.tag}>${p.value}</${p.tag}>`);
|
||||||
} else if (p.value || p.tag === 'KeyCount' || p.tag === 'MaxKeys') {
|
} else if (p.value || p.tag === 'KeyCount') {
|
||||||
xml.push(`<${p.tag}>${escapeXmlFn(p.value)}</${p.tag}>`);
|
xml.push(`<${p.tag}>${escapeXmlFn(p.value)}</${p.tag}>`);
|
||||||
} else if (p.tag !== 'NextMarker' &&
|
} else if (p.tag !== 'NextMarker' &&
|
||||||
p.tag !== 'EncodingType' &&
|
p.tag !== 'EncodingType' &&
|
||||||
p.tag !== 'Delimiter' &&
|
p.tag !== 'Delimiter' &&
|
||||||
p.tag !== 'StartAfter' &&
|
p.tag !== 'StartAfter') {
|
||||||
p.tag !== 'NextContinuationToken') {
|
|
||||||
xml.push(`<${p.tag}/>`);
|
xml.push(`<${p.tag}/>`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -263,7 +262,8 @@ function handleResult(listParams, requestMaxKeys, encoding, authInfo,
|
||||||
listParams.encoding = encoding;
|
listParams.encoding = encoding;
|
||||||
let res;
|
let res;
|
||||||
if (listParams.listingType === 'DelimiterVersions') {
|
if (listParams.listingType === 'DelimiterVersions') {
|
||||||
res = processVersions(bucketName, listParams, list);
|
res = processVersions(bucketName, listParams, list,
|
||||||
|
config.versionIdEncodingType);
|
||||||
} else {
|
} else {
|
||||||
res = processMasterVersions(bucketName, listParams, list);
|
res = processMasterVersions(bucketName, listParams, list);
|
||||||
}
|
}
|
||||||
|
@ -322,19 +322,15 @@ function bucketGet(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGet',
|
requestType: 'bucketGet',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const listParams = {
|
const listParams = {
|
||||||
listingType: 'DelimiterMaster',
|
listingType: 'DelimiterMaster',
|
||||||
maxKeys: actualMaxKeys,
|
maxKeys: actualMaxKeys,
|
||||||
|
delimiter: params.delimiter,
|
||||||
prefix: params.prefix,
|
prefix: params.prefix,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (params.delimiter) {
|
|
||||||
listParams.delimiter = params.delimiter;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (v2) {
|
if (v2) {
|
||||||
listParams.v2 = true;
|
listParams.v2 = true;
|
||||||
listParams.startAfter = params['start-after'];
|
listParams.startAfter = params['start-after'];
|
||||||
|
@ -345,7 +341,7 @@ function bucketGet(authInfo, request, log, callback) {
|
||||||
listParams.marker = params.marker;
|
listParams.marker = params.marker;
|
||||||
}
|
}
|
||||||
|
|
||||||
standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucket);
|
request.method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
const aclUtils = require('../utilities/aclUtils');
|
const aclUtils = require('../utilities/aclUtils');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const vault = require('../auth/vault');
|
const vault = require('../auth/vault');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
|
@ -44,7 +44,7 @@ function bucketGetACL(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetACL',
|
requestType: 'bucketGetACL',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const grantInfo = {
|
const grantInfo = {
|
||||||
|
@ -55,7 +55,7 @@ function bucketGetACL(authInfo, request, log, callback) {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucket);
|
request.method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
|
|
|
@ -39,8 +39,7 @@ function bucketGetCors(authInfo, request, log, callback) {
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucket);
|
request.method, bucket);
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request)) {
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketGetCors',
|
method: 'bucketGetCors',
|
||||||
|
|
|
@ -4,7 +4,7 @@ const async = require('async');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const { checkExpectedBucketOwner } = require('./apiUtils/authorization/bucketOwner');
|
const { checkExpectedBucketOwner } = require('./apiUtils/authorization/bucketOwner');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const escapeForXml = s3middleware.escapeForXml;
|
const escapeForXml = s3middleware.escapeForXml;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -22,12 +22,12 @@ function bucketGetEncryption(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetEncryption',
|
requestType: 'bucketGetEncryption',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next),
|
next => metadataValidateBucket(metadataValParams, log, next),
|
||||||
(bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)),
|
(bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)),
|
||||||
(bucket, next) => {
|
(bucket, next) => {
|
||||||
// If sseInfo is present but the `mandatory` flag is not set
|
// If sseInfo is present but the `mandatory` flag is not set
|
||||||
|
|
|
@ -2,7 +2,7 @@ const { errors } = require('arsenal');
|
||||||
const LifecycleConfiguration =
|
const LifecycleConfiguration =
|
||||||
require('arsenal').models.LifecycleConfiguration;
|
require('arsenal').models.LifecycleConfiguration;
|
||||||
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
|
@ -21,10 +21,10 @@ function bucketGetLifecycle(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetLifecycle',
|
requestType: 'bucketGetLifecycle',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error processing request', {
|
log.debug('error processing request', {
|
||||||
|
|
|
@ -41,8 +41,7 @@ function bucketGetLocation(authInfo, request, log, callback) {
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucket);
|
request.method, bucket);
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request)) {
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
|
||||||
log.debug('access denied for account on bucket', {
|
log.debug('access denied for account on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketGetLocation',
|
method: 'bucketGetLocation',
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const { NotificationConfiguration } = require('arsenal').models;
|
const { NotificationConfiguration } = require('arsenal').models;
|
||||||
|
@ -37,11 +37,11 @@ function bucketGetNotification(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetNotification',
|
requestType: 'bucketGetNotification',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error processing request', {
|
log.debug('error processing request', {
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
const { errors } = require('arsenal');
|
const { errors } = require('arsenal');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const ObjectLockConfiguration =
|
const ObjectLockConfiguration =
|
||||||
|
@ -33,10 +33,10 @@ function bucketGetObjectLock(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetObjectLock',
|
requestType: 'bucketGetObjectLock',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error processing request', {
|
log.debug('error processing request', {
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
const { errors } = require('arsenal');
|
const { errors } = require('arsenal');
|
||||||
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { metadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -17,11 +17,11 @@ function bucketGetPolicy(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetPolicy',
|
requestType: 'bucketGetPolicy',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error processing request', {
|
log.debug('error processing request', {
|
||||||
|
|
|
@ -1,58 +0,0 @@
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* bucketGetQuota - Get the bucket quota
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {object} log - Werelogs logger
|
|
||||||
* @param {function} callback - callback to server
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function bucketGetQuota(authInfo, request, log, callback) {
|
|
||||||
log.debug('processing request', { method: 'bucketGetQuota' });
|
|
||||||
const { bucketName, headers, method } = request;
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: request.apiMethods || 'bucketGetQuota',
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
const xml = [];
|
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', {
|
|
||||||
error: err,
|
|
||||||
method: 'bucketGetQuota',
|
|
||||||
});
|
|
||||||
return callback(err, null, corsHeaders);
|
|
||||||
}
|
|
||||||
xml.push(
|
|
||||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
||||||
'<GetBucketQuota>',
|
|
||||||
'<Name>', bucket.getName(), '</Name>',
|
|
||||||
);
|
|
||||||
const bucketQuota = bucket.getQuota();
|
|
||||||
if (!bucketQuota) {
|
|
||||||
log.debug('bucket has no quota', {
|
|
||||||
method: 'bucketGetQuota',
|
|
||||||
});
|
|
||||||
return callback(errors.NoSuchQuota, null,
|
|
||||||
corsHeaders);
|
|
||||||
}
|
|
||||||
xml.push('<Quota>', bucketQuota, '</Quota>',
|
|
||||||
'</GetBucketQuota>');
|
|
||||||
|
|
||||||
pushMetric('getBucketQuota', log, {
|
|
||||||
authInfo,
|
|
||||||
bucket: bucketName,
|
|
||||||
});
|
|
||||||
return callback(null, xml.join(''), corsHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = bucketGetQuota;
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue