Compare commits

..

3 Commits

Author SHA1 Message Date
Rahul Padigela 0319e6627c
Merge pull request #3003 from yutaka-oishi/ft/restore-object
FT: Provide RestoreObject API to support cold storages
2020-11-17 17:52:38 -08:00
yutaka-oishi 0ecf7be4b7
Merge branch 'development/8.2' into ft/restore-object 2020-09-15 14:55:08 +09:00
Yutaka Oishi 3eefcd87be FT: Provide RestoreObject API to support cold storages 2020-09-11 16:20:24 +09:00
481 changed files with 16698 additions and 41864 deletions

View File

@ -1,10 +1,6 @@
{ {
"extends": "scality", "extends": "scality",
"plugins": [
"mocha"
],
"rules": { "rules": {
"import/extensions": "off",
"lines-around-directive": "off", "lines-around-directive": "off",
"no-underscore-dangle": "off", "no-underscore-dangle": "off",
"indent": "off", "indent": "off",
@ -45,10 +41,6 @@
"no-restricted-properties": "off", "no-restricted-properties": "off",
"new-parens": "off", "new-parens": "off",
"no-multi-spaces": "off", "no-multi-spaces": "off",
"quote-props": "off", "quote-props": "off"
"mocha/no-exclusive-tests": "error",
},
"parserOptions": {
"ecmaVersion": 2020
} }
} }

View File

@ -1,43 +0,0 @@
---
name: "Setup CI environment"
description: "Setup Cloudserver CI environment"
runs:
using: composite
steps:
- name: Setup etc/hosts
shell: bash
run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
- name: Setup Credentials
shell: bash
run: bash .github/scripts/credentials.bash
- name: Setup job artifacts directory
shell: bash
run: |-
set -exu;
mkdir -p /tmp/artifacts/${JOB_NAME}/;
- uses: actions/setup-node@v4
with:
node-version: '16'
cache: 'yarn'
- name: install dependencies
shell: bash
run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
- uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Setup python2 test environment
shell: bash
run: |
sudo apt-get install -y libdigest-hmac-perl
pip install 's3cmd==2.3.0'
- name: fix sproxyd.conf permissions
shell: bash
run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
- name: ensure fuse kernel module is loaded (for sproxyd)
shell: bash
run: sudo modprobe fuse

View File

@ -1,36 +0,0 @@
azurebackend_AZURE_STORAGE_ACCESS_KEY
azurebackend_AZURE_STORAGE_ACCOUNT_NAME
azurebackend_AZURE_STORAGE_ENDPOINT
azurebackend2_AZURE_STORAGE_ACCESS_KEY
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME
azurebackend2_AZURE_STORAGE_ENDPOINT
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME
azurebackendmismatch_AZURE_STORAGE_ENDPOINT
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT
azuretest_AZURE_BLOB_ENDPOINT
b2backend_B2_ACCOUNT_ID
b2backend_B2_STORAGE_ACCESS_KEY
GOOGLE_SERVICE_EMAIL
GOOGLE_SERVICE_KEY
AWS_S3_BACKEND_ACCESS_KEY
AWS_S3_BACKEND_SECRET_KEY
AWS_S3_BACKEND_ACCESS_KEY_2
AWS_S3_BACKEND_SECRET_KEY_2
AWS_GCP_BACKEND_ACCESS_KEY
AWS_GCP_BACKEND_SECRET_KEY
AWS_GCP_BACKEND_ACCESS_KEY_2
AWS_GCP_BACKEND_SECRET_KEY_2
b2backend_B2_STORAGE_ENDPOINT
gcpbackend2_GCP_SERVICE_EMAIL
gcpbackend2_GCP_SERVICE_KEY
gcpbackend2_GCP_SERVICE_KEYFILE
gcpbackend_GCP_SERVICE_EMAIL
gcpbackend_GCP_SERVICE_KEY
gcpbackendmismatch_GCP_SERVICE_EMAIL
gcpbackendmismatch_GCP_SERVICE_KEY
gcpbackend_GCP_SERVICE_KEYFILE
gcpbackendmismatch_GCP_SERVICE_KEYFILE
gcpbackendnoproxy_GCP_SERVICE_KEYFILE

View File

@ -1,92 +0,0 @@
services:
cloudserver:
image: ${CLOUDSERVER_IMAGE}
command: sh -c "yarn start > /artifacts/s3.log"
network_mode: "host"
volumes:
- /tmp/ssl:/ssl
- /tmp/ssl-kmip:/ssl-kmip
- ${HOME}/.aws/credentials:/root/.aws/credentials
- /tmp/artifacts/${JOB_NAME}:/artifacts
environment:
- CI=true
- ENABLE_LOCAL_CACHE=true
- REDIS_HOST=0.0.0.0
- REDIS_PORT=6379
- REPORT_TOKEN=report-token-1
- REMOTE_MANAGEMENT_DISABLE=1
- HEALTHCHECKS_ALLOWFROM=0.0.0.0/0
- DATA_HOST=0.0.0.0
- METADATA_HOST=0.0.0.0
- S3BACKEND
- S3DATA
- S3METADATA
- MPU_TESTING
- S3VAULT
- S3_LOCATION_FILE
- ENABLE_UTAPI_V2
- BUCKET_DENY_FILTER
- S3KMS
- S3KMIP_PORT
- S3KMIP_HOSTS
- S3KMIP-COMPOUND_CREATE
- S3KMIP_BUCKET_ATTRIBUTE_NAME
- S3KMIP_PIPELINE_DEPTH
- S3KMIP_KEY
- S3KMIP_CERT
- S3KMIP_CA
- MONGODB_HOSTS=0.0.0.0:27018
- MONGODB_RS=rs0
- DEFAULT_BUCKET_KEY_FORMAT
- METADATA_MAX_CACHED_BUCKETS
- ENABLE_NULL_VERSION_COMPAT_MODE
- SCUBA_HOST
- SCUBA_PORT
- SCUBA_HEALTHCHECK_FREQUENCY
- S3QUOTA
- QUOTA_ENABLE_INFLIGHTS
env_file:
- creds.env
depends_on:
- redis
extra_hosts:
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
- "pykmip.local:127.0.0.1"
redis:
image: redis:alpine
network_mode: "host"
squid:
network_mode: "host"
profiles: ['ci-proxy']
image: scality/ci-squid
command: >-
sh -c 'mkdir -p /ssl &&
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-keyout /ssl/myca.pem -out /ssl/myca.pem &&
cp /ssl/myca.pem /ssl/CA.pem &&
squid -f /etc/squid/squid.conf -N -z &&
squid -f /etc/squid/squid.conf -NYCd 1'
volumes:
- /tmp/ssl:/ssl
pykmip:
network_mode: "host"
profiles: ['pykmip']
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
volumes:
- /tmp/artifacts/${JOB_NAME}:/artifacts
mongo:
network_mode: "host"
profiles: ['mongo', 'ceph']
image: ${MONGODB_IMAGE}
ceph:
network_mode: "host"
profiles: ['ceph']
image: ghcr.io/scality/cloudserver/ci-ceph
sproxyd:
network_mode: "host"
profiles: ['sproxyd']
image: sproxyd-standalone
build: ./sproxyd
user: 0:0
privileged: yes

View File

@ -1,28 +0,0 @@
FROM mongo:5.0.21
ENV USER=scality \
HOME_DIR=/home/scality \
CONF_DIR=/conf \
DATA_DIR=/data
# Set up directories and permissions
RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
mkdir /logs; \
adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
# Set up environment variables and directories for scality user
RUN mkdir ${CONF_DIR} && \
chown -R ${USER} ${CONF_DIR} && \
chown -R ${USER} ${DATA_DIR}
# copy the mongo config file
COPY /conf/mongod.conf /conf/mongod.conf
COPY /conf/mongo-run.sh /conf/mongo-run.sh
COPY /conf/initReplicaSet /conf/initReplicaSet.js
EXPOSE 27017/tcp
EXPOSE 27018
# Set up CMD
ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
CMD ["bash", "/conf/mongo-run.sh"]

View File

@ -1,4 +0,0 @@
rs.initiate({
_id: "rs0",
members: [{ _id: 0, host: "127.0.0.1:27018" }]
});

View File

@ -1,10 +0,0 @@
#!/bin/bash
set -exo pipefail
init_RS() {
sleep 5
mongo --port 27018 /conf/initReplicaSet.js
}
init_RS &
mongod --bind_ip_all --config=/conf/mongod.conf

View File

@ -1,15 +0,0 @@
storage:
journal:
enabled: true
engine: wiredTiger
dbPath: "/data/db"
processManagement:
fork: false
net:
port: 27018
bindIp: 0.0.0.0
replication:
replSetName: "rs0"
enableMajorityReadConcern: true
security:
authorization: disabled

View File

@ -1,3 +0,0 @@
FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
RUN chown root:root /conf/sproxyd0.conf

View File

@ -1,26 +0,0 @@
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
#fastcgi_param SCRIPT_NAME $fastcgi_script_name;
fastcgi_param SCRIPT_NAME /var/www;
fastcgi_param PATH_INFO $document_uri;
fastcgi_param REQUEST_URI $request_uri;
fastcgi_param DOCUMENT_URI $document_uri;
fastcgi_param DOCUMENT_ROOT $document_root;
fastcgi_param SERVER_PROTOCOL $server_protocol;
fastcgi_param HTTPS $https if_not_empty;
fastcgi_param GATEWAY_INTERFACE CGI/1.1;
fastcgi_param SERVER_SOFTWARE nginx/$nginx_version;
fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param REMOTE_PORT $remote_port;
fastcgi_param SERVER_ADDR $server_addr;
fastcgi_param SERVER_PORT $server_port;
fastcgi_param SERVER_NAME $server_name;
# PHP only, required if PHP was built with --enable-force-cgi-redirect
fastcgi_param REDIRECT_STATUS 200;

View File

@ -1,88 +0,0 @@
worker_processes 1;
error_log /logs/error.log;
user root root;
events {
worker_connections 1000;
reuse_port on;
multi_accept on;
}
worker_rlimit_nofile 20000;
http {
root /var/www/;
upstream sproxyds {
least_conn;
keepalive 40;
server 127.0.0.1:20000;
}
server {
client_max_body_size 0;
client_body_timeout 150;
client_header_timeout 150;
postpone_output 0;
client_body_postpone_size 0;
keepalive_requests 1100;
keepalive_timeout 300s;
server_tokens off;
default_type application/octet-stream;
gzip off;
tcp_nodelay on;
tcp_nopush on;
sendfile on;
listen 81;
server_name localhost;
rewrite ^/arc/(.*)$ /dc1/$1 permanent;
location ~* ^/proxy/(.*)$ {
rewrite ^/proxy/(.*)$ /$1 last;
}
allow 127.0.0.1;
deny all;
set $usermd '-';
set $sentusermd '-';
set $elapsed_ms '-';
set $now '-';
log_by_lua '
if not(ngx.var.http_x_scal_usermd == nil) and string.len(ngx.var.http_x_scal_usermd) > 2 then
ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
end
if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
end
local elapsed_ms = tonumber(ngx.var.request_time)
if not ( elapsed_ms == nil) then
elapsed_ms = elapsed_ms * 1000
ngx.var.elapsed_ms = tostring(elapsed_ms)
end
local time = tonumber(ngx.var.msec) * 1000
ngx.var.now = time
';
log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
'"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
'"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
'"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
'"contentType":"$content_type","s3Address":"$remote_addr",'
'"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
'"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
'"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
'"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
'"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
'"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
access_log /dev/stdout irm;
error_log /dev/stdout error;
location / {
proxy_request_buffering off;
fastcgi_request_buffering off;
fastcgi_no_cache 1;
fastcgi_cache_bypass 1;
fastcgi_buffering off;
fastcgi_ignore_client_abort on;
fastcgi_keep_conn on;
include fastcgi_params;
fastcgi_pass sproxyds;
fastcgi_next_upstream error timeout;
fastcgi_send_timeout 285s;
fastcgi_read_timeout 285s;
}
}
}

View File

@ -1,12 +0,0 @@
{
"general": {
"ring": "DATA",
"port": 20000,
"syslog_facility": "local0"
},
"ring_driver:0": {
"alias": "dc1",
"type": "local",
"queue_path": "/tmp/ring-objs"
},
}

View File

@ -1,43 +0,0 @@
[supervisord]
nodaemon = true
loglevel = info
logfile = %(ENV_LOG_DIR)s/supervisord.log
pidfile = %(ENV_SUP_RUN_DIR)s/supervisord.pid
logfile_maxbytes = 20MB
logfile_backups = 2
[unix_http_server]
file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
[program:nginx]
directory=%(ENV_SUP_RUN_DIR)s
command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
stdout_logfile_maxbytes=100MB
stdout_logfile_backups=7
stderr_logfile_maxbytes=100MB
stderr_logfile_backups=7
autorestart=true
autostart=true
user=root
[program:sproxyd]
directory=%(ENV_SUP_RUN_DIR)s
process_name=%(program_name)s-%(process_num)s
numprocs=1
numprocs_start=0
command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
stdout_logfile_maxbytes=100MB
stdout_logfile_backups=7
redirect_stderr=true
autorestart=true
autostart=true
user=root

View File

@ -1,18 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIC6zCCAdOgAwIBAgIUPIpMY95b4HjKAk+FyydZApAEFskwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJDEQMA4GA1UECgwHU2NhbGl0eTEQ
MA4GA1UEAwwHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AKqLFEsWtfRTxnoZrQe63tq+rQnVgninHMahRmXkzyjK/uNhoKnIh8bXdTC/eCZ6
FBROqBYNL0TJb0HDv1FzcZS1UCUldRqTlvr6wZb0pfrp40fvztsqQgAh1t/Blg5i
Zv5+ESSlNs5rWbFTxtq+FbMW/ERYTrVfnMkBiLg4Gq0HwID9a5jvJatzrrno2s1m
OfZCT3HaE3tMZ6vvYuoamvLNdvdH+9KeTmBCursfNejt0rSGjIqfi6DvFJSayydQ
is5DMSTbCLGdKQmA85VfEQmlQ8v0232WDSd6gVfp2tthDEDHnCbgWkEd1vsTyS85
ubdt5v4CWGOWV+mu3bf8xM0CAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkq
hkiG9w0BAQsFAAOCAQEARTjc2zV/ol1/LsSzZy6l1R0uFBmR2KumH+Se1Yq2vKpY
Dv6xmrvmjOUr5RBO77nRhIgdcQA+LyAg8ii2Dfzc8r1RTD+j1bYOxESXctBOBcXM
Chy6FEBydR6m7S8qQyL+caJWO1WZWp2tapcm6sUG1oRVznWtK1/SHKIzOBwsmJ07
79KsCJ6wf9tzD05EDTI2QhAObE9/thy+zc8l8cmv9A6p3jKkx9rwXUttSUqTn0CW
w45bgKg6+DDcrhZ+MATbzuTfhuA4NFUTzK7KeX9sMuOV03Zs8SA3VhAOXmu063M3
0f9X7P/0RmGTTp7GGCqEINcZdbLh3k7CpFb2Ox998Q==
-----END CERTIFICATE-----

View File

@ -1,18 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIC2zCCAcOgAwIBAgIUIlE8UAkqQ+6mbJDtrt9kkmi8aJYwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowKTEQMA4GA1UECgwHU2NhbGl0eTEV
MBMGA1UEAwwMcHlrbWlwLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEAtxr7pq/lnzVeZz4z52Yc3DeaPqjNfRSyW5cPUlT7ABXFb7+tja7K2C7u
DYVK+Q+2yJCQwYJY47aKJB++ewam9t2V8Xy0Z8S+0I2ImCwuyeihaD/f6uJZRzms
ycdECH22BA6tCPlQLnlboRiZzI6rcIvXAbUMvLvFm3nyYIs9qidExRnfyMjISknM
V+83LT5QW4IcHgKYqzdz2ZmOnk+f4wmMmitcivTdIZCL8Z0cxr7BJlOh5JZ/V5uj
WUXeNa+ttW0RKKBlg9T+wj0JvwoJBPZTmsMAy3tI9tjLg3DwGYKsflbFeU2tebXI
gncGFZ/dFxj331GGtq3kz1PzAUYf2wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB1
8HgJ0fu6/pCrDxAm90eESFjmaTFyTN8q00zhq4Cb3zAT9KMWzAygkZ9n4ZFgELPo
7kBE2H6RcDdoBmjVYd8HnBloDdYzYbncKgt5YBvxRaMSF4/l65BM8wjatyXErqnH
QLLTRe5AuF0/F0KtPeDQ2JFVu8dZ35W3fyKGPRsEdVOSCTHROmqpGhZCpscyUP4W
Hb0dBTESQ9mQHw14OCaaahARd0X5WdcA/E+m0fpGqj1rQCXS+PrRcSLe1E1hqPlK
q/hXSXD5nybwipktELvJCbB7l4HmJr2pIpldeR5+ef68Cs8hqs6DRlsJX9sK2ng+
TFe5v6SCarqZ9kFvr6Yp
-----END CERTIFICATE-----

View File

@ -1,18 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIC8zCCAdugAwIBAgIUBs6nVXQXhrFbClub3aSLg72/DiYwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJTEQMA4GA1UECgwHU2NhbGl0eTER
MA8GA1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQC6neSYoBoWh/i2mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4L
jLLRsPGt9KkJlUhHGWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jt
UDnQUwshj+hRFe0iKAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsH
PVju9yZBYzYhwAMyYFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFip
yR2Fh3WGSwWh45HgMT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+p
SMLm0T4iNxedQWBtDM7ts4EjAgMBAAGjGjAYMBYGA1UdJQEB/wQMMAoGCCsGAQUF
BwMCMA0GCSqGSIb3DQEBCwUAA4IBAQCMi9HEhZc5jHJMj18Wq00fZy4O9XtjCe0J
nntW9tzi3rTQcQWKA7i9uVdDoCg+gMFVxWMvV7luFEUc/VYV1v8hFfbIFygzFsZY
xwv4GQaIwbsgzD+oziia53w0FSuNL0uE0MeKvrt3yzHxCxylHyl+TQd/UdAtAo+k
RL1sI0mBZx5qo6d1J7ZMCxzAGaT7KjnJvziFr/UbfSNnwDsxsUwGaI1ZeAxJN8DI
zTrg3f3lrrmHcauEgKnuQwIqaMZR6veG6RkjtcYSlJYID1irkE6njs7+wivOAkzt
fBt/0PD76FmAI0VArgU/zDB8dGyYzrq39W749LuEfm1TPmlnUtDr
-----END CERTIFICATE-----

View File

@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6neSYoBoWh/i2
mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4LjLLRsPGt9KkJlUhH
GWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jtUDnQUwshj+hRFe0i
KAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsHPVju9yZBYzYhwAMy
YFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFipyR2Fh3WGSwWh45Hg
MT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+pSMLm0T4iNxedQWBt
DM7ts4EjAgMBAAECggEANNXdUeUKXdSzcycPV/ea/c+0XFcy8e9B46lfQTpTqQOx
xD8GbWD1L/gdk6baJgT43+ukEWdSsJbmdtLXti29Ta8OF2VtIDhIbCVtvs3dq3zt
vrvugsiVDr8nkP306qOrKrNIVIFE+igmEmSaXsu/h/33ladxeeV9/s2DC7NOOjWN
Mu4KYr5BBbu3qAavdzbrcz7Sch+GzsYqK/pBounCTQu3o9E4TSUcmcsasWmtHN3u
e6G2UjObdzEW7J0wWvvtJ0wHQUVRueHfqwqKf0dymcZ3xOlx3ZPhKPz5n4F1UGUt
RQaNazqs5SzZpUgDuPw4k8h/aCHK21Yexw/l4+O9KQKBgQD1WZSRK54zFoExBQgt
OZSBNZW3Ibti5lSiF0M0g+66yNZSWfPuABEH0tu5CXopdPDXo4kW8NLGEqQStWTX
RGK0DE9buEL3eebOfjIdS2IZ3t3dX3lMypplVCj4HzAgITlweSH1LLTyAtaaOpwa
jksqfcn5Zw+XGkyc6GBBVaZetQKBgQDCt6Xf/g26+zjvHscjdzsfBhnYvTOrr6+F
xqFFxOEOocGr+mL7UTAs+a9m/6lOWhlagk+m+TIZNL8o3IN7KFTYxPYPxTiewgVE
rIm3JBmPxRiPn01P3HrtjaqfzsXF30j3ele7ix5OxieZq4vsW7ZXP3GZE34a08Ov
12sE1DlvdwKBgQDzpYQOLhyqazzcqzyVfMrnDYmiFVN7QXTmiudobWRUBUIhAcdl
oJdJB7K/rJOuO704x+RJ7dnCbZyWH6EGzZifaGIemXuXO21jvpqR0NyZCGOXhUp2
YfS1j8AntwEZxyS9du2sBjui4gKvomiHTquChOxgSmKHEcznPTTpbN8MyQKBgF5F
LVCZniolkLXsL7tS8VOez4qoZ0i6wP7CYLf3joJX+/z4N023S9yqcaorItvlMRsp
tciAIyoi6F2vDRTmPNXJ3dtav4PVKVnLMs1w89MwOCjoljSQ6Q7zpGTEZenbpWbz
W2BYBS9cLjXu4MpoyInLFINo9YeleLs8TvrCiKAXAoGBANsduqLnlUW/f5zDb5Fe
SB51+KhBjsVIeYmU+8xtur9Z7IxZXK28wpoEsm7LmX7Va5dERjI+tItBiJ5+Unu1
Xs2ljDg35ARKHs0dWBJGpbnZg4dbT6xpIL4YMPXm1Zu++PgRpxPIMn646xqd8GlH
bavm6Km/fXNG58xus+EeLpV5
-----END PRIVATE KEY-----

View File

@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC3Gvumr+WfNV5n
PjPnZhzcN5o+qM19FLJblw9SVPsAFcVvv62NrsrYLu4NhUr5D7bIkJDBgljjtook
H757Bqb23ZXxfLRnxL7QjYiYLC7J6KFoP9/q4llHOazJx0QIfbYEDq0I+VAueVuh
GJnMjqtwi9cBtQy8u8WbefJgiz2qJ0TFGd/IyMhKScxX7zctPlBbghweApirN3PZ
mY6eT5/jCYyaK1yK9N0hkIvxnRzGvsEmU6Hkln9Xm6NZRd41r621bREooGWD1P7C
PQm/CgkE9lOawwDLe0j22MuDcPAZgqx+VsV5Ta15tciCdwYVn90XGPffUYa2reTP
U/MBRh/bAgMBAAECggEABCvcMcbuDztzBB0Zp5re63Fk1SqZS9Et4wJE+hYvhaf5
UHtoY8LoohYnnC0+MQBXpKgOdCoZBk8BRKNofnr/UL5pjQ/POFH2GuAujXDsO/NN
wgc6fapcaE/7DLm6ZgsfG2aOMJclaXmgScI6trtFUpIM+t/6A06vyMP1bpeddwPW
Fqu7NvpDiEcTRUGd+z1JooYgUhGgC7peYUx5+9zqFrwoDBKxnUOnz3BkDsXBy3qm
65Vu0BSjuJzf6vVMpNGUHY6JXjopVNWku+JAX0wD+iikOd5sziNVdIj1fnZ+IHIf
7G5h5owHpvSGzJFQ18/g5VHtJdCm+4WQSnbSJRsCAQKBgQDu4IH8yspyeH44fhoS
PAp/OtILqSP+Da0zAp2LbhrOgyzyuSTdEAYyptqjqHS6QkB1Bu1H44FS0BYUxRXc
iu2e9AndiLVCGngsE7TpA/ZVLN1B0LEZEHjM6p4d6zZM6iveKVnPAOkTWTBAgzCt
b31nj4jL8PdlPKQil1AMrOlRAQKBgQDEOwshzIdr2Iy6B/n4CuBViEtwnbAd5f/c
atA9bcfF8kCahokJsI4eCCLgBwDZpYKD+v0AwOBlacF6t6TX+vdlJsi5EP7uxZ22
ILsuWqVm/0H77PACuckc5/qLZoGGC81l0DhnpoeMEb6r/TKOo5xAK1gxdlwNNrq+
nP1zdZnU2wKBgBAS92xFUR4m0YeHpMV5WNN658t1FEDyNqdqE6PgQtmGpi2nG73s
aB5cb/X3TfOCpce6MZlWy8sAyZuYL4Jprte1YDySCHBsS43bvZ64b4kHvdPB8UjY
fOh9GSq2Oy8tysnmSm7NhuGQbNjKeyoQiIXBeNkQW/VqATl6qR5RPFoBAoGACNqV
JQBCd/Y8W0Ry3eM3vgQ5SyqCQMcY5UwYez0Rz3efvJknY72InAhH8o2+VxOlsOjJ
M5iAR3MfHLdeg7Q6J2E5m0gOCJ34ALi3WV8TqXMI+iH1rlnNnjVFU7bbTz4HFXnw
oZSc9w/x53a0KkVtjmOmRg0OGDaI9ILG2MfMmhMCgYB8ZqJtX8qZ2TqKU3XdLZ4z
T2N7xMFuKohWP420r5jKm3Xw85IC+y1SUTB9XGcL79r2eJzmzmdKQ3A3sf3oyUH3
RdYWxtKcZ5PAE8hVRtn1ETZqUgxASGOUn/6w0npkYSOXPU5bc0W6RSLkjES0i+c3
fv3OMNI8qpmQhEjpHHQS1g==
-----END PRIVATE KEY-----

View File

@ -1,35 +0,0 @@
name: Test alerts
on:
push:
branches-ignore:
- 'development/**'
- 'q/*/**'
jobs:
run-alert-tests:
runs-on: ubuntu-latest
strategy:
matrix:
tests:
- name: 1 minute interval tests
file: monitoring/alerts.test.yaml
- name: 10 seconds interval tests
file: monitoring/alerts.10s.test.yaml
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Render and test ${{ matrix.tests.name }}
uses: scality/action-prom-render-test@1.0.3
with:
alert_file_path: monitoring/alerts.yaml
test_file_path: ${{ matrix.tests.file }}
alert_inputs: |
namespace=zenko
service=artesca-data-connector-s3api-metrics
reportJob=artesca-data-ops-report-handler
replicas=3
github_token: ${{ secrets.GITHUB_TOKEN }}

View File

@ -1,25 +0,0 @@
---
name: codeQL
on:
push:
branches: [w/**, q/*]
pull_request:
branches: [development/*, stabilization/*, hotfix/*]
workflow_dispatch:
jobs:
analyze:
name: Static analysis with CodeQL
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: javascript, python, ruby
- name: Build and analyze
uses: github/codeql-action/analyze@v3

View File

@ -1,16 +0,0 @@
---
name: dependency review
on:
pull_request:
branches: [development/*, stabilization/*, hotfix/*]
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@v4
- name: 'Dependency Review'
uses: actions/dependency-review-action@v4

View File

@ -1,80 +0,0 @@
---
name: release
run-name: release ${{ inputs.tag }}
on:
workflow_dispatch:
inputs:
tag:
description: 'Tag to be released'
required: true
env:
PROJECT_NAME: ${{ github.event.repository.name }}
jobs:
build-federation-image:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Build and push image for federation
uses: docker/build-push-action@v5
with:
push: true
context: .
file: images/svc-base/Dockerfile
tags: |
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
cache-from: type=gha,scope=federation
cache-to: type=gha,mode=max,scope=federation
release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildk
uses: docker/setup-buildx-action@v3
- name: Login to Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Push dashboards into the production namespace
run: |
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
dashboard.json:application/grafana-dashboard+json \
alerts.yaml:application/prometheus-alerts+yaml
working-directory: monitoring
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Create Release
uses: softprops/action-gh-release@v2
env:
GITHUB_TOKEN: ${{ github.token }}
with:
name: Release ${{ github.event.inputs.tag }}
tag_name: ${{ github.event.inputs.tag }}
generate_release_notes: true
target_commitish: ${{ github.sha }}

View File

@ -1,533 +0,0 @@
---
name: tests
on:
workflow_dispatch:
push:
branches-ignore:
- 'development/**'
- 'q/*/**'
env:
# Secrets
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurebackend_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
b2backend_B2_STORAGE_ACCESS_KEY: >-
${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
gcpbackendmismatch_GCP_SERVICE_KEY: >-
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
# Configs
ENABLE_LOCAL_CACHE: "true"
REPORT_TOKEN: "report-token-1"
REMOTE_MANAGEMENT_DISABLE: "1"
# https://github.com/git-lfs/git-lfs/issues/5749
GIT_CLONE_PROTECTION_ACTIVE: 'false'
jobs:
linting-coverage:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '16'
cache: yarn
- name: install dependencies
run: yarn install --frozen-lockfile --network-concurrency 1
- uses: actions/setup-python@v5
with:
python-version: '3.9'
- uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
- name: Install python deps
run: pip install flake8
- name: Lint Javascript
run: yarn run --silent lint -- --max-warnings 0
- name: Lint Markdown
run: yarn run --silent lint_md
- name: Lint python
run: flake8 $(git ls-files "*.py")
- name: Lint Yaml
run: yamllint -c yamllint.yml $(git ls-files "*.yml")
- name: Unit Coverage
run: |
set -ex
mkdir -p $CIRCLE_TEST_REPORTS/unit
yarn test
yarn run test_legacy_location
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
CIRCLE_TEST_REPORTS: /tmp
CIRCLE_ARTIFACTS: /tmp
CI_REPORTS: /tmp
- name: Unit Coverage logs
run: find /tmp/unit -exec cat {} \;
- name: preparing junit files for upload
run: |
mkdir -p artifacts/junit
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
if: always()
- name: Upload files to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: artifacts
if: always()
build:
runs-on: ubuntu-20.04
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Build and push cloudserver image
uses: docker/build-push-action@v5
with:
push: true
context: .
provenance: false
tags: |
ghcr.io/${{ github.repository }}:${{ github.sha }}
labels: |
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=cloudserver
cache-to: type=gha,mode=max,scope=cloudserver
- name: Build and push pykmip image
uses: docker/build-push-action@v5
with:
push: true
context: .github/pykmip
tags: |
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
labels: |
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=pykmip
cache-to: type=gha,mode=max,scope=pykmip
- name: Build and push MongoDB
uses: docker/build-push-action@v5
with:
push: true
context: .github/docker/mongodb
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
cache-from: type=gha,scope=mongodb
cache-to: type=gha,mode=max,scope=mongodb
multiple-backend:
runs-on: ubuntu-latest
needs: build
env:
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
S3BACKEND: mem
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
S3DATA: multiple
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Login to Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile sproxyd up -d
working-directory: .github/docker
- name: Run multiple backend test
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 81 40
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
mongo-v0-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
MPU_TESTING: "yes"
S3METADATA: mongodb
S3KMS: file
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
DEFAULT_BUCKET_KEY_FORMAT: v0
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run functional tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
mongo-v1-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
MPU_TESTING: "yes"
S3METADATA: mongodb
S3KMS: file
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
DEFAULT_BUCKET_KEY_FORMAT: v1
METADATA_MAX_CACHED_BUCKETS: 1
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run functional tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
file-ft-tests:
strategy:
matrix:
include:
- job-name: file-ft-tests
name: ${{ matrix.job-name }}
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: file
S3VAULT: mem
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
MPU_TESTING: "yes"
JOB_NAME: ${{ matrix.job-name }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup matrix job artifacts directory
shell: bash
run: |
set -exu
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
- name: Setup CI services
run: docker compose up -d
working-directory: .github/docker
- name: Run file ft tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
utapi-v2-tests:
runs-on: ubuntu-latest
needs: build
env:
ENABLE_UTAPI_V2: t
S3BACKEND: mem
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose up -d
working-directory: .github/docker
- name: Run file utapi v2 tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
quota-tests:
runs-on: ubuntu-latest
needs: build
strategy:
matrix:
inflights:
- name: "With Inflights"
value: "true"
- name: "Without Inflights"
value: "false"
env:
S3METADATA: mongodb
S3BACKEND: mem
S3QUOTA: scuba
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
SCUBA_HOST: localhost
SCUBA_PORT: 8100
SCUBA_HEALTHCHECK_FREQUENCY: 100
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run quota tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
kmip-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: file
S3VAULT: mem
MPU_TESTING: "yes"
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Copy KMIP certs
run: cp -r ./certs /tmp/ssl-kmip
working-directory: .github/pykmip
- name: Setup CI services
run: docker compose --profile pykmip up -d
working-directory: .github/docker
- name: Run file KMIP tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 5696 40
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
ceph-backend-test:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
S3DATA: multiple
S3KMS: file
CI_CEPH: 'true'
MPU_TESTING: "yes"
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- uses: ruby/setup-ruby@v1
with:
ruby-version: '2.5.9'
- name: Install Ruby dependencies
run: |
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
- name: Install Java dependencies
run: |
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
- name: Setup CI services
run: docker compose --profile ceph up -d
working-directory: .github/docker
env:
S3METADATA: mongodb
- name: Run Ceph multiple backend tests
run: |-
set -ex -o pipefail;
bash .github/ceph/wait_for_ceph.sh
bash wait_for_local_port.bash 27018 40
bash wait_for_local_port.bash 8000 40
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
S3METADATA: mem
- name: Run Java tests
run: |-
set -ex -o pipefail;
mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
working-directory: tests/functional/jaws
- name: Run Ruby tests
run: |-
set -ex -o pipefail;
rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
working-directory: tests/functional/fog
- name: Run Javascript AWS SDK tests
run: |-
set -ex -o pipefail;
yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
S3BACKEND: file
S3VAULT: mem
S3METADATA: mongodb
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()

View File

@ -1,60 +1,40 @@
ARG NODE_VERSION=16.20-bullseye-slim FROM node:10.22.0-slim
MAINTAINER Giorgio Regni <gr@scality.com>
FROM node:${NODE_VERSION} as builder
WORKDIR /usr/src/app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
curl \
git \
gnupg2 \
jq \
python3 \
ssh \
wget \
libffi-dev \
zlib1g-dev \
&& apt-get clean \
&& mkdir -p /root/ssh \
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
ENV PYTHON=python3
COPY package.json yarn.lock /usr/src/app/
RUN npm install typescript -g
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
################################################################################
FROM node:${NODE_VERSION}
RUN apt-get update && \
apt-get install -y --no-install-recommends \
jq \
&& rm -rf /var/lib/apt/lists/*
ENV NO_PROXY localhost,127.0.0.1 ENV NO_PROXY localhost,127.0.0.1
ENV no_proxy localhost,127.0.0.1 ENV no_proxy localhost,127.0.0.1
EXPOSE 8000 EXPOSE 8000
EXPOSE 8002
RUN apt-get update && \ COPY ./package.json /usr/src/app/
apt-get install -y --no-install-recommends \ COPY ./yarn.lock /usr/src/app/
jq \
tini \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /usr/src/app WORKDIR /usr/src/app
# Keep the .git directory in order to properly report version RUN apt-get update \
COPY . /usr/src/app && apt-get install -y \
COPY --from=builder /usr/src/app/node_modules ./node_modules/ curl \
gnupg2
RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
RUN apt-get update \
&& apt-get install -y jq python git build-essential ssh --no-install-recommends yarn \
&& mkdir -p /root/ssh \
&& ssh-keyscan -H github.com > /root/ssh/known_hosts \
&& yarn cache clean \
&& yarn install --frozen-lockfile --production --ignore-optional \
&& apt-get autoremove --purge -y python git build-essential \
&& rm -rf /var/lib/apt/lists/* \
&& yarn cache clean \
&& rm -rf ~/.node-gyp \
&& rm -rf /tmp/yarn-*
COPY . /usr/src/app
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"] VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"] ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
CMD [ "yarn", "start" ] CMD [ "yarn", "start" ]

View File

@ -1,7 +1,6 @@
# S3 Healthcheck # S3 Healthcheck
Scality S3 exposes a healthcheck route `/live` on the port used Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
for the metrics (defaults to port 8002) which returns a
response with HTTP code response with HTTP code
- 200 OK - 200 OK

164
README.md
View File

@ -1,7 +1,10 @@
# Zenko CloudServer with Vitastor Backend # Zenko CloudServer
![Zenko CloudServer logo](res/scality-cloudserver-logo.png) ![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/zenko/cloudserver)
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
## Overview ## Overview
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
@ -11,71 +14,126 @@ Scalitys Open Source Multi-Cloud Data Controller.
CloudServer provides a single AWS S3 API interface to access multiple CloudServer provides a single AWS S3 API interface to access multiple
backend data storage both on-premise or public in the cloud. backend data storage both on-premise or public in the cloud.
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor) CloudServer is useful for Developers, either to run as part of a
backend support. continous integration test environment to emulate the AWS S3 service locally
or as an abstraction layer to develop object storage enabled
application on the go.
## Quick Start with Vitastor ## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
Vitastor Backend is in experimental status, however you can already try to ## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
it works too 😊.
Installation instructions: ## Docker
### Install Vitastor [Run your Zenko CloudServer with Docker](https://hub.docker.com/r/zenko/cloudserver/)
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md). ## Contributing
### Install Zenko with Vitastor Backend In order to contribute, please follow the
[Contributing Guidelines](
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor` ## Installation
- Install dependencies: `npm install --omit dev` or just `npm install`
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
### Install and Configure MongoDB ### Dependencies
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/). Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
. Up-to-date versions can be found at
[Nodesource](https://github.com/nodesource/distributions).
### Setup Zenko ### Clone source code
- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data` ```shell
- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data` git clone https://github.com/scality/S3.git
- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
access keys, but it's not published, so let's use a file for now.
- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
in this file.
Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
### Start Zenko
Start the S3 server with: `node index.js`
If you use default settings, Zenko CloudServer starts on port 8000.
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
Now you can access your S3 with `s3cmd` or `geesefs`:
```
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
``` ```
``` ### Install js dependencies
AWS_ACCESS_KEY_ID=accessKey1 \
AWS_SECRET_ACCESS_KEY=verySecretKey1 \ Go to the ./S3 folder,
geesefs --endpoint http://localhost:8000 testbucket mountdir
```shell
yarn install --frozen-lockfile
``` ```
# Author & License If you get an error regarding installation of the diskUsage module,
please install g++.
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0) If you get an error regarding level-down bindings, try clearing your yarn cache:
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way) ```shell
yarn cache clean
```
## Run it with a file backend
```shell
yarn start
```
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
9991 are also open locally for internal transfer of metadata and data,
respectively.
The default access key is accessKey1 with
a secret key of verySecretKey1.
By default the metadata files will be saved in the
localMetadata directory and the data files will be saved
in the localData directory within the ./S3 directory on your
machine. These directories have been pre-created within the
repository. If you would like to save the data or metadata in
different locations of your choice, you must specify them with absolute paths.
So, when starting the server:
```shell
mkdir -m 700 $(pwd)/myFavoriteDataPath
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
yarn start
```
## Run it with multiple data backends
```shell
export S3DATA='multiple'
yarn start
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
With multiple backends, you have the ability to
choose where each object will be saved by setting
the following header with a locationConstraint on
a PUT request:
```shell
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
```
If no header is sent with a PUT object request, the
location constraint of the bucket will determine
where the data is saved. If the bucket has no location
constraint, the endpoint of the PUT request will be
used to determine location.
See the Configuration section in our documentation
[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
to learn how to set location constraints.
## Run it with an in-memory backend
```shell
yarn run mem_backend
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae

View File

@ -1,2 +1,2 @@
--- ---
theme: jekyll-theme-modernist theme: jekyll-theme-minimal

46
bin/metrics_server.js Executable file
View File

@ -0,0 +1,46 @@
#!/usr/bin/env node
'use strict'; // eslint-disable-line strict
const {
startWSManagementClient,
startPushConnectionHealthCheckServer,
} = require('../lib/management/push');
const logger = require('../lib/utilities/logger');
const {
PUSH_ENDPOINT: pushEndpoint,
INSTANCE_ID: instanceId,
MANAGEMENT_TOKEN: managementToken,
} = process.env;
if (!pushEndpoint) {
logger.error('missing push endpoint env var');
process.exit(1);
}
if (!instanceId) {
logger.error('missing instance id env var');
process.exit(1);
}
if (!managementToken) {
logger.error('missing management token env var');
process.exit(1);
}
startPushConnectionHealthCheckServer(err => {
if (err) {
logger.error('could not start healthcheck server', { error: err });
process.exit(1);
}
const url = `${pushEndpoint}/${instanceId}/ws?metrics=1`;
startWSManagementClient(url, managementToken, err => {
if (err) {
logger.error('connection failed, exiting', { error: err });
process.exit(1);
}
logger.info('no more connection, exiting');
process.exit(0);
});
});

View File

@ -13,26 +13,20 @@ function _performSearch(host,
port, port,
bucketName, bucketName,
query, query,
listVersions,
accessKey, accessKey,
secretKey, secretKey,
sessionToken,
verbose, ssl) { verbose, ssl) {
const escapedSearch = encodeURIComponent(query); const escapedSearch = encodeURIComponent(query);
const options = { const options = {
host, host,
port, port,
method: 'GET', method: 'GET',
path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`, path: `/${bucketName}/?search=${escapedSearch}`,
headers: { headers: {
'Content-Length': 0, 'Content-Length': 0,
}, },
rejectUnauthorized: false, rejectUnauthorized: false,
versions: '',
}; };
if (sessionToken) {
options.headers['x-amz-security-token'] = sessionToken;
}
const transport = ssl ? https : http; const transport = ssl ? https : http;
const request = transport.request(options, response => { const request = transport.request(options, response => {
if (verbose) { if (verbose) {
@ -61,9 +55,9 @@ function _performSearch(host,
// generateV4Headers exepects request object with path that does not // generateV4Headers exepects request object with path that does not
// include query // include query
request.path = `/${bucketName}`; request.path = `/${bucketName}`;
const requestData = listVersions ? { search: query, versions: '' } : { search: query }; auth.client.generateV4Headers(request, { search: query },
auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3'); accessKey, secretKey, 's3');
request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`; request.path = `/${bucketName}?search=${escapedSearch}`;
if (verbose) { if (verbose) {
logger.info('request headers', { headers: request._headers }); logger.info('request headers', { headers: request._headers });
} }
@ -82,17 +76,15 @@ function searchBucket() {
.version('0.0.1') .version('0.0.1')
.option('-a, --access-key <accessKey>', 'Access key id') .option('-a, --access-key <accessKey>', 'Access key id')
.option('-k, --secret-key <secretKey>', 'Secret access key') .option('-k, --secret-key <secretKey>', 'Secret access key')
.option('-t, --session-token <sessionToken>', 'Session token')
.option('-b, --bucket <bucket>', 'Name of the bucket') .option('-b, --bucket <bucket>', 'Name of the bucket')
.option('-q, --query <query>', 'Search query') .option('-q, --query <query>', 'Search query')
.option('-h, --host <host>', 'Host of the server') .option('-h, --host <host>', 'Host of the server')
.option('-p, --port <port>', 'Port of the server') .option('-p, --port <port>', 'Port of the server')
.option('-s', '--ssl', 'Enable ssl') .option('-s', '--ssl', 'Enable ssl')
.option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
'otherwise only list the latest version')
.option('-v, --verbose') .option('-v, --verbose')
.parse(process.argv); .parse(process.argv);
const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
const { host, port, accessKey, secretKey, bucket, query, verbose, ssl } =
commander; commander;
if (!host || !port || !accessKey || !secretKey || !bucket || !query) { if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
@ -101,7 +93,7 @@ function searchBucket() {
process.exit(1); process.exit(1);
} }
_performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose, _performSearch(host, port, bucket, query, accessKey, secretKey, verbose,
ssl); ssl);
} }

46
bin/secure_channel_proxy.js Executable file
View File

@ -0,0 +1,46 @@
#!/usr/bin/env node
'use strict'; // eslint-disable-line strict
const {
startWSManagementClient,
startPushConnectionHealthCheckServer,
} = require('../lib/management/push');
const logger = require('../lib/utilities/logger');
const {
PUSH_ENDPOINT: pushEndpoint,
INSTANCE_ID: instanceId,
MANAGEMENT_TOKEN: managementToken,
} = process.env;
if (!pushEndpoint) {
logger.error('missing push endpoint env var');
process.exit(1);
}
if (!instanceId) {
logger.error('missing instance id env var');
process.exit(1);
}
if (!managementToken) {
logger.error('missing management token env var');
process.exit(1);
}
startPushConnectionHealthCheckServer(err => {
if (err) {
logger.error('could not start healthcheck server', { error: err });
process.exit(1);
}
const url = `${pushEndpoint}/${instanceId}/ws?proxy=1`;
startWSManagementClient(url, managementToken, err => {
if (err) {
logger.error('connection failed, exiting', { error: err });
process.exit(1);
}
logger.info('no more connection, exiting');
process.exit(0);
});
});

View File

@ -1,10 +1,7 @@
{ {
"port": 8000, "port": 8000,
"listenOn": [], "listenOn": [],
"metricsPort": 8002,
"metricsListenOn": [],
"replicationGroupId": "RG001", "replicationGroupId": "RG001",
"workers": 4,
"restEndpoints": { "restEndpoints": {
"localhost": "us-east-1", "localhost": "us-east-1",
"127.0.0.1": "us-east-1", "127.0.0.1": "us-east-1",
@ -102,14 +99,6 @@
"readPreference": "primary", "readPreference": "primary",
"database": "metadata" "database": "metadata"
}, },
"authdata": "authdata.json",
"backends": {
"auth": "file",
"data": "file",
"metadata": "mongodb",
"kms": "file",
"quota": "none"
},
"externalBackends": { "externalBackends": {
"aws_s3": { "aws_s3": {
"httpAgent": { "httpAgent": {

View File

@ -1,71 +0,0 @@
{
"port": 8000,
"listenOn": [],
"metricsPort": 8002,
"metricsListenOn": [],
"replicationGroupId": "RG001",
"restEndpoints": {
"localhost": "STANDARD",
"127.0.0.1": "STANDARD",
"yourhostname.ru": "STANDARD"
},
"websiteEndpoints": [
"static.yourhostname.ru"
],
"replicationEndpoints": [ {
"site": "zenko",
"servers": ["127.0.0.1:8000"],
"default": true
} ],
"log": {
"logLevel": "info",
"dumpLevel": "error"
},
"healthChecks": {
"allowFrom": ["127.0.0.1/8", "::1"]
},
"backends": {
"metadata": "mongodb"
},
"mongodb": {
"replicaSetHosts": "127.0.0.1:27017",
"writeConcern": "majority",
"replicaSet": "rs0",
"readPreference": "primary",
"database": "s3",
"authCredentials": {
"username": "s3",
"password": ""
}
},
"externalBackends": {
"aws_s3": {
"httpAgent": {
"keepAlive": false,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
},
"gcp": {
"httpAgent": {
"keepAlive": true,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
}
},
"requests": {
"viaProxy": false,
"trustedProxyCIDRs": [],
"extractClientIPFromHeader": ""
},
"bucketNotificationDestinations": [
{
"resource": "target1",
"type": "dummy",
"host": "localhost:6000"
}
]
}

View File

@ -86,9 +86,6 @@ const constants = {
// In testing, AWS seems to allow up to 88 more bytes, so we do the same. // In testing, AWS seems to allow up to 88 more bytes, so we do the same.
maximumMetaHeadersSize: 2136, maximumMetaHeadersSize: 2136,
// Maximum HTTP headers size allowed
maxHttpHeadersSize: 14122,
// hex digest of sha256 hash of empty string: // hex digest of sha256 hash of empty string:
emptyStringHash: crypto.createHash('sha256') emptyStringHash: crypto.createHash('sha256')
.update('', 'binary').digest('hex'), .update('', 'binary').digest('hex'),
@ -98,26 +95,28 @@ const constants = {
unsupportedQueries: [ unsupportedQueries: [
'accelerate', 'accelerate',
'analytics', 'analytics',
'encryption',
'inventory', 'inventory',
'logging', 'logging',
'metrics', 'metrics',
'policyStatus', 'policyStatus',
'publicAccessBlock', 'publicAccessBlock',
'requestPayment', 'requestPayment',
'restore',
'torrent', 'torrent',
], ],
// Headers supported by AWS that we do not currently support. // Headers supported by AWS that we do not currently support.
unsupportedHeaders: [ unsupportedHeaders: [
'x-amz-server-side-encryption-customer-algorithm', 'x-amz-server-side-encryption-customer-algorithm',
'x-amz-server-side-encryption-aws-kms-key-id',
'x-amz-server-side-encryption-context', 'x-amz-server-side-encryption-context',
'x-amz-server-side-encryption-customer-key', 'x-amz-server-side-encryption-customer-key',
'x-amz-server-side-encryption-customer-key-md5', 'x-amz-server-side-encryption-customer-key-md5',
], ],
// user metadata header to set object locationConstraint // user metadata header to set object locationConstraint
objectLocationConstraintHeader: 'x-amz-storage-class', objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
legacyLocations: ['sproxyd', 'legacy'], legacyLocations: ['sproxyd', 'legacy'],
// declare here all existing service accounts and their properties // declare here all existing service accounts and their properties
// (if any, otherwise an empty object) // (if any, otherwise an empty object)
@ -130,7 +129,7 @@ const constants = {
}, },
}, },
/* eslint-disable camelcase */ /* eslint-disable camelcase */
externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true }, externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true },
// some of the available data backends (if called directly rather // some of the available data backends (if called directly rather
// than through the multiple backend gateway) need a key provided // than through the multiple backend gateway) need a key provided
// as a string as first parameter of the get/delete methods. // as a string as first parameter of the get/delete methods.
@ -176,8 +175,6 @@ const constants = {
'objectDeleteTagging', 'objectDeleteTagging',
'objectGetTagging', 'objectGetTagging',
'objectPutTagging', 'objectPutTagging',
'objectPutLegalHold',
'objectPutRetention',
], ],
// response header to be sent when there are invalid // response header to be sent when there are invalid
// user metadata in the object's metadata // user metadata in the object's metadata
@ -185,64 +182,8 @@ const constants = {
// Bucket specific queries supported by AWS that we do not currently support // Bucket specific queries supported by AWS that we do not currently support
// these queries may or may not be supported at object level // these queries may or may not be supported at object level
unsupportedBucketQueries: [ unsupportedBucketQueries: [
'tagging',
], ],
suppressedUtapiEventFields: [
'object',
'location',
'versionId',
],
allowedUtapiEventFilterFields: [
'operationId',
'location',
'account',
'user',
'bucket',
],
arrayOfAllowed: [
'objectPutTagging',
'objectPutLegalHold',
'objectPutRetention',
],
allowedUtapiEventFilterStates: ['allow', 'deny'],
allowedRestoreObjectRequestTierValues: ['Standard'],
lifecycleListing: {
CURRENT_TYPE: 'current',
NON_CURRENT_TYPE: 'noncurrent',
ORPHAN_DM_TYPE: 'orphan',
},
multiObjectDeleteConcurrency: 50,
maxScannedLifecycleListingEntries: 10000,
overheadField: [
'content-length',
'owner-id',
'versionId',
'isNull',
'isDeleteMarker',
],
unsupportedSignatureChecksums: new Set([
'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
]),
supportedSignatureChecksums: new Set([
'UNSIGNED-PAYLOAD',
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
]),
ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
// The AWS assumed Role resource type
assumedRoleArnResourceType: 'assumed-role',
// Session name of the backbeat lifecycle assumed role session.
backbeatLifecycleSessionName: 'backbeat-lifecycle',
actionsToConsiderAsObjectPut: [
'initiateMultipartUpload',
'objectPutPart',
'completeMultipartUpload',
],
// if requester is not bucket owner, bucket policy actions should be denied with
// MethodNotAllowed error
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
}; };
module.exports = constants; module.exports = constants;

View File

@ -195,14 +195,6 @@ if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
fi fi
if [[ -n "$BUCKET_DENY_FILTER" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
fi
if [[ "$TESTING_MODE" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
fi
if [[ $JQ_FILTERS_CONFIG != "." ]]; then if [[ $JQ_FILTERS_CONFIG != "." ]]; then
jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
mv config.json.tmp config.json mv config.json.tmp config.json

View File

@ -295,51 +295,3 @@ Should force path-style requests even though v3 advertises it does by default.
$client->createBucket(array( $client->createBucket(array(
'Bucket' => 'bucketphp', 'Bucket' => 'bucketphp',
)); ));
Go
~~
`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. code:: go
package main
import (
"context"
"fmt"
"log"
"os"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
func main() {
os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
endpoint := "http://localhost:8000"
timeout := time.Duration(10) * time.Second
sess := session.Must(session.NewSession())
// Create a context with a timeout that will abort the upload if it takes
// more than the passed in timeout.
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
svc := s3.New(sess, &aws.Config{
Region: aws.String(endpoints.UsEast1RegionID),
Endpoint: &endpoint,
})
out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
if err != nil {
log.Fatal(err)
} else {
fmt.Println(out)
}
}

View File

@ -91,17 +91,6 @@ This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1. The secret key is verySecretKey1. The default access key is accessKey1. The secret key is verySecretKey1.
Run CloudServer with Vault User Management
------------------------------------------
.. code:: shell
export S3VAULT=vault
yarn start
Note: Vault is proprietary and must be accessed separately.
This starts a Zenko CloudServer using Vault for user management.
Run CloudServer for Continuous Integration Testing or in Production with Docker Run CloudServer for Continuous Integration Testing or in Production with Docker
------------------------------------------------------------------------------- -------------------------------------------------------------------------------

View File

@ -1,73 +0,0 @@
# Cloudserver Release Plan
## Docker Image Generation
Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
CloudServer has a few images there:
* Cloudserver container image: ghcr.io/scality/cloudserver
* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
With every CI build, the CI will push images, tagging the
content with the developer branch's short SHA-1 commit hash.
This allows those images to be used by developers, CI builds,
build chain and so on.
Tagged versions of cloudserver will be stored in the production namespace.
## How to Pull Docker Images
```sh
docker pull ghcr.io/scality/cloudserver:<commit hash>
docker pull ghcr.io/scality/cloudserver:<tag>
```
## Release Process
To release a production image:
* Create a PR to bump the package version
Update Cloudserver's `package.json` by bumping it to the relevant next
version in a new PR. Per example if the last released version was
`8.4.7`, the next version would be `8.4.8`.
```js
{
"name": "cloudserver",
"version": "8.4.8", <--- Here
[...]
}
```
* Review & merge the PR
* Create the release on GitHub
* Go the Release tab (https://github.com/scality/cloudserver/releases);
* Click on the `Draft new release button`;
* In the `tag` field, type the name of the release (`8.4.8`), and confirm
to create the tag on publish;
* Click on `Generate release notes` button to fill the fields;
* Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
* Click to `Publish the release` to create the GitHub release and git tag
Notes:
* the Git tag will be created automatically.
* this should be done as soon as the PR is merged, so that the tag
is put on the "version bump" commit.
* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
* Branch Name: The one used for the tag earlier. In this example `development/8.4`
* Override Stage: 'release'
* Extra properties:
* name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
* Release the release version on Jira
* Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
* Create a next version
* Name: `[next version]`, in this example `8.4.9`
* Click `...` and select `Release` on the recently released version (`8.4.8`)
* Fill in the field to move incomplete version to the next one

13
eve/get_product_version.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
script_full_path=$(readlink -f "$0")
file_dir=$(dirname "$script_full_path")/..
PACKAGE_VERSION=$(cat $file_dir/package.json \
| grep version \
| head -1 \
| awk -F: '{ print $2 }' \
| sed 's/[",]//g' \
| tr -d '[[:space:]]')
echo $PACKAGE_VERSION

458
eve/main.yml Normal file
View File

@ -0,0 +1,458 @@
---
version: 0.2
branches:
feature/*, documentation/*, improvement/*, bugfix/*, w/*, q/*, hotfix/*:
stage: pre-merge
development/*:
stage: post-merge
models:
- env: &global-env
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurebackend_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key_2)s
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name_2)s
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint_2)s
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azuretest_AZURE_BLOB_ENDPOINT: "%(secret:azure_storage_endpoint)s"
b2backend_B2_ACCOUNT_ID: "%(secret:b2backend_b2_account_id)s"
b2backend_B2_STORAGE_ACCESS_KEY: >-
%(secret:b2backend_b2_storage_access_key)s
GOOGLE_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
GOOGLE_SERVICE_KEY: "%(secret:gcp_service_key)s"
AWS_S3_BACKEND_ACCESS_KEY: "%(secret:aws_s3_backend_access_key)s"
AWS_S3_BACKEND_SECRET_KEY: "%(secret:aws_s3_backend_secret_key)s"
AWS_S3_BACKEND_ACCESS_KEY_2: "%(secret:aws_s3_backend_access_key_2)s"
AWS_S3_BACKEND_SECRET_KEY_2: "%(secret:aws_s3_backend_secret_key_2)s"
AWS_GCP_BACKEND_ACCESS_KEY: "%(secret:aws_gcp_backend_access_key)s"
AWS_GCP_BACKEND_SECRET_KEY: "%(secret:aws_gcp_backend_secret_key)s"
AWS_GCP_BACKEND_ACCESS_KEY_2: "%(secret:aws_gcp_backend_access_key_2)s"
AWS_GCP_BACKEND_SECRET_KEY_2: "%(secret:aws_gcp_backend_secret_key_2)s"
b2backend_B2_STORAGE_ENDPOINT: "%(secret:b2backend_b2_storage_endpoint)s"
gcpbackend2_GCP_SERVICE_EMAIL: "%(secret:gcp2_service_email)s"
gcpbackend2_GCP_SERVICE_KEY: "%(secret:gcp2_service_key)s"
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackend_GCP_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
gcpbackend_GCP_SERVICE_KEY: "%(secret:gcp_service_key)s"
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
%(secret:gcpbackendmismatch_gcp_service_email)s
gcpbackendmismatch_GCP_SERVICE_KEY: >-
%(secret:gcpbackendmismatch_gcp_service_key)s
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
- env: &mongo-vars
S3BACKEND: "mem"
MPU_TESTING: "yes"
S3METADATA: mongodb
- env: &multiple-backend-vars
S3BACKEND: "mem"
S3DATA: "multiple"
- env: &file-mem-mpu
S3BACKEND: "file"
S3VAULT: "mem"
MPU_TESTING: "yes"
- Git: &clone
name: Pull repo
repourl: '%(prop:git_reference)s'
shallow: true
retryFetch: true
haltOnFailure: true
- ShellCommand: &credentials
name: Setup Credentials
command: bash eve/workers/build/credentials.bash
haltOnFailure: true
env: *global-env
- ShellCommand: &yarn-install
name: install modules
command: yarn install
haltOnFailure: true
- Upload: &upload-artifacts
source: /artifacts
urls:
- "*"
- ShellCommand: &follow-s3-log
logfiles:
s3:
filename: /artifacts/s3.log
follow: true
- ShellCommand: &follow-s3-ceph-logs
logfiles:
ceph:
filename: /artifacts/ceph.log
follow: true
s3:
filename: /artifacts/s3.log
follow: true
- ShellCommand: &add-hostname
name: add hostname
command: |
echo "127.0.0.1 testrequestbucket.localhost" >> /etc/hosts
echo \
"127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" \
>> /etc/hosts
haltOnFailure: true
- ShellCommand: &setup-junit-upload
name: preparing junit files for upload
command: |
mkdir -p artifacts/junit
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
alwaysRun: true
- Upload: &upload-junits
source: artifacts
urls:
- "*"
alwaysRun: true
stages:
pre-merge:
worker:
type: local
steps:
- TriggerStages:
name: Launch all workers
stage_names:
- linting-coverage
- file-ft-tests
- multiple-backend-test
- mongo-ft-tests
- ceph-backend-tests
- kmip-ft-tests
waitForFinish: true
haltOnFailure: true
linting-coverage:
worker:
type: docker
path: eve/workers/build
volumes: &default_volumes
- '/home/eve/workspace'
steps:
- Git: *clone
- ShellCommand: *yarn-install
- ShellCommand: *add-hostname
- ShellCommand: *credentials
- ShellCommand:
name: Unit Coverage mandatory file
command: |
set -ex
test -f .git/HEAD
- ShellCommand:
name: Linting
command: |
set -ex
yarn run --silent lint -- --max-warnings 0
yarn run --silent lint_md
flake8 $(git ls-files "*.py")
yamllint $(git ls-files "*.yml")
- ShellCommand:
name: Unit Coverage
command: |
set -ex
unset HTTP_PROXY HTTPS_PROXY NO_PROXY
unset http_proxy https_proxy no_proxy
mkdir -p $CIRCLE_TEST_REPORTS/unit
yarn test
yarn run test_legacy_location
env: &shared-vars
<<: *global-env
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
CIRCLE_TEST_REPORTS: /tmp
CIRCLE_ARTIFACTS: /tmp
CI_REPORTS: /tmp
- ShellCommand:
name: Unit Coverage logs
command: find /tmp/unit -exec cat {} \;
- ShellCommand: *setup-junit-upload
- Upload: *upload-junits
multiple-backend-test:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMem: "2560Mi"
s3Mem: "2560Mi"
env:
<<: *multiple-backend-vars
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *yarn-install
- ShellCommand:
command: |
bash -c "
source /root/.aws/exports &> /dev/null
set -ex
bash wait_for_local_port.bash 8000 40
yarn run multiple_backend_test
yarn run ft_awssdk_external_backends"
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
<<: *global-env
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- ShellCommand:
command: mvn test
workdir: build/tests/functional/jaws
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
- ShellCommand:
command: rspec tests.rb
workdir: build/tests/functional/fog
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
ceph-backend-tests:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
ceph: eve/workers/ceph
vars:
aggressorMem: "2500Mi"
s3Mem: "2560Mi"
redis: enabled
env:
<<: *multiple-backend-vars
<<: *global-env
CI_CEPH: "true"
MPU_TESTING: "yes"
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *yarn-install
- ShellCommand:
command: |
bash -c "
source /root/.aws/exports &> /dev/null
set -ex
bash eve/workers/ceph/wait_for_ceph.sh
bash wait_for_local_port.bash 8000 40
yarn run multiple_backend_test"
env:
<<: *multiple-backend-vars
<<: *global-env
<<: *follow-s3-ceph-logs
- ShellCommand:
command: mvn test
workdir: build/tests/functional/jaws
<<: *follow-s3-ceph-logs
env:
<<: *multiple-backend-vars
- ShellCommand:
command: rspec tests.rb
workdir: build/tests/functional/fog
<<: *follow-s3-ceph-logs
env:
<<: *multiple-backend-vars
- ShellCommand:
command: |
yarn run ft_awssdk &&
yarn run ft_s3cmd
env:
<<: *file-mem-mpu
<<: *global-env
S3_LOCATION_FILE: "/kube_pod-prod-cloudserver-backend-0/\
build/tests/locationConfig/locationConfigCeph.json"
<<: *follow-s3-ceph-logs
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
mongo-ft-tests:
worker: &s3-pod
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMem: "2Gi"
s3Mem: "1664Mi"
redis: enabled
env:
<<: *mongo-vars
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *yarn-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
yarn run ft_test
<<: *follow-s3-log
env:
<<: *mongo-vars
<<: *global-env
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
file-ft-tests:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMem: "3Gi"
s3Mem: "2560Mi"
redis: enabled
env:
<<: *file-mem-mpu
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *yarn-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
yarn run ft_test
<<: *follow-s3-log
env:
<<: *file-mem-mpu
<<: *global-env
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
kmip-ft-tests:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
pykmip: eve/workers/pykmip
vars:
aggressorMem: "2Gi"
s3Mem: "1664Mi"
redis: enabled
pykmip: enabled
env:
<<: *file-mem-mpu
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *yarn-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 5696 40
yarn run ft_kmip
logfiles:
pykmip:
filename: /artifacts/pykmip.log
follow: true
s3:
filename: /artifacts/s3.log
follow: true
env:
<<: *file-mem-mpu
<<: *global-env
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
post-merge:
worker:
type: kube_pod
path: eve/workers/release/pod.yaml
images:
release: eve/workers/release
steps:
- Git: *clone
- ShellCommand: &wait_docker_daemon
name: Wait for Docker daemon to be ready
command: |
bash -c '
for i in {1..150}
do
docker info &> /dev/null && exit
sleep 2
done
echo "Could not reach Docker daemon from buildbot worker" >&2
exit 1'
haltOnFailure: true
- ShellCommand: &docker_login
name: Private Registry Login
command: >
docker login
-u '%(secret:harbor_login)s'
-p '%(secret:harbor_password)s'
registry.scality.com
- SetProperty: &registry_url
name: registry
property: registry_url
value: registry.scality.com/zenko/cloudserver
- SetProperty: &docker_tag_revision
name: Set docker tag revision property
property: docker_tag_revision
value: "%(prop:registry_url)s:%(prop:commit_short_revision)s"
- SetProperty: &docker_tag_latest
name: Set docker tag latest property
property: docker_tag_latest
value: "%(prop:registry_url)s:latest-%(prop:product_version)s"
- SetPropertyFromCommand: &short_version
name: Set short version
property: short_version
command: echo "%(prop:product_version)s" | grep -o '^[0-9]*\.[0-9]*'
- SetProperty: &docker_tag_short_version
name: Set docker tag latest short version property
property: docker_tag_short_version
value: "%(prop:registry_url)s:latest-%(prop:short_version)s"
- ShellCommand:
name: Build docker image
command: >-
docker build
--no-cache
-t %(prop:docker_tag_revision)s
-t %(prop:docker_tag_latest)s
-t %(prop:docker_tag_short_version)s
.
- ShellCommand:
name: Push images
command: |
docker push %(prop:docker_tag_revision)s
docker push %(prop:docker_tag_latest)s
docker push %(prop:docker_tag_short_version)s

View File

@ -0,0 +1,58 @@
FROM buildpack-deps:xenial-curl
#
# Install packages needed by the buildchain
#
ENV LANG C.UTF-8
COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
&& apt-get update \
&& apt-get install -y yarn \
&& cat /tmp/*packages.list | xargs apt-get install -y \
&& git clone https://github.com/tj/n.git \
&& make -C ./n \
&& n 10.22.0 \
&& pip install pip==9.0.1 \
&& rm -rf ./n \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /tmp/packages.list
#
# Add user eve
#
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
&& adduser eve sudo \
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
#
# Install Dependencies
#
# Install RVM and gems
ENV RUBY_VERSION="2.4.1"
COPY ./gems.list /tmp/
RUN cat /tmp/gems.list | xargs gem install
#RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
# && curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
# && usermod -a -G rvm eve
#RUN /bin/bash -l -c "\
# source /usr/local/rvm/scripts/rvm \
# && cat /tmp/gems.list | xargs gem install \
# && rm /tmp/gems.list"
# Install Pip packages
COPY ./pip_packages.list /tmp/
RUN cat /tmp/pip_packages.list | xargs pip install \
&& rm -f /tmp/pip_packages.list \
&& mkdir /home/eve/.aws \
&& chown eve /home/eve/.aws
#
# Run buildbot-worker on startup
#
ARG BUILDBOT_VERSION
RUN pip install buildbot-worker==$BUILDBOT_VERSION
CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS && buildbot-worker start --nodaemon"]

View File

@ -0,0 +1,13 @@
ca-certificates
git
gnupg
libffi-dev
libssl-dev
python-pip
python2.7
python2.7-dev
software-properties-common
sudo
tcl
wget
procps

View File

@ -2,9 +2,9 @@
set -x #echo on set -x #echo on
set -e #exit at the first error set -e #exit at the first error
mkdir -p $HOME/.aws mkdir -p ~/.aws
cat >>$HOME/.aws/credentials <<EOF cat >>/root/.aws/credentials <<EOF
[default] [default]
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY

View File

@ -0,0 +1,4 @@
fog-aws:1.3.0
json
mime-types:3.1
rspec:3.5

View File

@ -0,0 +1,3 @@
flake8
s3cmd==1.6.1
yamllint

View File

@ -0,0 +1,11 @@
build-essential
curl
default-jdk
libdigest-hmac-perl
lsof
maven
netcat
redis-server
ruby-full
yarn
zlib1g-dev

233
eve/workers/pod.yaml Normal file
View File

@ -0,0 +1,233 @@
---
apiVersion: v1
kind: Pod
metadata:
name: "proxy-ci-test-pod"
spec:
restartPolicy: Never
terminationGracePeriodSeconds: 10
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com"
- "testrequestbucket.localhost"
- "pykmip.local"
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
initContainers:
- name: kmip-certs-installer
image: {{ images.pykmip }}
command: [ 'sh', '-c', 'cp /ssl/* /ssl-kmip/']
volumeMounts:
- name: kmip-certs
readOnly: false
mountPath: /ssl-kmip
{%- endif %}
containers:
{% if vars.env.S3METADATA is defined and vars.env.S3METADATA == "mongodb" -%}
- name: mongo
image: scality/ci-mongo:3.6.8
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: 500m
memory: 1Gi
{%- endif %}
- name: aggressor
image: {{ images.aggressor }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: "1"
memory: {{ vars.aggressorMem }}
limits:
cpu: "1"
memory: {{ vars.aggressorMem }}
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
- name: artifacts
readOnly: true
mountPath: /artifacts
command:
- bash
- -lc
- |
buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS
buildbot-worker start --nodaemon
env:
- name: CI
value: "true"
- name: ENABLE_LOCAL_CACHE
value: "true"
- name: REPORT_TOKEN
value: "report-token-1"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
{% for key, value in vars.env.items() %}
- name: {{ key }}
value: "{{ value }}"
{% endfor %}
- name: s3
image: {{ images.s3 }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: "2"
memory: {{ vars.s3Mem }}
limits:
cpu: "2"
memory: {{ vars.s3Mem }}
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
- name: certs
readOnly: false
mountPath: /tmp
- name: artifacts
readOnly: false
mountPath: /artifacts
- name: kmip-certs
readOnly: false
mountPath: /ssl-kmip
command:
- bash
- -ec
- |
sleep 10 # wait for mongo
/usr/src/app/docker-entrypoint.sh yarn start | tee -a /artifacts/s3.log
env:
{% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" and vars.env.CI_CEPH is not defined -%}
- name: S3_LOCATION_FILE
value: "/usr/src/app/tests/locationConfig/locationConfigTests.json"
{%- endif %}
{% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" and vars.env.CI_CEPH is defined and vars.env.CI_CEPH == "true" -%}
- name: S3_LOCATION_FILE
value: "/usr/src/app/tests/locationConfig/locationConfigCeph.json"
{%- endif %}
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
- name: S3KMS
value: kmip
- name: S3KMIP_PORT
value: "5696"
- name: S3KMIP_HOSTS
value: "pykmip.local"
- name: S3KMIP_COMPOUND_CREATE
value: "false"
- name: S3KMIP_BUCKET_ATTRIBUTE_NAME
value: ''
- name: S3KMIP_PIPELINE_DEPTH
value: "8"
- name: S3KMIP_KEY
value: /ssl-kmip/kmip-client-key.pem
- name: S3KMIP_CERT
value: /ssl-kmip/kmip-client-cert.pem
- name: S3KMIP_CA
value: /ssl-kmip/kmip-ca.pem
{%- endif %}
- name: CI
value: "true"
- name: ENABLE_LOCAL_CACHE
value: "true"
- name: MONGODB_HOSTS
value: "localhost:27018"
- name: MONGODB_RS
value: "rs0"
- name: REDIS_HOST
value: "localhost"
- name: REDIS_PORT
value: "6379"
- name: REPORT_TOKEN
value: "report-token-1"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
- name: HEALTHCHECKS_ALLOWFROM
value: "0.0.0.0/0"
{% for key, value in vars.env.items() %}
- name: {{ key }}
value: "{{ value }}"
{% endfor %}
{% if vars.redis is defined and vars.redis == "enabled" -%}
- name: redis
image: redis:alpine
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 200m
memory: 128Mi
limits:
cpu: 200m
memory: 128Mi
{%- endif %}
{% if vars.env.CI_PROXY is defined and vars.env.CI_PROXY == "true" -%}
- name: squid
image: scality/ci-squid
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 250m
memory: 128Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: certs
readOnly: false
mountPath: /ssl
command:
- sh
- -exc
- |
mkdir -p /ssl
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-keyout /ssl/myca.pem -out /ssl/myca.pem
cp /ssl/myca.pem /ssl/CA.pem
squid -f /etc/squid/squid.conf -N -z
squid -f /etc/squid/squid.conf -NYCd 1
{%- endif %}
{% if vars.env.CI_CEPH is defined and vars.env.CI_CEPH == "true" -%}
- name: ceph
image: {{ images.ceph }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 500m
memory: 1536Mi
limits:
cpu: 500m
memory: 1536Mi
volumeMounts:
- name: artifacts
readOnly: false
mountPath: /artifacts
{%- endif %}
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
- name: pykmip
image: {{ images.pykmip }}
imagePullPolicy: IfNotPresent
volumeMounts:
- name: artifacts
readOnly: false
mountPath: /artifacts
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 100m
memory: 128Mi
{%- endif %}
volumes:
- name: creds
emptyDir: {}
- name: certs
emptyDir: {}
- name: artifacts
emptyDir: {}
- name: kmip-certs
emptyDir: {}

View File

@ -1,4 +1,5 @@
FROM python:3.10-alpine FROM python:3-alpine
RUN apk add --no-cache \ RUN apk add --no-cache \
libressl && \ libressl && \
@ -7,14 +8,8 @@ RUN apk add --no-cache \
libffi-dev \ libffi-dev \
libressl-dev \ libressl-dev \
sqlite-dev \ sqlite-dev \
build-base \ build-base && \
curl pip install pykmip requests && \
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN pip3 install -U pip && \
pip3 install pykmip requests && \
apk del .build-deps && \ apk del .build-deps && \
mkdir /pykmip mkdir /pykmip

View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC6zCCAdOgAwIBAgIUOf68qXxLlhpqa94YFjWARaY09VIwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMDA0
MDcxNzUwNThaGA8yMTIwMDMxNDE3NTA1OFowJDEQMA4GA1UECgwHU2NhbGl0eTEQ
MA4GA1UEAwwHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AOPoCXWg+Mpwd2ltSvgicehcaeW6BoNRm3+icx6OmJHjQL/iApfBJJ54A9ef/Myw
NjBDd2RQBdu2nnEKRar06p/w+ubTbRU189mfAAOGC9m1gkl16gNlbByYikmC0grQ
UDeQ3uKnJ8LFwpu/A+MLpK4Os1CH8fqFIse5w2AT9BscwzF0aHlgO5vm48P5rR9d
EKYTsPlfuCNYIWQ6x75fk//2o9/mCRc9bFdY/ASNwZTTlnSPZ9DQ+g7zaExGG5ah
eJXoKK4skups+GdzLMQj53/lR6fZjn8pX5Mmv4Ex9OvlpFeV85HyZcKnjGqM7eGl
6TcbGZlHw0GvebXIeg4VHe8CAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkq
hkiG9w0BAQsFAAOCAQEApYYIRlu8VWPCm2HJ/k119/ADPgFI6fAmbojNoZo9lLW+
i6JWkohptV5c0oMUjxTrzBRYWyMWdScHUfxjV6AY7QFoIrNBqEDTARfRu+ekFkfH
qfnSp45aLi0gf53WJNTCQJAILGxcXkvSBuw50Fm5aBSgG70Oczf/bgJY4uJTVUra
3GdmDXqLZ7WpSCS5EyzJchDo75TFFRnNAy7+YoKh+11TIwdrrs2qEiAjJ/2zAuIg
r5Vnx6AEEnhqFNbyTLOfk2/MN87FEKqH07vSAHA6M4tprWU8kArcTjMlmVaH8cgO
47lVJ6blCUiJTJLnzNw83dzJTfFmm9CQx7JyTpEBag==
-----END CERTIFICATE-----

View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC2TCCAcGgAwIBAgIUYXclvoYJK/U2Gh4ji0yRevvj3F0wDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAeFw0yMDA0
MDcxNzUwNThaFw0yMTA0MDcxNzUwNThaMCkxEDAOBgNVBAoMB1NjYWxpdHkxFTAT
BgNVBAMMDHB5a21pcC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
ggEBALtrAMYY1VaaJYWPIF630dPH/Pt4kefSjvr0NlKyKv2gpUy3L3Dm1CTd0Ay6
yQWR6jVyKvGWFdyA4GfP2+/96i7DLTI4ePZq/catho8hOSuPBOK1h+N153LlZ9Yu
YKm5lNRjchQ/Di4JykhAgQlNHh7ziR3hrV1QrgjDAgDHKvumKBcRc8oOYuW4ATYl
35C/4h5hzMucnlmwZjq03wBZyhuLKgMT6dmrXv67ZexIWo881ANaBpiUq36wPDJo
FoeS3tXvEJRy3nFL+AEEHpqEpP6GMgt8rnPVJJgUb0OFIpx3kpAaEY7I9hNQsjl3
B5okt1cRHONFOl25KXO+QKf8KuECAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAkesK
fVNo2pGaOybzf9sbmUf/MJx6vPZzqkfsED3KStjBaTKndjvqBvXmxI7hczLvs8Jo
ALbwRJxjLwT3NbxqEwj0nCtid3wzl70f9fSYLh/FC0Nus/1kYJA/JKiZKbEiiaCM
ZULLqkrK26baCxoVpa3cWSo3zR2F3h0Px54abLXuBFoq6QNt5u/3+WIPY1wJYHUk
iCcjLaL7VM9snmK0jF3CNz4+6ZFPAT6H54ILuJ/V427zB1TYMDLOVZumzPLg6DEz
YxMyz99Rjwf4BHEI186csIoZZx6GOVauRoiw9KAct2KOKdcOoiyyKINL/NGlH04e
GDazMx7Yt7xHIjFmlw==
-----END CERTIFICATE-----

View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC8TCCAdmgAwIBAgIUOuxeluKdf49aeSl3tZ+BJEZilaswDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAeFw0yMDA0
MDcxNzUwNThaFw0yMTA0MDcxNzUwNThaMCUxEDAOBgNVBAoMB1NjYWxpdHkxETAP
BgNVBAMMCEpvaG4gRG9lMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
xk8i8m8YEyAqT8S1Hh3VzKLt8EQf6YqnjgunCuTi5PvYd5JtKnneIBu5+JsIXt4/
TlaLUjtT8RiXLtXxDglcxKcEX1M8wwQR35dj3Jx3xQbYv6Pr1Mqp6+egIvxyuJYW
9pc7N5geQAATbJ6iqttY/+6l8KUKF8V28kocgaxhvVUlf62SKaFcau1DCBQu8TwB
sk7JCQ/kUt75cBDV6EDpiShZ7XYAo9hy3eyumMufKO5tTIl1/T2HYaVabbzYZBbY
W2kr4stBDLjls1W0JVmt4V9pn+TrrwRfhWPIXo3KlxYRUbsW2Z223QB+jCndGqcf
7dZSaoi7pVS+tKzIn3P+HQIDAQABoxowGDAWBgNVHSUBAf8EDDAKBggrBgEFBQcD
AjANBgkqhkiG9w0BAQsFAAOCAQEAXCAJBfm4/SUFJr5jgrerHNfDFS+PUo5PHexs
q1Yie1Q6MDqlPTc/ZcWT/DoqLy3aq+tqqr66OY+2oARq+SgVnsSzgWQ/LosSQikl
3fwcHZfelbbvPIofM0DKtGJLyTTGvtwjj9t3tzx0+7b6PXXWm91p0Xzpp9i0llJ3
YBAxIW43LAFaIBJonxgnR9TLVlP+XatnYBzWbOg44uxeNDQkudbEUsGb2j4SShk4
Pw1fgyKmyQRikvgst7uCgS5S9Vzu6O9CC6nvqnEfzPLWMA9GRUiTjOeZ0BlMqrlH
orzpJpoFNelpqgh9LxA026a4TeK/pdfx/NJE6gTN+CVxHU40iQ==
-----END CERTIFICATE-----

View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDGTyLybxgTICpP
xLUeHdXMou3wRB/piqeOC6cK5OLk+9h3km0qed4gG7n4mwhe3j9OVotSO1PxGJcu
1fEOCVzEpwRfUzzDBBHfl2PcnHfFBti/o+vUyqnr56Ai/HK4lhb2lzs3mB5AABNs
nqKq21j/7qXwpQoXxXbyShyBrGG9VSV/rZIpoVxq7UMIFC7xPAGyTskJD+RS3vlw
ENXoQOmJKFntdgCj2HLd7K6Yy58o7m1MiXX9PYdhpVptvNhkFthbaSviy0EMuOWz
VbQlWa3hX2mf5OuvBF+FY8hejcqXFhFRuxbZnbbdAH6MKd0apx/t1lJqiLulVL60
rMifc/4dAgMBAAECggEABbdIOZasKfj2X0A7PDf97p0PoKpGBTRC6hw5312DkLgV
oDSvQtcqaOCDtr+5OQrM5lQmReOB4uQjj20JOq9YZi6uOJUsni5i2YACl9xGs34k
BzoRVRvWU/9kJT4DjIB+/vKS+WJAFPYrmSjlZWlXImFdlRccuFyvtgIe2jn+wzdB
ErYirqHNnSJdiNBA58VmLR2ge3Ilo/S9YTgB/+aKtElhitGqSkVB8QKNWt2u5oFw
tJ0n1ikz1fiD16/kcpaOT9TWm1kkhIthSwZd65+HMSuxpO+KqmwgmXPXhDnTrlSd
y5r//+ggAnDhuUmNmFwxHn8prThYGLKlLpUFtivJIQKBgQDqWMOf8Y4Im7nW7b+r
r00gSu+XpvBBnC2Dpr0bw90OQYdLhDcdeqeJ2fFpQqXaevNhmqnAy6PFeqdoSscu
gc1XkejZsPIQNSa7e0RC7kveVp3gkzcQ222vgJY7++R46/S3Du3oQ3FAlB7/0VOi
+9CBgdSI+d3u1rcXOW63uQRb1QKBgQDYofBl2a/SQmWcMFA20fN+z1ft4j3YXb1X
KgluiJpBFGP1lWXEZ/+v7L2PI5fGAl1FJgjYYuzVH3T1f0kgZmuIUuZPMB7d8gLw
1qyfKH4MKTItwh38K28/dRckBlMj10Uc+aXMe1LTcl15jp6LLI4Mx09qy1QF/c/y
76Cvkc2lKQKBgERpfVJn9grVSz9PULESD/XpamBfP6wnp7HTL0m3uAS9ZH3LLzvP
3rEDitIrvrc1RW+s8vlxeXHhCJYNnnAZPJTf55YFbeUFXzVEGv1fC47wwk4ZK+4j
4LVnWHRSaLRUTbBTD2jKp3kuxI3x0fS2hnwIJr+GEh/zVqfVAFlqDbexAoGABf1U
PDysk8+qJ4tebGWZqePptnYO57CPz50l7ZxxR2Nc8ClVSvzlIOQWyaJeS+c81PCc
Rf9WNP5NqYv/ZZnvVzGTlJTsBY7vbeFBnJTuB0AMVx+K3LIGvWZrYV+bZN5K1uZA
I0s1mwsKcpXy5D4zHz9TfsxoYlIGMd1WQARz/yECgYB5JH9wFDKjMAhCGG9hxMSx
drMeZ/Ypya/DTyN+1p3DnG+Tw0Mh6Hmpm04D6iDKrafLkmlD+ZduzhzfEqn0lDDg
uIFFvWO6BOo2KALtMpcuDGYFKjJJ0S47EiWhKHUn/OsnAzk2lKlEOcPfrc9lY5AM
6P6Jg/Q21kr1j5gQedj7Cw==
-----END PRIVATE KEY-----

View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7awDGGNVWmiWF
jyBet9HTx/z7eJHn0o769DZSsir9oKVMty9w5tQk3dAMuskFkeo1cirxlhXcgOBn
z9vv/eouwy0yOHj2av3GrYaPITkrjwTitYfjdedy5WfWLmCpuZTUY3IUPw4uCcpI
QIEJTR4e84kd4a1dUK4IwwIAxyr7pigXEXPKDmLluAE2Jd+Qv+IeYczLnJ5ZsGY6
tN8AWcobiyoDE+nZq17+u2XsSFqPPNQDWgaYlKt+sDwyaBaHkt7V7xCUct5xS/gB
BB6ahKT+hjILfK5z1SSYFG9DhSKcd5KQGhGOyPYTULI5dweaJLdXERzjRTpduSlz
vkCn/CrhAgMBAAECggEAVo7o4JT/kuvGgJTF4nkLU8B9urbIzESW/JhlrnPTHyhe
r7u7EW3KdOxs9jQeO5BUlzKPWtxyZFCxU8DQV1ryGX7TFOq4Ezb+1g+2ocw6Vz/K
hdpJVGhT6ODCwEzTIBAyCJWVTnyA1Ap5fj0sW1temfToUwCzzPFCL5HBDxLtayNb
DvAdqP6nvzF85oGlyJlwdY9Zz8bWxd+kDeFAp+23rcLrwPwUASilLwp9v1erc6Hs
Mg1UkF7liuIIaSQahOF93YTMJAOMNft8wN+QQqQ12zMCD/fOcZTRBrLwHSaGJ+/Y
ACq9yzVYpzRnNOgkmpBkxXUzddkPPom/LdGaY5aE8QKBgQDwpbQaIf1VQZRyVEVL
VeW9fh1f4rzI0Yd+/BvuSzVLAuZCWDWjTC5zqpLeQ9mk7ziF0ck/MlcSlumZ+f0Q
aMUNxjtpGeKmAygCx5mZRCqKz+oxP0vdHkP5PwQQIUAwWXnWiSAuCKRX1TnjW9Dk
7Qq4Qfz2FS/PmY7Z3sS4ugId9wKBgQDHX/LYRuaoGS8gqR72DOqCJMkMduz5DGtD
fTUvSP+OZHAMRgTnANAOagy/drj6nNM3zDHTocEMfcC4HbiRU7c6Ys1iuiJ4bP4l
OkJajl9O3LJsJwLsHeMnZM27/9/mD8mrPzbNUHXZCU26uah9qLHwAwexp+rzJJNX
iTHkT/Gn5wKBgQCCNseblGTGKzQuIRdVymcEACfY6JGKgIY22igq6xstOaZqo9xy
PhiskdHi3wf3zVHiZz/kKFMhRfOlU7XxmR93cppXJqCTgAW4a1TbsBzs+9AXUc61
GVlilwyVxcg74U6iHZUCE78Jn+Ew+0+vb+xrA5njdldmmArKLVZ5Nn1KxQKBgFZs
COAnG6SSBhOqO3l8b8qqF1wH0QDDmVtP0tYEVoJqlwc68rUPbSBSZ+Q2mkhH4ma1
ZIPQAdZgTEGC4JZeK3ZrjYvWE0sQM7n/XvPR8w5ELDMlVebzrZtN3sA3Ud5vyYMp
i5/D2NGTbtYZ1CdkEH1xUsx3dSigGh4/ohjNbnrRAoGANWNVP+WK/iC6H49UGbtC
f+0Q5UL1hEh+9W8WqYiY38mA1xO3A07RQhBjk004CSqcd/ZJgniWYTZOhfd9lwBp
5FIg95xaJbklGOxL8jYAYqO4D/QURXuYpLVpVS/Xvy0d5n2U1tISSAvhs4u6/7Fv
M0iIzoBaJY36ZSK4LRgiHjM=
-----END PRIVATE KEY-----

View File

@ -4,7 +4,7 @@ port=5696
certificate_path=/ssl/kmip-cert.pem certificate_path=/ssl/kmip-cert.pem
key_path=/ssl/kmip-key.pem key_path=/ssl/kmip-key.pem
ca_path=/ssl/kmip-ca.pem ca_path=/ssl/kmip-ca.pem
auth_suite=TLS1.2 auth_suite=Basic
policy_path=/etc/pykmip/policies policy_path=/etc/pykmip/policies
enable_tls_client_auth=True enable_tls_client_auth=True
database_path=/pykmip/pykmip.db database_path=/pykmip/pykmip.db

View File

@ -0,0 +1,24 @@
FROM ubuntu:bionic
WORKDIR /home/eve/workspace
RUN apt-get update && apt-get install -y \
git \
gnupg2 \
lsb-release \
curl \
python3-buildbot-worker \
&& adduser -u 1042 --home /home/eve eve \
&& chown -R eve:eve /home/eve
# Install docker cli
ARG DOCKER_VERSION=18.06.1~ce~3-0~ubuntu
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
&& echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
| tee -a /etc/apt/sources.list.d/docker-ce.list \
&& apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
docker-ce=${DOCKER_VERSION} \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt
USER eve

View File

@ -0,0 +1,45 @@
---
apiVersion: v1
kind: Pod
metadata:
name: release-worker
spec:
containers:
- name: release-worker
image: "{{ images.release }}"
command: ["/bin/sh"]
args: ["-c", "buildbot-worker create-worker . ${BUILDMASTER}:${BUILDMASTER_PORT} ${WORKERNAME} ${WORKERPASS} && buildbot-worker start --nodaemon"]
resources:
requests:
cpu: "250m"
memory: 1Gi
limits:
cpu: "1"
memory: 1Gi
env:
- name: DOCKER_HOST
value: localhost:2375
volumeMounts:
- name: worker-workspace
mountPath: /home/eve/workspace
- name: dind-daemon
image: docker:18.06.1-dind
resources:
requests:
cpu: "500m"
memory: 1Gi
limits:
cpu: "1"
memory: 1Gi
securityContext:
privileged: true
volumeMounts:
- name: docker-storage
mountPath: /var/lib/docker
- name: worker-workspace
mountPath: /home/eve/workspace
volumes:
- name: docker-storage
emptyDir: {}
- name: worker-workspace
emptyDir: {}

View File

@ -48,7 +48,7 @@ signed_headers = 'host;x-amz-content-sha256;x-amz-date'
canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \ canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
.format(method, canonical_uri, canonical_querystring, canonical_headers, .format(method, canonical_uri, canonical_querystring, canonical_headers,
signed_headers, payload_hash) signed_headers, payload_hash)
print(canonical_request) print canonical_request
credential_scope = '{0}/{1}/{2}/aws4_request' \ credential_scope = '{0}/{1}/{2}/aws4_request' \
.format(date_stamp, region, service) .format(date_stamp, region, service)
@ -76,4 +76,4 @@ headers = {
endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
r = requests.get(endpoint, headers=headers) r = requests.get(endpoint, headers=headers)
print(r.text) print (r.text)

View File

@ -1,28 +0,0 @@
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
COPY . ${HOME_DIR}/s3
RUN chown -R ${USER} ${HOME_DIR}
RUN pip3 install redis===3.5.3 requests==2.27.1 && \
apt-get install -y git-lfs
USER ${USER}
WORKDIR ${HOME_DIR}/s3
RUN rm -f ~/.gitconfig && \
git config --global --add safe.directory . && \
git lfs install && \
GIT_LFS_SKIP_SMUDGE=1 && \
yarn global add typescript && \
yarn install --frozen-lockfile --production --network-concurrency 1 && \
yarn cache clean --all && \
yarn global remove typescript
# run symlinking separately to avoid yarn installation errors
# we might have to check if the symlinking is really needed!
RUN ln -sf /scality-kms node_modules
EXPOSE 8000
CMD bash -c "source ${CONF_DIR}/env && export && supervisord -c ${CONF_DIR}/supervisord.conf"

View File

@ -1,10 +1,3 @@
'use strict'; // eslint-disable-line strict 'use strict'; // eslint-disable-line strict
require('werelogs').stderrUtils.catchAndTimestampStderr(
undefined,
// Do not exit as workers have their own listener that will exit
// But primary don't have another listener
require('cluster').isPrimary ? 1 : null,
);
require('./lib/server.js')(); require('./lib/server.js')();

View File

@ -5,21 +5,15 @@ const path = require('path');
const url = require('url'); const url = require('url');
const crypto = require('crypto'); const crypto = require('crypto');
const { v4: uuidv4 } = require('uuid'); const uuid = require('node-uuid');
const cronParser = require('cron-parser'); const cronParser = require('cron-parser');
const joi = require('@hapi/joi'); const joi = require('@hapi/joi');
const { s3routes, auth: arsenalAuth, s3middleware } = require('arsenal');
const { isValidBucketName } = s3routes.routesUtils; const { isValidBucketName } = require('arsenal').s3routes.routesUtils;
const validateAuthConfig = arsenalAuth.inMemory.validateAuthConfig; const validateAuthConfig = require('arsenal').auth.inMemory.validateAuthConfig;
const { buildAuthDataAccount } = require('./auth/in_memory/builder'); const { buildAuthDataAccount } = require('./auth/in_memory/builder');
const validExternalBackends = require('../constants').externalBackends; const validExternalBackends = require('../constants').externalBackends;
const { azureAccountNameRegex, base64Regex, const { azureAccountNameRegex, base64Regex } = require('../constants');
allowedUtapiEventFilterFields, allowedUtapiEventFilterStates,
} = require('../constants');
const { utapiVersion } = require('utapi');
const { scaleMsPerDay } = s3middleware.objectUtils;
const constants = require('../constants');
// config paths // config paths
const configSearchPaths = [ const configSearchPaths = [
@ -107,47 +101,6 @@ function parseSproxydConfig(configSproxyd) {
return joi.attempt(configSproxyd, joiSchema, 'bad config'); return joi.attempt(configSproxyd, joiSchema, 'bad config');
} }
function parseRedisConfig(redisConfig) {
const joiSchema = joi.object({
password: joi.string().allow(''),
host: joi.string(),
port: joi.number(),
retry: joi.object({
connectBackoff: joi.object({
min: joi.number().required(),
max: joi.number().required(),
jitter: joi.number().required(),
factor: joi.number().required(),
deadline: joi.number().required(),
}),
}),
// sentinel config
sentinels: joi.alternatives().try(
joi.string()
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
.custom(hosts => hosts.split(',').map(item => {
const [host, port] = item.split(':');
return { host, port: Number.parseInt(port, 10) };
})),
joi.array().items(
joi.object({
host: joi.string().required(),
port: joi.number().required(),
})
).min(1),
),
name: joi.string(),
sentinelPassword: joi.string().allow(''),
})
.and('host', 'port')
.and('sentinels', 'name')
.xor('host', 'sentinels')
.without('sentinels', ['host', 'port'])
.without('host', ['sentinels', 'sentinelPassword']);
return joi.attempt(redisConfig, joiSchema, 'bad config');
}
function restEndpointsAssert(restEndpoints, locationConstraints) { function restEndpointsAssert(restEndpoints, locationConstraints) {
assert(typeof restEndpoints === 'object', assert(typeof restEndpoints === 'object',
'bad config: restEndpoints must be an object of endpoints'); 'bad config: restEndpoints must be an object of endpoints');
@ -178,71 +131,26 @@ function gcpLocationConstraintAssert(location, locationObj) {
}); });
} }
function azureGetStorageAccountName(location, locationDetails) { function azureLocationConstraintAssert(location, locationObj) {
const { azureStorageAccountName } = locationDetails; const {
azureStorageEndpoint,
azureStorageAccountName,
azureStorageAccessKey,
azureContainerName,
} = locationObj.details;
const storageEndpointFromEnv =
process.env[`${location}_AZURE_STORAGE_ENDPOINT`];
const storageAccountNameFromEnv = const storageAccountNameFromEnv =
process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`]; process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`];
return storageAccountNameFromEnv || azureStorageAccountName; const storageAccessKeyFromEnv =
} process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`];
function azureGetLocationCredentials(location, locationDetails) {
const storageAccessKey =
process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`] ||
locationDetails.azureStorageAccessKey;
const sasToken =
process.env[`${location}_AZURE_SAS_TOKEN`] ||
locationDetails.sasToken;
const clientKey =
process.env[`${location}_AZURE_CLIENT_KEY`] ||
locationDetails.clientKey;
const authMethod =
process.env[`${location}_AZURE_AUTH_METHOD`] ||
locationDetails.authMethod ||
(storageAccessKey && 'shared-key') ||
(sasToken && 'shared-access-signature') ||
(clientKey && 'client-secret') ||
'shared-key';
switch (authMethod) {
case 'shared-key':
default:
return {
authMethod,
storageAccountName:
azureGetStorageAccountName(location, locationDetails),
storageAccessKey,
};
case 'shared-access-signature':
return {
authMethod,
sasToken,
};
case 'client-secret':
return {
authMethod,
tenantId:
process.env[`${location}_AZURE_TENANT_ID`] ||
locationDetails.tenantId,
clientId:
process.env[`${location}_AZURE_CLIENT_ID`] ||
locationDetails.clientId,
clientKey,
};
}
}
function azureLocationConstraintAssert(location, locationObj) {
const locationParams = { const locationParams = {
...azureGetLocationCredentials(location, locationObj.details), azureStorageEndpoint: storageEndpointFromEnv || azureStorageEndpoint,
azureStorageEndpoint: azureStorageAccountName:
process.env[`${location}_AZURE_STORAGE_ENDPOINT`] || storageAccountNameFromEnv || azureStorageAccountName,
locationObj.details.azureStorageEndpoint, azureStorageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
azureContainerName: locationObj.details.azureContainerName, azureContainerName,
}; };
Object.keys(locationParams).forEach(param => { Object.keys(locationParams).forEach(param => {
const value = locationParams[param]; const value = locationParams[param];
assert.notEqual(value, undefined, assert.notEqual(value, undefined,
@ -252,16 +160,13 @@ function azureLocationConstraintAssert(location, locationObj) {
`bad location constraint: "${location}" ${param} ` + `bad location constraint: "${location}" ${param} ` +
`"${value}" must be a string`); `"${value}" must be a string`);
}); });
assert(azureAccountNameRegex.test(locationParams.azureStorageAccountName),
if (locationParams.authMethod === 'shared-key') {
assert(azureAccountNameRegex.test(locationParams.storageAccountName),
`bad location constraint: "${location}" azureStorageAccountName ` + `bad location constraint: "${location}" azureStorageAccountName ` +
`"${locationParams.storageAccountName}" is an invalid value`); `"${locationParams.storageAccountName}" is an invalid value`);
assert(base64Regex.test(locationParams.storageAccessKey), assert(base64Regex.test(locationParams.azureStorageAccessKey),
`bad location constraint: "${location}" ` + `bad location constraint: "${location}" ` +
'azureStorageAccessKey is not a valid base64 string'); 'azureStorageAccessKey is not a valid base64 string');
} assert(isValidBucketName(azureContainerName, []),
assert(isValidBucketName(locationParams.azureContainerName, []),
`bad location constraint: "${location}" ` + `bad location constraint: "${location}" ` +
'azureContainerName is an invalid container name'); 'azureContainerName is an invalid container name');
} }
@ -280,104 +185,10 @@ function hdClientLocationConstraintAssert(configHd) {
return hdclientFields; return hdclientFields;
} }
function azureArchiveLocationConstraintAssert(locationObj) {
const checkedFields = [
'azureContainerName',
'azureStorageEndpoint',
];
if (Object.keys(locationObj.details).length === 0 ||
!checkedFields.every(field => field in locationObj.details)) {
return;
}
const {
azureContainerName,
azureStorageEndpoint,
} = locationObj.details;
const stringFields = [
azureContainerName,
azureStorageEndpoint,
];
stringFields.forEach(field => {
assert(typeof field === 'string',
`bad config: ${field} must be a string`);
});
let hasAuthMethod = false;
if (locationObj.details.sasToken !== undefined) {
assert(typeof locationObj.details.sasToken === 'string',
`bad config: ${locationObj.details.sasToken} must be a string`);
hasAuthMethod = true;
}
if (locationObj.details.azureStorageAccountName !== undefined &&
locationObj.details.azureStorageAccessKey !== undefined) {
assert(typeof locationObj.details.azureStorageAccountName === 'string',
`bad config: ${locationObj.details.azureStorageAccountName} must be a string`);
assert(typeof locationObj.details.azureStorageAccessKey === 'string',
`bad config: ${locationObj.details.azureStorageAccessKey} must be a string`);
assert(!hasAuthMethod, 'Multiple authentication methods are not allowed');
hasAuthMethod = true;
}
if (locationObj.details.tenantId !== undefined &&
locationObj.details.clientId !== undefined &&
locationObj.details.clientKey !== undefined) {
assert(typeof locationObj.details.tenantId === 'string',
`bad config: ${locationObj.details.tenantId} must be a string`);
assert(typeof locationObj.details.clientId === 'string',
`bad config: ${locationObj.details.clientId} must be a string`);
assert(typeof locationObj.details.clientKey === 'string',
`bad config: ${locationObj.details.clientKey} must be a string`);
assert(!hasAuthMethod, 'Multiple authentication methods are not allowed');
hasAuthMethod = true;
}
assert(hasAuthMethod, 'Missing authentication method');
}
function dmfLocationConstraintAssert(locationObj) {
const checkedFields = [
'endpoint',
'username',
'password',
'repoId',
'nsId',
];
if (Object.keys(locationObj.details).length === 0 ||
!checkedFields.every(field => field in locationObj.details)) {
return;
}
const {
endpoint,
username,
password,
repoId,
nsId,
} = locationObj.details;
const stringFields = [
endpoint,
username,
password,
nsId,
];
stringFields.forEach(field => {
assert(typeof field === 'string',
`bad config: ${field} must be a string`);
});
assert.strictEqual(Array.isArray(repoId), true);
repoId.forEach(rId => {
assert(typeof rId === 'string',
`bad config: ${rId} must be a string`);
});
assert(repoId.every(
r => typeof r === 'string'),
'bad config: each repoId must be a string',
);
}
function locationConstraintAssert(locationConstraints) { function locationConstraintAssert(locationConstraints) {
const supportedBackends = const supportedBackends =
['mem', 'file', 'scality', ['mem', 'file', 'scality',
'mongodb', 'dmf', 'azure_archive', 'vitastor'].concat(Object.keys(validExternalBackends)); 'mongodb'].concat(Object.keys(validExternalBackends));
assert(typeof locationConstraints === 'object', assert(typeof locationConstraints === 'object',
'bad config: locationConstraints must be an object'); 'bad config: locationConstraints must be an object');
Object.keys(locationConstraints).forEach(l => { Object.keys(locationConstraints).forEach(l => {
@ -485,12 +296,6 @@ function locationConstraintAssert(locationConstraints) {
if (locationConstraints[l].type === 'gcp') { if (locationConstraints[l].type === 'gcp') {
gcpLocationConstraintAssert(l, locationConstraints[l]); gcpLocationConstraintAssert(l, locationConstraints[l]);
} }
if (locationConstraints[l].type === 'dmf') {
dmfLocationConstraintAssert(locationConstraints[l]);
}
if (locationConstraints[l].type === 'azure_archive') {
azureArchiveLocationConstraintAssert(locationConstraints[l]);
}
if (locationConstraints[l].type === 'pfs') { if (locationConstraints[l].type === 'pfs') {
assert(typeof details.pfsDaemonEndpoint === 'object', assert(typeof details.pfsDaemonEndpoint === 'object',
'bad config: pfsDaemonEndpoint is mandatory and must be an object'); 'bad config: pfsDaemonEndpoint is mandatory and must be an object');
@ -502,33 +307,26 @@ function locationConstraintAssert(locationConstraints) {
locationConstraints[l].details.connector.hdclient); locationConstraints[l].details.connector.hdclient);
} }
}); });
assert(Object.keys(locationConstraints)
.includes('us-east-1'), 'bad locationConfig: must ' +
'include us-east-1 as a locationConstraint');
} }
function parseUtapiReindex(config) { function parseUtapiReindex({ enabled, schedule, sentinel, bucketd }) {
const {
enabled,
schedule,
redis,
bucketd,
onlyCountLatestWhenObjectLocked,
} = config;
assert(typeof enabled === 'boolean', assert(typeof enabled === 'boolean',
'bad config: utapi.reindex.enabled must be a boolean'); 'bad config: utapi.reindex.enabled must be a boolean');
assert(typeof sentinel === 'object',
const parsedRedis = parseRedisConfig(redis); 'bad config: utapi.reindex.sentinel must be an object');
assert(Array.isArray(parsedRedis.sentinels), assert(typeof sentinel.port === 'number',
'bad config: utapi reindex redis config requires a list of sentinels'); 'bad config: utapi.reindex.sentinel.port must be a number');
assert(typeof sentinel.name === 'string',
'bad config: utapi.reindex.sentinel.name must be a string');
assert(typeof bucketd === 'object', assert(typeof bucketd === 'object',
'bad config: utapi.reindex.bucketd must be an object'); 'bad config: utapi.reindex.bucketd must be an object');
assert(typeof bucketd.port === 'number', assert(typeof bucketd.port === 'number',
'bad config: utapi.reindex.bucketd.port must be a number'); 'bad config: utapi.reindex.bucketd.port must be a number');
assert(typeof schedule === 'string', assert(typeof schedule === 'string',
'bad config: utapi.reindex.schedule must be a string'); 'bad config: utapi.reindex.schedule must be a string');
if (onlyCountLatestWhenObjectLocked !== undefined) {
assert(typeof onlyCountLatestWhenObjectLocked === 'boolean',
'bad config: utapi.reindex.onlyCountLatestWhenObjectLocked must be a boolean');
}
try { try {
cronParser.parseExpression(schedule); cronParser.parseExpression(schedule);
} catch (e) { } catch (e) {
@ -536,13 +334,6 @@ function parseUtapiReindex(config) {
'bad config: utapi.reindex.schedule must be a valid ' + 'bad config: utapi.reindex.schedule must be a valid ' +
`cron schedule. ${e.message}.`); `cron schedule. ${e.message}.`);
} }
return {
enabled,
schedule,
redis: parsedRedis,
bucketd,
onlyCountLatestWhenObjectLocked,
};
} }
function requestsConfigAssert(requestsConfig) { function requestsConfigAssert(requestsConfig) {
@ -630,6 +421,7 @@ class Config extends EventEmitter {
// Read config automatically // Read config automatically
this._getLocationConfig(); this._getLocationConfig();
this._getConfig(); this._getConfig();
this._configureBackends();
} }
_getLocationConfig() { _getLocationConfig() {
@ -681,35 +473,6 @@ class Config extends EventEmitter {
return tlsFileContent; return tlsFileContent;
} }
/**
* Parse list of endpoints.
* @param {string[] | undefined} listenOn - List of string of the form "ip:port"
* @param {string} fieldName - Name of field being parsed
* @returns {{ip: string, port: number}[]} - List of ip/port pairs
*/
_parseEndpoints(listenOn, fieldName) {
let result = [];
if (listenOn !== undefined) {
assert(Array.isArray(listenOn)
&& listenOn.every(e => typeof e === 'string'),
`bad config: ${fieldName} must be a list of strings`);
result = listenOn.map(item => {
const lastColon = item.lastIndexOf(':');
// if address is IPv6 format, it includes brackets
// that have to be removed from the final IP address
const ipAddress = item.indexOf(']') > 0 ?
item.substr(1, lastColon - 2) :
item.substr(0, lastColon);
// the port should not include the colon
const port = item.substr(lastColon + 1);
assert(Number.parseInt(port, 10),
`bad config: ${fieldName} port must be a positive integer`);
return { ip: ipAddress, port };
});
}
return result;
}
_getConfig() { _getConfig() {
let config; let config;
try { try {
@ -738,17 +501,26 @@ class Config extends EventEmitter {
this.port = config.port; this.port = config.port;
} }
this.listenOn = this._parseEndpoints(config.listenOn, 'listenOn'); this.listenOn = [];
if (config.listenOn !== undefined) {
this.metricsPort = 8002; assert(Array.isArray(config.listenOn)
if (config.metricsPort !== undefined) { && config.listenOn.every(e => typeof e === 'string'),
assert(Number.isInteger(config.metricsPort) && config.metricsPort > 0, 'bad config: listenOn must be a list of strings');
'bad config: metricsPort must be a positive integer'); config.listenOn.forEach(item => {
this.metricsPort = config.metricsPort; const lastColon = item.lastIndexOf(':');
// if address is IPv6 format, it includes brackets
// that have to be removed from the final IP address
const ipAddress = item.indexOf(']') > 0 ?
item.substr(1, lastColon - 2) :
item.substr(0, lastColon);
// the port should not include the colon
const port = item.substr(lastColon + 1);
assert(Number.parseInt(port, 10),
'bad config: listenOn port must be a positive integer');
this.listenOn.push({ ip: ipAddress, port });
});
} }
this.metricsListenOn = this._parseEndpoints(config.metricsListenOn, 'metricsListenOn');
if (config.replicationGroupId) { if (config.replicationGroupId) {
assert(typeof config.replicationGroupId === 'string', assert(typeof config.replicationGroupId === 'string',
'bad config: replicationGroupId must be a string'); 'bad config: replicationGroupId must be a string');
@ -762,6 +534,12 @@ class Config extends EventEmitter {
const { replicationEndpoints } = config; const { replicationEndpoints } = config;
assert(replicationEndpoints instanceof Array, 'bad config: ' + assert(replicationEndpoints instanceof Array, 'bad config: ' +
'`replicationEndpoints` property must be an array'); '`replicationEndpoints` property must be an array');
if (replicationEndpoints.length > 1) {
const hasDefault = replicationEndpoints.some(
replicationEndpoint => replicationEndpoint.default);
assert(hasDefault, 'bad config: `replicationEndpoints` must ' +
'contain a default endpoint');
}
replicationEndpoints.forEach(replicationEndpoint => { replicationEndpoints.forEach(replicationEndpoint => {
assert.strictEqual(typeof replicationEndpoint, 'object', assert.strictEqual(typeof replicationEndpoint, 'object',
'bad config: `replicationEndpoints` property must be an ' + 'bad config: `replicationEndpoints` property must be an ' +
@ -779,7 +557,7 @@ class Config extends EventEmitter {
assert(validExternalBackends[type], 'bad config: `type` ' + assert(validExternalBackends[type], 'bad config: `type` ' +
'property of `replicationEndpoints` object must be ' + 'property of `replicationEndpoints` object must be ' +
'a valid external backend (one of: "' + 'a valid external backend (one of: "' +
`${Object.keys(validExternalBackends).join('", "')}")`); `${Object.keys(validExternalBackends).join('", "')})`);
} else { } else {
assert.notStrictEqual(servers, undefined, 'bad config: ' + assert.notStrictEqual(servers, undefined, 'bad config: ' +
'each object of `replicationEndpoints` array that is ' + 'each object of `replicationEndpoints` array that is ' +
@ -841,11 +619,11 @@ class Config extends EventEmitter {
this.websiteEndpoints = config.websiteEndpoints; this.websiteEndpoints = config.websiteEndpoints;
} }
this.workers = false; this.clusters = false;
if (config.workers !== undefined) { if (config.clusters !== undefined) {
assert(Number.isInteger(config.workers) && config.workers > 0, assert(Number.isInteger(config.clusters) && config.clusters > 0,
'bad config: workers must be a positive integer'); 'bad config: clusters must be a positive integer');
this.workers = config.workers; this.clusters = config.clusters;
} }
if (config.usEastBehavior !== undefined) { if (config.usEastBehavior !== undefined) {
@ -1083,7 +861,8 @@ class Config extends EventEmitter {
assert(typeof config.localCache.port === 'number', assert(typeof config.localCache.port === 'number',
'config: bad port for localCache. port must be a number'); 'config: bad port for localCache. port must be a number');
if (config.localCache.password !== undefined) { if (config.localCache.password !== undefined) {
assert(typeof config.localCache.password === 'string', assert(
this._verifyRedisPassword(config.localCache.password),
'config: vad password for localCache. password must' + 'config: vad password for localCache. password must' +
' be a string'); ' be a string');
} }
@ -1109,60 +888,64 @@ class Config extends EventEmitter {
} }
if (config.redis) { if (config.redis) {
this.redis = parseRedisConfig(config.redis); if (config.redis.sentinels) {
this.redis = { sentinels: [], name: null };
assert(typeof config.redis.name === 'string',
'bad config: redis sentinel name must be a string');
this.redis.name = config.redis.name;
assert(Array.isArray(config.redis.sentinels) ||
typeof config.redis.sentinels === 'string',
'bad config: redis sentinels must be an array or string');
if (typeof config.redis.sentinels === 'string') {
config.redis.sentinels.split(',').forEach(item => {
const [host, port] = item.split(':');
this.redis.sentinels.push({ host,
port: Number.parseInt(port, 10) });
});
} else if (Array.isArray(config.redis.sentinels)) {
config.redis.sentinels.forEach(item => {
const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.redis.sentinels.push({ host, port });
});
} }
if (config.scuba) {
this.scuba = {}; if (config.redis.sentinelPassword !== undefined) {
if (config.scuba.host) { assert(
assert(typeof config.scuba.host === 'string', this._verifyRedisPassword(config.redis.sentinelPassword));
'bad config: scuba host must be a string'); this.redis.sentinelPassword = config.redis.sentinelPassword;
this.scuba.host = config.scuba.host;
} }
if (config.scuba.port) { } else {
assert(Number.isInteger(config.scuba.port) // check for standalone configuration
&& config.scuba.port > 0, this.redis = {};
'bad config: scuba port must be a positive integer'); assert(typeof config.redis.host === 'string',
this.scuba.port = config.scuba.port; 'bad config: redis.host must be a string');
assert(typeof config.redis.port === 'number',
'bad config: redis.port must be a number');
this.redis.host = config.redis.host;
this.redis.port = config.redis.port;
}
if (config.redis.password !== undefined) {
assert(
this._verifyRedisPassword(config.redis.password),
'bad config: invalid password for redis. password must ' +
'be a string');
this.redis.password = config.redis.password;
} }
} }
if (process.env.SCUBA_HOST && process.env.SCUBA_PORT) {
assert(typeof process.env.SCUBA_HOST === 'string',
'bad config: scuba host must be a string');
assert(Number.isInteger(Number(process.env.SCUBA_PORT))
&& Number(process.env.SCUBA_PORT) > 0,
'bad config: scuba port must be a positive integer');
this.scuba = {
host: process.env.SCUBA_HOST,
port: Number(process.env.SCUBA_PORT),
};
}
if (this.scuba) {
this.quotaEnabled = true;
}
const maxStaleness = Number(process.env.QUOTA_MAX_STALENESS_MS) ||
config.quota?.maxStatenessMS ||
24 * 60 * 60 * 1000;
assert(Number.isInteger(maxStaleness), 'bad config: maxStalenessMS must be an integer');
const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' ||
config.quota?.enableInflights || false;
this.quota = {
maxStaleness,
enableInflights,
};
if (config.utapi) { if (config.utapi) {
this.utapi = { component: 's3' }; this.utapi = { component: 's3' };
if (config.utapi.host) {
assert(typeof config.utapi.host === 'string',
'bad config: utapi host must be a string');
this.utapi.host = config.utapi.host;
}
if (config.utapi.port) { if (config.utapi.port) {
assert(Number.isInteger(config.utapi.port) assert(Number.isInteger(config.utapi.port)
&& config.utapi.port > 0, && config.utapi.port > 0,
'bad config: utapi port must be a positive integer'); 'bad config: utapi port must be a positive integer');
this.utapi.port = config.utapi.port; this.utapi.port = config.utapi.port;
} }
if (utapiVersion === 1) {
if (config.utapi.workers !== undefined) { if (config.utapi.workers !== undefined) {
assert(Number.isInteger(config.utapi.workers) assert(Number.isInteger(config.utapi.workers)
&& config.utapi.workers > 0, && config.utapi.workers > 0,
@ -1173,21 +956,52 @@ class Config extends EventEmitter {
// config duplication. // config duplication.
assert(config.localCache, 'missing required property of utapi ' + assert(config.localCache, 'missing required property of utapi ' +
'configuration: localCache'); 'configuration: localCache');
this.utapi.localCache = this.localCache; this.utapi.localCache = config.localCache;
assert(config.redis, 'missing required property of utapi ' + assert(config.redis, 'missing required property of utapi ' +
'configuration: redis'); 'configuration: redis');
if (config.utapi.redis) { if (config.utapi.redis) {
this.utapi.redis = parseRedisConfig(config.utapi.redis); if (config.utapi.redis.sentinels) {
if (this.utapi.redis.retry === undefined) { this.utapi.redis = { sentinels: [], name: null };
this.utapi.redis.retry = {
connectBackoff: { assert(typeof config.utapi.redis.name === 'string',
min: 10, 'bad config: redis sentinel name must be a string');
max: 1000, this.utapi.redis.name = config.utapi.redis.name;
jitter: 0.1,
factor: 1.5, assert(Array.isArray(config.utapi.redis.sentinels),
deadline: 10000, 'bad config: redis sentinels must be an array');
}, config.utapi.redis.sentinels.forEach(item => {
}; const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.utapi.redis.sentinels.push({ host, port });
});
} else {
// check for standalone configuration
this.utapi.redis = {};
assert(typeof config.utapi.redis.host === 'string',
'bad config: redis.host must be a string');
assert(typeof config.utapi.redis.port === 'number',
'bad config: redis.port must be a number');
this.utapi.redis.host = config.utapi.redis.host;
this.utapi.redis.port = config.utapi.redis.port;
}
if (config.utapi.redis.password !== undefined) {
assert(
this._verifyRedisPassword(config.utapi.redis.password),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.password = config.utapi.redis.password;
}
if (config.utapi.redis.sentinelPassword !== undefined) {
assert(
this._verifyRedisPassword(
config.utapi.redis.sentinelPassword),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.sentinelPassword =
config.utapi.redis.sentinelPassword;
} }
} }
if (config.utapi.metrics) { if (config.utapi.metrics) {
@ -1258,27 +1072,8 @@ class Config extends EventEmitter {
} }
if (config.utapi && config.utapi.reindex) { if (config.utapi && config.utapi.reindex) {
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex); parseUtapiReindex(config.utapi.reindex);
} this.utapi.reindex = config.utapi.reindex;
}
if (utapiVersion === 2 && config.utapi.filter) {
const { filter: filterConfig } = config.utapi;
const utapiResourceFilters = {};
allowedUtapiEventFilterFields.forEach(
field => allowedUtapiEventFilterStates.forEach(
state => {
const resources = (filterConfig[state] && filterConfig[state][field]) || null;
if (resources) {
assert.strictEqual(utapiResourceFilters[field], undefined,
`bad config: utapi.filter.${state}.${field} can't define an allow and a deny list`);
assert(resources.every(r => typeof r === 'string'),
`bad config: utapi.filter.${state}.${field} must be an array of strings`);
utapiResourceFilters[field] = { [state]: new Set(resources) };
}
}
));
this.utapi.filter = utapiResourceFilters;
} }
} }
if (Object.keys(this.locationConstraints).some( if (Object.keys(this.locationConstraints).some(
@ -1303,8 +1098,6 @@ class Config extends EventEmitter {
} }
} }
this.authdata = config.authdata || 'authdata.json';
this.kms = {}; this.kms = {};
if (config.kms) { if (config.kms) {
assert(typeof config.kms.userName === 'string'); assert(typeof config.kms.userName === 'string');
@ -1524,13 +1317,32 @@ class Config extends EventEmitter {
this.outboundProxy.certs = certObj.certs; this.outboundProxy.certs = certObj.certs;
} }
this.managementAgent = {};
this.managementAgent.port = 8010;
this.managementAgent.host = 'localhost';
if (config.managementAgent !== undefined) {
if (config.managementAgent.port !== undefined) {
assert(Number.isInteger(config.managementAgent.port)
&& config.managementAgent.port > 0,
'bad config: managementAgent port must be a positive ' +
'integer');
this.managementAgent.port = config.managementAgent.port;
}
if (config.managementAgent.host !== undefined) {
assert.strictEqual(typeof config.managementAgent.host, 'string',
'bad config: management agent host must ' +
'be a string');
this.managementAgent.host = config.managementAgent.host;
}
}
// Ephemeral token to protect the reporting endpoint: // Ephemeral token to protect the reporting endpoint:
// try inherited from parent first, then hardcoded in conf file, // try inherited from parent first, then hardcoded in conf file,
// then create a fresh one as last resort. // then create a fresh one as last resort.
this.reportToken = this.reportToken =
process.env.REPORT_TOKEN || process.env.REPORT_TOKEN ||
config.reportToken || config.reportToken ||
uuidv4(); uuid.v4().toString();
// External backends // External backends
// Currently supports configuring httpAgent(s) for keepAlive // Currently supports configuring httpAgent(s) for keepAlive
@ -1576,114 +1388,38 @@ class Config extends EventEmitter {
requestsConfigAssert(config.requests); requestsConfigAssert(config.requests);
this.requests = config.requests; this.requests = config.requests;
} }
// CLDSRV-378: on 8.x branches, null version compatibility
// mode is enforced because null keys are not supported by the
// MongoDB backend.
this.nullVersionCompatMode = true;
if (config.bucketNotificationDestinations) { if (config.bucketNotificationDestinations) {
this.bucketNotificationDestinations = bucketNotifAssert(config.bucketNotificationDestinations); this.bucketNotificationDestinations = bucketNotifAssert(config.bucketNotificationDestinations);
} }
this.lifecycleRoleName = config.lifecycleRoleName || null;
// Version of the configuration we're running under
this.overlayVersion = config.overlayVersion || 0;
this._setTimeOptions();
this.multiObjectDeleteConcurrency = constants.multiObjectDeleteConcurrency;
const extractedNumber = Number.parseInt(config.multiObjectDeleteConcurrency, 10);
if (!isNaN(extractedNumber) && extractedNumber > 0 && extractedNumber < 1000) {
this.multiObjectDeleteConcurrency = extractedNumber;
} }
this.multiObjectDeleteEnableOptimizations = true; _configureBackends() {
if (config.multiObjectDeleteEnableOptimizations === false) {
this.multiObjectDeleteEnableOptimizations = false;
}
this.testingMode = config.testingMode || false;
this.maxScannedLifecycleListingEntries = constants.maxScannedLifecycleListingEntries;
if (config.maxScannedLifecycleListingEntries !== undefined) {
// maxScannedLifecycleListingEntries > 2 is required as a minimum because we must
// scan at least three entries to determine version eligibility.
// Two entries representing the master key and the following one representing the non-current version.
assert(Number.isInteger(config.maxScannedLifecycleListingEntries) &&
config.maxScannedLifecycleListingEntries > 2,
'bad config: maxScannedLifecycleListingEntries must be greater than 2');
this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
}
this._configureBackends(config);
}
_setTimeOptions() {
// NOTE: EXPIRE_ONE_DAY_EARLIER and TRANSITION_ONE_DAY_EARLIER are deprecated in favor of
// TIME_PROGRESSION_FACTOR which decreases the weight attributed to a day in order to among other things
// expedite the lifecycle of objects.
// moves lifecycle expiration deadlines 1 day earlier, mostly for testing
const expireOneDayEarlier = process.env.EXPIRE_ONE_DAY_EARLIER === 'true';
// moves lifecycle transition deadlines 1 day earlier, mostly for testing
const transitionOneDayEarlier = process.env.TRANSITION_ONE_DAY_EARLIER === 'true';
// decreases the weight attributed to a day in order to expedite the lifecycle of objects.
const timeProgressionFactor = Number.parseInt(process.env.TIME_PROGRESSION_FACTOR, 10) || 1;
const isIncompatible = (expireOneDayEarlier || transitionOneDayEarlier) && (timeProgressionFactor > 1);
assert(!isIncompatible, 'The environment variables "EXPIRE_ONE_DAY_EARLIER" or ' +
'"TRANSITION_ONE_DAY_EARLIER" are not compatible with the "TIME_PROGRESSION_FACTOR" variable.');
// The scaledMsPerDay value is initially set to the number of milliseconds per day
// (24 * 60 * 60 * 1000) as the default value.
// However, during testing, if the timeProgressionFactor is defined and greater than 1,
// the scaledMsPerDay value is decreased. This adjustment allows for simulating actions occurring
// earlier in time.
const scaledMsPerDay = scaleMsPerDay(timeProgressionFactor);
this.timeOptions = {
expireOneDayEarlier,
transitionOneDayEarlier,
timeProgressionFactor,
scaledMsPerDay,
};
}
getTimeOptions() {
return this.timeOptions;
}
_getAuthData() {
return JSON.parse(fs.readFileSync(findConfigFile(process.env.S3AUTH_CONFIG || this.authdata), { encoding: 'utf-8' }));
}
_configureBackends(config) {
const backends = config.backends || {};
/** /**
* Configure the backends for Authentication, Data and Metadata. * Configure the backends for Authentication, Data and Metadata.
*/ */
let auth = backends.auth || 'mem'; let auth = 'mem';
let data = backends.data || 'multiple'; let data = 'multiple';
let metadata = backends.metadata || 'file'; let metadata = 'file';
let kms = backends.kms || 'file'; let kms = 'file';
let quota = backends.quota || 'none';
if (process.env.S3BACKEND) { if (process.env.S3BACKEND) {
const validBackends = ['mem', 'file', 'scality', 'cdmi']; const validBackends = ['mem', 'file', 'scality', 'cdmi'];
assert(validBackends.indexOf(process.env.S3BACKEND) > -1, assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
'bad environment variable: S3BACKEND environment variable ' + 'bad environment variable: S3BACKEND environment variable ' +
'should be one of mem/file/scality/cdmi' 'should be one of mem/file/scality/cdmi'
); );
auth = process.env.S3BACKEND == 'scality' ? 'scality' : 'mem'; auth = process.env.S3BACKEND;
data = process.env.S3BACKEND; data = process.env.S3BACKEND;
metadata = process.env.S3BACKEND; metadata = process.env.S3BACKEND;
kms = process.env.S3BACKEND; kms = process.env.S3BACKEND;
} }
if (process.env.S3VAULT) { if (process.env.S3VAULT) {
auth = process.env.S3VAULT; auth = process.env.S3VAULT;
auth = (auth === 'file' || auth === 'mem' || auth === 'cdmi' ? 'mem' : auth);
} }
if (auth === 'file' || auth === 'mem' || auth === 'cdmi') { if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
// Auth only checks for 'mem' since mem === file // Auth only checks for 'mem' since mem === file
auth = 'mem';
const authfile = findConfigFile(process.env.S3AUTH_CONFIG ||
'authdata.json');
let authData; let authData;
if (process.env.SCALITY_ACCESS_KEY_ID && if (process.env.SCALITY_ACCESS_KEY_ID &&
process.env.SCALITY_SECRET_ACCESS_KEY) { process.env.SCALITY_SECRET_ACCESS_KEY) {
@ -1691,20 +1427,13 @@ class Config extends EventEmitter {
process.env.SCALITY_ACCESS_KEY_ID, process.env.SCALITY_ACCESS_KEY_ID,
process.env.SCALITY_SECRET_ACCESS_KEY); process.env.SCALITY_SECRET_ACCESS_KEY);
} else { } else {
authData = this._getAuthData(); authData = require(authfile);
} }
if (validateAuthConfig(authData)) { if (validateAuthConfig(authData)) {
throw new Error('bad config: invalid auth config file.'); throw new Error('bad config: invalid auth config file.');
} }
this.authData = authData; this.authData = authData;
} else if (auth === 'multiple') {
const authData = this._getAuthData();
if (validateAuthConfig(authData)) {
throw new Error('bad config: invalid auth config file.');
} }
this.authData = authData;
}
if (process.env.S3DATA) { if (process.env.S3DATA) {
const validData = ['mem', 'file', 'scality', 'multiple']; const validData = ['mem', 'file', 'scality', 'multiple'];
assert(validData.indexOf(process.env.S3DATA) > -1, assert(validData.indexOf(process.env.S3DATA) > -1,
@ -1712,10 +1441,10 @@ class Config extends EventEmitter {
'should be one of mem/file/scality/multiple' 'should be one of mem/file/scality/multiple'
); );
data = process.env.S3DATA; data = process.env.S3DATA;
}
if (data === 'scality' || data === 'multiple') { if (data === 'scality' || data === 'multiple') {
data = 'multiple'; data = 'multiple';
} }
}
assert(this.locationConstraints !== undefined && assert(this.locationConstraints !== undefined &&
this.restEndpoints !== undefined, this.restEndpoints !== undefined,
'bad config: locationConstraints and restEndpoints must be set' 'bad config: locationConstraints and restEndpoints must be set'
@ -1727,18 +1456,18 @@ class Config extends EventEmitter {
if (process.env.S3KMS) { if (process.env.S3KMS) {
kms = process.env.S3KMS; kms = process.env.S3KMS;
} }
if (process.env.S3QUOTA) {
quota = process.env.S3QUOTA;
}
this.backends = { this.backends = {
auth, auth,
data, data,
metadata, metadata,
kms, kms,
quota,
}; };
} }
_verifyRedisPassword(password) {
return typeof password === 'string';
}
setAuthDataAccounts(accounts) { setAuthDataAccounts(accounts) {
this.authData.accounts = accounts; this.authData.accounts = accounts;
this.emit('authdata-update'); this.emit('authdata-update');
@ -1786,7 +1515,8 @@ class Config extends EventEmitter {
getAzureEndpoint(locationConstraint) { getAzureEndpoint(locationConstraint) {
let azureStorageEndpoint = let azureStorageEndpoint =
process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] || process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] ||
this.locationConstraints[locationConstraint].details.azureStorageEndpoint; this.locationConstraints[locationConstraint]
.details.azureStorageEndpoint;
if (!azureStorageEndpoint.endsWith('/')) { if (!azureStorageEndpoint.endsWith('/')) {
// append the trailing slash // append the trailing slash
azureStorageEndpoint = `${azureStorageEndpoint}/`; azureStorageEndpoint = `${azureStorageEndpoint}/`;
@ -1795,40 +1525,23 @@ class Config extends EventEmitter {
} }
getAzureStorageAccountName(locationConstraint) { getAzureStorageAccountName(locationConstraint) {
const accountName = azureGetStorageAccountName( const { azureStorageAccountName } =
locationConstraint, this.locationConstraints[locationConstraint].details;
this.locationConstraints[locationConstraint].details const storageAccountNameFromEnv =
); process.env[`${locationConstraint}_AZURE_STORAGE_ACCOUNT_NAME`];
if (accountName) { return storageAccountNameFromEnv || azureStorageAccountName;
return accountName;
}
// For SAS & ServicePrincipal, retrieve the accountName from the endpoint
const endpoint = this.getAzureEndpoint(locationConstraint);
const url = new URL(endpoint);
const fragments = url.hostname.split('.', 3);
if (fragments.length === 3 && fragments[1] === 'blob') {
return fragments[0];
}
// We should always reach here, though it may not be the case for "mock" servers,
// where the accoutName is in the path
const path = url.pathname.replace(/^\//, '').replace(/\/$/, '');
if (path) {
return path;
}
// We should not reach here; if that happens, use the endpoint itself, which
// should be close-enough since this function is used for detecting when two
// locations actually point to the same account
return endpoint;
} }
getAzureStorageCredentials(locationConstraint) { getAzureStorageCredentials(locationConstraint) {
return azureGetLocationCredentials( const { azureStorageAccessKey } =
locationConstraint, this.locationConstraints[locationConstraint].details;
this.locationConstraints[locationConstraint].details const storageAccessKeyFromEnv =
); process.env[`${locationConstraint}_AZURE_STORAGE_ACCESS_KEY`];
return {
storageAccountName:
this.getAzureStorageAccountName(locationConstraint),
storageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
};
} }
getPfsDaemonEndpoint(locationConstraint) { getPfsDaemonEndpoint(locationConstraint) {
@ -1847,7 +1560,7 @@ class Config extends EventEmitter {
return azureSrcAccount === azureDestAccount; return azureSrcAccount === azureDestAccount;
} }
isAWSServerSideEncryption(locationConstraint) { isAWSServerSideEncrytion(locationConstraint) {
return this.locationConstraints[locationConstraint].details return this.locationConstraints[locationConstraint].details
.serverSideEncryption === true; .serverSideEncryption === true;
} }
@ -1861,25 +1574,13 @@ class Config extends EventEmitter {
.update(instanceId) .update(instanceId)
.digest('hex'); .digest('hex');
} }
isQuotaEnabled() {
return !!this.quotaEnabled;
}
isQuotaInflightEnabled() {
return this.quota.enableInflights;
}
} }
module.exports = { module.exports = {
parseSproxydConfig, parseSproxydConfig,
parseRedisConfig,
locationConstraintAssert, locationConstraintAssert,
ConfigObject: Config, ConfigObject: Config,
config: new Config(), config: new Config(),
requestsConfigAssert, requestsConfigAssert,
bucketNotifAssert, bucketNotifAssert,
azureGetStorageAccountName,
azureGetLocationCredentials,
azureArchiveLocationConstraintAssert,
}; };

View File

@ -1,14 +1,12 @@
const { auth, errors, policies } = require('arsenal'); const { auth, errors } = require('arsenal');
const async = require('async'); const async = require('async');
const bucketDelete = require('./bucketDelete'); const bucketDelete = require('./bucketDelete');
const bucketDeleteCors = require('./bucketDeleteCors'); const bucketDeleteCors = require('./bucketDeleteCors');
const bucketDeleteEncryption = require('./bucketDeleteEncryption');
const bucketDeleteWebsite = require('./bucketDeleteWebsite'); const bucketDeleteWebsite = require('./bucketDeleteWebsite');
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle'); const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
const bucketDeletePolicy = require('./bucketDeletePolicy'); const bucketDeletePolicy = require('./bucketDeletePolicy');
const bucketDeleteQuota = require('./bucketDeleteQuota'); const bucketGet = require('./bucketGet');
const { bucketGet } = require('./bucketGet');
const bucketGetACL = require('./bucketGetACL'); const bucketGetACL = require('./bucketGetACL');
const bucketGetCors = require('./bucketGetCors'); const bucketGetCors = require('./bucketGetCors');
const bucketGetVersioning = require('./bucketGetVersioning'); const bucketGetVersioning = require('./bucketGetVersioning');
@ -18,24 +16,17 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
const bucketGetNotification = require('./bucketGetNotification'); const bucketGetNotification = require('./bucketGetNotification');
const bucketGetObjectLock = require('./bucketGetObjectLock'); const bucketGetObjectLock = require('./bucketGetObjectLock');
const bucketGetPolicy = require('./bucketGetPolicy'); const bucketGetPolicy = require('./bucketGetPolicy');
const bucketGetQuota = require('./bucketGetQuota');
const bucketGetEncryption = require('./bucketGetEncryption');
const bucketHead = require('./bucketHead'); const bucketHead = require('./bucketHead');
const { bucketPut } = require('./bucketPut'); const { bucketPut } = require('./bucketPut');
const bucketPutACL = require('./bucketPutACL'); const bucketPutACL = require('./bucketPutACL');
const bucketPutCors = require('./bucketPutCors'); const bucketPutCors = require('./bucketPutCors');
const bucketPutVersioning = require('./bucketPutVersioning'); const bucketPutVersioning = require('./bucketPutVersioning');
const bucketPutTagging = require('./bucketPutTagging');
const bucketDeleteTagging = require('./bucketDeleteTagging');
const bucketGetTagging = require('./bucketGetTagging');
const bucketPutWebsite = require('./bucketPutWebsite'); const bucketPutWebsite = require('./bucketPutWebsite');
const bucketPutReplication = require('./bucketPutReplication'); const bucketPutReplication = require('./bucketPutReplication');
const bucketPutLifecycle = require('./bucketPutLifecycle'); const bucketPutLifecycle = require('./bucketPutLifecycle');
const bucketPutNotification = require('./bucketPutNotification'); const bucketPutNotification = require('./bucketPutNotification');
const bucketPutEncryption = require('./bucketPutEncryption');
const bucketPutPolicy = require('./bucketPutPolicy'); const bucketPutPolicy = require('./bucketPutPolicy');
const bucketPutObjectLock = require('./bucketPutObjectLock'); const bucketPutObjectLock = require('./bucketPutObjectLock');
const bucketUpdateQuota = require('./bucketUpdateQuota');
const bucketGetReplication = require('./bucketGetReplication'); const bucketGetReplication = require('./bucketGetReplication');
const bucketDeleteReplication = require('./bucketDeleteReplication'); const bucketDeleteReplication = require('./bucketDeleteReplication');
const corsPreflight = require('./corsPreflight'); const corsPreflight = require('./corsPreflight');
@ -43,11 +34,10 @@ const completeMultipartUpload = require('./completeMultipartUpload');
const initiateMultipartUpload = require('./initiateMultipartUpload'); const initiateMultipartUpload = require('./initiateMultipartUpload');
const listMultipartUploads = require('./listMultipartUploads'); const listMultipartUploads = require('./listMultipartUploads');
const listParts = require('./listParts'); const listParts = require('./listParts');
const metadataSearch = require('./metadataSearch');
const { multiObjectDelete } = require('./multiObjectDelete'); const { multiObjectDelete } = require('./multiObjectDelete');
const multipartDelete = require('./multipartDelete'); const multipartDelete = require('./multipartDelete');
const objectCopy = require('./objectCopy'); const objectCopy = require('./objectCopy');
const { objectDelete } = require('./objectDelete'); const objectDelete = require('./objectDelete');
const objectDeleteTagging = require('./objectDeleteTagging'); const objectDeleteTagging = require('./objectDeleteTagging');
const objectGet = require('./objectGet'); const objectGet = require('./objectGet');
const objectGetACL = require('./objectGetACL'); const objectGetACL = require('./objectGetACL');
@ -67,48 +57,18 @@ const prepareRequestContexts
= require('./apiUtils/authorization/prepareRequestContexts'); = require('./apiUtils/authorization/prepareRequestContexts');
const serviceGet = require('./serviceGet'); const serviceGet = require('./serviceGet');
const vault = require('../auth/vault'); const vault = require('../auth/vault');
const website = require('./website'); const websiteGet = require('./websiteGet');
const websiteHead = require('./websiteHead');
const writeContinue = require('../utilities/writeContinue'); const writeContinue = require('../utilities/writeContinue');
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders'); const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
const parseCopySource = require('./apiUtils/object/parseCopySource'); const parseCopySource = require('./apiUtils/object/parseCopySource');
const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys'); const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
auth.setHandler(vault); auth.setHandler(vault);
/* eslint-disable no-param-reassign */ /* eslint-disable no-param-reassign */
const api = { const api = {
callApiMethod(apiMethod, request, response, log, callback) { callApiMethod(apiMethod, request, response, log, callback) {
// Attach the apiMethod method to the request, so it can used by monitoring in the server
// eslint-disable-next-line no-param-reassign
request.apiMethod = apiMethod;
// Array of end of API callbacks, used to perform some logic
// at the end of an API.
// eslint-disable-next-line no-param-reassign
request.finalizerHooks = [];
const actionLog = monitoringMap[apiMethod];
if (!actionLog &&
apiMethod !== 'websiteGet' &&
apiMethod !== 'websiteHead' &&
apiMethod !== 'corsPreflight') {
log.error('callApiMethod(): No actionLog for this api method', {
apiMethod,
});
}
log.addDefaultFields({
service: 's3',
action: actionLog,
bucketName: request.bucketName,
});
if (request.objectKey) {
log.addDefaultFields({
objectKey: request.objectKey,
});
}
let returnTagCount = true; let returnTagCount = true;
const validationRes = validateQueryAndHeaders(request, log); const validationRes = validateQueryAndHeaders(request, log);
@ -123,7 +83,6 @@ const api = {
// no need to check auth on website or cors preflight requests // no need to check auth on website or cors preflight requests
if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' || if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
apiMethod === 'corsPreflight') { apiMethod === 'corsPreflight') {
request.actionImplicitDenies = false;
return this[apiMethod](request, log, callback); return this[apiMethod](request, log, callback);
} }
@ -136,35 +95,17 @@ const api = {
return process.nextTick(callback, parsingError); return process.nextTick(callback, parsingError);
} }
const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers);
if (httpHeadersSizeError) {
log.debug('http header size limit exceeded', {
error: httpHeadersSizeError,
});
return process.nextTick(callback, httpHeadersSizeError);
}
const requestContexts = prepareRequestContexts(apiMethod, request, const requestContexts = prepareRequestContexts(apiMethod, request,
sourceBucket, sourceObject, sourceVersionId); sourceBucket, sourceObject, sourceVersionId);
// Extract all the _apiMethods and store them in an array
const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : [];
// Attach the names to the current request
// eslint-disable-next-line no-param-reassign
request.apiMethods = apiMethods;
function checkAuthResults(authResults) { function checkAuthResults(authResults) {
let returnTagCount = true; let returnTagCount = true;
const isImplicitDeny = {};
let isOnlyImplicitDeny = true;
if (apiMethod === 'objectGet') { if (apiMethod === 'objectGet') {
// first item checks s3:GetObject(Version) action // first item checks s3:GetObject(Version) action
if (!authResults[0].isAllowed && !authResults[0].isImplicit) { if (!authResults[0].isAllowed) {
log.trace('get object authorization denial from Vault'); log.trace('get object authorization denial from Vault');
return errors.AccessDenied; return errors.AccessDenied;
} }
// TODO add support for returnTagCount in the bucket policy
// checks
isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
// second item checks s3:GetObject(Version)Tagging action // second item checks s3:GetObject(Version)Tagging action
if (!authResults[1].isAllowed) { if (!authResults[1].isAllowed) {
log.trace('get tagging authorization denial ' + log.trace('get tagging authorization denial ' +
@ -173,55 +114,49 @@ const api = {
} }
} else { } else {
for (let i = 0; i < authResults.length; i++) { for (let i = 0; i < authResults.length; i++) {
isImplicitDeny[authResults[i].action] = true; if (!authResults[i].isAllowed) {
if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
// Any explicit deny rejects the current API call
log.trace('authorization denial from Vault'); log.trace('authorization denial from Vault');
return errors.AccessDenied; return errors.AccessDenied;
} }
if (authResults[i].isAllowed) {
// If the action is allowed, the result is not implicit
// Deny.
isImplicitDeny[authResults[i].action] = false;
isOnlyImplicitDeny = false;
} }
} }
} return returnTagCount;
// These two APIs cannot use ACLs or Bucket Policies, hence, any
// implicit deny from vault must be treated as an explicit deny.
if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
return errors.AccessDenied;
}
return { returnTagCount, isImplicitDeny };
} }
return async.waterfall([ return async.waterfall([
next => auth.server.doAuth( next => auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) =>
request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => { next(err, userInfo, authorizationResults, streamingV4Params), 's3', requestContexts),
(userInfo, authorizationResults, streamingV4Params, next) => {
if (authorizationResults) {
const checkedResults = checkAuthResults(authorizationResults);
if (checkedResults instanceof Error) {
return next(checkedResults);
}
returnTagCount = checkedResults;
}
return tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log,
(err, tagAuthResults, updatedContexts) =>
next(err, tagAuthResults, authorizationResults, userInfo, streamingV4Params, updatedContexts));
},
], (err, tagAuthResults, authorizationResults, userInfo, streamingV4Params, updatedContexts) => {
if (err) { if (err) {
// VaultClient returns standard errors, but the route requires
// Arsenal errors
const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
log.trace('authentication error', { error: err }); log.trace('authentication error', { error: err });
return next(arsenalError); return callback(err);
} }
return next(null, userInfo, authorizationResults, streamingV4Params, infos); if (tagAuthResults) {
}, 's3', requestContexts), const checkedResults = checkAuthResults(tagAuthResults);
(userInfo, authorizationResults, streamingV4Params, infos, next) => { if (checkedResults instanceof Error) {
const authNames = { accountName: userInfo.getAccountDisplayName() }; return callback(checkedResults);
if (userInfo.isRequesterAnIAMUser()) {
authNames.userName = userInfo.getIAMdisplayName();
} }
if (isRequesterASessionUser(userInfo)) { returnTagCount = checkedResults;
authNames.sessionName = userInfo.getShortid().split(':')[1];
}
log.addDefaultFields(authNames);
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
} }
// issue 100 Continue to the client // issue 100 Continue to the client
writeContinue(request, response); writeContinue(request, response);
const MAX_POST_LENGTH = request.method === 'POST' ? if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
return this[apiMethod](userInfo, request, streamingV4Params,
log, callback, authorizationResults);
}
const MAX_POST_LENGTH = request.method.toUpperCase() === 'POST' ?
1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB 1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
const post = []; const post = [];
let postLength = 0; let postLength = 0;
@ -231,85 +166,56 @@ const api = {
if (postLength <= MAX_POST_LENGTH) { if (postLength <= MAX_POST_LENGTH) {
post.push(chunk); post.push(chunk);
} }
return undefined;
}); });
request.on('error', err => { request.on('error', err => {
log.trace('error receiving request', { log.trace('error receiving request', {
error: err, error: err,
}); });
return next(errors.InternalError); return callback(errors.InternalError);
}); });
request.on('end', () => { request.on('end', () => {
if (postLength > MAX_POST_LENGTH) { if (postLength > MAX_POST_LENGTH) {
log.error('body length is too long for request type', log.error('body length is too long for request type',
{ postLength }); { postLength });
return next(errors.InvalidRequest); return callback(errors.InvalidRequest);
} }
// Convert array of post buffers into one string // Convert array of post buffers into one string
request.post = Buffer.concat(post, postLength).toString(); request.post = Buffer.concat(post, postLength).toString();
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
}); // IAM policy -Tag condition keys require information from CloudServer for evaluation
return undefined; return tagConditionKeyAuth(authorizationResults, request, (updatedContexts || requestContexts),
}, apiMethod, log, (err, tagAuthResults) => {
// Tag condition keys require information from CloudServer for evaluation
(userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
authorizationResults,
request,
requestContexts,
apiMethod,
log,
(err, authResultsWithTags) => {
if (err) { if (err) {
log.trace('tag authentication error', { error: err }); log.trace('tag authentication error', { error: err });
return next(err);
}
return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
},
),
], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
if (err) {
return callback(err); return callback(err);
} }
request.accountQuotas = infos?.accountQuota; if (tagAuthResults) {
if (authorizationResults) { const checkedResults = checkAuthResults(tagAuthResults);
const checkedResults = checkAuthResults(authorizationResults);
if (checkedResults instanceof Error) { if (checkedResults instanceof Error) {
return callback(checkedResults); return callback(checkedResults);
} }
returnTagCount = checkedResults.returnTagCount; returnTagCount = checkedResults;
request.actionImplicitDenies = checkedResults.isImplicitDeny;
} else {
// create an object of keys apiMethods with all values to false:
// for backward compatibility, all apiMethods are allowed by default
// thus it is explicitly allowed, so implicit deny is false
request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
acc[curr] = false;
return acc;
}, {});
} }
const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5, if (apiMethod === 'objectCopy' ||
(hook, done) => hook(err, done), apiMethod === 'objectPutCopyPart') {
() => callback(err, ...results));
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
request._response = response;
return this[apiMethod](userInfo, request, streamingV4Params,
log, methodCallback, authorizationResults);
}
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
return this[apiMethod](userInfo, request, sourceBucket, return this[apiMethod](userInfo, request, sourceBucket,
sourceObject, sourceVersionId, log, methodCallback); sourceObject, sourceVersionId, log, callback);
} }
if (apiMethod === 'objectGet') { if (apiMethod === 'objectGet') {
return this[apiMethod](userInfo, request, returnTagCount, log, callback); return this[apiMethod](userInfo, request,
returnTagCount, log, callback);
} }
return this[apiMethod](userInfo, request, log, methodCallback); return this[apiMethod](userInfo, request, log, callback);
});
});
return undefined;
}); });
}, },
bucketDelete, bucketDelete,
bucketDeleteCors, bucketDeleteCors,
bucketDeleteEncryption,
bucketDeleteWebsite, bucketDeleteWebsite,
bucketGet, bucketGet,
bucketGetACL, bucketGetACL,
@ -318,38 +224,29 @@ const api = {
bucketGetVersioning, bucketGetVersioning,
bucketGetWebsite, bucketGetWebsite,
bucketGetLocation, bucketGetLocation,
bucketGetEncryption,
bucketHead, bucketHead,
bucketPut, bucketPut,
bucketPutACL, bucketPutACL,
bucketPutCors, bucketPutCors,
bucketPutVersioning, bucketPutVersioning,
bucketPutTagging,
bucketDeleteTagging,
bucketGetTagging,
bucketPutWebsite, bucketPutWebsite,
bucketPutReplication, bucketPutReplication,
bucketGetReplication, bucketGetReplication,
bucketDeleteReplication, bucketDeleteReplication,
bucketDeleteQuota,
bucketPutLifecycle, bucketPutLifecycle,
bucketUpdateQuota,
bucketGetLifecycle, bucketGetLifecycle,
bucketDeleteLifecycle, bucketDeleteLifecycle,
bucketPutPolicy, bucketPutPolicy,
bucketGetPolicy, bucketGetPolicy,
bucketGetQuota,
bucketDeletePolicy, bucketDeletePolicy,
bucketPutObjectLock, bucketPutObjectLock,
bucketPutNotification, bucketPutNotification,
bucketGetNotification, bucketGetNotification,
bucketPutEncryption,
corsPreflight, corsPreflight,
completeMultipartUpload, completeMultipartUpload,
initiateMultipartUpload, initiateMultipartUpload,
listMultipartUploads, listMultipartUploads,
listParts, listParts,
metadataSearch,
multiObjectDelete, multiObjectDelete,
multipartDelete, multipartDelete,
objectDelete, objectDelete,
@ -368,10 +265,10 @@ const api = {
objectPutPart, objectPutPart,
objectPutCopyPart, objectPutCopyPart,
objectPutRetention, objectPutRetention,
objectRestore,
serviceGet, serviceGet,
websiteGet: website, websiteGet,
websiteHead: website, websiteHead,
objectRestore,
}; };
module.exports = api; module.exports = api;

View File

@ -1,29 +0,0 @@
const { errors } = require('arsenal');
const vault = require('../../../auth/vault');
function checkExpectedBucketOwner(headers, bucket, log, cb) {
const expectedOwner = headers['x-amz-expected-bucket-owner'];
if (expectedOwner === undefined) {
return cb();
}
const bucketOwner = bucket.getOwner();
return vault.getAccountIds([bucketOwner], log, (error, res) => {
if (error) {
log.error('error fetch accountId from vault', {
method: 'checkExpectedBucketOwner',
error,
});
}
if (error || res[bucketOwner] !== expectedOwner) {
return cb(errors.AccessDenied);
}
return cb();
});
}
module.exports = {
checkExpectedBucketOwner,
};

View File

@ -1,23 +1,49 @@
const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies; const { evaluators } = require('arsenal').policies;
const { errors } = require('arsenal');
const { parseCIDR, isValid } = require('ipaddr.js');
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { config } = require('../../../Config');
const { const actionMap = {
allAuthedUsersId, multipartDelete: 's3:AbortMultipartUpload',
bucketOwnerActions, bucketDelete: 's3:DeleteBucket',
logId, bucketDeletePolicy: 's3:DeleteBucketPolicy',
publicId, bucketDeleteWebsite: 's3:DeleteBucketWebsite',
arrayOfAllowed, objectDelete: 's3:DeleteObject',
assumedRoleArnResourceType, objectDeleteTagging: 's3:DeleteObjectTagging',
backbeatLifecycleSessionName, bucketGetACL: 's3:GetBucketAcl',
actionsToConsiderAsObjectPut, bucketGetCors: 's3:GetBucketCORS',
} = constants; bucketGetLocation: 's3:GetBucketLocation',
bucketGetObjectLock: 's3:GetBucketObjectLockConfiguration',
bucketGetPolicy: 's3:GetBucketPolicy',
bucketGetVersioning: 's3:GetBucketVersioning',
bucketGetWebsite: 's3:GetBucketWebsite',
bucketGetLifecycle: 's3:GetLifecycleConfiguration',
objectGet: 's3:GetObject',
objectGetACL: 's3:GetObjectAcl',
objectGetLegalHold: 's3:GetObjectLegalHold',
objectGetRetention: 's3:GetObjectRetention',
objectGetTagging: 's3:GetObjectTagging',
bucketGetReplication: 's3:GetReplicationConfiguration',
bucketHead: 's3:ListBucket',
bucketGet: 's3:ListBucket',
listMultipartUploads: 's3:ListBucketMultipartUploads',
listParts: 's3:ListMultipartUploadParts',
bucketPutACL: 's3:PutBucketAcl',
bucketPutCors: 's3:PutBucketCORS',
bucketPutObjectLock: 's3:PutBucketObjectLockConfiguration',
bucketPutPolicy: 's3:PutBucketPolicy',
bucketPutVersioning: 's3:PutBucketVersioning',
bucketPutWebsite: 's3:PutBucketWebsite',
bucketPutLifecycle: 's3:PutLifecycleConfiguration',
objectPut: 's3:PutObject',
objectPutACL: 's3:PutObjectAcl',
objectPutTagging: 's3:PutObjectTagging',
objectPutLegalHold: 's3:PutObjectLegalHold',
objectPutRetention: 's3:PutObjectRetention',
bucketPutReplication: 's3:PutReplicationConfiguration',
};
// whitelist buckets to allow public read on objects // whitelist buckets to allow public read on objects
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ?
? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : []; process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
function getServiceAccountProperties(canonicalID) { function getServiceAccountProperties(canonicalID) {
const canonicalIDArray = canonicalID.split('/'); const canonicalIDArray = canonicalID.split('/');
@ -29,109 +55,51 @@ function isServiceAccount(canonicalID) {
return getServiceAccountProperties(canonicalID) !== undefined; return getServiceAccountProperties(canonicalID) !== undefined;
} }
function isRequesterASessionUser(authInfo) { function checkBucketAcls(bucket, requestType, canonicalID) {
const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
return regexpAssumedRoleArn.test(authInfo.getArn());
}
function isRequesterNonAccountUser(authInfo) {
return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
}
/**
* Checks the access control for a given bucket based on the request type and user's canonical ID.
*
* @param {Bucket} bucket - The bucket to check access control for.
* @param {string} requestType - The list of s3 actions to check within the API call.
* @param {string} canonicalID - The canonical ID of the user making the request.
* @param {string} mainApiCall - The main API call (first item of the requestType).
*
* @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
*/
function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
// Same logic applies on the Versioned APIs, so let's simplify it.
let requestTypeParsed = requestType.endsWith('Version') ?
requestType.slice(0, 'Version'.length * -1) : requestType;
requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
'objectPut' : requestTypeParsed;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
'objectPut' : mainApiCall;
if (bucket.getOwner() === canonicalID) { if (bucket.getOwner() === canonicalID) {
return true; return true;
} }
if (parsedMainApiCall === 'objectGet') {
if (requestTypeParsed === 'objectGetTagging') {
return true;
}
}
if (parsedMainApiCall === 'objectPut') {
if (arrayOfAllowed.includes(requestTypeParsed)) {
return true;
}
}
// any requestType outside of those checked in this function is
// outside the scope of ACL permissions and will be denied unless
// allowed by a different permissions granter
const bucketAcl = bucket.getAcl(); const bucketAcl = bucket.getAcl();
if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') { if (requestType === 'bucketGet' || requestType === 'bucketHead') {
if (bucketAcl.Canned === 'public-read' if (bucketAcl.Canned === 'public-read'
|| bucketAcl.Canned === 'public-read-write' || bucketAcl.Canned === 'public-read-write'
|| (bucketAcl.Canned === 'authenticated-read' || (bucketAcl.Canned === 'authenticated-read'
&& canonicalID !== publicId)) { && canonicalID !== constants.publicId)) {
return true; return true;
} else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ.indexOf(canonicalID) > -1) { || bucketAcl.READ.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.READ.indexOf(publicId) > -1
|| (bucketAcl.READ.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'bucketGetACL') { if (requestType === 'bucketGetACL') {
if ((bucketAcl.Canned === 'log-delivery-write' if ((bucketAcl.Canned === 'log-delivery-write'
&& canonicalID === logId) && canonicalID === constants.logId)
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ_ACP.indexOf(canonicalID) > -1) { || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.READ_ACP.indexOf(publicId) > -1
|| (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'bucketPutACL') { if (requestType === 'bucketPutACL') {
if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) { || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1
|| (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') { if (requestType === 'bucketDelete' && bucket.getOwner() === canonicalID) {
return true;
}
if (requestType === 'objectDelete' || requestType === 'objectPut') {
if (bucketAcl.Canned === 'public-read-write' if (bucketAcl.Canned === 'public-read-write'
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE.indexOf(canonicalID) > -1) { || bucketAcl.WRITE.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.WRITE.indexOf(publicId) > -1
|| (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
// Note that an account can have the ability to do objectPutACL, // Note that an account can have the ability to do objectPutACL,
@ -140,43 +108,29 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
// objectPutACL, objectGetACL, objectHead or objectGet, the bucket // objectPutACL, objectGetACL, objectHead or objectGet, the bucket
// authorization check should just return true so can move on to check // authorization check should just return true so can move on to check
// rights at the object level. // rights at the object level.
return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL' return (requestType === 'objectPutACL' || requestType === 'objectGetACL' ||
|| requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); requestType === 'objectGet' || requestType === 'objectHead');
} }
function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser, function checkObjectAcls(bucket, objectMD, requestType, canonicalID) {
isUserUnauthenticated, mainApiCall) {
const bucketOwner = bucket.getOwner(); const bucketOwner = bucket.getOwner();
const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
'objectPut' : requestType;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
'objectPut' : mainApiCall;
// acls don't distinguish between users and accounts, so both should be allowed // acls don't distinguish between users and accounts, so both should be allowed
if (bucketOwnerActions.includes(requestTypeParsed) if (constants.bucketOwnerActions.includes(requestType)
&& (bucketOwner === canonicalID)) { && (bucketOwner === canonicalID)) {
return true; return true;
} }
if (objectMD['owner-id'] === canonicalID) { if (objectMD['owner-id'] === canonicalID) {
return true; return true;
} }
// Backward compatibility
if (parsedMainApiCall === 'objectGet') {
if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
&& requestTypeParsed === 'objectGetTagging') {
return true;
}
}
if (!objectMD.acl) { if (!objectMD.acl) {
return false; return false;
} }
if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') { if (requestType === 'objectGet' || requestType === 'objectHead') {
if (objectMD.acl.Canned === 'public-read' if (objectMD.acl.Canned === 'public-read'
|| objectMD.acl.Canned === 'public-read-write' || objectMD.acl.Canned === 'public-read-write'
|| (objectMD.acl.Canned === 'authenticated-read' || (objectMD.acl.Canned === 'authenticated-read'
&& canonicalID !== publicId)) { && canonicalID !== constants.publicId)) {
return true; return true;
} else if (objectMD.acl.Canned === 'bucket-owner-read' } else if (objectMD.acl.Canned === 'bucket-owner-read'
&& bucketOwner === canonicalID) { && bucketOwner === canonicalID) {
@ -186,100 +140,54 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ.indexOf(canonicalID) > -1) { || objectMD.acl.READ.indexOf(canonicalID) > -1) {
return true; return true;
} else if (objectMD.acl.READ.indexOf(publicId) > -1
|| (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
// User is already authorized on the bucket for FULL_CONTROL or WRITE or // User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write // bucket has canned ACL public-read-write
if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') { if (requestType === 'objectPut' || requestType === 'objectDelete') {
return true; return true;
} }
if (requestTypeParsed === 'objectPutACL') { if (requestType === 'objectPutACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control' if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID) && bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) { || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1
|| (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'objectGetACL') { if (requestType === 'objectGetACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control' if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID) && bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) { || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1
|| (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
// allow public reads on buckets that are whitelisted for anonymous reads // allow public reads on buckets that are whitelisted for anonymous reads
// TODO: remove this after bucket policies are implemented // TODO: remove this after bucket policies are implemented
const bucketAcl = bucket.getAcl(); const bucketAcl = bucket.getAcl();
const allowPublicReads = publicReadBuckets.includes(bucket.getName()) const allowPublicReads = publicReadBuckets.includes(bucket.getName()) &&
&& bucketAcl.Canned === 'public-read' bucketAcl.Canned === 'public-read' &&
&& (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); (requestType === 'objectGet' || requestType === 'objectHead');
if (allowPublicReads) { if (allowPublicReads) {
return true; return true;
} }
return false; return false;
} }
function _checkBucketPolicyActions(requestType, actions, log) { function _checkActions(requestType, actions, log) {
const mappedAction = actionMaps.actionMapBP[requestType]; const mappedAction = actionMap[requestType];
// Deny any action that isn't in list of controlled actions // if requestType isn't in list of controlled actions
if (!mappedAction) { if (!mappedAction) {
return false; return true;
} }
return evaluators.isActionApplicable(mappedAction, actions, log); return evaluators.isActionApplicable(mappedAction, actions, log);
} }
function _checkBucketPolicyResources(request, resource, log) {
if (!request || (Array.isArray(resource) && resource.length === 0)) {
return true;
}
// build request context from the request!
const requestContext = new RequestContext(request.headers, request.query,
request.bucketName, request.objectKey, null,
request.connection.encrypted, request.resourceType, 's3');
return evaluators.isResourceApplicable(requestContext, resource, log);
}
function _checkBucketPolicyConditions(request, conditions, log) {
const ip = request ? requestUtils.getClientIp(request, config) : undefined;
if (!conditions) {
return true;
}
// build request context from the request!
const requestContext = new RequestContext(request.headers, request.query,
request.bucketName, request.objectKey, ip,
request.connection.encrypted, request.resourceType, 's3', null, null,
null, null, null, null, null, null, null, null, null,
request.objectLockRetentionDays);
return evaluators.meetConditions(requestContext, conditions, log);
}
function _getAccountId(arn) { function _getAccountId(arn) {
// account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc... // account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
return arn.substr(13, 12); return arn.substr(13, 12);
@ -324,26 +232,24 @@ function _checkPrincipals(canonicalID, arn, principal) {
return false; return false;
} }
function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) { function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log) {
let permission = 'defaultDeny'; let permission = 'defaultDeny';
// if requester is user within bucket owner account, actions should be // if requester is user within bucket owner account, actions should be
// allowed unless explicitly denied (assumes allowed by IAM policy) // allowed unless explicitly denied (assumes allowed by IAM policy)
if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) { if (bucketOwner === canonicalID) {
permission = 'allow'; permission = 'allow';
} }
let copiedStatement = JSON.parse(JSON.stringify(policy.Statement)); let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
while (copiedStatement.length > 0) { while (copiedStatement.length > 0) {
const s = copiedStatement[0]; const s = copiedStatement[0];
const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal); const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log); const actionMatch = _checkActions(requestType, s.Action, log);
const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') { if (principalMatch && actionMatch && s.Effect === 'Deny') {
// explicit deny trumps any allows, so return immediately // explicit deny trumps any allows, so return immediately
return 'explicitDeny'; return 'explicitDeny';
} }
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') { if (principalMatch && actionMatch && s.Effect === 'Allow') {
permission = 'allow'; permission = 'allow';
} }
copiedStatement = copiedStatement.splice(1); copiedStatement = copiedStatement.splice(1);
@ -351,141 +257,73 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l
return permission; return permission;
} }
function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log, function isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log) {
request, aclPermission, results, actionImplicitDenies) {
const bucketPolicy = bucket.getBucketPolicy();
let processedResult = results[requestType];
if (!bucketPolicy) {
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
} else {
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
bucketOwner, log, request, actionImplicitDenies);
if (bucketPolicyPermission === 'explicitDeny') {
processedResult = false;
} else if (bucketPolicyPermission === 'allow') {
processedResult = true;
} else {
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
}
}
return processedResult;
}
function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
actionImplicitDeniesInput = {}, isWebsite = false) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const mainApiCall = requestTypes[0];
const results = {};
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
// Check to see if user is authorized to perform a // Check to see if user is authorized to perform a
// particular action on bucket based on ACLs. // particular action on bucket based on ACLs.
// TODO: Add IAM checks // TODO: Add IAM checks
let requesterIsNotUser = true; let requesterIsNotUser = true;
let arn = null; let arn = null;
if (authInfo) { if (authInfo) {
requesterIsNotUser = !isRequesterNonAccountUser(authInfo); requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
arn = authInfo.getArn(); arn = authInfo.getArn();
} }
// if the bucket owner is an account, users should not have default access // if the bucket owner is an account, users should not have default access
if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) { if (((bucket.getOwner() === canonicalID) && requesterIsNotUser)
results[_requestType] = actionImplicitDenies[_requestType] === false; || isServiceAccount(canonicalID)) {
return results[_requestType]; return true;
} }
const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall); const aclPermission = checkBucketAcls(bucket, requestType, canonicalID);
// In case of error bucket access is checked with bucketGet const bucketPolicy = bucket.getBucketPolicy();
// For website, bucket policy only uses objectGet and ignores bucketGet if (!bucketPolicy) {
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html return aclPermission;
// bucketGet should be used to check acl but switched to objectGet for bucket policy
if (isWebsite && _requestType === 'bucketGet') {
// eslint-disable-next-line no-param-reassign
_requestType = 'objectGet';
actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
} }
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log, const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
request, aclPermission, results, actionImplicitDenies); canonicalID, arn, bucket.getOwner(), log);
}); if (bucketPolicyPermission === 'explicitDeny') {
return false;
}
return (aclPermission || (bucketPolicyPermission === 'allow'));
} }
function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {}, function isObjAuthorized(bucket, objectMD, requestType, canonicalID, authInfo, log) {
log, request) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const results = {};
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
let arn = null;
if (authInfo) {
arn = authInfo.getArn();
}
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
request, true, results, actionImplicitDenies);
});
}
function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
actionImplicitDeniesInput = {}, isWebsite = false) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const results = {};
const mainApiCall = requestTypes[0];
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
const parsedMethodName = _requestType.endsWith('Version')
? _requestType.slice(0, -7) : _requestType;
const bucketOwner = bucket.getOwner(); const bucketOwner = bucket.getOwner();
if (!objectMD) { if (!objectMD) {
// check bucket has read access return false;
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
let permission = 'bucketGet';
if (actionsToConsiderAsObjectPut.includes(_requestType)) {
permission = 'objectPut';
}
results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
actionImplicitDenies, isWebsite);
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write
if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
&& results[_requestType] === false) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
}
return results[_requestType];
} }
let requesterIsNotUser = true; let requesterIsNotUser = true;
let arn = null; let arn = null;
let isUserUnauthenticated = false;
if (authInfo) { if (authInfo) {
requesterIsNotUser = !isRequesterNonAccountUser(authInfo); requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
arn = authInfo.getArn(); arn = authInfo.getArn();
isUserUnauthenticated = arn === undefined;
} }
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) { if (objectMD['owner-id'] === canonicalID && requesterIsNotUser) {
results[_requestType] = actionImplicitDenies[_requestType] === false; return true;
return results[_requestType]; }
if (isServiceAccount(canonicalID)) {
return true;
} }
// account is authorized if: // account is authorized if:
// - requesttype is included in bucketOwnerActions and // - requesttype is included in bucketOwnerActions and
// - account is the bucket owner // - account is the bucket owner
// - requester is account, not user // - requester is account, not user
if (bucketOwnerActions.includes(parsedMethodName) if (constants.bucketOwnerActions.includes(requestType)
&& (bucketOwner === canonicalID) && (bucketOwner === canonicalID)
&& requesterIsNotUser) { && requesterIsNotUser) {
results[_requestType] = actionImplicitDenies[_requestType] === false; return true;
return results[_requestType];
} }
const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName, const aclPermission = checkObjectAcls(bucket, objectMD, requestType,
canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall); canonicalID);
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner, const bucketPolicy = bucket.getBucketPolicy();
log, request, aclPermission, results, actionImplicitDenies); if (!bucketPolicy) {
}); return aclPermission;
}
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
canonicalID, arn, bucket.getOwner(), log);
if (bucketPolicyPermission === 'explicitDeny') {
return false;
}
return (aclPermission || (bucketPolicyPermission === 'allow'));
} }
function _checkResource(resource, bucketArn) { function _checkResource(resource, bucketArn) {
@ -514,128 +352,12 @@ function validatePolicyResource(bucketName, policy) {
}); });
} }
function checkIp(value) {
const errString = 'Invalid IP address in Conditions';
const values = Array.isArray(value) ? value : [value];
for (let i = 0; i < values.length; i++) {
// these preliminary checks are validating the provided
// ip address against ipaddr.js, the library we use when
// evaluating IP condition keys. It ensures compatibility,
// but additional checks are required to enforce the right
// notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
// we would accept different ip formats, which is not
// standard in an AWS use case.
try {
try {
parseCIDR(values[i]);
} catch (err) {
isValid(values[i]);
}
} catch (err) {
return errString;
}
// Apply the existing IP validation logic to each element
const validateIpRegex = ip => {
if (constants.ipv4Regex.test(ip)) {
return ip.split('.').every(part => parseInt(part, 10) <= 255);
}
if (constants.ipv6Regex.test(ip)) {
return ip.split(':').every(part => part.length <= 4);
}
return false;
};
if (validateIpRegex(values[i]) !== true) {
return errString;
}
}
// If the function hasn't returned by now, all elements are valid
return null;
}
// This function checks all bucket policy conditions if the values provided
// are valid for the condition type. If not it returns a relevant Malformed policy error string
function validatePolicyConditions(policy) {
const validConditions = [
{ conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
{ conditionKey: 's3:object-lock-remaining-retention-days' },
];
// keys where value type does not seem to be checked by AWS:
// - s3:object-lock-remaining-retention-days
if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
return null;
}
// there can be multiple statements in the policy, each with a Condition enclosure
for (let i = 0; i < policy.Statement.length; i++) {
const s = policy.Statement[i];
if (s.Condition) {
const conditionOperators = Object.keys(s.Condition);
// there can be multiple condition operations in the Condition enclosure
// eslint-disable-next-line no-restricted-syntax
for (const conditionOperator of conditionOperators) {
const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
const conditionValue = s.Condition[conditionOperator][conditionKey];
const validCondition = validConditions.find(validCondition =>
validCondition.conditionKey === conditionKey
);
// AWS returns does not return an error if the condition starts with 'aws:'
// so we reproduce this behaviour
if (!validCondition && !conditionKey.startsWith('aws:')) {
return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
}
if (validCondition && validCondition.conditionValueTypeChecker) {
const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
if (conditionValueTypeError) {
return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
}
}
}
}
}
return null;
}
/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
* @param {string} arn - Amazon resource name - example:
* arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
* @return {boolean} true if Lifecycle assumed role session arn, false if not.
*/
function isLifecycleSession(arn) {
if (!arn) {
return false;
}
const arnSplits = arn.split(':');
const service = arnSplits[2];
const resourceNames = arnSplits[arnSplits.length - 1].split('/');
const resourceType = resourceNames[0];
const sessionName = resourceNames[resourceNames.length - 1];
return (service === 'sts'
&& resourceType === assumedRoleArnResourceType
&& sessionName === backbeatLifecycleSessionName);
}
module.exports = { module.exports = {
isBucketAuthorized, isBucketAuthorized,
isObjAuthorized, isObjAuthorized,
getServiceAccountProperties, getServiceAccountProperties,
isServiceAccount, isServiceAccount,
isRequesterASessionUser,
isRequesterNonAccountUser,
checkBucketAcls, checkBucketAcls,
checkObjectAcls, checkObjectAcls,
validatePolicyResource, validatePolicyResource,
validatePolicyConditions,
isLifecycleSession,
evaluateBucketPolicyWithIAM,
}; };

View File

@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
apiMethod, 's3'); apiMethod, 's3');
} }
if (apiMethod === 'bucketPut') { if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
return null; return null;
} }
@ -65,17 +65,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
const requestContexts = []; const requestContexts = [];
if (apiMethod === 'multiObjectDelete') { if (apiMethodAfterVersionCheck === 'objectCopy'
// MultiObjectDelete does not require any authorization when evaluating
// the API. Instead, we authorize each object passed.
// But in order to get any relevant information from the authorization service
// for example, the account quota, we must send a request context object
// with no `specificResource`. We expect the result to be an implicit deny.
// In the API, we then ignore these authorization results, and we can use
// any information returned, e.g., the quota.
const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
requestContexts.push(requestContextMultiObjectDelete);
} else if (apiMethodAfterVersionCheck === 'objectCopy'
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') { || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
const objectGetAction = sourceVersionId ? 'objectGetVersion' : const objectGetAction = sourceVersionId ? 'objectGetVersion' :
'objectGet'; 'objectGet';
@ -107,63 +97,12 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
const objectGetTaggingAction = (request.query && const objectGetTaggingAction = (request.query &&
request.query.versionId) ? 'objectGetTaggingVersion' : request.query.versionId) ? 'objectGetTaggingVersion' :
'objectGetTagging'; 'objectGetTagging';
if (request.headers['x-amz-version-id']) {
const objectGetVersionAction = 'objectGetVersion';
const getVersionResourceVersion =
generateRequestContext(objectGetVersionAction);
requestContexts.push(getVersionResourceVersion);
}
const getRequestContext = const getRequestContext =
generateRequestContext(apiMethodAfterVersionCheck); generateRequestContext(apiMethodAfterVersionCheck);
const getTaggingRequestContext = const getTaggingRequestContext =
generateRequestContext(objectGetTaggingAction); generateRequestContext(objectGetTaggingAction);
requestContexts.push(getRequestContext, getTaggingRequestContext); requestContexts.push(getRequestContext, getTaggingRequestContext);
} else if (apiMethodAfterVersionCheck === 'objectGetTagging') {
const objectGetTaggingAction = 'objectGetTagging';
const getTaggingResourceVersion =
generateRequestContext(objectGetTaggingAction);
requestContexts.push(getTaggingResourceVersion);
if (request.headers['x-amz-version-id']) {
const objectGetTaggingVersionAction = 'objectGetTaggingVersion';
const getTaggingVersionResourceVersion =
generateRequestContext(objectGetTaggingVersionAction);
requestContexts.push(getTaggingVersionResourceVersion);
}
} else if (apiMethodAfterVersionCheck === 'objectHead') {
const objectHeadAction = 'objectHead';
const headObjectAction =
generateRequestContext(objectHeadAction);
requestContexts.push(headObjectAction);
if (request.headers['x-amz-version-id']) {
const objectHeadVersionAction = 'objectGetVersion';
const headObjectVersion =
generateRequestContext(objectHeadVersionAction);
requestContexts.push(headObjectVersion);
}
} else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
const putObjectTaggingRequestContext =
generateRequestContext('objectPutTagging');
requestContexts.push(putObjectTaggingRequestContext);
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
} else if (apiMethodAfterVersionCheck === 'objectPutCopyPart') {
const putObjectRequestContext =
generateRequestContext('objectPut');
requestContexts.push(putObjectRequestContext);
const getObjectRequestContext =
generateRequestContext('objectGet');
requestContexts.push(getObjectRequestContext);
} else if (apiMethodAfterVersionCheck === 'objectPut') { } else if (apiMethodAfterVersionCheck === 'objectPut') {
// if put object with version
if (request.headers['x-scal-s3-version-id'] ||
request.headers['x-scal-s3-version-id'] === '') {
const putVersionRequestContext =
generateRequestContext('objectPutVersion');
requestContexts.push(putVersionRequestContext);
} else {
const putRequestContext = const putRequestContext =
generateRequestContext(apiMethodAfterVersionCheck); generateRequestContext(apiMethodAfterVersionCheck);
requestContexts.push(putRequestContext); requestContexts.push(putRequestContext);
@ -173,60 +112,12 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
generateRequestContext('objectPutTagging'); generateRequestContext('objectPutTagging');
requestContexts.push(putTaggingRequestContext); requestContexts.push(putTaggingRequestContext);
} }
if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
const putLegalHoldStatusAction =
generateRequestContext('objectPutLegalHold');
requestContexts.push(putLegalHoldStatusAction);
}
// if put object (versioning) with ACL // if put object (versioning) with ACL
if (isHeaderAcl(request.headers)) { if (isHeaderAcl(request.headers)) {
const putAclRequestContext = const putAclRequestContext =
generateRequestContext('objectPutACL'); generateRequestContext('objectPutACL');
requestContexts.push(putAclRequestContext); requestContexts.push(putAclRequestContext);
} }
if (request.headers['x-amz-object-lock-mode']) {
const putObjectLockRequestContext =
generateRequestContext('objectPutRetention');
requestContexts.push(putObjectLockRequestContext);
}
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
}
} else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
apiMethodAfterVersionCheck === 'objectPutPart' ||
apiMethodAfterVersionCheck === 'completeMultipartUpload'
) {
if (request.headers['x-scal-s3-version-id'] ||
request.headers['x-scal-s3-version-id'] === '') {
const putVersionRequestContext =
generateRequestContext('objectPutVersion');
requestContexts.push(putVersionRequestContext);
} else {
const putRequestContext =
generateRequestContext(apiMethodAfterVersionCheck);
requestContexts.push(putRequestContext);
}
// if put object (versioning) with ACL
if (isHeaderAcl(request.headers)) {
const putAclRequestContext =
generateRequestContext('objectPutACL');
requestContexts.push(putAclRequestContext);
}
if (request.headers['x-amz-object-lock-mode']) {
const putObjectLockRequestContext =
generateRequestContext('objectPutRetention');
requestContexts.push(putObjectLockRequestContext);
}
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
} else { } else {
const requestContext = const requestContext =
generateRequestContext(apiMethodAfterVersionCheck); generateRequestContext(apiMethodAfterVersionCheck);

View File

@ -1,5 +1,5 @@
const assert = require('assert');
const async = require('async'); const async = require('async');
const { auth, s3middleware } = require('arsenal'); const { auth, s3middleware } = require('arsenal');
const metadata = require('../../../metadata/wrapper'); const metadata = require('../../../metadata/wrapper');
const { decodeVersionId } = require('../object/versioning'); const { decodeVersionId } = require('../object/versioning');
@ -12,24 +12,29 @@ function makeTagQuery(tags) {
.join('&'); .join('&');
} }
function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) { function updateRequestContexts(request, requestContexts, apiMethod, log, cb) {
async.waterfall([ requestContexts.forEach(rc => {
rc.setNeedTagEval(true);
async.series([
next => { next => {
if (request.headers['x-amz-tagging']) { if (request.headers['x-amz-tagging']) {
return next(null, request.headers['x-amz-tagging']); rc.setRequestObjTags(request.headers['x-amz-tagging']);
} process.nextTick(() => next());
if (request.post && apiMethod === 'objectPutTagging') { } else if (request.post && apiMethod === 'objectPutTagging') {
return parseTagXml(request.post, log, (err, tags) => { parseTagXml(request.post, log, (err, tags) => {
if (err) { if (err) {
log.trace('error parsing request tags'); log.trace('error parsing request tags');
return next(err); return next(err);
} }
return next(null, makeTagQuery(tags)); rc.setRequestObjTags(makeTagQuery(tags));
return next();
}); });
} else {
process.nextTick(() => next());
} }
return next(null, null);
}, },
(requestTagsQuery, next) => { next => {
const objectKey = request.objectKey; const objectKey = request.objectKey;
const bucketName = request.bucketName; const bucketName = request.bucketName;
const decodedVidResult = decodeVersionId(request.query); const decodedVidResult = decodeVersionId(request.query);
@ -38,40 +43,32 @@ function updateRequestContextsWithTags(request, requestContexts, apiMethod, log,
versionId: request.query.versionId, versionId: request.query.versionId,
error: decodedVidResult, error: decodedVidResult,
}); });
return next(decodedVidResult); return process.nextTick(() => next(decodedVidResult));
} }
const reqVersionId = decodedVidResult; const reqVersionId = decodedVidResult;
return metadata.getObjectMD( return metadata.getObjectMD(bucketName, objectKey, { versionId: reqVersionId }, log,
bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => { (err, objMD) => {
if (err) { if (err) {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err.NoSuchKey) { if (err.NoSuchKey) {
return next(null, requestTagsQuery, null); return next();
} }
log.trace('error getting request object tags'); log.trace('error getting request object tags');
return next(err); return next(err);
} }
const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags); const existingTags = objMD.tags;
return next(null, requestTagsQuery, existingTagsQuery); if (existingTags) {
rc.setExistingObjTag(makeTagQuery(existingTags));
}
return next();
}); });
}, },
], (err, requestTagsQuery, existingTagsQuery) => { ], err => {
if (err) { if (err) {
log.trace('error processing tag condition key evaluation'); log.trace('error processing tag condition key evaluation');
return cb(err); return cb(err);
} }
// FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter return cb(null, requestContexts);
// eslint-disable-next-line no-restricted-syntax });
for (const rc of requestContexts) {
rc.setNeedTagEval(true);
if (requestTagsQuery) {
rc.setRequestObjTags(requestTagsQuery);
}
if (existingTagsQuery) {
rc.setExistingObjTag(existingTagsQuery);
}
}
return cb();
}); });
} }
@ -80,20 +77,23 @@ function tagConditionKeyAuth(authorizationResults, request, requestContexts, api
return cb(); return cb();
} }
if (!authorizationResults.some(authRes => authRes.checkTagConditions)) { if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
return cb(null, authorizationResults); return cb();
} }
return updateRequestContextsWithTags(request, requestContexts, apiMethod, log, err => { return updateRequestContexts(request, requestContexts, apiMethod, log, (err, updatedContexts) => {
if (err) { if (err) {
return cb(err); return cb(err);
} }
if (assert.deepStrictEqual(requestContexts, updatedContexts)) {
return cb();
}
return auth.server.doAuth(request, log, return auth.server.doAuth(request, log,
(err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts); (err, userInfo, tagAuthResults) => cb(err, tagAuthResults), 's3', updatedContexts);
}); });
} }
module.exports = { module.exports = {
tagConditionKeyAuth, tagConditionKeyAuth,
updateRequestContextsWithTags, updateRequestContexts,
makeTagQuery, makeTagQuery,
}; };

View File

@ -6,7 +6,6 @@ const acl = require('../../../metadata/acl');
const BucketInfo = require('arsenal').models.BucketInfo; const BucketInfo = require('arsenal').models.BucketInfo;
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const createKeyForUserBucket = require('./createKeyForUserBucket'); const createKeyForUserBucket = require('./createKeyForUserBucket');
const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
const metadata = require('../../../metadata/wrapper'); const metadata = require('../../../metadata/wrapper');
const kms = require('../../../kms/wrapper'); const kms = require('../../../kms/wrapper');
const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior'); const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
@ -24,7 +23,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
// Get new format usersBucket to see if it exists // Get new format usersBucket to see if it exists
return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => { return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => {
if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) { if (err && !err.NoSuchBucket && !err.BucketAlreadyExists) {
return cb(err); return cb(err);
} }
const splitter = usersBucketAttrs ? const splitter = usersBucketAttrs ?
@ -41,7 +40,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
usersBucket : oldUsersBucket; usersBucket : oldUsersBucket;
return metadata.putObjectMD(usersBucketBeingCalled, key, return metadata.putObjectMD(usersBucketBeingCalled, key,
omVal, {}, log, err => { omVal, {}, log, err => {
if (err?.is?.NoSuchBucket) { if (err && err.NoSuchBucket) {
// There must be no usersBucket so createBucket // There must be no usersBucket so createBucket
// one using the new format // one using the new format
log.trace('users bucket does not exist, ' + log.trace('users bucket does not exist, ' +
@ -61,8 +60,9 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
// from getting a BucketAlreadyExists // from getting a BucketAlreadyExists
// error with respect // error with respect
// to the usersBucket. // to the usersBucket.
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err &&
if (err && !err.BucketAlreadyExists) { err !==
errors.BucketAlreadyExists) {
log.error('error from metadata', { log.error('error from metadata', {
error: err, error: err,
}); });
@ -223,7 +223,6 @@ function createBucket(authInfo, bucketName, headers,
}, },
getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) { getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
metadata.getBucket(bucketName, log, (err, data) => { metadata.getBucket(bucketName, log, (err, data) => {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err && err.NoSuchBucket) { if (err && err.NoSuchBucket) {
return callback(null, 'NoBucketYet'); return callback(null, 'NoBucketYet');
} }
@ -248,9 +247,8 @@ function createBucket(authInfo, bucketName, headers,
} }
const newBucketMD = results.prepareNewBucketMD; const newBucketMD = results.prepareNewBucketMD;
if (existingBucketMD === 'NoBucketYet') { if (existingBucketMD === 'NoBucketYet') {
const sseConfig = parseBucketEncryptionHeaders(headers);
return bucketLevelServerSideEncryption( return bucketLevelServerSideEncryption(
bucketName, sseConfig, log, bucketName, headers, log,
(err, sseInfo) => { (err, sseInfo) => {
if (err) { if (err) {
return cb(err); return cb(err);

View File

@ -3,7 +3,6 @@ const async = require('async');
const { errors } = require('arsenal'); const { errors } = require('arsenal');
const abortMultipartUpload = require('../object/abortMultipartUpload'); const abortMultipartUpload = require('../object/abortMultipartUpload');
const { pushMetric } = require('../../../utapi/utilities');
const { splitter, oldSplitter, mpuBucketPrefix } = const { splitter, oldSplitter, mpuBucketPrefix } =
require('../../../../constants'); require('../../../../constants');
@ -16,7 +15,6 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
`${mpuBucketPrefix}${destinationBucketName}`; `${mpuBucketPrefix}${destinationBucketName}`;
return metadata.deleteBucket(mpuBucketName, log, err => { return metadata.deleteBucket(mpuBucketName, log, err => {
// If the mpu bucket does not exist, just move on // If the mpu bucket does not exist, just move on
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err && err.NoSuchBucket) { if (err && err.NoSuchBucket) {
return cb(); return cb();
} }
@ -24,23 +22,14 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
}); });
} }
function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) { function _deleteOngoingMPUs(authInfo, bucketName, mpus, log, cb) {
async.mapLimit(mpus, 1, (mpu, next) => { async.mapLimit(mpus, 1, (mpu, next) => {
const splitterChar = mpu.key.includes(oldSplitter) ? const splitterChar = mpu.key.includes(oldSplitter) ?
oldSplitter : splitter; oldSplitter : splitter;
// `overview${splitter}${objectKey}${splitter}${uploadId} // `overview${splitter}${objectKey}${splitter}${uploadId}
const [, objectKey, uploadId] = mpu.key.split(splitterChar); const [, objectKey, uploadId] = mpu.key.split(splitterChar);
abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log, abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
(err, destBucket, partSizeSum) => { next);
pushMetric('abortMultipartUpload', log, {
authInfo,
canonicalID: bucketMD.getOwner(),
bucket: bucketName,
keys: [objectKey],
byteLength: partSizeSum,
});
next(err);
}, request);
}, cb); }, cb);
} }
/** /**
@ -49,13 +38,11 @@ function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log,
* @param {object} bucketMD - bucket attributes/metadata * @param {object} bucketMD - bucket attributes/metadata
* @param {string} bucketName - bucket in which objectMetadata is stored * @param {string} bucketName - bucket in which objectMetadata is stored
* @param {string} canonicalID - account canonicalID of requester * @param {string} canonicalID - account canonicalID of requester
* @param {object} request - request object given by router
* including normalized headers
* @param {object} log - Werelogs logger * @param {object} log - Werelogs logger
* @param {function} cb - callback from async.waterfall in bucketDelete * @param {function} cb - callback from async.waterfall in bucketDelete
* @return {undefined} * @return {undefined}
*/ */
function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, cb) { function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, log, cb) {
log.trace('deleting bucket from metadata'); log.trace('deleting bucket from metadata');
assert.strictEqual(typeof bucketName, 'string'); assert.strictEqual(typeof bucketName, 'string');
assert.strictEqual(typeof canonicalID, 'string'); assert.strictEqual(typeof canonicalID, 'string');
@ -93,7 +80,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
log, (err, objectsListRes) => { log, (err, objectsListRes) => {
// If no shadow bucket ever created, no ongoing MPU's, so // If no shadow bucket ever created, no ongoing MPU's, so
// continue with deletion // continue with deletion
if (err?.is.NoSuchBucket) { if (err && err.NoSuchBucket) {
return next(); return next();
} }
if (err) { if (err) {
@ -102,7 +89,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
} }
if (objectsListRes.Contents.length) { if (objectsListRes.Contents.length) {
return _deleteOngoingMPUs(authInfo, bucketName, return _deleteOngoingMPUs(authInfo, bucketName,
bucketMD, objectsListRes.Contents, request, log, err => { objectsListRes.Contents, log, err => {
if (err) { if (err) {
return next(err); return next(err);
} }

View File

@ -1,255 +0,0 @@
const { errors } = require('arsenal');
const metadata = require('../../../metadata/wrapper');
const kms = require('../../../kms/wrapper');
const { parseString } = require('xml2js');
/**
* ServerSideEncryptionInfo - user configuration for server side encryption
* @typedef {Object} ServerSideEncryptionInfo
* @property {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
* @property {string} masterKeyId - Key id for the kms key used to encrypt data keys.
* @property {string} configuredMasterKeyId - User configured master key id.
* @property {boolean} mandatory - Whether a default encryption policy has been enabled.
*/
/**
* @callback ServerSideEncryptionInfo~callback
* @param {Object} error - Instance of Arsenal error
* @param {ServerSideEncryptionInfo} - SSE configuration
*/
/**
* parseEncryptionXml - Parses and validates a ServerSideEncryptionConfiguration xml document
* @param {object} xml - ServerSideEncryptionConfiguration doc
* @param {object} log - logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function parseEncryptionXml(xml, log, cb) {
return parseString(xml, (err, parsed) => {
if (err) {
log.trace('xml parsing failed', {
error: err,
method: 'parseEncryptionXml',
});
log.debug('invalid xml', { xml });
return cb(errors.MalformedXML);
}
if (!parsed
|| !parsed.ServerSideEncryptionConfiguration
|| !parsed.ServerSideEncryptionConfiguration.Rule) {
log.trace('error in sse config, invalid ServerSideEncryptionConfiguration section', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const { Rule } = parsed.ServerSideEncryptionConfiguration;
if (!Array.isArray(Rule)
|| Rule.length > 1
|| !Rule[0]
|| !Rule[0].ApplyServerSideEncryptionByDefault
|| !Rule[0].ApplyServerSideEncryptionByDefault[0]) {
log.trace('error in sse config, invalid ApplyServerSideEncryptionByDefault section', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const [encConfig] = Rule[0].ApplyServerSideEncryptionByDefault;
if (!encConfig.SSEAlgorithm || !encConfig.SSEAlgorithm[0]) {
log.trace('error in sse config, no SSEAlgorithm provided', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const [algorithm] = encConfig.SSEAlgorithm;
if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
log.trace('error in sse config, unknown SSEAlgorithm', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const result = { algorithm, mandatory: true };
if (encConfig.KMSMasterKeyID) {
if (algorithm === 'AES256') {
log.trace('error in sse config, can not specify KMSMasterKeyID when using AES256', {
method: 'parseEncryptionXml',
});
return cb(errors.InvalidArgument.customizeDescription(
'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'));
}
if (!encConfig.KMSMasterKeyID[0] || typeof encConfig.KMSMasterKeyID[0] !== 'string') {
log.trace('error in sse config, invalid KMSMasterKeyID', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
result.configuredMasterKeyId = encConfig.KMSMasterKeyID[0];
}
return cb(null, result);
});
}
/**
* hydrateEncryptionConfig - Constructs a ServerSideEncryptionInfo object from arguments
* ensuring no invalid or undefined keys are added
*
* @param {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
* @param {string} configuredMasterKeyId - User configured master key id.
* @param {boolean} [mandatory] - Whether a default encryption policy has been enabled.
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function hydrateEncryptionConfig(algorithm, configuredMasterKeyId, mandatory = null) {
if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
return {
algorithm: null,
};
}
const sseConfig = { algorithm, mandatory };
if (algorithm === 'aws:kms' && configuredMasterKeyId) {
sseConfig.configuredMasterKeyId = configuredMasterKeyId;
}
if (mandatory !== null) {
sseConfig.mandatory = mandatory;
}
return sseConfig;
}
/**
* parseBucketEncryptionHeaders - retrieves bucket level sse configuration from request headers
* @param {object} headers - Request headers
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function parseBucketEncryptionHeaders(headers) {
const sseAlgorithm = headers['x-amz-scal-server-side-encryption'];
const configuredMasterKeyId = headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] || null;
return hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId, true);
}
/**
* parseObjectEncryptionHeaders - retrieves bucket level sse configuration from request headers
* @param {object} headers - Request headers
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function parseObjectEncryptionHeaders(headers) {
const sseAlgorithm = headers['x-amz-server-side-encryption'];
const configuredMasterKeyId = headers['x-amz-server-side-encryption-aws-kms-key-id'] || null;
if (sseAlgorithm && sseAlgorithm !== 'AES256' && sseAlgorithm !== 'aws:kms') {
return {
error: errors.InvalidArgument.customizeDescription('The encryption method specified is not supported'),
};
}
if (sseAlgorithm !== 'aws:kms' && configuredMasterKeyId) {
return {
error: errors.InvalidArgument.customizeDescription(
'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'),
};
}
return { objectSSE: hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId) };
}
/**
* createDefaultBucketEncryptionMetadata - Creates master key and sets up default server side encryption configuration
* @param {BucketInfo} bucket - bucket metadata
* @param {object} log - werelogs logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function createDefaultBucketEncryptionMetadata(bucket, log, cb) {
return kms.bucketLevelEncryption(
bucket.getName(),
{ algorithm: 'AES256', mandatory: false },
log,
(error, sseConfig) => {
if (error) {
return cb(error);
}
bucket.setServerSideEncryption(sseConfig);
return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig));
});
}
/**
*
* @param {object} headers - request headers
* @param {BucketInfo} bucket - BucketInfo model
* @param {*} log - werelogs logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function getObjectSSEConfiguration(headers, bucket, log, cb) {
const bucketSSE = bucket.getServerSideEncryption();
const { error, objectSSE } = parseObjectEncryptionHeaders(headers);
if (error) {
return cb(error);
}
// If a per object sse algo has been passed through
// x-amz-server-side-encryption
if (objectSSE.algorithm) {
// If aws:kms and a custom key id
// pass it through without updating the bucket md
if (objectSSE.algorithm === 'aws:kms' && objectSSE.configuredMasterKeyId) {
return cb(null, objectSSE);
}
// If the client has not specified a key id,
// and we have a default config, then we reuse
// it and pass it through
if (!objectSSE.configuredMasterKeyId && bucketSSE) {
// The default configs algo is overridden with the one passed in the
// request headers. Our implementations of AES256 and aws:kms are the
// same underneath so this is only cosmetic change.
const sseConfig = Object.assign({}, bucketSSE, { algorithm: objectSSE.algorithm });
return cb(null, sseConfig);
}
// If the client has not specified a key id, and we
// don't have a default config, generate it
if (!objectSSE.configuredMasterKeyId && !bucketSSE) {
return createDefaultBucketEncryptionMetadata(bucket, log, (error, sseConfig) => {
if (error) {
return cb(error);
}
// Override the algorithm, for the same reasons as above.
Object.assign(sseConfig, { algorithm: objectSSE.algorithm });
return cb(null, sseConfig);
});
}
}
// If the bucket has a default encryption config, and it is mandatory
// (created with putBucketEncryption or legacy headers)
// pass it through
if (bucketSSE && bucketSSE.mandatory) {
return cb(null, bucketSSE);
}
// No encryption config
return cb(null, null);
}
module.exports = {
createDefaultBucketEncryptionMetadata,
getObjectSSEConfiguration,
hydrateEncryptionConfig,
parseEncryptionXml,
parseBucketEncryptionHeaders,
parseObjectEncryptionHeaders,
};

View File

@ -30,9 +30,6 @@ function bucketShield(bucket, requestType) {
// Otherwise return an error to the client // Otherwise return an error to the client
if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) && if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
(requestType !== 'objectPut' && (requestType !== 'objectPut' &&
requestType !== 'initiateMultipartUpload' &&
requestType !== 'objectPutPart' &&
requestType !== 'completeMultipartUpload' &&
requestType !== 'bucketPutACL' && requestType !== 'bucketPutACL' &&
requestType !== 'bucketDelete')) { requestType !== 'bucketDelete')) {
return true; return true;

View File

@ -11,16 +11,15 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => { metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
// If the object representing the bucket is not in the // If the object representing the bucket is not in the
// users bucket just continue // users bucket just continue
if (error?.is.NoSuchKey) { if (error && error.NoSuchKey) {
return cb(null); return cb(null);
// BACKWARDS COMPATIBILITY: Remove this once no longer // BACKWARDS COMPATIBILITY: Remove this once no longer
// have old user bucket format // have old user bucket format
} else if (error?.is.NoSuchBucket) { } else if (error && error.NoSuchBucket) {
const keyForUserBucket2 = createKeyForUserBucket(canonicalID, const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
oldSplitter, bucketName); oldSplitter, bucketName);
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2, return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
{}, log, error => { {}, log, error => {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (error && !error.NoSuchKey) { if (error && !error.NoSuchKey) {
log.error('from metadata while deleting user bucket', log.error('from metadata while deleting user bucket',
{ error }); { error });

View File

@ -15,21 +15,10 @@ function getNotificationConfiguration(parsedXml) {
} }
const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource)); const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource));
const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]); const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]);
// getting invalid targets if (!notifConfigTargets.every(t => targets.has(t))) {
const invalidTargets = []; return { error: errors.MalformedXML.customizeDescription(
notifConfigTargets.forEach((t, i) => { 'queue arn target is not included in Cloudserver bucket notification config'),
if (!targets.has(t)) { };
invalidTargets.push({
ArgumentName: notifConfig.queueConfig[i].queueArn,
ArgumentValue: 'The destination queue does not exist',
});
}
});
if (invalidTargets.length > 0) {
const errDesc = 'Unable to validate the following destination configurations';
let error = errors.InvalidArgument.customizeDescription(errDesc);
error = error.addMetadataEntry('invalidArguments', invalidTargets);
return { error };
} }
return notifConfig; return notifConfig;
} }

View File

@ -3,7 +3,7 @@ const async = require('async');
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { data } = require('../../../data/wrapper'); const { data } = require('../../../data/wrapper');
const locationConstraintCheck = require('../object/locationConstraintCheck'); const locationConstraintCheck = require('../object/locationConstraintCheck');
const { standardMetadataValidateBucketAndObj } = const { metadataValidateBucketAndObj } =
require('../../../metadata/metadataUtils'); require('../../../metadata/metadataUtils');
const services = require('../../../services'); const services = require('../../../services');
@ -14,19 +14,17 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
bucketName, bucketName,
objectKey, objectKey,
uploadId, uploadId,
preciseRequestType: request.apiMethods || 'multipartDelete', preciseRequestType: 'multipartDelete',
request,
}; };
// For validating the request at the destinationBucket level // For validating the request at the destinationBucket level
// params are the same as validating at the MPU level // params are the same as validating at the MPU level
// but the requestType is the more general 'objectDelete' // but the requestType is the more general 'objectDelete'
const metadataValParams = Object.assign({}, metadataValMPUparams); const metadataValParams = Object.assign({}, metadataValMPUparams);
metadataValParams.requestType = 'objectPut'; metadataValParams.requestType = 'objectPut';
const authzIdentityResult = request ? request.actionImplicitDenies : false;
async.waterfall([ async.waterfall([
function checkDestBucketVal(next) { function checkDestBucketVal(next) {
standardMetadataValidateBucketAndObj(metadataValParams, authzIdentityResult, log, metadataValidateBucketAndObj(metadataValParams, log,
(err, destinationBucket) => { (err, destinationBucket) => {
if (err) { if (err) {
return next(err, destinationBucket); return next(err, destinationBucket);
@ -57,14 +55,9 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket, function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket,
next) { next) {
const location = mpuOverviewObj.controllingLocationConstraint; const location = mpuOverviewObj.controllingLocationConstraint;
const originalIdentityAuthzResults = request.actionImplicitDenies;
// eslint-disable-next-line no-param-reassign
delete request.actionImplicitDenies;
return data.abortMPU(objectKey, uploadId, location, bucketName, return data.abortMPU(objectKey, uploadId, location, bucketName,
request, destBucket, locationConstraintCheck, log, request, destBucket, locationConstraintCheck, log,
(err, skipDataDelete) => { (err, skipDataDelete) => {
// eslint-disable-next-line no-param-reassign
request.actionImplicitDenies = originalIdentityAuthzResults;
if (err) { if (err) {
return next(err, destBucket); return next(err, destBucket);
} }

View File

@ -0,0 +1,29 @@
/**
* Checks the object encryption request against bucket encryption for matching
* SSE-S3 configuration
* @param {object} objectRequest - http request
* @param {object} serverSideEncryption - bucket encryption info
* @return {boolean} returns true if the object request has correct SSE-S3
* configuration
*/
function isValidSSES3(objectRequest, serverSideEncryption) {
// x-amz-server-side-encryption is allowed only if bucket
// encryption is enabled and if the value is AES256
// NOTE: object level encryption is not supported, but we allow
// encryption headers in the object request headers!
const sseHeader = objectRequest.headers['x-amz-server-side-encryption'];
const encryptionAlgorithm = 'AES256';
const result = ((!serverSideEncryption && sseHeader) ||
(serverSideEncryption && sseHeader
&& sseHeader === encryptionAlgorithm
&& serverSideEncryption.algorithm !== encryptionAlgorithm) ||
(serverSideEncryption && sseHeader
&& sseHeader !== encryptionAlgorithm
&& serverSideEncryption.algorithm === encryptionAlgorithm));
return !result;
}
module.exports = {
isValidSSES3,
};

View File

@ -1,25 +0,0 @@
const { errors } = require('arsenal');
const { maxHttpHeadersSize } = require('../../../../constants');
/**
* Checks the size of the HTTP headers
* @param {object} requestHeaders - HTTP request headers
* @return {object} object with error or null
*/
function checkHttpHeadersSize(requestHeaders) {
let httpHeadersSize = 0;
Object.keys(requestHeaders).forEach(header => {
httpHeadersSize += Buffer.byteLength(header, 'utf8') +
Buffer.byteLength(requestHeaders[header], 'utf8');
});
if (httpHeadersSize > maxHttpHeadersSize) {
return {
httpHeadersSizeError: errors.HttpHeadersTooLarge,
};
}
return {};
}
module.exports = checkHttpHeadersSize;

View File

@ -1,195 +1,91 @@
/* const coldstorage = require('../../../coldstorage/wrapper');
* Code based on Yutaka Oishi (Fujifilm) contributions const { metadataGetObject } = require('../../../metadata/metadataUtils');
* Date: 11 Sep 2020
*/
const { ObjectMDArchive } = require('arsenal').models;
const errors = require('arsenal').errors; const errors = require('arsenal').errors;
const { config } = require('../../../Config');
const { locationConstraints } = config;
const { scaledMsPerDay } = config.getTimeOptions();
/** /**
* Get response header "x-amz-restore" * Get response header "x-amz-restore"
* Be called by objectHead.js * Be called by objectHead.js
* @param {object} objMD - object's metadata * @param {object} objMD - object's metadata
* @returns {string|undefined} x-amz-restore * @returns {string} x-amz-restore
*/ */
function getAmzRestoreResHeader(objMD) { function getAmzRestoreResHeader(objMD){
if (objMD.archive &&
objMD.archive.restoreRequestedAt && let value;
!objMD.archive.restoreCompletedAt) { if(objMD['x-amz-restore']){
// Avoid race condition by relying on the `archive` MD of the object if(objMD['x-amz-restore']['ongoing-request']){
// and return the right header after a RESTORE request. value = `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}"`;
// eslint-disable-next-line
return `ongoing-request="true"`;
} }
if (objMD['x-amz-restore']) {
// expiry-date is transformed to format of RFC2822
if (objMD['x-amz-restore']['expiry-date']) { if (objMD['x-amz-restore']['expiry-date']) {
const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString(); const utcDateTime = alDateUtils.toUTCString(new Date(objMD['x-amz-restore']['expiry-date']));
// eslint-disable-next-line value = `${value}, ${expiry-date}="${utcDateTime}"`;
return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
} }
} }
return undefined;
return value;
} }
/** /**
* Check if restore can be done. * Check object metadata if GET request is possible
* * Be called by objectGet.js
* @param {ObjectMD} objectMD - object metadata * @param {object} objMD - object's metadata
* @param {object} log - werelogs logger * @return {boolean} true if the GET request is accepted, false if not
* @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
*/ */
function _validateStartRestore(objectMD, log) { function validateAmzRestoreForGet(objMD){
if (objectMD.archive?.restoreCompletedAt) {
if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) { if(!objMD) {
// return InvalidObjectState error if the restored object is expired return false;
// but restore info md of this object has not yet been cleared
log.debug('The restored object already expired.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
} }
// If object is already restored, no further check is needed if(objMD['x-amz-storage-class'] === 'GLACIER'){
// Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have return false;
// been reset.
return undefined;
} }
const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
if (!isLocationCold) { if(objMD['x-amz-restore']['ongoing-request']){
// return InvalidObjectState error if the object is not in cold storage, return false;
// not in cold storage means either location cold flag not exists or cold flag is explicit false
log.debug('The bucket of the object is not in a cold storage location.',
{
isLocationCold,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
} }
if (objectMD.archive?.restoreRequestedAt) {
// return RestoreAlreadyInProgress error if the object is currently being restored return true;
// check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
log.debug('The object is currently being restored.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.RestoreAlreadyInProgress;
}
return undefined;
} }
/** /**
* Check if "put version id" is allowed * Start to archive to GLACIER
* * ( Be called by Lifecycle batch? )
* @param {ObjectMD} objMD - object metadata
* @param {string} versionId - object's version id
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} - undefined if "put version id" is allowed
*/ */
function validatePutVersionId(objMD, versionId, log) { function startGlacier(bucketName, objName, versionId, log, cb){
if (!objMD) {
const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
return err;
}
if (objMD.isDeleteMarker) { return completeGlacier(bucketName, objName, versionId, log, cb);
log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
return errors.MethodNotAllowed;
}
const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
if (!isLocationCold) {
log.error('The object data is not stored in a cold storage location.',
{
isLocationCold,
dataStoreName: objMD.dataStoreName,
method: 'validatePutVersionId',
});
return errors.InvalidObjectState;
}
// make sure object archive restoration is in progress
// NOTE: we do not use putObjectVersion to update the restoration period.
if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
|| objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
log.error('object archive restoration is not in progress',
{ method: 'validatePutVersionId', versionId });
return errors.InvalidObjectState;
}
return undefined;
} }
/** /**
* Check if the object is already restored, and update the expiration date accordingly: * Complete to archive to GLACIER
* > After restoring an archived object, you can update the restoration period by reissuing the * ( Be called by Lifecycle batch? )
* > request with a new period. Amazon S3 updates the restoration period relative to the current * update x-amz-storage-class to "GLACIER".
* > time.
*
* @param {ObjectMD} objectMD - object metadata
* @param {object} log - werelogs logger
* @return {boolean} - true if the object is already restored
*/ */
function _updateObjectExpirationDate(objectMD, log) { function completeGlacier(bucketName, objName, versionId, log, cb){
// Check if restoreCompletedAt field exists
// Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
// checked earlier in the process, so checking again here would create weird states
const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
log.debug('The restore status of the object.', {
isObjectAlreadyRestored,
method: 'isObjectAlreadyRestored'
});
if (isObjectAlreadyRestored) {
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
/* eslint-disable no-param-reassign */ metadataGetObject(bucketName, objectKey, versionId, log,
objectMD.archive.restoreWillExpireAt = expiryDate; (err, objMD) => {
objectMD['x-amz-restore'] = { if(err){
'ongoing-request': false, log.trace('error processing get metadata', {
'expiry-date': expiryDate, error: err,
}; method: 'metadataGetObject',
/* eslint-enable no-param-reassign */ });
return cb(err);
} }
return isObjectAlreadyRestored;
}
/** const storageClass = 'GLACIER';
* update restore expiration date.
* // FIXME: return error NotImplemented when using "ColdStorageFileInterface"
* @param {ObjectMD} objectMD - objectMD instance coldstorage.updateAmzStorageClass(bucketName, objName, objMD, storageClass, log, cb);
* @param {object} restoreParam - restore param
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} internal error if object MD is not valid
*
*/
function _updateRestoreInfo(objectMD, restoreParam, log) {
if (!objectMD.archive) {
log.debug('objectMD.archive doesn\'t exits', {
objectMD,
method: '_updateRestoreInfo'
});
return errors.InternalError.customizeDescription('Archive metadata is missing.');
} }
/* eslint-disable no-param-reassign */ );
objectMD.archive.restoreRequestedAt = new Date();
objectMD.archive.restoreRequestedDays = restoreParam.days;
objectMD.originOp = 's3:ObjectRestore:Post';
/* eslint-enable no-param-reassign */
if (!ObjectMDArchive.isValid(objectMD.archive)) {
log.debug('archive is not valid', {
archive: objectMD.archive,
method: '_updateRestoreInfo'
});
return errors.InternalError.customizeDescription('Invalid archive metadata.');
}
return undefined;
} }
/** /**
@ -198,50 +94,92 @@ function _updateRestoreInfo(objectMD, restoreParam, log) {
* calculate restore expiry-date and add it to objectMD. * calculate restore expiry-date and add it to objectMD.
* Be called by objectRestore.js * Be called by objectRestore.js
* *
* @param {ObjectMD} objectMD - objectMd instance * FIXME: After restore is started, there is no timing to update restore parameter to the content of complete restore.
* @param {object} restoreParam - bucket name
* @param {object} log - werelogs logger
* @param {function} cb - bucket name
* @return {undefined}
*
*/ */
function startRestore(objectMD, restoreParam, log, cb) { function startRestore(bucketName, objName, objectMD, restoreParam, cb){
log.info('Validating if restore can be done or not.');
const checkResultError = _validateStartRestore(objectMD, log); let checkResult = _validateStartRestore(objectMD);
if (checkResultError) { if(checkResult instanceof errors){
return cb(checkResultError); return cb(checkResult);
} };
log.info('Updating restore information.');
const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log); // update restore parameter to the content of doing restore.
if (updateResultError) { _updateRestoreExpiration(bucketName, objName, objMD, restoreParam, log, cb);
return cb(updateResultError);
}
const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log); return cb(objectMD, restoreParam);
return cb(null, isObjectAlreadyRestored);
} }
/** /**
* checks if object data is available or if it's in cold storage * complete to restore object.
* @param {ObjectMD} objMD Object metadata * Update restore-ongoing to false.
* @returns {ArsenalError|null} error if object data is not available * ( Be called by batch to check if the restore is complete? )
*
*/ */
function verifyColdObjectAvailable(objMD) { function completeRestore(bucketName, objName, objMD){
// return error when object is cold
if (objMD.archive && const updateParam = false;
// Object is in cold backend
(!objMD.archive.restoreRequestedAt || // FIXME: return error NotImplemented when using "ColdStorageFileInterface"
// Object is being restored return coldstorage.updateRestoreOngoing(bucketName, objName, objMD, updateParam, log, cb);
(objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
const err = errors.InvalidObjectState
.customizeDescription('The operation is not valid for the object\'s storage class');
return err;
}
return null;
} }
/**
* expire to restore object.
* Delete x-amz-restore.
* ( Be called by batch to check if the restore is expire? )
*/
function expireRestore(bucketName, objName, objMD){
// FIXME: return error NotImplemented when using "ColdStorageFileInterface"
return coldstorage.deleteAmzRestore(bucketName, objName, objMD, log, cb);
}
/**
* Check if restore has already started.
*/
function _validateStartRestore(objectMD){
if(objectMD['x-amz-restore'] && objMD['x-amz-restore']['ongoing-request']){
return errors.RestoreAlreadyInProgress;
}
else{
return undefined;
}
}
/**
* update restore expiration date.
*/
function _updateRestoreExpiration(bucketName, objName, objMD, restoreParam, log, cb){
if(objMD['x-amz-restore'] && !objMD['x-amz-restore']['ongoing-request']){
// FIXME: return error NotImplemented when using "ColdStorageFileInterface"
return coldstorage.updateRestoreExpiration(bucketName, objName, objMD, restoreParam, log, cb);
}
else{
log.debug('do not updateRestoreExpiration', { method: '_updateRestoreExpiration' });
return undefined;
}
}
module.exports = { module.exports = {
startRestore,
getAmzRestoreResHeader, getAmzRestoreResHeader,
validatePutVersionId, validateAmzRestoreForGet,
verifyColdObjectAvailable, startGlacier,
completeGlacier,
startRestore,
completeRestore,
expireRestore,
}; };

View File

@ -5,9 +5,10 @@ const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { data } = require('../../../data/wrapper'); const { data } = require('../../../data/wrapper');
const services = require('../../../services'); const services = require('../../../services');
const logger = require('../../../utilities/logger');
const { dataStore } = require('./storeObject'); const { dataStore } = require('./storeObject');
const locationConstraintCheck = require('./locationConstraintCheck'); const locationConstraintCheck = require('./locationConstraintCheck');
const { versioningPreprocessing, overwritingVersioning } = require('./versioning'); const { versioningPreprocessing } = require('./versioning');
const removeAWSChunked = require('./removeAWSChunked'); const removeAWSChunked = require('./removeAWSChunked');
const getReplicationInfo = require('./getReplicationInfo'); const getReplicationInfo = require('./getReplicationInfo');
const { config } = require('../../../Config'); const { config } = require('../../../Config');
@ -20,7 +21,7 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' +
'a versioned object to a location-constraint of type Azure or GCP.'; 'a versioned object to a location-constraint of type Azure or GCP.';
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
metadataStoreParams, dataToDelete, log, requestMethod, callback) { metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
services.metadataStoreObject(bucketName, dataGetInfo, services.metadataStoreObject(bucketName, dataGetInfo,
cipherBundle, metadataStoreParams, (err, result) => { cipherBundle, metadataStoreParams, (err, result) => {
if (err) { if (err) {
@ -30,7 +31,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
const newDataStoreName = Array.isArray(dataGetInfo) ? const newDataStoreName = Array.isArray(dataGetInfo) ?
dataGetInfo[0].dataStoreName : null; dataGetInfo[0].dataStoreName : null;
return data.batchDelete(dataToDelete, requestMethod, return data.batchDelete(dataToDelete, requestMethod,
newDataStoreName, log, err => callback(err, result)); newDataStoreName, deleteLog, err => callback(err, result));
} }
return callback(null, result); return callback(null, result);
}); });
@ -50,9 +51,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
* @param {(object|null)} streamingV4Params - if v4 auth, object containing * @param {(object|null)} streamingV4Params - if v4 auth, object containing
* accessKey, signatureFromRequest, region, scopeDate, timestamp, and * accessKey, signatureFromRequest, region, scopeDate, timestamp, and
* credentialScope (to be used for streaming v4 auth if applicable) * credentialScope (to be used for streaming v4 auth if applicable)
* @param {(object|null)} overheadField - fields to be included in metadata overhead
* @param {RequestLogger} log - logger instance * @param {RequestLogger} log - logger instance
* @param {string} originOp - Origin operation
* @param {function} callback - callback function * @param {function} callback - callback function
* @return {undefined} and call callback with (err, result) - * @return {undefined} and call callback with (err, result) -
* result.contentMD5 - content md5 of new object or version * result.contentMD5 - content md5 of new object or version
@ -60,10 +59,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
*/ */
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params, canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
overheadField, log, originOp, callback) { log, callback) {
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
const size = isDeleteMarker ? 0 : request.parsedContentLength; const size = isDeleteMarker ? 0 : request.parsedContentLength;
// although the request method may actually be 'DELETE' if creating a // although the request method may actually be 'DELETE' if creating a
// delete marker, for our purposes we consider this to be a 'PUT' // delete marker, for our purposes we consider this to be a 'PUT'
@ -116,7 +112,6 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
isDeleteMarker, isDeleteMarker,
replicationInfo: getReplicationInfo( replicationInfo: getReplicationInfo(
objectKey, bucketMD, false, size, null, null, authInfo), objectKey, bucketMD, false, size, null, null, authInfo),
overheadField,
log, log,
}; };
@ -134,6 +129,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
} }
} }
if (!isDeleteMarker) { if (!isDeleteMarker) {
metadataStoreParams.contentType = request.headers['content-type']; metadataStoreParams.contentType = request.headers['content-type'];
metadataStoreParams.cacheControl = request.headers['cache-control']; metadataStoreParams.cacheControl = request.headers['cache-control'];
@ -143,7 +139,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
removeAWSChunked(request.headers['content-encoding']); removeAWSChunked(request.headers['content-encoding']);
metadataStoreParams.expires = request.headers.expires; metadataStoreParams.expires = request.headers.expires;
metadataStoreParams.tagging = request.headers['x-amz-tagging']; metadataStoreParams.tagging = request.headers['x-amz-tagging'];
metadataStoreParams.originOp = originOp; metadataStoreParams.originOp = 's3:ObjectCreated:Put';
const defaultObjectLockConfiguration const defaultObjectLockConfiguration
= bucketMD.getObjectLockConfiguration(); = bucketMD.getObjectLockConfiguration();
if (defaultObjectLockConfiguration) { if (defaultObjectLockConfiguration) {
@ -158,7 +154,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
// eslint-disable-next-line no-param-reassign // eslint-disable-next-line no-param-reassign
request.headers[constants.objectLocationConstraintHeader] = request.headers[constants.objectLocationConstraintHeader] =
objMD[constants.objectLocationConstraintHeader]; objMD[constants.objectLocationConstraintHeader];
metadataStoreParams.originOp = originOp; metadataStoreParams.originOp = 's3:ObjectRemoved:DeleteMarkerCreated';
} }
const backendInfoObj = const backendInfoObj =
@ -189,17 +185,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
} }
} }
if (objMD && objMD.uploadId) {
metadataStoreParams.oldReplayId = objMD.uploadId;
}
/* eslint-disable camelcase */ /* eslint-disable camelcase */
const dontSkipBackend = externalBackends; const dontSkipBackend = externalBackends;
/* eslint-enable camelcase */ /* eslint-enable camelcase */
const requestLogger =
logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
const mdOnlyHeader = request.headers['x-amz-meta-mdonly']; const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
const mdOnlySize = request.headers['x-amz-meta-size']; const mdOnlySize = request.headers['x-amz-meta-size'];
return async.waterfall([ return async.waterfall([
function storeData(next) { function storeData(next) {
if (size === 0) { if (size === 0) {
@ -264,17 +257,12 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
return next(null, dataGetInfoArr); return next(null, dataGetInfoArr);
}, },
function getVersioningInfo(infoArr, next) { function getVersioningInfo(infoArr, next) {
// if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
if (isPutVersion) {
const options = overwritingVersioning(objMD, metadataStoreParams);
return process.nextTick(() => next(null, options, infoArr));
}
return versioningPreprocessing(bucketName, bucketMD, return versioningPreprocessing(bucketName, bucketMD,
metadataStoreParams.objectKey, objMD, log, (err, options) => { metadataStoreParams.objectKey, objMD, log, (err, options) => {
if (err) { if (err) {
// TODO: check AWS error when user requested a specific // TODO: check AWS error when user requested a specific
// version before any versions have been put // version before any versions have been put
const logLvl = err.is.BadRequest ? const logLvl = err === errors.BadRequest ?
'debug' : 'error'; 'debug' : 'error';
log[logLvl]('error getting versioning info', { log[logLvl]('error getting versioning info', {
error: err, error: err,
@ -288,13 +276,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
metadataStoreParams.versionId = options.versionId; metadataStoreParams.versionId = options.versionId;
metadataStoreParams.versioning = options.versioning; metadataStoreParams.versioning = options.versioning;
metadataStoreParams.isNull = options.isNull; metadataStoreParams.isNull = options.isNull;
metadataStoreParams.deleteNullKey = options.deleteNullKey; metadataStoreParams.nullVersionId = options.nullVersionId;
if (options.extraMD) {
Object.assign(metadataStoreParams, options.extraMD);
}
return _storeInMDandDeleteData(bucketName, infoArr, return _storeInMDandDeleteData(bucketName, infoArr,
cipherBundle, metadataStoreParams, cipherBundle, metadataStoreParams,
options.dataToDelete, log, requestMethod, next); options.dataToDelete, requestLogger, requestMethod, next);
}, },
], callback); ], callback);
} }

View File

@ -1,18 +0,0 @@
/**
* _bucketRequiresOplogUpdate - DELETE an object from a bucket
* @param {BucketInfo} bucket - bucket object
* @return {boolean} whether objects require oplog updates on deletion, or not
*/
function _bucketRequiresOplogUpdate(bucket) {
// Default behavior is to require an oplog update
if (!bucket || !bucket.getLifecycleConfiguration || !bucket.getNotificationConfiguration) {
return true;
}
// If the bucket has lifecycle configuration or notification configuration
// set, we also require an oplog update
return bucket.getLifecycleConfiguration() || bucket.getNotificationConfiguration();
}
module.exports = {
_bucketRequiresOplogUpdate,
};

View File

@ -1,140 +0,0 @@
const { supportedLifecycleRules } = require('arsenal').constants;
const { LifecycleConfiguration } = require('arsenal').models;
const {
LifecycleDateTime,
LifecycleUtils,
} = require('arsenal').s3middleware.lifecycleHelpers;
const { config } = require('../../../Config');
const {
expireOneDayEarlier,
transitionOneDayEarlier,
timeProgressionFactor,
scaledMsPerDay,
} = config.getTimeOptions();
const lifecycleDateTime = new LifecycleDateTime({
transitionOneDayEarlier,
expireOneDayEarlier,
timeProgressionFactor,
});
const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor);
function calculateDate(objDate, expDays, datetime) {
return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay));
}
function formatExpirationHeader(date, id) {
return `expiry-date="${date}", rule-id="${encodeURIComponent(id)}"`;
}
// format: x-amz-expiration: expiry-date="Fri, 21 Dec 2012 00:00:00 GMT", rule-id="id"
const AMZ_EXP_HEADER = 'x-amz-expiration';
// format: x-amz-abort-date: "Fri, 21 Dec 2012 00:00:00 GMT"
const AMZ_ABORT_DATE_HEADER = 'x-amz-abort-date';
// format: x-amz-abort-rule-id: "rule id"
const AMZ_ABORT_ID_HEADER = 'x-amz-abort-rule-id';
function _generateExpHeadersObjects(rules, params, datetime) {
const tags = {
TagSet: params.tags
? Object.keys(params.tags)
.map(key => ({ Key: key, Value: params.tags[key] }))
: [],
};
const objectInfo = { Key: params.key };
const filteredRules = lifecycleUtils.filterRules(rules, objectInfo, tags);
const applicable = lifecycleUtils.getApplicableRules(filteredRules, objectInfo, datetime);
if (applicable.Expiration) {
const rule = applicable.Expiration;
if (rule.Days === undefined && rule.Date === undefined) {
return {};
}
if (rule.Date) {
return {
[AMZ_EXP_HEADER]: formatExpirationHeader(rule.Date, rule.ID),
};
}
const date = calculateDate(params.date, rule.Days, datetime);
return {
[AMZ_EXP_HEADER]: formatExpirationHeader(date.toUTCString(), rule.ID),
};
}
return {};
}
function _generateExpHeadresMPU(rules, params, datetime) {
const noTags = { TagSet: [] };
const objectInfo = { Key: params.key };
const filteredRules = lifecycleUtils.filterRules(rules, objectInfo, noTags);
const applicable = lifecycleUtils.getApplicableRules(filteredRules, {}, datetime);
if (applicable.AbortIncompleteMultipartUpload) {
const rule = applicable.AbortIncompleteMultipartUpload;
const date = calculateDate(
params.date,
rule.DaysAfterInitiation,
datetime
);
return {
[AMZ_ABORT_ID_HEADER]: encodeURIComponent(rule.ID),
[AMZ_ABORT_DATE_HEADER]: date.toUTCString(),
};
}
return {};
}
/**
* generate response expiration headers
* @param {object} params - params
* @param {LifecycleDateTime} datetime - lifecycle datetime object
* @returns {object} - expiration response headers
*/
function generateExpirationHeaders(params, datetime) {
const { lifecycleConfig, objectParams, mpuParams, isVersionedReq } = params;
if (!lifecycleConfig || isVersionedReq) {
return {};
}
const lcfg = LifecycleConfiguration.getConfigJson(lifecycleConfig);
if (objectParams) {
return _generateExpHeadersObjects(lcfg.Rules, objectParams, datetime);
}
if (mpuParams) {
return _generateExpHeadresMPU(lcfg.Rules, mpuParams, datetime);
}
return {};
}
/**
* set response expiration headers to target header object
* @param {object} headers - target header object
* @param {object} params - params
* @returns {undefined}
*/
function setExpirationHeaders(headers, params) {
const expHeaders = generateExpirationHeaders(params, lifecycleDateTime);
Object.assign(headers, expHeaders);
}
module.exports = {
lifecycleDateTime,
generateExpirationHeaders,
setExpirationHeaders,
};

View File

@ -23,12 +23,12 @@ function _getStorageClasses(rule) {
} }
const { replicationEndpoints } = s3config; const { replicationEndpoints } = s3config;
// If no storage class, use the given default endpoint or the sole endpoint // If no storage class, use the given default endpoint or the sole endpoint
if (replicationEndpoints.length > 0) { if (replicationEndpoints.length > 1) {
const endPoint = const endPoint =
replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0]; replicationEndpoints.find(endpoint => endpoint.default);
return [endPoint.site]; return [endPoint.site];
} }
return undefined; return [replicationEndpoints[0].site];
} }
function _getReplicationInfo(rule, replicationConfig, content, operationType, function _getReplicationInfo(rule, replicationConfig, content, operationType,
@ -36,9 +36,6 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
const storageTypes = []; const storageTypes = [];
const backends = []; const backends = [];
const storageClasses = _getStorageClasses(rule); const storageClasses = _getStorageClasses(rule);
if (!storageClasses) {
return undefined;
}
storageClasses.forEach(storageClass => { storageClasses.forEach(storageClass => {
const storageClassName = const storageClassName =
storageClass.endsWith(':preferred_read') ? storageClass.endsWith(':preferred_read') ?

View File

@ -1,190 +0,0 @@
const { versioning } = require('arsenal');
const versionIdUtils = versioning.VersionID;
const { lifecycleListing } = require('../../../../constants');
const { CURRENT_TYPE, NON_CURRENT_TYPE, ORPHAN_DM_TYPE } = lifecycleListing;
function _makeTags(tags) {
const res = [];
Object.entries(tags).forEach(([key, value]) =>
res.push(
{
Key: key,
Value: value,
}
));
return res;
}
function processCurrents(bucketName, listParams, isBucketVersioned, list) {
const data = {
Name: bucketName,
Prefix: listParams.prefix,
MaxKeys: listParams.maxKeys,
MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
IsTruncated: !!list.IsTruncated,
Marker: listParams.marker,
BeforeDate: listParams.beforeDate,
NextMarker: list.NextMarker,
Contents: [],
};
list.Contents.forEach(item => {
const v = item.value;
const content = {
Key: item.key,
LastModified: v.LastModified,
ETag: `"${v.ETag}"`,
Size: v.Size,
Owner: {
ID: v.Owner.ID,
DisplayName: v.Owner.DisplayName,
},
StorageClass: v.StorageClass,
TagSet: _makeTags(v.tags),
IsLatest: true, // for compatibility with AWS ListObjectVersions.
DataStoreName: v.dataStoreName,
ListType: CURRENT_TYPE,
};
// NOTE: The current versions listed to be lifecycle should include version id
// if the bucket is versioned.
if (isBucketVersioned) {
const versionId = (v.IsNull || v.VersionId === undefined) ?
'null' : versionIdUtils.encode(v.VersionId);
content.VersionId = versionId;
}
data.Contents.push(content);
});
return data;
}
function _encodeVersionId(vid) {
let versionId = vid;
if (versionId && versionId !== 'null') {
versionId = versionIdUtils.encode(versionId);
}
return versionId;
}
function processNonCurrents(bucketName, listParams, list) {
const nextVersionIdMarker = _encodeVersionId(list.NextVersionIdMarker);
const versionIdMarker = _encodeVersionId(listParams.versionIdMarker);
const data = {
Name: bucketName,
Prefix: listParams.prefix,
MaxKeys: listParams.maxKeys,
MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
IsTruncated: !!list.IsTruncated,
KeyMarker: listParams.keyMarker,
VersionIdMarker: versionIdMarker,
BeforeDate: listParams.beforeDate,
NextKeyMarker: list.NextKeyMarker,
NextVersionIdMarker: nextVersionIdMarker,
Contents: [],
};
list.Contents.forEach(item => {
const v = item.value;
const versionId = (v.IsNull || v.VersionId === undefined) ?
'null' : versionIdUtils.encode(v.VersionId);
const content = {
Key: item.key,
LastModified: v.LastModified,
ETag: `"${v.ETag}"`,
Size: v.Size,
Owner: {
ID: v.Owner.ID,
DisplayName: v.Owner.DisplayName,
},
StorageClass: v.StorageClass,
TagSet: _makeTags(v.tags),
staleDate: v.staleDate, // lowerCamelCase to be compatible with existing lifecycle.
VersionId: versionId,
DataStoreName: v.dataStoreName,
ListType: NON_CURRENT_TYPE,
};
data.Contents.push(content);
});
return data;
}
function processOrphans(bucketName, listParams, list) {
const data = {
Name: bucketName,
Prefix: listParams.prefix,
MaxKeys: listParams.maxKeys,
MaxScannedLifecycleListingEntries: listParams.maxScannedLifecycleListingEntries,
IsTruncated: !!list.IsTruncated,
Marker: listParams.marker,
BeforeDate: listParams.beforeDate,
NextMarker: list.NextMarker,
Contents: [],
};
list.Contents.forEach(item => {
const v = item.value;
const versionId = (v.IsNull || v.VersionId === undefined) ?
'null' : versionIdUtils.encode(v.VersionId);
data.Contents.push({
Key: item.key,
LastModified: v.LastModified,
Owner: {
ID: v.Owner.ID,
DisplayName: v.Owner.DisplayName,
},
VersionId: versionId,
IsLatest: true, // for compatibility with AWS ListObjectVersions.
ListType: ORPHAN_DM_TYPE,
});
});
return data;
}
function getLocationConstraintErrorMessage(locationName) {
return 'value of the location you are attempting to set ' +
`- ${locationName} - is not listed in the locationConstraint config`;
}
/**
* validateMaxScannedEntries - Validates and returns the maximum scanned entries value.
*
* @param {object} params - Query parameters
* @param {object} config - CloudServer configuration
* @param {number} min - Minimum number of entries to be scanned
* @returns {Object} - An object indicating the validation result:
* - isValid (boolean): Whether the validation is successful.
* - maxScannedLifecycleListingEntries (number): The validated maximum scanned entries value if isValid is true.
*/
function validateMaxScannedEntries(params, config, min) {
let maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
if (params['max-scanned-lifecycle-listing-entries']) {
const maxEntriesParams = Number.parseInt(params['max-scanned-lifecycle-listing-entries'], 10);
if (Number.isNaN(maxEntriesParams) || maxEntriesParams < min ||
maxEntriesParams > maxScannedLifecycleListingEntries) {
return { isValid: false };
}
maxScannedLifecycleListingEntries = maxEntriesParams;
}
return { isValid: true, maxScannedLifecycleListingEntries };
}
module.exports = {
processCurrents,
processNonCurrents,
processOrphans,
getLocationConstraintErrorMessage,
validateMaxScannedEntries,
};

View File

@ -1,34 +0,0 @@
/**
* Check if all keys that exist in the current list which will be used
* in composing object are not present in the old object's list.
*
* This method can be used to check against accidentally removing data
* keys due to instability from the metadata layer, or for replay
* detection in general.
*
* @param {array|string|null} prev - list of keys from the object being
* overwritten
* @param {array|null} curr - list of keys to be used in composing
* current object
* @returns {boolean} true if no key in `curr` is present in `prev`,
* false otherwise
*/
function locationKeysHaveChanged(prev, curr) {
if (!prev || prev.length === 0 || !curr) {
return true;
}
// backwards compatibility check if object is of model version 2
if (typeof prev === 'string') {
return curr.every(v => v.key !== prev);
}
const keysMap = {};
prev.forEach(v => {
if (!keysMap[v.dataStoreType]) {
keysMap[v.dataStoreType] = {};
}
keysMap[v.dataStoreType][v.key] = true;
});
return curr.every(v => !(keysMap[v.dataStoreType] && keysMap[v.dataStoreType][v.key]));
}
module.exports = locationKeysHaveChanged;

View File

@ -0,0 +1,24 @@
/**
* Check keys that exist in the current list which will be used in composing
* object. This method checks against accidentally removing data keys due to
* instability from the metadata layer. The check returns true if there was no
* match and false if at least one key from the previous list exists in the
* current list
* @param {array|string} prev - list of keys from the object being overwritten
* @param {array} curr - list of keys to be used in composing current object
* @returns {array} list of keys that can be deleted
*/
function locationKeysSanityCheck(prev, curr) {
if (!prev || prev.length === 0) {
return true;
}
// backwards compatibility check if object is of model version 2
if (typeof prev === 'string') {
return curr.every(v => v.key !== prev);
}
const keysMap = {};
prev.forEach(v => { keysMap[v.key] = true; });
return curr.every(v => !keysMap[v.key]);
}
module.exports = locationKeysSanityCheck;

View File

@ -1,11 +1,5 @@
const { errors, auth, policies } = require('arsenal'); const { errors } = require('arsenal');
const moment = require('moment'); const moment = require('moment');
const { config } = require('../../../Config');
const vault = require('../../../auth/vault');
const { evaluateBucketPolicyWithIAM } = require('../authorization/permissionChecks');
const { scaledMsPerDay } = config.getTimeOptions();
/** /**
* Calculates retain until date for the locked object version * Calculates retain until date for the locked object version
* @param {object} retention - includes days or years retention period * @param {object} retention - includes days or years retention period
@ -13,17 +7,11 @@ const { scaledMsPerDay } = config.getTimeOptions();
*/ */
function calculateRetainUntilDate(retention) { function calculateRetainUntilDate(retention) {
const { days, years } = retention; const { days, years } = retention;
if (!days && !years) {
return undefined;
}
const date = moment(); const date = moment();
// Calculate the number of days to retain the lock on the object // Calculate the number of days to retain the lock on the object
const retainUntilDays = days || years * 365; const retainUntilDays = days || years * 365;
const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay;
const retainUntilDate const retainUntilDate
= date.add(retainUntilDaysInMs, 'ms'); = date.add(retainUntilDays, 'days');
return retainUntilDate.toISOString(); return retainUntilDate.toISOString();
} }
/** /**
@ -50,7 +38,7 @@ function validateHeaders(bucket, headers, log) {
!(objectLockMode && objectLockDate)) { !(objectLockMode && objectLockDate)) {
return errors.InvalidArgument.customizeDescription( return errors.InvalidArgument.customizeDescription(
'x-amz-object-lock-retain-until-date and ' + 'x-amz-object-lock-retain-until-date and ' +
'x-amz-object-lock-mode must both be supplied', 'x-amz-object-lock-mode must both be supplied'
); );
} }
const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']); const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']);
@ -79,25 +67,17 @@ function validateHeaders(bucket, headers, log) {
*/ */
function compareObjectLockInformation(headers, defaultRetention) { function compareObjectLockInformation(headers, defaultRetention) {
const objectLockInfoToSave = {}; const objectLockInfoToSave = {};
if (defaultRetention && defaultRetention.rule) {
const defaultMode = defaultRetention.rule.mode;
const defaultTime = calculateRetainUntilDate(defaultRetention.rule);
if (defaultMode && defaultTime) {
objectLockInfoToSave.retentionInfo = {
mode: defaultMode,
date: defaultTime,
};
}
}
if (headers) { if (headers) {
const headerMode = headers['x-amz-object-lock-mode']; const headerMode = headers['x-amz-object-lock-mode'];
const headerDate = headers['x-amz-object-lock-retain-until-date']; const headerDate = headers['x-amz-object-lock-retain-until-date'];
if (headerMode && headerDate) { const objectRetention = headerMode && headerDate;
if (objectRetention || defaultRetention) {
const mode = headerMode || defaultRetention.rule.mode;
const date = headerDate
|| calculateRetainUntilDate(defaultRetention.rule);
objectLockInfoToSave.retentionInfo = { objectLockInfoToSave.retentionInfo = {
mode: headerMode, mode,
date: headerDate, date,
}; };
} }
const headerLegalHold = headers['x-amz-object-lock-legal-hold']; const headerLegalHold = headers['x-amz-object-lock-legal-hold'];
@ -106,7 +86,6 @@ function compareObjectLockInformation(headers, defaultRetention) {
objectLockInfoToSave.legalHold = legalHold; objectLockInfoToSave.legalHold = legalHold;
} }
} }
return objectLockInfoToSave; return objectLockInfoToSave;
} }
@ -133,216 +112,42 @@ function setObjectLockInformation(headers, md, defaultRetention) {
} }
/** /**
* Helper class for check object lock state checks * isObjectLocked - checks whether object is locked or not
* @param {obect} bucket - bucket metadata
* @param {object} objectMD - object metadata
* @param {array} headers - request headers
* @return {boolean} - indicates whether object is locked or not
*/ */
class ObjectLockInfo { function isObjectLocked(bucket, objectMD, headers) {
/** if (bucket.isObjectLockEnabled()) {
* const objectLegalHold = objectMD.legalHold;
* @param {object} retentionInfo - The object lock retention policy if (objectLegalHold) {
* @param {"GOVERNANCE" | "COMPLIANCE" | null} retentionInfo.mode - Retention policy mode.
* @param {string} retentionInfo.date - Expiration date of retention policy. A string in ISO-8601 format
* @param {bool} retentionInfo.legalHold - Whether a legal hold is enable for the object
*/
constructor(retentionInfo) {
this.mode = retentionInfo.mode || null;
this.date = retentionInfo.date || null;
this.legalHold = retentionInfo.legalHold || false;
}
/**
* ObjectLockInfo.isLocked
* @returns {bool} - Whether the retention policy is active and protecting the object
*/
isLocked() {
if (this.legalHold) {
return true; return true;
} }
const retentionMode = objectMD.retentionMode;
if (!this.mode || !this.date) { const retentionDate = objectMD.retentionDate;
if (!retentionMode || !retentionDate) {
return false; return false;
} }
if (retentionMode === 'GOVERNANCE' &&
return !this.isExpired(); headers['x-amz-bypass-governance-retention']) {
return false;
} }
const objectDate = moment(retentionDate);
/**
* ObjectLockInfo.isGovernanceMode
* @returns {bool} - true if retention mode is GOVERNANCE
*/
isGovernanceMode() {
return this.mode === 'GOVERNANCE';
}
/**
* ObjectLockInfo.isComplianceMode
* @returns {bool} - True if retention mode is COMPLIANCE
*/
isComplianceMode() {
return this.mode === 'COMPLIANCE';
}
/**
* ObjectLockInfo.isExpired
* @returns {bool} - True if the retention policy has expired
*/
isExpired() {
const now = moment(); const now = moment();
return this.date === null || now.isSameOrAfter(this.date); // indicates retain until date has expired
} if (now.isSameOrAfter(objectDate)) {
/**
* ObjectLockInfo.isExtended
* @param {string} timestamp - Timestamp in ISO-8601 format
* @returns {bool} - True if the given timestamp is after the policy expiration date or if no expiration date is set
*/
isExtended(timestamp) {
return timestamp !== undefined && (this.date === null || moment(timestamp).isSameOrAfter(this.date));
}
/**
* ObjectLockInfo.canModifyObject
* @param {bool} hasGovernanceBypass - Whether to bypass governance retention policies
* @returns {bool} - True if the retention policy allows the objects data to be modified (overwritten/deleted)
*/
canModifyObject(hasGovernanceBypass) {
// can modify object if object is not locked
// cannot modify object in any cases if legal hold is enabled
// if no legal hold, can only modify object if bypassing governance when locked
if (!this.isLocked()) {
return true;
}
return !this.legalHold && this.isGovernanceMode() && !!hasGovernanceBypass;
}
/**
* ObjectLockInfo.canModifyPolicy
* @param {object} policyChanges - Proposed changes to the retention policy
* @param {"GOVERNANCE" | "COMPLIANCE" | undefined} policyChanges.mode - Retention policy mode.
* @param {string} policyChanges.date - Expiration date of retention policy. A string in ISO-8601 format
* @param {bool} hasGovernanceBypass - Whether to bypass governance retention policies
* @returns {bool} - True if the changes are allowed to be applied to the retention policy
*/
canModifyPolicy(policyChanges, hasGovernanceBypass) {
// If an object does not have a retention policy or it is expired then all changes are allowed
if (!this.isLocked()) {
return true;
}
// The only allowed change in compliance mode is extending the retention period
if (this.isComplianceMode()) {
if (policyChanges.mode === 'COMPLIANCE' && this.isExtended(policyChanges.date)) {
return true;
}
}
if (this.isGovernanceMode()) {
// Extensions are always allowed in governance mode
if (policyChanges.mode === 'GOVERNANCE' && this.isExtended(policyChanges.date)) {
return true;
}
// All other changes in governance mode require a bypass
if (hasGovernanceBypass) {
return true;
}
}
return false; return false;
} }
} return true;
/**
*
* @param {object} headers - s3 request headers
* @returns {bool} - True if the headers is present and === "true"
*/
function hasGovernanceBypassHeader(headers) {
const bypassHeader = headers['x-amz-bypass-governance-retention'] || '';
return bypassHeader.toLowerCase() === 'true';
}
/**
* checkUserGovernanceBypass
*
* Checks for the presence of the s3:BypassGovernanceRetention permission for a given user
*
* @param {object} request - Incoming s3 request
* @param {object} authInfo - s3 authentication info
* @param {object} bucketMD - bucket metadata
* @param {string} objectKey - object key
* @param {object} log - Werelogs logger
* @param {function} cb - callback returns errors.AccessDenied if the authorization fails
* @returns {undefined} -
*/
function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log, cb) {
log.trace(
'object in GOVERNANCE mode and is user, checking for attached policies',
{ method: 'checkUserPolicyGovernanceBypass' },
);
const authParams = auth.server.extractParams(request, log, 's3', request.query);
const ip = policies.requestUtils.getClientIp(request, config);
const requestContextParams = {
constantParams: {
headers: request.headers,
query: request.query,
generalResource: bucketMD.getName(),
specificResource: { key: objectKey },
requesterIp: ip,
sslEnabled: request.connection.encrypted,
apiMethod: 'bypassGovernanceRetention',
awsService: 's3',
locationConstraint: bucketMD.getLocationConstraint(),
requesterInfo: authInfo,
signatureVersion: authParams.params.data.signatureVersion,
authType: authParams.params.data.authType,
signatureAge: authParams.params.data.signatureAge,
},
};
return vault.checkPolicies(requestContextParams,
authInfo.getArn(), log, (err, authorizationResults) => {
if (err) {
return cb(err);
} }
const explicitDenyExists = authorizationResults.some( return false;
authzResult => authzResult.isAllowed === false && !authzResult.isImplicit);
if (explicitDenyExists) {
log.trace('authorization check failed for user',
{
'method': 'checkUserPolicyGovernanceBypass',
's3:BypassGovernanceRetention': false,
});
return cb(errors.AccessDenied);
}
// Convert authorization results into an easier to handle format
const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => {
const apiMethod = authorizationResults[idx].action;
// eslint-disable-next-line no-param-reassign
acc[apiMethod] = curr.isImplicit;
return acc;
}, {});
// Evaluate against the bucket policies
const areAllActionsAllowed = evaluateBucketPolicyWithIAM(
bucketMD,
Object.keys(actionImplicitDenies),
authInfo.getCanonicalID(),
authInfo,
actionImplicitDenies,
log,
request);
return cb(areAllActionsAllowed === true ? null : errors.AccessDenied);
});
} }
module.exports = { module.exports = {
calculateRetainUntilDate, calculateRetainUntilDate,
compareObjectLockInformation, compareObjectLockInformation,
setObjectLockInformation, setObjectLockInformation,
isObjectLocked,
validateHeaders, validateHeaders,
hasGovernanceBypassHeader,
checkUserGovernanceBypass,
ObjectLockInfo,
}; };

View File

@ -1,172 +1,316 @@
const async = require('async'); const async = require('async');
const { errors, s3middleware } = require('arsenal');
const { allowedRestoreObjectRequestTierValues } = require('../../../../constants'); const { errors } = require('arsenal');
const ObjectMD = require('arsenal').models.ObjectMD;
const coldStorage = require('./coldStorage'); const coldStorage = require('./coldStorage');
const monitoring = require('../../../utilities/monitoringHandler');
const { pushMetric } = require('../../../utapi/utilities');
const { decodeVersionId } = require('./versioning');
const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
const { parseRestoreRequestXml } = s3middleware.objectRestore;
const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
/**
* Check if tier is supported const METHOD = 'objectRestore';
* @param {object} restoreInfo - restore information
* @returns {ArsenalError|undefined} return NotImplemented error if tier not support
*/
function checkTierSupported(restoreInfo) {
if (!allowedRestoreObjectRequestTierValues.includes(restoreInfo.tier)) {
return errors.NotImplemented;
}
return undefined;
}
/** /**
* POST Object restore process * POST Object restore process
* *
* @param {MetadataWrapper} metadata - metadata wrapper * @param {MetadataWrapper} metadata metadata wrapper
* @param {object} mdUtils - utility object to treat metadata * @param {object} mdUtils utility object to treat metadata
* @param {AuthInfo} userInfo - Instance of AuthInfo class with requester's info * @param {object} func object with a reference to each function of cloudserver
* @param {IncomingMessage} request - request info * @param {function(object):string|Error} func.decodeVersionId
* @param {object} log - Werelogs logger * @param {function(object, string, BucketInfo):object} func.collectCorsHeaders
* @param {function} callback callback function * @param {function(object, object):string} func.getVersionIdResHeader
* @return {undefined} * @param {AuthInfo} userInfo Instance of AuthInfo class with requester's info
* @param {IncomingMessage} request request info
* @param {werelogs.Logger} log Werelogs instance
* @param {module:api/objectRestore~NoBodyResultCallback} callback callback function
* @return {void}
*/ */
function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { function objectRestore(metadata, mdUtils, func, userInfo, request, log, callback) {
const METHOD = 'objectRestore';
const { bucketName, objectKey } = request; const { bucketName, objectKey } = request;
const requestedAt = request['x-sdt-requested-at'];
log.debug('processing request', { method: METHOD }); log.debug('processing request', { method: METHOD });
const decodedVidResult = decodeVersionId(request.query); const decodedVidResult = func.decodeVersionId(request.query);
if (decodedVidResult instanceof Error) { if (decodedVidResult instanceof Error) {
log.trace('invalid versionId query', log.trace('invalid versionId query',
{ { method: METHOD, versionId: request.query.versionId, error: decodedVidResult });
method: METHOD, return callback(decodedVidResult, decodedVidResult.code);
versionId: request.query.versionId,
error: decodedVidResult,
});
return process.nextTick(() => callback(decodedVidResult));
} }
let isObjectRestored = false; const reqVersionId = decodedVidResult;
const mdValueParams = { const mdValueParams = {
authInfo: userInfo, authInfo: userInfo,
bucketName, bucketName,
objectKey, objectKey,
versionId: decodedVidResult, versionId: reqVersionId,
requestType: request.apiMethods || 'restoreObject', requestType: 'bucketOwnerAction',
/**
* Restoring an object might not cause any impact on
* the storage, if the object is already restored: in
* this case, the duration is extended. We disable the
* quota evaluation and trigger it manually.
*/
checkQuota: false,
request,
}; };
return async.waterfall([ return async.waterfall([
// get metadata of bucket and object // get metadata of bucket and object
function validateBucketAndObject(next) { function validateBucketAndObject(next) {
return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies,
log, (err, bucketMD, objectMD) => { return mdUtils.metadataValidateBucketAndObj(mdValueParams, log, (err, bucketMD, objectMD) => {
if (err) { if (err) {
log.trace('request authorization failed', { method: METHOD, error: err }); log.trace('request authorization failed', { method: METHOD, error: err });
return next(err); return next(err);
} }
// Call back error if object metadata could not be obtained // Call back error if object metadata could not be obtained
if (!objectMD) { if (!objectMD) {
const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey; const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey;
log.trace('error no object metadata found', { method: METHOD, error: err }); log.trace('error no object metadata found', { method: METHOD, error: err });
return next(err, bucketMD); return next(err, bucketMD);
} }
const instance = new ObjectMD(objectMD);
// If object metadata is delete marker, // If object metadata is delete marker,
// call back NoSuchKey or MethodNotAllowed depending on specifying versionId // call back NoSuchKey or MethodNotAllowed depending on specifying versionId
if (objectMD.isDeleteMarker) { if (objectMD.isDeleteMarker) {
let err = errors.NoSuchKey; let err = errors.NoSuchKey;
if (decodedVidResult) { if (reqVersionId) {
err = errors.MethodNotAllowed; err = errors.MethodNotAllowed;
} }
log.trace('version is a delete marker', { method: METHOD, error: err }); log.trace('version is a delete marker', { method: METHOD, error: err });
return next(err, bucketMD, objectMD); return next(err, bucketMD, instance);
} }
log.info('it acquired the object metadata.', { log.info('it acquired the object metadata.', {
'method': METHOD, 'method': METHOD,
'x-coldstorage-uuid': instance.getColdstorageUuid(),
'x-coldstorage-zenko-id': instance.getColdstorageZenkoId(),
}); });
return next(null, bucketMD, objectMD);
return next(null, bucketMD, instance);
}); });
}, },
// generate restore param obj from xml of request body and check tier validity // generate restore param obj from xml of request body
function parseRequestXmlAndCheckTier(bucketMD, objectMD, next) { function parseRequestXml(bucketMD, objectMD, next) {
log.trace('parsing object restore information');
return parseRestoreRequestXml(request.post, log, (err, restoreInfo) => { return parsePostObjectRestoreXml(request.post, log, (err, params) => {
if (err) { if (err) {
return next(err, bucketMD, objectMD, restoreInfo);
}
log.info('it parsed xml of the request body.', { method: METHOD, value: restoreInfo });
const checkTierResult = checkTierSupported(restoreInfo);
if (checkTierResult instanceof Error) {
return next(checkTierResult);
}
return next(null, bucketMD, objectMD, restoreInfo);
});
},
// start restore process
function startRestore(bucketMD, objectMD, restoreInfo, next) {
return coldStorage.startRestore(objectMD, restoreInfo, log,
(err, _isObjectRestored) => {
isObjectRestored = _isObjectRestored;
return next(err, bucketMD, objectMD); return next(err, bucketMD, objectMD);
}
log.info('it parsed xml of the request body.', { method: METHOD, value: params });
return next(null, bucketMD, objectMD, params);
}); });
}, },
function evaluateQuotas(bucketMD, objectMD, next) {
if (isObjectRestored) { // start restore process
return next(null, bucketMD, objectMD); function startRestore(bucketMD, objectMD, next) {
} return coldStorage.startRestore(bucketName, objectKey, objectMD, params,
const actions = Array.isArray(mdValueParams.requestType) ? (err, result) => next(err, bucketMD, objectMD, result));
mdValueParams.requestType : [mdValueParams.requestType];
const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
false, log, err => next(err, bucketMD, objectMD));
},
function updateObjectMD(bucketMD, objectMD, next) {
const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
log, err => next(err, bucketMD, objectMD));
}, },
], ],
(err, bucketMD) => { (err, bucketMD, objectMD, result) => {
// generate CORS response header // generate CORS response header
const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD); const responseHeaders = func.collectCorsHeaders(request.headers.origin, request.method, bucketMD);
if (err) { if (err) {
log.trace('error processing request', log.trace('error processing request', { method: METHOD, error: err });
{
method: METHOD, // If object metadata is delete marker and error is MethodNotAllowed,
error: err, // set response header of x-amz-delete-marker and x-amz-version-id (S3 API compliant)
}); if (objectMD && objectMD.getIsDeleteMarker() && err.MethodNotAllowed) {
monitoring.promMetrics( const vConfig = bucketMD.getVersioningConfiguration();
'POST', bucketName, err.code, 'restoreObject'); responseHeaders['x-amz-delete-marker'] = true;
responseHeaders['x-amz-version-id'] = func.getVersionIdResHeader(vConfig, objectMD.getValue());
}
return callback(err, err.code, responseHeaders); return callback(err, err.code, responseHeaders);
} }
pushMetric('restoreObject', log, {
userInfo, // If versioning configuration is setting, set response header of x-amz-version-id
bucket: bucketName, const vConfig = bucketMD.getVersioningConfiguration();
responseHeaders['x-amz-version-id'] = func.getVersionIdResHeader(vConfig, objectMD.getValue());
return callback(null, result.statusCode, responseHeaders);
}); });
if (isObjectRestored) {
monitoring.promMetrics( /**
'POST', bucketName, '200', 'restoreObject'); * Generate request parameter object by parsing XML ofrequest body
return callback(null, 200, responseHeaders); *
* @param {convertableToString} xml XML of request body
* @param {werelogs.Logger} log logger
* @param {module:api/utils~ObjectResultCallback} callback callback function
* @returns {void}
*/
function parsePostObjectRestoreXml(xml, log, callback) {
log.debug('parsing xml string of request body.', alCreateLogParams(
this, this.parsePostObjectRestoreXml, {
xmlString: xml,
// eslint-disable-next-line comma-dangle
} }
monitoring.promMetrics( ));
'POST', bucketName, '202', 'restoreObject');
return callback(null, 202, responseHeaders); return xml2js.parseString(xml, { explicitArray: false }, (err, result) => {
// If cause an error, callback MalformedXML
if (err) {
log.info('parse xml string of request body was failed.', { error: err });
return callback(errors.MalformedXML);
}
// If restore parameter is invalid, callback MalformedXML
const validateResult = validateRestoreRequestParameters(result);
if (validateResult) {
log.info('invalid restore parameters.', { error: validateResult.message });
return callback(errors.MalformedXML);
}
// normalize restore request parameters
const normalizedResult = normalizeRestoreRequestParameters(result);
log.debug('parse xml string of request body.', alCreateLogParams(
this, this.parsePostObjectRestoreXml, {
resultObject: normalizedResult,
// eslint-disable-next-line comma-dangle
}
));
return callback(null, normalizedResult);
}); });
};
/**
* validate restore parameter object
*
* @private
* @param {object} params restore parameter object
* @returns {Error} Error instance
*/
function validateRestoreRequestParameters(params) {
if (!params) {
return new Error('request body is required.');
}
const rootElem = getSafeValue(params, 'RestoreRequest');
if (!rootElem) {
return new Error('RestoreRequest element is required.');
}
if (!rootElem['Days']) {
return new Error('RestoreRequest.Days element is required.');
}
// RestoreRequest.Days must be greater than or equal to 1
const daysValue = Number.parseInt(rootElem['Days'], 10);
if (Number.isNaN(daysValue)) {
return new Error(`RestoreRequest.Days is invalid type. [${rootElem['Days']}]`);
}
if (daysValue < 1) {
return new Error(`RestoreRequest.Days must be greater than 0. [${rootElem['Days']}]`);
}
if (daysValue > 2147483647) {
return new Error(`RestoreRequest.Days must be less than 2147483648. [${rootElem['Days']}]`);
}
// If RestoreRequest.GlacierJobParameters.Tier is specified,
// Must be "Expedited" or "Standard" or "Bulk"
const tierValue = getSafeValue(rootElem,
'GlacierJobParameters', 'Tier');
const tierList = {
EXPEDITED: 'Expedited',
STANDARD: 'Standard',
BULK: 'Bulk',
}
const tierConstants = getValues(tierList);
if (tierValue && !tierConstants.includes(tierValue)) {
return new Error(`RestoreRequest.GlacierJobParameters.Tier is invalid value. [${tierValue}]`);
}
return undefined;
}
/**
* Normalize restore request parameters.
*
* @private
* @param {object} params restore request parameters object
* @return {object} restore request parameters object(normalized)
*/
function normalizeRestoreRequestParameters(params) {
const normalizedParams = {};
// set RestoreRequest.Days
const rootElem = getSafeValue(params, 'RestoreRequest');
const daysValue = Number.parseInt(rootElem['Days'], 10);
setSafeValue(normalizedParams, daysValue, 'Days');
// set RestoreRequest.GlacierJobParameters.Tier
// If do not specify, set "Standard"
const tierValue = getSafeValue(rootElem,
'GlacierJobParameters', 'Tier')
|| 'Standard';
setSafeValue(normalizedParams, tierValue,
'GlacierJobParameters', 'Tier');
return normalizedParams;
}
/**
* Attribute values that the object has are returned as an array.
* Node v6 does not support Object.values, so prepare a function with the same result.
*
* @param {object} obj object
* @returns {Array<object>} UTC date infomation(string)
*/
function getValues(obj) {
const results = [];
Object.keys(obj).forEach(key => {
results.push(obj[key]);
});
return results;
}
/**
* For layered objects, safely get the value corresponding to the key passed in the variable length argument.
*
* @param {object} obj object
* @param {...string} args array of keys
* @returns {object}
*/
function getSafeValue(obj, ...args) {
let result = obj;
if (!result || !Array.isArray(args) || args.length === 0) {
return result;
}
args.some(value => {
result = result[value];
return !result;
});
return result;
}
} }
module.exports = objectRestore; module.exports = {
objectRestore,
};

View File

@ -54,21 +54,7 @@ function getPartSize(objMD, partNumber) {
return size; return size;
} }
/**
* Gets parts count if object was put with mpu
* @param {object} objMD - object metadata
* @return {(Integer|null)} - number of parts of mpu object or null
*/
function getPartCountFromMd5(objMD) {
const md5 = objMD['content-md5'];
if (md5.includes('-')) {
return md5.split('-')[1];
}
return null;
}
module.exports = { module.exports = {
getPartNumber, getPartNumber,
getPartSize, getPartSize,
getPartCountFromMd5,
}; };

View File

@ -2,7 +2,7 @@ const { errors } = require('arsenal');
const { const {
parseRangeSpec, parseRangeSpec,
parseRange, parseRange,
} = require('arsenal').network.http.utils; } = require('arsenal/lib/network/http/utils');
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const setPartRanges = require('./setPartRanges'); const setPartRanges = require('./setPartRanges');

View File

@ -1,32 +0,0 @@
const { errors } = require('arsenal');
const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants');
function validateChecksumHeaders(headers) {
// If the x-amz-trailer header is present the request is using one of the
// trailing checksum algorithms, which are not supported.
if (headers['x-amz-trailer'] !== undefined) {
return errors.BadRequest.customizeDescription('trailing checksum is not supported');
}
const signatureChecksum = headers['x-amz-content-sha256'];
if (signatureChecksum === undefined) {
return null;
}
if (supportedSignatureChecksums.has(signatureChecksum)) {
return null;
}
// If the value is not one of the possible checksum algorithms
// the only other valid value is the actual sha256 checksum of the payload.
// Do a simple sanity check of the length to guard against future algos.
// If the value is an unknown algo, then it will fail checksum validation.
if (!unsupportedSignatureChecksums.has(signatureChecksum) && signatureChecksum.length === 64) {
return null;
}
return errors.BadRequest.customizeDescription('unsupported checksum algorithm');
}
module.exports = validateChecksumHeaders;

View File

@ -4,40 +4,13 @@ const async = require('async');
const metadata = require('../../../metadata/wrapper'); const metadata = require('../../../metadata/wrapper');
const { config } = require('../../../Config'); const { config } = require('../../../Config');
const { scaledMsPerDay } = config.getTimeOptions();
const versionIdUtils = versioning.VersionID; const versionIdUtils = versioning.VersionID;
// Use Arsenal function to generate a version ID used internally by metadata // Use Arsenal function to generate a version ID used internally by metadata
// for null versions that are created before bucket versioning is configured // for null versions that are created before bucket versioning is configured
const nonVersionedObjId = const nonVersionedObjId =
versionIdUtils.getInfVid(config.replicationGroupId); versionIdUtils.getInfVid(config.replicationGroupId);
/** decodeVID - decode the version id /** decodedVidResult - decode the version id from a query object
* @param {string} versionId - version ID
* @return {(Error|string|undefined)} - return Invalid Argument if decryption
* fails due to improper format, otherwise undefined or the decoded version id
*/
function decodeVID(versionId) {
if (versionId === 'null') {
return versionId;
}
let decoded;
const invalidErr = errors.InvalidArgument.customizeDescription('Invalid version id specified');
try {
decoded = versionIdUtils.decode(versionId);
} catch (err) {
return invalidErr;
}
if (decoded instanceof Error) {
return invalidErr;
}
return decoded;
}
/** decodeVersionId - decode the version id from a query object
* @param {object} [reqQuery] - request query object * @param {object} [reqQuery] - request query object
* @param {string} [reqQuery.versionId] - version ID sent in request query * @param {string} [reqQuery.versionId] - version ID sent in request query
* @return {(Error|string|undefined)} - return Invalid Argument if decryption * @return {(Error|string|undefined)} - return Invalid Argument if decryption
@ -47,7 +20,16 @@ function decodeVersionId(reqQuery) {
if (!reqQuery || !reqQuery.versionId) { if (!reqQuery || !reqQuery.versionId) {
return undefined; return undefined;
} }
return decodeVID(reqQuery.versionId); let versionId = reqQuery.versionId;
if (versionId === 'null') {
return versionId;
}
versionId = versionIdUtils.decode(versionId);
if (versionId instanceof Error) {
return errors.InvalidArgument
.customizeDescription('Invalid version id specified');
}
return versionId;
} }
/** getVersionIdResHeader - return encrypted version ID if appropriate /** getVersionIdResHeader - return encrypted version ID if appropriate
@ -58,7 +40,7 @@ function decodeVersionId(reqQuery) {
*/ */
function getVersionIdResHeader(verCfg, objectMD) { function getVersionIdResHeader(verCfg, objectMD) {
if (verCfg) { if (verCfg) {
if (objectMD.isNull || !objectMD.versionId) { if (objectMD.isNull || (objectMD && !objectMD.versionId)) {
return 'null'; return 'null';
} }
return versionIdUtils.encode(objectMD.versionId); return versionIdUtils.encode(objectMD.versionId);
@ -79,34 +61,17 @@ function checkQueryVersionId(query) {
return undefined; return undefined;
} }
function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb) { function _storeNullVersionMD(bucketName, objKey, objMD, options, log, cb) {
// In compatibility mode, create null versioned keys instead of null keys metadata.putObjectMD(bucketName, objKey, objMD, options, log, err => {
let versionId;
let nullVersionMD;
if (config.nullVersionCompatMode) {
versionId = nullVersionId;
nullVersionMD = Object.assign({}, objMD, {
versionId: nullVersionId,
isNull: true,
});
} else {
versionId = 'null';
nullVersionMD = Object.assign({}, objMD, {
versionId: nullVersionId,
isNull: true,
isNull2: true,
});
}
metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => {
if (err) { if (err) {
log.debug('error from metadata storing null version as new version', log.debug('error from metadata storing null version as new version',
{ error: err }); { error: err });
} }
cb(err); cb(err, options);
}); });
} }
/** check existence and get location of null version data for deletion /** get location of null version data for deletion
* @param {string} bucketName - name of bucket * @param {string} bucketName - name of bucket
* @param {string} objKey - name of object key * @param {string} objKey - name of object key
* @param {object} options - metadata options for getting object MD * @param {object} options - metadata options for getting object MD
@ -117,179 +82,100 @@ function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb)
* @param {function} cb - callback * @param {function} cb - callback
* @return {undefined} - and call callback with (err, dataToDelete) * @return {undefined} - and call callback with (err, dataToDelete)
*/ */
function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) { function _getNullVersionsToDelete(bucketName, objKey, options, mst, log, cb) {
const nullOptions = {};
if (!options.deleteData) {
return process.nextTick(cb, null, nullOptions);
}
if (options.versionId === mst.versionId) { if (options.versionId === mst.versionId) {
// no need to get another key as the master is the target // no need to get delete location, we already have the master's metadata
nullOptions.dataToDelete = mst.objLocation; const dataToDelete = mst.objLocation;
return process.nextTick(cb, null, nullOptions); return process.nextTick(cb, null, dataToDelete);
}
if (options.versionId === 'null') {
// deletion of the null key will be done by the main metadata
// PUT via this option
nullOptions.deleteNullKey = true;
} }
return metadata.getObjectMD(bucketName, objKey, options, log, return metadata.getObjectMD(bucketName, objKey, options, log,
(err, versionMD) => { (err, versionMD) => {
if (err) { if (err) {
// the null key may not exist, hence it's a normal log.debug('err from metadata getting specified version', {
// situation to have a NoSuchKey error, in which case
// there is nothing to delete
if (err.is.NoSuchKey) {
log.debug('null version does not exist', {
method: '_prepareNullVersionDeletion',
});
} else {
log.warn('could not get null version metadata', {
error: err, error: err,
method: '_prepareNullVersionDeletion', method: '_getNullVersionsToDelete',
}); });
}
return cb(err); return cb(err);
} }
if (versionMD.location) { if (!versionMD.location) {
return cb();
}
const dataToDelete = Array.isArray(versionMD.location) ? const dataToDelete = Array.isArray(versionMD.location) ?
versionMD.location : [versionMD.location]; versionMD.location : [versionMD.location];
nullOptions.dataToDelete = dataToDelete; return cb(null, dataToDelete);
}
return cb(null, nullOptions);
}); });
} }
function _deleteNullVersionMD(bucketName, objKey, options, log, cb) { function _deleteNullVersionMD(bucketName, objKey, options, mst, log, cb) {
return metadata.deleteObjectMD(bucketName, objKey, options, log, err => { return _getNullVersionsToDelete(bucketName, objKey, options, mst, log,
(err, nullDataToDelete) => {
if (err) { if (err) {
log.warn('metadata error deleting null versioned key', log.warn('could not find null version metadata', {
{ bucketName, objKey, error: err, method: '_deleteNullVersionMD' }); error: err,
} method: '_deleteNullVersionMD',
});
return cb(err); return cb(err);
}
return metadata.deleteObjectMD(bucketName, objKey, options, log,
err => {
if (err) {
log.warn('metadata error deleting null version',
{ error: err, method: '_deleteNullVersionMD' });
return cb(err);
}
return cb(null, nullDataToDelete);
});
}); });
} }
/** function processVersioningState(mst, vstat, cb) {
* Process state from the master version of an object and the bucket const options = {};
* versioning configuration, return a set of options objects const storeOptions = {};
* const delOptions = {};
* @param {object} mst - state of master version, as returned by // object does not exist or is not versioned (before versioning)
* getMasterState() if (mst.versionId === undefined || mst.isNull) {
* @param {string} vstat - bucket versioning status: 'Enabled' or 'Suspended' // versioning is suspended, overwrite existing master version
* @param {boolean} nullVersionCompatMode - if true, behaves in null if (vstat === 'Suspended') {
* version compatibility mode and return appropriate values: this mode options.versionId = '';
* does not attempt to create null keys but create null versioned keys options.isNull = true;
* instead
*
* @return {object} result object with the following attributes:
* - {object} options: versioning-related options to pass to the
services.metadataStoreObject() call
* - {object} [options.extraMD]: extra attributes to set in object metadata
* - {string} [nullVersionId]: null version key to create, if needed
* - {object} [delOptions]: options for metadata to delete the null
version key, if needed
*/
function processVersioningState(mst, vstat, nullVersionCompatMode) {
const versioningSuspended = (vstat === 'Suspended');
const masterIsNull = mst.exists && (mst.isNull || !mst.versionId);
if (versioningSuspended) {
// versioning is suspended: overwrite the existing null version
const options = { versionId: '', isNull: true };
if (masterIsNull) {
// if the null version exists, clean it up prior to put
if (mst.objLocation) {
options.dataToDelete = mst.objLocation; options.dataToDelete = mst.objLocation;
// if null version exists, clean it up prior to put
if (mst.isNull) {
delOptions.versionId = mst.versionId;
return cb(null, options, null, delOptions);
} }
// backward-compat: a null version key may exist even with return cb(null, options);
// a null master (due to S3C-7526), if so, delete it (its
// data will be deleted as part of the master cleanup, so
// no "deleteData" param is needed)
//
// "isNull2" attribute is set in master metadata when
// null keys are used, which is used as an optimization to
// avoid having to check the versioned key since there can
// be no more versioned key to clean up
if (mst.isNull && mst.versionId && !mst.isNull2) {
const delOptions = { versionId: mst.versionId };
return { options, delOptions };
} }
return { options }; // versioning is enabled, create a new version
options.versioning = true;
if (mst.exists) {
// store master version in a new key
const versionId = mst.isNull ? mst.versionId : nonVersionedObjId;
storeOptions.versionId = versionId;
storeOptions.isNull = true;
options.nullVersionId = versionId;
return cb(null, options, storeOptions);
} }
if (mst.nullVersionId) { return cb(null, options);
// backward-compat: delete the null versioned key and data
const delOptions = { versionId: mst.nullVersionId, deleteData: true };
if (mst.nullUploadId) {
delOptions.replayId = mst.nullUploadId;
} }
return { options, delOptions }; // master is versioned and is not a null version
const nullVersionId = mst.nullVersionId;
if (vstat === 'Suspended') {
// versioning is suspended, overwrite the existing master version
options.versionId = '';
options.isNull = true;
if (nullVersionId === undefined) {
return cb(null, options);
} }
// clean up the eventual null key's location data prior to put delOptions.versionId = nullVersionId;
return cb(null, options, null, delOptions);
// NOTE: due to metadata v1 internal format, we cannot guess
// from the master key whether there is an associated null
// key, because the master key may be removed whenever the
// latest version becomes a delete marker. Hence we need to
// pessimistically try to get the null key metadata and delete
// it if it exists.
const delOptions = { versionId: 'null', deleteData: true };
return { options, delOptions };
} }
// versioning is enabled, put the new version
// versioning is enabled: create a new version options.versioning = true;
const options = { versioning: true }; options.nullVersionId = nullVersionId;
if (masterIsNull) { return cb(null, options);
// if master is a null version or a non-versioned key,
// copy it to a new null key
const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
if (nullVersionCompatMode) {
options.extraMD = {
nullVersionId,
};
if (mst.uploadId) {
options.extraMD.nullUploadId = mst.uploadId;
}
return { options, nullVersionId };
}
if (mst.isNull && !mst.isNull2) {
// if master null version was put with an older
// Cloudserver (or in compat mode), there is a
// possibility that it also has a null versioned key
// associated, so we need to delete it as we write the
// null key
const delOptions = {
versionId: nullVersionId,
};
return { options, nullVersionId, delOptions };
}
return { options, nullVersionId };
}
// backward-compat: keep a reference to the existing null
// versioned key
if (mst.nullVersionId) {
options.extraMD = {
nullVersionId: mst.nullVersionId,
};
if (mst.nullUploadId) {
options.extraMD.nullUploadId = mst.nullUploadId;
}
}
return { options };
} }
/**
* Build the state of the master version from its object metadata
*
* @param {object} objMD - object metadata parsed from JSON
*
* @return {object} state of master version, with the following attributes:
* - {boolean} exists - true if the object exists (i.e. if `objMD` is truish)
* - {string} versionId - version ID of the master key
* - {boolean} isNull - whether the master version is a null version
* - {string} nullVersionId - if not a null version, reference to the
* null version ID
* - {array} objLocation - array of data locations
*/
function getMasterState(objMD) { function getMasterState(objMD) {
if (!objMD) { if (!objMD) {
return {}; return {};
@ -297,11 +183,8 @@ function getMasterState(objMD) {
const mst = { const mst = {
exists: true, exists: true,
versionId: objMD.versionId, versionId: objMD.versionId,
uploadId: objMD.uploadId,
isNull: objMD.isNull, isNull: objMD.isNull,
isNull2: objMD.isNull2,
nullVersionId: objMD.nullVersionId, nullVersionId: objMD.nullVersionId,
nullUploadId: objMD.nullUploadId,
}; };
if (objMD.location) { if (objMD.location) {
mst.objLocation = Array.isArray(objMD.location) ? mst.objLocation = Array.isArray(objMD.location) ?
@ -323,113 +206,62 @@ function getMasterState(objMD) {
* ('' overwrites the master version) * ('' overwrites the master version)
* options.versioning - (true/undefined) metadata instruction to create new ver * options.versioning - (true/undefined) metadata instruction to create new ver
* options.isNull - (true/undefined) whether new version is null or not * options.isNull - (true/undefined) whether new version is null or not
* options.nullVersionId - if storing a null version in version history, the
* version id of the null version
* options.deleteNullVersionData - whether to delete the data of the null ver
*/ */
function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
log, callback) { log, callback) {
const options = {};
const mst = getMasterState(objMD); const mst = getMasterState(objMD);
const vCfg = bucketMD.getVersioningConfiguration(); const vCfg = bucketMD.getVersioningConfiguration();
// bucket is not versioning configured // bucket is not versioning configured
if (!vCfg) { if (!vCfg) {
const options = { dataToDelete: mst.objLocation }; options.dataToDelete = mst.objLocation;
return process.nextTick(callback, null, options); return process.nextTick(callback, null, options);
} }
// bucket is versioning configured // bucket is versioning configured
const { options, nullVersionId, delOptions } = return async.waterfall([
processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode); function processState(next) {
return async.series([ processVersioningState(mst, vCfg.Status,
function storeNullVersionMD(next) { (err, options, storeOptions, delOptions) => {
if (!nullVersionId) { process.nextTick(next, err, options, storeOptions,
return process.nextTick(next); delOptions);
} });
return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
}, },
function prepareNullVersionDeletion(next) { function storeVersion(options, storeOptions, delOptions, next) {
if (!storeOptions) {
return process.nextTick(next, null, options, delOptions);
}
const versionMD = Object.assign({}, objMD, storeOptions);
const params = { versionId: storeOptions.versionId };
return _storeNullVersionMD(bucketName, objectKey, versionMD,
params, log, err => next(err, options, delOptions));
},
function deleteNullVersion(options, delOptions, next) {
if (!delOptions) { if (!delOptions) {
return process.nextTick(next); return process.nextTick(next, null, options);
} }
return _prepareNullVersionDeletion( return _deleteNullVersionMD(bucketName, objectKey, delOptions, mst,
bucketName, objectKey, delOptions, mst, log, log, (err, nullDataToDelete) => {
(err, nullOptions) => {
if (err) { if (err) {
return next(err); log.warn('unexpected error deleting null version md', {
error: err,
method: 'versioningPreprocessing',
});
// it's possible there was a concurrent request to
// delete the null version, so proceed with putting a
// new version
if (err === errors.NoSuchKey) {
return next(null, options);
} }
Object.assign(options, nullOptions); return next(errors.InternalError);
return next(); }
Object.assign(options, { dataToDelete: nullDataToDelete });
return next(null, options);
}); });
}, },
function deleteNullVersionMD(next) { ], (err, options) => callback(err, options));
if (delOptions &&
delOptions.versionId &&
delOptions.versionId !== 'null') {
// backward-compat: delete old null versioned key
return _deleteNullVersionMD(
bucketName, objectKey, { versionId: delOptions.versionId }, log, next);
}
return process.nextTick(next);
},
], err => {
// it's possible there was a prior request that deleted the
// null version, so proceed with putting a new version
if (err && err.is.NoSuchKey) {
return callback(null, options);
}
return callback(err, options);
});
}
/** Return options to pass to Metadata layer for version-specific
* operations with the given requested version ID
*
* @param {object} objectMD - object metadata
* @param {boolean} nullVersionCompatMode - if true, behaves in null
* version compatibility mode
* @return {object} options object with params:
* {string} [options.versionId] - specific versionId to update
* {boolean} [options.isNull=true|false|undefined] - if set, tells the
* Metadata backend if we're updating or deleting a new-style null
* version (stored in master or null key), or not a null version.
*/
function getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode) {
// Use the internal versionId if it is a "real" null version (not
// non-versioned)
//
// If the target object is non-versioned: do not specify a
// "versionId" attribute nor "isNull"
//
// If the target version is a null version, i.e. has the "isNull"
// attribute:
//
// - send the "isNull=true" param to Metadata if the version is
// already a null key put by a non-compat mode Cloudserver, to
// let Metadata know that the null key is to be updated or
// deleted. This is the case if the "isNull2" metadata attribute
// exists
//
// - otherwise, do not send the "isNull" parameter to hint
// Metadata that it is a legacy null version
//
// If the target version is not a null version and is versioned:
//
// - send the "isNull=false" param to Metadata in non-compat
// mode (mandatory for v1 format)
//
// - otherwise, do not send the "isNull" parameter to hint
// Metadata that an existing null version may not be stored in a
// null key
//
//
if (objectMD.versionId === undefined) {
return {};
}
const options = { versionId: objectMD.versionId };
if (objectMD.isNull) {
if (objectMD.isNull2) {
options.isNull = true;
}
} else if (!nullVersionCompatMode) {
options.isNull = false;
}
return options;
} }
/** preprocessingVersioningDelete - return versioning information for S3 to /** preprocessingVersioningDelete - return versioning information for S3 to
@ -438,124 +270,59 @@ function getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode) {
* @param {object} bucketMD - bucket metadata * @param {object} bucketMD - bucket metadata
* @param {object} objectMD - obj metadata * @param {object} objectMD - obj metadata
* @param {string} [reqVersionId] - specific version ID sent as part of request * @param {string} [reqVersionId] - specific version ID sent as part of request
* @param {boolean} nullVersionCompatMode - if true, behaves in null version compatibility mode * @param {RequestLogger} log - logger instance
* @return {object} options object with params: * @param {function} callback - callback
* {boolean} [options.deleteData=true|undefined] - whether to delete data (if undefined * @return {undefined} and call callback with params (err, options):
* options.deleteData - (true/undefined) whether to delete data (if undefined
* means creating a delete marker instead) * means creating a delete marker instead)
* {string} [options.versionId] - specific versionId to delete * options.versionId - specific versionId to delete
* {boolean} [options.isNull=true|false|undefined] - if set, tells the
* Metadata backend if we're deleting a new-style null version (stored
* in master or null key), or not a null version.
*/ */
function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersionId, nullVersionCompatMode) { function preprocessingVersioningDelete(bucketName, bucketMD, objectMD,
let options = {}; reqVersionId, log, callback) {
if (bucketMD.getVersioningConfiguration() && reqVersionId) { const options = {};
options = getVersionSpecificMetadataOptions(objectMD, nullVersionCompatMode); // bucket is not versioning enabled
} if (!bucketMD.getVersioningConfiguration()) {
if (!bucketMD.getVersioningConfiguration() || reqVersionId) {
// delete data if bucket is non-versioned or the request
// deletes a specific version
options.deleteData = true; options.deleteData = true;
return callback(null, options);
} }
return options; // bucket is versioning enabled
} if (reqVersionId && reqVersionId !== 'null') {
// deleting a specific version
/** options.deleteData = true;
* Keep metadatas when the object is restored from cold storage options.versionId = reqVersionId;
* but remove the specific ones we don't want to keep return callback(null, options);
* @param {object} objMD - obj metadata
* @param {object} metadataStoreParams - custom built object containing resource details.
* @return {undefined}
*/
function restoreMetadata(objMD, metadataStoreParams) {
/* eslint-disable no-param-reassign */
const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt'];
// We need to keep user metadata and tags
Object.keys(objMD).forEach(key => {
if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) {
metadataStoreParams.metaHeaders[key] = objMD[key];
} }
}); if (reqVersionId) {
// deleting the 'null' version if it exists
if (objMD['x-amz-website-redirect-location']) { if (objectMD.versionId === undefined) {
if (!metadataStoreParams.headers) { // object is not versioned, deleting it
metadataStoreParams.headers = {}; options.deleteData = true;
return callback(null, options);
} }
metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location']; if (objectMD.isNull) {
// master is the null version
options.deleteData = true;
options.versionId = objectMD.versionId;
return callback(null, options);
} }
if (objectMD.nullVersionId) {
if (objMD.replicationInfo) { // null version exists, deleting it
metadataStoreParams.replicationInfo = objMD.replicationInfo; options.deleteData = true;
options.versionId = objectMD.nullVersionId;
return callback(null, options);
} }
// null version does not exist, no deletion
if (objMD.legalHold) { // TODO check AWS behaviour for no deletion (seems having no error)
metadataStoreParams.legalHold = objMD.legalHold; return callback(errors.NoSuchKey);
} }
// not deleting any specific version, making a delete marker instead
if (objMD.acl) { return callback(null, options);
metadataStoreParams.acl = objMD.acl;
}
metadataStoreParams.creationTime = objMD['creation-time'];
metadataStoreParams.lastModifiedDate = objMD['last-modified'];
metadataStoreParams.taggingCopy = objMD.tags;
}
/** overwritingVersioning - return versioning information for S3 to handle
* storing version metadata with a specific version id.
* @param {object} objMD - obj metadata
* @param {object} metadataStoreParams - custom built object containing resource details.
* @return {object} options
* options.versionId - specific versionId to overwrite in metadata
* options.isNull - (true/undefined) whether new version is null or not
* options.nullVersionId - if storing a null version in version history, the
* version id of the null version
*/
function overwritingVersioning(objMD, metadataStoreParams) {
metadataStoreParams.updateMicroVersionId = true;
metadataStoreParams.amzStorageClass = objMD['x-amz-storage-class'];
// set correct originOp
metadataStoreParams.originOp = 's3:ObjectRestore:Completed';
// update restore
const days = objMD.archive?.restoreRequestedDays;
const now = Date.now();
metadataStoreParams.archive = {
archiveInfo: objMD.archive?.archiveInfo,
restoreRequestedAt: objMD.archive?.restoreRequestedAt,
restoreRequestedDays: objMD.archive?.restoreRequestedDays,
restoreCompletedAt: new Date(now),
restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
};
/* eslint-enable no-param-reassign */
const versionId = objMD.versionId || undefined;
const options = {
versionId,
isNull: objMD.isNull,
};
if (objMD.nullVersionId) {
options.extraMD = {
nullVersionId: objMD.nullVersionId,
};
}
restoreMetadata(objMD, metadataStoreParams);
return options;
} }
module.exports = { module.exports = {
decodeVersionId, decodeVersionId,
getVersionIdResHeader, getVersionIdResHeader,
checkQueryVersionId, checkQueryVersionId,
processVersioningState,
getMasterState,
versioningPreprocessing, versioningPreprocessing,
getVersionSpecificMetadataOptions,
preprocessingVersioningDelete, preprocessingVersioningDelete,
overwritingVersioning,
decodeVID,
}; };

View File

@ -101,33 +101,8 @@ function validateWebsiteHeader(header) {
header.startsWith('http://') || header.startsWith('https://')); header.startsWith('http://') || header.startsWith('https://'));
} }
/**
* appendWebsiteIndexDocument - append index to objectKey if necessary
* @param {object} request - normalized request object
* @param {string} indexDocumentSuffix - index document from website config
* @param {boolean} force - flag to force append index
* @return {undefined}
*/
function appendWebsiteIndexDocument(request, indexDocumentSuffix, force = false) {
const reqObjectKey = request.objectKey ? request.objectKey : '';
/* eslint-disable no-param-reassign */
// find index document if "directory" sent in request
if (reqObjectKey.endsWith('/')) {
request.objectKey += indexDocumentSuffix;
// find index document if no key provided
} else if (reqObjectKey === '') {
request.objectKey = indexDocumentSuffix;
// force for redirect 302 on folder without trailing / that has an index
} else if (force) {
request.objectKey += `/${indexDocumentSuffix}`;
}
/* eslint-enable no-param-reassign */
}
module.exports = { module.exports = {
findRoutingRule, findRoutingRule,
extractRedirectInfo, extractRedirectInfo,
validateWebsiteHeader, validateWebsiteHeader,
appendWebsiteIndexDocument,
}; };

View File

@ -1,314 +0,0 @@
const async = require('async');
const { errors } = require('arsenal');
const monitoring = require('../../../utilities/monitoringHandler');
const {
actionNeedQuotaCheckCopy,
actionNeedQuotaCheck,
actionWithDataDeletion,
} = require('arsenal').policies;
const { config } = require('../../../Config');
const QuotaService = require('../../../quotas/quotas');
/**
* Process the bytes to write based on the request and object metadata
* @param {string} apiMethod - api method
* @param {BucketInfo} bucket - bucket info
* @param {string} versionId - version id of the object
* @param {number} contentLength - content length of the object
* @param {object} objMD - object metadata
* @param {object} destObjMD - destination object metadata
* @return {number} processed content length
*/
function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
let bytes = contentLength;
if (apiMethod === 'objectRestore') {
// object is being restored
bytes = Number.parseInt(objMD['content-length'], 10);
} else if (!bytes && objMD?.['content-length']) {
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
if (!destObjMD || bucket.isVersioningEnabled()) {
// object is being copied
bytes = Number.parseInt(objMD['content-length'], 10);
} else if (!bucket.isVersioningEnabled()) {
// object is being copied and replaces the target
bytes = Number.parseInt(objMD['content-length'], 10) -
Number.parseInt(destObjMD['content-length'], 10);
}
} else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
// object is being deleted
bytes = -Number.parseInt(objMD['content-length'], 10);
}
} else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
// object is being replaced: store the diff, if the bucket is not versioned
bytes = bytes - Number.parseInt(objMD['content-length'], 10);
}
return bytes || 0;
}
/**
* Checks if a metric is stale based on the provided parameters.
*
* @param {Object} metric - The metric object to check.
* @param {string} resourceType - The type of the resource.
* @param {string} resourceName - The name of the resource.
* @param {string} action - The action being performed.
* @param {number} inflight - The number of inflight requests.
* @param {Object} log - The logger object.
* @returns {boolean} Returns true if the metric is stale, false otherwise.
*/
function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
if (metric.date && Date.now() - new Date(metric.date).getTime() >
QuotaService.maxStaleness) {
log.warn('Stale metrics from the quota service, allowing the request', {
resourceType,
resourceName,
action,
inflight,
});
monitoring.requestWithQuotaMetricsUnavailable.inc();
return true;
}
return false;
}
/**
* Evaluates quotas for a bucket and an account and update inflight count.
*
* @param {number} bucketQuota - The quota limit for the bucket.
* @param {number} accountQuota - The quota limit for the account.
* @param {object} bucket - The bucket object.
* @param {object} account - The account object.
* @param {number} inflight - The number of inflight requests.
* @param {number} inflightForCheck - The number of inflight requests for checking quotas.
* @param {string} action - The action being performed.
* @param {object} log - The logger object.
* @param {function} callback - The callback function to be called when evaluation is complete.
* @returns {object} - The result of the evaluation.
*/
function _evaluateQuotas(
bucketQuota,
accountQuota,
bucket,
account,
inflight,
inflightForCheck,
action,
log,
callback,
) {
let bucketQuotaExceeded = false;
let accountQuotaExceeded = false;
const creationDate = new Date(bucket.getCreationDate()).getTime();
return async.parallel({
bucketQuota: parallelDone => {
if (bucketQuota > 0) {
return QuotaService.getUtilizationMetrics('bucket',
`${bucket.getName()}_${creationDate}`, null, {
action,
inflight,
}, (err, bucketMetrics) => {
if (err || inflight < 0) {
return parallelDone(err);
}
if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
log.debug('Bucket quota exceeded', {
bucket: bucket.getName(),
action,
inflight,
quota: bucketQuota,
bytesTotal: bucketMetrics.bytesTotal,
});
bucketQuotaExceeded = true;
}
return parallelDone();
});
}
return parallelDone();
},
accountQuota: parallelDone => {
if (accountQuota > 0 && account?.account) {
return QuotaService.getUtilizationMetrics('account',
account.account, null, {
action,
inflight,
}, (err, accountMetrics) => {
if (err || inflight < 0) {
return parallelDone(err);
}
if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
log.debug('Account quota exceeded', {
accountId: account.account,
action,
inflight,
quota: accountQuota,
bytesTotal: accountMetrics.bytesTotal,
});
accountQuotaExceeded = true;
}
return parallelDone();
});
}
return parallelDone();
},
}, err => {
if (err) {
log.warn('Error evaluating quotas', {
error: err.name,
description: err.message,
isInflightDeletion: inflight < 0,
});
}
return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
});
}
/**
* Monitors the duration of quota evaluation for a specific API method.
*
* @param {string} apiMethod - The name of the API method being monitored.
* @param {string} type - The type of quota being evaluated.
* @param {string} code - The code associated with the quota being evaluated.
* @param {number} duration - The duration of the quota evaluation in nanoseconds.
* @returns {undefined} - Returns nothing.
*/
function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
monitoring.quotaEvaluationDuration.labels({
action: apiMethod,
type,
code,
}).observe(duration / 1e9);
}
/**
*
* @param {Request} request - request object
* @param {BucketInfo} bucket - bucket object
* @param {Account} account - account object
* @param {array} apiNames - action names: operations to authorize
* @param {string} apiMethod - the main API call
* @param {number} inflight - inflight bytes
* @param {boolean} isStorageReserved - Flag to check if the current quota, minus
* the incoming bytes, are under the limit.
* @param {Logger} log - logger
* @param {function} callback - callback function
* @returns {boolean} - true if the quota is valid, false otherwise
*/
function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
return callback(null);
}
let type;
let bucketQuotaExceeded = false;
let accountQuotaExceeded = false;
let quotaEvaluationDuration;
const requestStartTime = process.hrtime.bigint();
const bucketQuota = bucket.getQuota();
const accountQuota = account?.quota || 0;
const shouldSendInflights = config.isQuotaInflightEnabled();
if (bucketQuota && accountQuota) {
type = 'bucket+account';
} else if (bucketQuota) {
type = 'bucket';
} else {
type = 'account';
}
if (actionWithDataDeletion[apiMethod]) {
type = 'delete';
}
if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
if (bucketQuota > 0 || accountQuota > 0) {
log.warn('quota is set for a bucket, but the quota service is disabled', {
bucketName: bucket.getName(),
});
monitoring.requestWithQuotaMetricsUnavailable.inc();
}
return callback(null);
}
if (isStorageReserved) {
// eslint-disable-next-line no-param-reassign
inflight = 0;
}
return async.forEach(apiNames, (apiName, done) => {
// Object copy operations first check the target object,
// meaning the source object, containing the current bytes,
// is checked second. This logic handles these APIs calls by
// ensuring the bytes are positives (i.e., not an object
// replacement).
if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
// eslint-disable-next-line no-param-reassign
inflight = Math.abs(inflight);
} else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
return done();
}
// When inflights are disabled, the sum of the current utilization metrics
// and the current bytes are compared with the quota. The current bytes
// are not sent to the utilization service. When inflights are enabled,
// the sum of the current utilization metrics only are compared with the
// quota. They include the current inflight bytes sent in the request.
let _inflights = shouldSendInflights ? inflight : undefined;
const inflightForCheck = shouldSendInflights ? 0 : inflight;
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
inflightForCheck, apiName, log,
(err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
if (err) {
return done(err);
}
bucketQuotaExceeded = _bucketQuotaExceeded;
accountQuotaExceeded = _accountQuotaExceeded;
// Inflights are inverted: in case of cleanup, we just re-issue
// the same API call.
if (_inflights) {
_inflights = -_inflights;
}
request.finalizerHooks.push((errorFromAPI, _done) => {
const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
const quotaCleanUpStartTime = process.hrtime.bigint();
// Quotas are cleaned only in case of error in the API
async.waterfall([
cb => {
if (errorFromAPI) {
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
null, apiName, log, cb);
}
return cb();
},
], () => {
monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
Number(process.hrtime.bigint() - quotaCleanUpStartTime));
return _done();
});
});
return done();
});
}, err => {
quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
if (err) {
log.warn('Error getting metrics from the quota service, allowing the request', {
error: err.name,
description: err.message,
});
}
if (!actionWithDataDeletion[apiMethod] &&
(bucketQuotaExceeded || accountQuotaExceeded)) {
return callback(errors.QuotaExceeded);
}
return callback();
});
}
module.exports = {
processBytesToWrite,
isMetricStale,
validateQuotas,
};

View File

@ -1,117 +0,0 @@
const { errors } = require('arsenal');
const constants = require('../../../constants');
const services = require('../../services');
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
const { pushMetric } = require('../../utapi/utilities');
const monitoring = require('../../utilities/monitoringHandler');
const { getLocationConstraintErrorMessage, processCurrents,
validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
const { config } = require('../../Config');
function handleResult(listParams, requestMaxKeys, authInfo,
bucketName, list, isBucketVersioned, log, callback) {
// eslint-disable-next-line no-param-reassign
listParams.maxKeys = requestMaxKeys;
const res = processCurrents(bucketName, listParams, isBucketVersioned, list);
pushMetric('listLifecycleCurrents', log, { authInfo, bucket: bucketName });
monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleCurrents');
return callback(null, res);
}
/**
* listLifecycleCurrents - Return list of current versions/masters in bucket
* @param {AuthInfo} authInfo - Instance of AuthInfo class with
* requester's info
* @param {array} locationConstraints - array of location contraint
* @param {object} request - http request object
* @param {function} log - Werelogs request logger
* @param {function} callback - callback to respond to http request
* with either error code or xml response body
* @return {undefined}
*/
function listLifecycleCurrents(authInfo, locationConstraints, request, log, callback) {
const params = request.query;
const bucketName = request.bucketName;
log.debug('processing request', { method: 'listLifecycleCurrents' });
const requestMaxKeys = params['max-keys'] ?
Number.parseInt(params['max-keys'], 10) : 1000;
if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
monitoring.promMetrics(
'GET', bucketName, 400, 'listLifecycleCurrents');
return callback(errors.InvalidArgument);
}
const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
const minEntriesToBeScanned = 1;
const { isValid, maxScannedLifecycleListingEntries } =
validateMaxScannedEntries(params, config, minEntriesToBeScanned);
if (!isValid) {
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
return callback(errors.InvalidArgument);
}
const excludedDataStoreName = params['excluded-data-store-name'];
if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
}
const metadataValParams = {
authInfo,
bucketName,
requestType: 'listLifecycleCurrents',
request,
};
const listParams = {
listingType: 'DelimiterCurrent',
maxKeys: actualMaxKeys,
prefix: params.prefix,
beforeDate: params['before-date'],
marker: params.marker,
excludedDataStoreName,
maxScannedLifecycleListingEntries,
};
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
if (err) {
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleCurrents');
return callback(err, null);
}
const vcfg = bucket.getVersioningConfiguration();
const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
if (!requestMaxKeys) {
const emptyList = {
Contents: [],
IsTruncated: false,
};
return handleResult(listParams, requestMaxKeys, authInfo,
bucketName, emptyList, isBucketVersioned, log, callback);
}
return services.getLifecycleListing(bucketName, listParams, log,
(err, list) => {
if (err) {
log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleCurrents');
return callback(err, null);
}
return handleResult(listParams, requestMaxKeys, authInfo,
bucketName, list, isBucketVersioned, log, callback);
});
});
}
module.exports = {
listLifecycleCurrents,
};

View File

@ -1,127 +0,0 @@
const { errors, versioning } = require('arsenal');
const constants = require('../../../constants');
const services = require('../../services');
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
const { pushMetric } = require('../../utapi/utilities');
const versionIdUtils = versioning.VersionID;
const monitoring = require('../../utilities/monitoringHandler');
const { getLocationConstraintErrorMessage, processNonCurrents,
validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
const { config } = require('../../Config');
function handleResult(listParams, requestMaxKeys, authInfo,
bucketName, list, log, callback) {
// eslint-disable-next-line no-param-reassign
listParams.maxKeys = requestMaxKeys;
const res = processNonCurrents(bucketName, listParams, list);
pushMetric('listLifecycleNonCurrents', log, { authInfo, bucket: bucketName });
monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleNonCurrents');
return callback(null, res);
}
/**
* listLifecycleNonCurrents - Return list of non-current versions in bucket
* @param {AuthInfo} authInfo - Instance of AuthInfo class with
* requester's info
* @param {array} locationConstraints - array of location contraint
* @param {object} request - http request object
* @param {function} log - Werelogs request logger
* @param {function} callback - callback to respond to http request
* with either error code or xml response body
* @return {undefined}
*/
function listLifecycleNonCurrents(authInfo, locationConstraints, request, log, callback) {
const params = request.query;
const bucketName = request.bucketName;
log.debug('processing request', { method: 'listLifecycleNonCurrents' });
const requestMaxKeys = params['max-keys'] ?
Number.parseInt(params['max-keys'], 10) : 1000;
if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
monitoring.promMetrics(
'GET', bucketName, 400, 'listLifecycleNonCurrents');
return callback(errors.InvalidArgument);
}
const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
// 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
// Two entries representing the master key and the following one representing the non-current version.
const minEntriesToBeScanned = 3;
const { isValid, maxScannedLifecycleListingEntries } =
validateMaxScannedEntries(params, config, minEntriesToBeScanned);
if (!isValid) {
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleNonCurrents');
return callback(errors.InvalidArgument);
}
const excludedDataStoreName = params['excluded-data-store-name'];
if (excludedDataStoreName && !locationConstraints[excludedDataStoreName]) {
const errMsg = getLocationConstraintErrorMessage(excludedDataStoreName);
log.error(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: excludedDataStoreName });
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents');
return callback(errors.InvalidLocationConstraint.customizeDescription(errMsg));
}
const metadataValParams = {
authInfo,
bucketName,
requestType: 'listLifecycleNonCurrents',
request,
};
const listParams = {
listingType: 'DelimiterNonCurrent',
maxKeys: actualMaxKeys,
prefix: params.prefix,
beforeDate: params['before-date'],
keyMarker: params['key-marker'],
excludedDataStoreName,
maxScannedLifecycleListingEntries,
};
listParams.versionIdMarker = params['version-id-marker'] ?
versionIdUtils.decode(params['version-id-marker']) : undefined;
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
if (err) {
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleNonCurrents');
return callback(err, null);
}
const vcfg = bucket.getVersioningConfiguration();
const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
if (!isBucketVersioned) {
log.debug('bucket is not versioned');
return callback(errors.InvalidRequest.customizeDescription(
'bucket is not versioned'), null);
}
if (!requestMaxKeys) {
const emptyList = {
Contents: [],
IsTruncated: false,
};
return handleResult(listParams, requestMaxKeys, authInfo,
bucketName, emptyList, log, callback);
}
return services.getLifecycleListing(bucketName, listParams, log,
(err, list) => {
if (err) {
log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleNonCurrents');
return callback(err, null);
}
return handleResult(listParams, requestMaxKeys, authInfo,
bucketName, list, log, callback);
});
});
}
module.exports = {
listLifecycleNonCurrents,
};

View File

@ -1,112 +0,0 @@
const { errors } = require('arsenal');
const constants = require('../../../constants');
const services = require('../../services');
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
const { pushMetric } = require('../../utapi/utilities');
const monitoring = require('../../utilities/monitoringHandler');
const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
const { config } = require('../../Config');
function handleResult(listParams, requestMaxKeys, authInfo,
bucketName, list, log, callback) {
// eslint-disable-next-line no-param-reassign
listParams.maxKeys = requestMaxKeys;
const res = processOrphans(bucketName, listParams, list);
pushMetric('listLifecycleOrphanDeleteMarkers', log, { authInfo, bucket: bucketName });
monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleOrphanDeleteMarkers');
return callback(null, res);
}
/**
* listLifecycleOrphanDeleteMarkers - Return list of expired object delete marker in bucket
* @param {AuthInfo} authInfo - Instance of AuthInfo class with
* requester's info
* @param {array} locationConstraints - array of location contraint
* @param {object} request - http request object
* @param {function} log - Werelogs request logger
* @param {function} callback - callback to respond to http request
* with either error code or xml response body
* @return {undefined}
*/
function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request, log, callback) {
const params = request.query;
const bucketName = request.bucketName;
log.debug('processing request', { method: 'listLifecycleOrphanDeleteMarkers' });
const requestMaxKeys = params['max-keys'] ?
Number.parseInt(params['max-keys'], 10) : 1000;
if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
monitoring.promMetrics(
'GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
return callback(errors.InvalidArgument);
}
const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
// 3 is required as a minimum because we must scan at least three entries to determine version eligibility.
// Two entries representing the master key and the following one representing the non-current version.
const minEntriesToBeScanned = 3;
const { isValid, maxScannedLifecycleListingEntries } =
validateMaxScannedEntries(params, config, minEntriesToBeScanned);
if (!isValid) {
monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers');
return callback(errors.InvalidArgument);
}
const metadataValParams = {
authInfo,
bucketName,
requestType: 'listLifecycleOrphanDeleteMarkers',
request,
};
const listParams = {
listingType: 'DelimiterOrphanDeleteMarker',
maxKeys: actualMaxKeys,
prefix: params.prefix,
beforeDate: params['before-date'],
marker: params.marker,
maxScannedLifecycleListingEntries,
};
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
if (err) {
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
return callback(err, null);
}
const vcfg = bucket.getVersioningConfiguration();
const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended');
if (!isBucketVersioned) {
log.debug('bucket is not versioned or suspended');
return callback(errors.InvalidRequest.customizeDescription(
'bucket is not versioned'), null);
}
if (!requestMaxKeys) {
const emptyList = {
Contents: [],
IsTruncated: false,
};
return handleResult(listParams, requestMaxKeys, authInfo,
bucketName, emptyList, log, callback);
}
return services.getLifecycleListing(bucketName, listParams, log,
(err, list) => {
if (err) {
log.debug('error processing request', { error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers');
return callback(err, null);
}
return handleResult(listParams, requestMaxKeys, authInfo,
bucketName, list, log, callback);
});
});
}
module.exports = {
listLifecycleOrphanDeleteMarkers,
};

Some files were not shown because too many files have changed in this diff Show More