Compare commits

..

1 Commits

Author SHA1 Message Date
Rahul Padigela 19b52b6840 Merge remote-tracking branch 'origin/development/7.5' into improvement/ZENKO-2062-port 2019-08-19 12:09:50 -07:00
600 changed files with 23604 additions and 55723 deletions

View File

@ -1,9 +1,3 @@
node_modules node_modules
localData/* localData/*
localMetadata/* localMetadata/*
# Keep the .git/HEAD file in order to properly report version
.git/objects
.github
.tox
coverage
.DS_Store

View File

@ -1,54 +1 @@
{ { "extends": "scality" }
"extends": "scality",
"plugins": [
"mocha"
],
"rules": {
"import/extensions": "off",
"lines-around-directive": "off",
"no-underscore-dangle": "off",
"indent": "off",
"object-curly-newline": "off",
"operator-linebreak": "off",
"function-paren-newline": "off",
"import/newline-after-import": "off",
"prefer-destructuring": "off",
"implicit-arrow-linebreak": "off",
"no-bitwise": "off",
"dot-location": "off",
"comma-dangle": "off",
"no-undef-init": "off",
"global-require": "off",
"import/no-dynamic-require": "off",
"class-methods-use-this": "off",
"no-plusplus": "off",
"no-else-return": "off",
"object-property-newline": "off",
"import/order": "off",
"no-continue": "off",
"no-tabs": "off",
"lines-between-class-members": "off",
"prefer-spread": "off",
"no-lonely-if": "off",
"no-useless-escape": "off",
"no-restricted-globals": "off",
"no-buffer-constructor": "off",
"import/no-extraneous-dependencies": "off",
"space-unary-ops": "off",
"no-useless-return": "off",
"no-unexpected-multiline": "off",
"no-mixed-operators": "off",
"newline-per-chained-call": "off",
"operator-assignment": "off",
"spaced-comment": "off",
"comma-style": "off",
"no-restricted-properties": "off",
"new-parens": "off",
"no-multi-spaces": "off",
"quote-props": "off",
"mocha/no-exclusive-tests": "error",
},
"parserOptions": {
"ecmaVersion": 2020
}
}

View File

@ -48,7 +48,7 @@ Describe the results you expected
- Node.js version, - Node.js version,
- Docker version, - Docker version,
- yarn version, - npm version,
- distribution/OS, - distribution/OS,
- optional: anything else you deem helpful to us. - optional: anything else you deem helpful to us.

View File

@ -1,43 +0,0 @@
---
name: "Setup CI environment"
description: "Setup Cloudserver CI environment"
runs:
using: composite
steps:
- name: Setup etc/hosts
shell: bash
run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
- name: Setup Credentials
shell: bash
run: bash .github/scripts/credentials.bash
- name: Setup job artifacts directory
shell: bash
run: |-
set -exu;
mkdir -p /tmp/artifacts/${JOB_NAME}/;
- uses: actions/setup-node@v4
with:
node-version: '16'
cache: 'yarn'
- name: install dependencies
shell: bash
run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
- uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Setup python2 test environment
shell: bash
run: |
sudo apt-get install -y libdigest-hmac-perl
pip install 's3cmd==2.3.0'
- name: fix sproxyd.conf permissions
shell: bash
run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
- name: ensure fuse kernel module is loaded (for sproxyd)
shell: bash
run: sudo modprobe fuse

View File

@ -1,25 +0,0 @@
FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
ENV CEPH_DAEMON demo
ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
ENV CEPH_DEMO_UID zenko
ENV CEPH_DEMO_ACCESS_KEY accessKey1
ENV CEPH_DEMO_SECRET_KEY verySecretKey1
ENV CEPH_DEMO_BUCKET zenkobucket
ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
ENV MON_IP 0.0.0.0
ENV NETWORK_AUTO_DETECT 4
ENV RGW_CIVETWEB_PORT 8001
RUN rm /etc/yum.repos.d/tcmu-runner.repo
ADD ./entrypoint-wrapper.sh /
RUN chmod +x /entrypoint-wrapper.sh && \
yum install -y python-pip && \
yum clean all && \
pip install awscli && \
rm -rf /root/.cache/pip
ENTRYPOINT [ "/entrypoint-wrapper.sh" ]

View File

@ -1,37 +0,0 @@
#!/bin/sh
touch /artifacts/ceph.log
mkfifo /tmp/entrypoint_output
# We run this in the background so that we can tail the RGW log after init,
# because entrypoint.sh never returns
# The next line will be needed when ceph builds 3.2.2 so I'll leave it here
# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
bash /entrypoint.sh > /tmp/entrypoint_output &
entrypoint_pid="$!"
while read -r line; do
echo $line
# When we find this line server has started
if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
break
fi
done < /tmp/entrypoint_output
# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
s3cmd mb s3://cephbucket s3://cephbucket2
mkdir /root/.aws
cat > /root/.aws/credentials <<EOF
[default]
aws_access_key_id = accessKey1
aws_secret_access_key = verySecretKey1
EOF
# Enable versioning on them
for bucket in cephbucket cephbucket2; do
echo "Enabling versiong for $bucket"
aws --endpoint http://127.0.0.1:8001 s3api put-bucket-versioning --bucket $bucket --versioning Status=Enabled
done
tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
wait $entrypoint_pid

View File

@ -1,11 +0,0 @@
#!/bin/sh
# This script is needed because RADOS Gateway
# will open the port before beginning to serve traffic
# causing wait_for_local_port.bash to exit immediately
echo 'Waiting for ceph'
while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
sleep 1
echo -n "."
done

View File

@ -1,10 +0,0 @@
---
version: 2
updates:
- package-ecosystem: npm
directory: "/"
schedule:
interval: daily
time: "13:00"
open-pull-requests-limit: 10
target-branch: "development/7.4"

View File

@ -1,36 +0,0 @@
azurebackend_AZURE_STORAGE_ACCESS_KEY
azurebackend_AZURE_STORAGE_ACCOUNT_NAME
azurebackend_AZURE_STORAGE_ENDPOINT
azurebackend2_AZURE_STORAGE_ACCESS_KEY
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME
azurebackend2_AZURE_STORAGE_ENDPOINT
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME
azurebackendmismatch_AZURE_STORAGE_ENDPOINT
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT
azuretest_AZURE_BLOB_ENDPOINT
b2backend_B2_ACCOUNT_ID
b2backend_B2_STORAGE_ACCESS_KEY
GOOGLE_SERVICE_EMAIL
GOOGLE_SERVICE_KEY
AWS_S3_BACKEND_ACCESS_KEY
AWS_S3_BACKEND_SECRET_KEY
AWS_S3_BACKEND_ACCESS_KEY_2
AWS_S3_BACKEND_SECRET_KEY_2
AWS_GCP_BACKEND_ACCESS_KEY
AWS_GCP_BACKEND_SECRET_KEY
AWS_GCP_BACKEND_ACCESS_KEY_2
AWS_GCP_BACKEND_SECRET_KEY_2
b2backend_B2_STORAGE_ENDPOINT
gcpbackend2_GCP_SERVICE_EMAIL
gcpbackend2_GCP_SERVICE_KEY
gcpbackend2_GCP_SERVICE_KEYFILE
gcpbackend_GCP_SERVICE_EMAIL
gcpbackend_GCP_SERVICE_KEY
gcpbackendmismatch_GCP_SERVICE_EMAIL
gcpbackendmismatch_GCP_SERVICE_KEY
gcpbackend_GCP_SERVICE_KEYFILE
gcpbackendmismatch_GCP_SERVICE_KEYFILE
gcpbackendnoproxy_GCP_SERVICE_KEYFILE

View File

@ -1,92 +0,0 @@
services:
cloudserver:
image: ${CLOUDSERVER_IMAGE}
command: sh -c "yarn start > /artifacts/s3.log"
network_mode: "host"
volumes:
- /tmp/ssl:/ssl
- /tmp/ssl-kmip:/ssl-kmip
- ${HOME}/.aws/credentials:/root/.aws/credentials
- /tmp/artifacts/${JOB_NAME}:/artifacts
environment:
- CI=true
- ENABLE_LOCAL_CACHE=true
- REDIS_HOST=0.0.0.0
- REDIS_PORT=6379
- REPORT_TOKEN=report-token-1
- REMOTE_MANAGEMENT_DISABLE=1
- HEALTHCHECKS_ALLOWFROM=0.0.0.0/0
- DATA_HOST=0.0.0.0
- METADATA_HOST=0.0.0.0
- S3BACKEND
- S3DATA
- S3METADATA
- MPU_TESTING
- S3VAULT
- S3_LOCATION_FILE
- ENABLE_UTAPI_V2
- BUCKET_DENY_FILTER
- S3KMS
- S3KMIP_PORT
- S3KMIP_HOSTS
- S3KMIP-COMPOUND_CREATE
- S3KMIP_BUCKET_ATTRIBUTE_NAME
- S3KMIP_PIPELINE_DEPTH
- S3KMIP_KEY
- S3KMIP_CERT
- S3KMIP_CA
- MONGODB_HOSTS=0.0.0.0:27018
- MONGODB_RS=rs0
- DEFAULT_BUCKET_KEY_FORMAT
- METADATA_MAX_CACHED_BUCKETS
- ENABLE_NULL_VERSION_COMPAT_MODE
- SCUBA_HOST
- SCUBA_PORT
- SCUBA_HEALTHCHECK_FREQUENCY
- S3QUOTA
- QUOTA_ENABLE_INFLIGHTS
env_file:
- creds.env
depends_on:
- redis
extra_hosts:
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
- "pykmip.local:127.0.0.1"
redis:
image: redis:alpine
network_mode: "host"
squid:
network_mode: "host"
profiles: ['ci-proxy']
image: scality/ci-squid
command: >-
sh -c 'mkdir -p /ssl &&
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-keyout /ssl/myca.pem -out /ssl/myca.pem &&
cp /ssl/myca.pem /ssl/CA.pem &&
squid -f /etc/squid/squid.conf -N -z &&
squid -f /etc/squid/squid.conf -NYCd 1'
volumes:
- /tmp/ssl:/ssl
pykmip:
network_mode: "host"
profiles: ['pykmip']
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
volumes:
- /tmp/artifacts/${JOB_NAME}:/artifacts
mongo:
network_mode: "host"
profiles: ['mongo', 'ceph']
image: ${MONGODB_IMAGE}
ceph:
network_mode: "host"
profiles: ['ceph']
image: ghcr.io/scality/cloudserver/ci-ceph
sproxyd:
network_mode: "host"
profiles: ['sproxyd']
image: sproxyd-standalone
build: ./sproxyd
user: 0:0
privileged: yes

View File

@ -1,28 +0,0 @@
FROM mongo:5.0.21
ENV USER=scality \
HOME_DIR=/home/scality \
CONF_DIR=/conf \
DATA_DIR=/data
# Set up directories and permissions
RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
mkdir /logs; \
adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
# Set up environment variables and directories for scality user
RUN mkdir ${CONF_DIR} && \
chown -R ${USER} ${CONF_DIR} && \
chown -R ${USER} ${DATA_DIR}
# copy the mongo config file
COPY /conf/mongod.conf /conf/mongod.conf
COPY /conf/mongo-run.sh /conf/mongo-run.sh
COPY /conf/initReplicaSet /conf/initReplicaSet.js
EXPOSE 27017/tcp
EXPOSE 27018
# Set up CMD
ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
CMD ["bash", "/conf/mongo-run.sh"]

View File

@ -1,4 +0,0 @@
rs.initiate({
_id: "rs0",
members: [{ _id: 0, host: "127.0.0.1:27018" }]
});

View File

@ -1,10 +0,0 @@
#!/bin/bash
set -exo pipefail
init_RS() {
sleep 5
mongo --port 27018 /conf/initReplicaSet.js
}
init_RS &
mongod --bind_ip_all --config=/conf/mongod.conf

View File

@ -1,15 +0,0 @@
storage:
journal:
enabled: true
engine: wiredTiger
dbPath: "/data/db"
processManagement:
fork: false
net:
port: 27018
bindIp: 0.0.0.0
replication:
replSetName: "rs0"
enableMajorityReadConcern: true
security:
authorization: disabled

View File

@ -1,3 +0,0 @@
FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
RUN chown root:root /conf/sproxyd0.conf

View File

@ -1,26 +0,0 @@
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
#fastcgi_param SCRIPT_NAME $fastcgi_script_name;
fastcgi_param SCRIPT_NAME /var/www;
fastcgi_param PATH_INFO $document_uri;
fastcgi_param REQUEST_URI $request_uri;
fastcgi_param DOCUMENT_URI $document_uri;
fastcgi_param DOCUMENT_ROOT $document_root;
fastcgi_param SERVER_PROTOCOL $server_protocol;
fastcgi_param HTTPS $https if_not_empty;
fastcgi_param GATEWAY_INTERFACE CGI/1.1;
fastcgi_param SERVER_SOFTWARE nginx/$nginx_version;
fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param REMOTE_PORT $remote_port;
fastcgi_param SERVER_ADDR $server_addr;
fastcgi_param SERVER_PORT $server_port;
fastcgi_param SERVER_NAME $server_name;
# PHP only, required if PHP was built with --enable-force-cgi-redirect
fastcgi_param REDIRECT_STATUS 200;

View File

@ -1,88 +0,0 @@
worker_processes 1;
error_log /logs/error.log;
user root root;
events {
worker_connections 1000;
reuse_port on;
multi_accept on;
}
worker_rlimit_nofile 20000;
http {
root /var/www/;
upstream sproxyds {
least_conn;
keepalive 40;
server 127.0.0.1:20000;
}
server {
client_max_body_size 0;
client_body_timeout 150;
client_header_timeout 150;
postpone_output 0;
client_body_postpone_size 0;
keepalive_requests 1100;
keepalive_timeout 300s;
server_tokens off;
default_type application/octet-stream;
gzip off;
tcp_nodelay on;
tcp_nopush on;
sendfile on;
listen 81;
server_name localhost;
rewrite ^/arc/(.*)$ /dc1/$1 permanent;
location ~* ^/proxy/(.*)$ {
rewrite ^/proxy/(.*)$ /$1 last;
}
allow 127.0.0.1;
deny all;
set $usermd '-';
set $sentusermd '-';
set $elapsed_ms '-';
set $now '-';
log_by_lua '
if not(ngx.var.http_x_scal_usermd == nil) and string.len(ngx.var.http_x_scal_usermd) > 2 then
ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
end
if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
end
local elapsed_ms = tonumber(ngx.var.request_time)
if not ( elapsed_ms == nil) then
elapsed_ms = elapsed_ms * 1000
ngx.var.elapsed_ms = tostring(elapsed_ms)
end
local time = tonumber(ngx.var.msec) * 1000
ngx.var.now = time
';
log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
'"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
'"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
'"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
'"contentType":"$content_type","s3Address":"$remote_addr",'
'"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
'"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
'"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
'"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
'"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
'"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
access_log /dev/stdout irm;
error_log /dev/stdout error;
location / {
proxy_request_buffering off;
fastcgi_request_buffering off;
fastcgi_no_cache 1;
fastcgi_cache_bypass 1;
fastcgi_buffering off;
fastcgi_ignore_client_abort on;
fastcgi_keep_conn on;
include fastcgi_params;
fastcgi_pass sproxyds;
fastcgi_next_upstream error timeout;
fastcgi_send_timeout 285s;
fastcgi_read_timeout 285s;
}
}
}

View File

@ -1,12 +0,0 @@
{
"general": {
"ring": "DATA",
"port": 20000,
"syslog_facility": "local0"
},
"ring_driver:0": {
"alias": "dc1",
"type": "local",
"queue_path": "/tmp/ring-objs"
},
}

View File

@ -1,43 +0,0 @@
[supervisord]
nodaemon = true
loglevel = info
logfile = %(ENV_LOG_DIR)s/supervisord.log
pidfile = %(ENV_SUP_RUN_DIR)s/supervisord.pid
logfile_maxbytes = 20MB
logfile_backups = 2
[unix_http_server]
file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
[program:nginx]
directory=%(ENV_SUP_RUN_DIR)s
command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
stdout_logfile_maxbytes=100MB
stdout_logfile_backups=7
stderr_logfile_maxbytes=100MB
stderr_logfile_backups=7
autorestart=true
autostart=true
user=root
[program:sproxyd]
directory=%(ENV_SUP_RUN_DIR)s
process_name=%(program_name)s-%(process_num)s
numprocs=1
numprocs_start=0
command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
stdout_logfile_maxbytes=100MB
stdout_logfile_backups=7
redirect_stderr=true
autorestart=true
autostart=true
user=root

View File

@ -1,18 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIC6zCCAdOgAwIBAgIUPIpMY95b4HjKAk+FyydZApAEFskwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJDEQMA4GA1UECgwHU2NhbGl0eTEQ
MA4GA1UEAwwHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AKqLFEsWtfRTxnoZrQe63tq+rQnVgninHMahRmXkzyjK/uNhoKnIh8bXdTC/eCZ6
FBROqBYNL0TJb0HDv1FzcZS1UCUldRqTlvr6wZb0pfrp40fvztsqQgAh1t/Blg5i
Zv5+ESSlNs5rWbFTxtq+FbMW/ERYTrVfnMkBiLg4Gq0HwID9a5jvJatzrrno2s1m
OfZCT3HaE3tMZ6vvYuoamvLNdvdH+9KeTmBCursfNejt0rSGjIqfi6DvFJSayydQ
is5DMSTbCLGdKQmA85VfEQmlQ8v0232WDSd6gVfp2tthDEDHnCbgWkEd1vsTyS85
ubdt5v4CWGOWV+mu3bf8xM0CAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkq
hkiG9w0BAQsFAAOCAQEARTjc2zV/ol1/LsSzZy6l1R0uFBmR2KumH+Se1Yq2vKpY
Dv6xmrvmjOUr5RBO77nRhIgdcQA+LyAg8ii2Dfzc8r1RTD+j1bYOxESXctBOBcXM
Chy6FEBydR6m7S8qQyL+caJWO1WZWp2tapcm6sUG1oRVznWtK1/SHKIzOBwsmJ07
79KsCJ6wf9tzD05EDTI2QhAObE9/thy+zc8l8cmv9A6p3jKkx9rwXUttSUqTn0CW
w45bgKg6+DDcrhZ+MATbzuTfhuA4NFUTzK7KeX9sMuOV03Zs8SA3VhAOXmu063M3
0f9X7P/0RmGTTp7GGCqEINcZdbLh3k7CpFb2Ox998Q==
-----END CERTIFICATE-----

View File

@ -1,18 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIC2zCCAcOgAwIBAgIUIlE8UAkqQ+6mbJDtrt9kkmi8aJYwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowKTEQMA4GA1UECgwHU2NhbGl0eTEV
MBMGA1UEAwwMcHlrbWlwLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEAtxr7pq/lnzVeZz4z52Yc3DeaPqjNfRSyW5cPUlT7ABXFb7+tja7K2C7u
DYVK+Q+2yJCQwYJY47aKJB++ewam9t2V8Xy0Z8S+0I2ImCwuyeihaD/f6uJZRzms
ycdECH22BA6tCPlQLnlboRiZzI6rcIvXAbUMvLvFm3nyYIs9qidExRnfyMjISknM
V+83LT5QW4IcHgKYqzdz2ZmOnk+f4wmMmitcivTdIZCL8Z0cxr7BJlOh5JZ/V5uj
WUXeNa+ttW0RKKBlg9T+wj0JvwoJBPZTmsMAy3tI9tjLg3DwGYKsflbFeU2tebXI
gncGFZ/dFxj331GGtq3kz1PzAUYf2wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB1
8HgJ0fu6/pCrDxAm90eESFjmaTFyTN8q00zhq4Cb3zAT9KMWzAygkZ9n4ZFgELPo
7kBE2H6RcDdoBmjVYd8HnBloDdYzYbncKgt5YBvxRaMSF4/l65BM8wjatyXErqnH
QLLTRe5AuF0/F0KtPeDQ2JFVu8dZ35W3fyKGPRsEdVOSCTHROmqpGhZCpscyUP4W
Hb0dBTESQ9mQHw14OCaaahARd0X5WdcA/E+m0fpGqj1rQCXS+PrRcSLe1E1hqPlK
q/hXSXD5nybwipktELvJCbB7l4HmJr2pIpldeR5+ef68Cs8hqs6DRlsJX9sK2ng+
TFe5v6SCarqZ9kFvr6Yp
-----END CERTIFICATE-----

View File

@ -1,18 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIC8zCCAdugAwIBAgIUBs6nVXQXhrFbClub3aSLg72/DiYwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJTEQMA4GA1UECgwHU2NhbGl0eTER
MA8GA1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQC6neSYoBoWh/i2mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4L
jLLRsPGt9KkJlUhHGWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jt
UDnQUwshj+hRFe0iKAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsH
PVju9yZBYzYhwAMyYFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFip
yR2Fh3WGSwWh45HgMT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+p
SMLm0T4iNxedQWBtDM7ts4EjAgMBAAGjGjAYMBYGA1UdJQEB/wQMMAoGCCsGAQUF
BwMCMA0GCSqGSIb3DQEBCwUAA4IBAQCMi9HEhZc5jHJMj18Wq00fZy4O9XtjCe0J
nntW9tzi3rTQcQWKA7i9uVdDoCg+gMFVxWMvV7luFEUc/VYV1v8hFfbIFygzFsZY
xwv4GQaIwbsgzD+oziia53w0FSuNL0uE0MeKvrt3yzHxCxylHyl+TQd/UdAtAo+k
RL1sI0mBZx5qo6d1J7ZMCxzAGaT7KjnJvziFr/UbfSNnwDsxsUwGaI1ZeAxJN8DI
zTrg3f3lrrmHcauEgKnuQwIqaMZR6veG6RkjtcYSlJYID1irkE6njs7+wivOAkzt
fBt/0PD76FmAI0VArgU/zDB8dGyYzrq39W749LuEfm1TPmlnUtDr
-----END CERTIFICATE-----

View File

@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6neSYoBoWh/i2
mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4LjLLRsPGt9KkJlUhH
GWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jtUDnQUwshj+hRFe0i
KAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsHPVju9yZBYzYhwAMy
YFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFipyR2Fh3WGSwWh45Hg
MT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+pSMLm0T4iNxedQWBt
DM7ts4EjAgMBAAECggEANNXdUeUKXdSzcycPV/ea/c+0XFcy8e9B46lfQTpTqQOx
xD8GbWD1L/gdk6baJgT43+ukEWdSsJbmdtLXti29Ta8OF2VtIDhIbCVtvs3dq3zt
vrvugsiVDr8nkP306qOrKrNIVIFE+igmEmSaXsu/h/33ladxeeV9/s2DC7NOOjWN
Mu4KYr5BBbu3qAavdzbrcz7Sch+GzsYqK/pBounCTQu3o9E4TSUcmcsasWmtHN3u
e6G2UjObdzEW7J0wWvvtJ0wHQUVRueHfqwqKf0dymcZ3xOlx3ZPhKPz5n4F1UGUt
RQaNazqs5SzZpUgDuPw4k8h/aCHK21Yexw/l4+O9KQKBgQD1WZSRK54zFoExBQgt
OZSBNZW3Ibti5lSiF0M0g+66yNZSWfPuABEH0tu5CXopdPDXo4kW8NLGEqQStWTX
RGK0DE9buEL3eebOfjIdS2IZ3t3dX3lMypplVCj4HzAgITlweSH1LLTyAtaaOpwa
jksqfcn5Zw+XGkyc6GBBVaZetQKBgQDCt6Xf/g26+zjvHscjdzsfBhnYvTOrr6+F
xqFFxOEOocGr+mL7UTAs+a9m/6lOWhlagk+m+TIZNL8o3IN7KFTYxPYPxTiewgVE
rIm3JBmPxRiPn01P3HrtjaqfzsXF30j3ele7ix5OxieZq4vsW7ZXP3GZE34a08Ov
12sE1DlvdwKBgQDzpYQOLhyqazzcqzyVfMrnDYmiFVN7QXTmiudobWRUBUIhAcdl
oJdJB7K/rJOuO704x+RJ7dnCbZyWH6EGzZifaGIemXuXO21jvpqR0NyZCGOXhUp2
YfS1j8AntwEZxyS9du2sBjui4gKvomiHTquChOxgSmKHEcznPTTpbN8MyQKBgF5F
LVCZniolkLXsL7tS8VOez4qoZ0i6wP7CYLf3joJX+/z4N023S9yqcaorItvlMRsp
tciAIyoi6F2vDRTmPNXJ3dtav4PVKVnLMs1w89MwOCjoljSQ6Q7zpGTEZenbpWbz
W2BYBS9cLjXu4MpoyInLFINo9YeleLs8TvrCiKAXAoGBANsduqLnlUW/f5zDb5Fe
SB51+KhBjsVIeYmU+8xtur9Z7IxZXK28wpoEsm7LmX7Va5dERjI+tItBiJ5+Unu1
Xs2ljDg35ARKHs0dWBJGpbnZg4dbT6xpIL4YMPXm1Zu++PgRpxPIMn646xqd8GlH
bavm6Km/fXNG58xus+EeLpV5
-----END PRIVATE KEY-----

View File

@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC3Gvumr+WfNV5n
PjPnZhzcN5o+qM19FLJblw9SVPsAFcVvv62NrsrYLu4NhUr5D7bIkJDBgljjtook
H757Bqb23ZXxfLRnxL7QjYiYLC7J6KFoP9/q4llHOazJx0QIfbYEDq0I+VAueVuh
GJnMjqtwi9cBtQy8u8WbefJgiz2qJ0TFGd/IyMhKScxX7zctPlBbghweApirN3PZ
mY6eT5/jCYyaK1yK9N0hkIvxnRzGvsEmU6Hkln9Xm6NZRd41r621bREooGWD1P7C
PQm/CgkE9lOawwDLe0j22MuDcPAZgqx+VsV5Ta15tciCdwYVn90XGPffUYa2reTP
U/MBRh/bAgMBAAECggEABCvcMcbuDztzBB0Zp5re63Fk1SqZS9Et4wJE+hYvhaf5
UHtoY8LoohYnnC0+MQBXpKgOdCoZBk8BRKNofnr/UL5pjQ/POFH2GuAujXDsO/NN
wgc6fapcaE/7DLm6ZgsfG2aOMJclaXmgScI6trtFUpIM+t/6A06vyMP1bpeddwPW
Fqu7NvpDiEcTRUGd+z1JooYgUhGgC7peYUx5+9zqFrwoDBKxnUOnz3BkDsXBy3qm
65Vu0BSjuJzf6vVMpNGUHY6JXjopVNWku+JAX0wD+iikOd5sziNVdIj1fnZ+IHIf
7G5h5owHpvSGzJFQ18/g5VHtJdCm+4WQSnbSJRsCAQKBgQDu4IH8yspyeH44fhoS
PAp/OtILqSP+Da0zAp2LbhrOgyzyuSTdEAYyptqjqHS6QkB1Bu1H44FS0BYUxRXc
iu2e9AndiLVCGngsE7TpA/ZVLN1B0LEZEHjM6p4d6zZM6iveKVnPAOkTWTBAgzCt
b31nj4jL8PdlPKQil1AMrOlRAQKBgQDEOwshzIdr2Iy6B/n4CuBViEtwnbAd5f/c
atA9bcfF8kCahokJsI4eCCLgBwDZpYKD+v0AwOBlacF6t6TX+vdlJsi5EP7uxZ22
ILsuWqVm/0H77PACuckc5/qLZoGGC81l0DhnpoeMEb6r/TKOo5xAK1gxdlwNNrq+
nP1zdZnU2wKBgBAS92xFUR4m0YeHpMV5WNN658t1FEDyNqdqE6PgQtmGpi2nG73s
aB5cb/X3TfOCpce6MZlWy8sAyZuYL4Jprte1YDySCHBsS43bvZ64b4kHvdPB8UjY
fOh9GSq2Oy8tysnmSm7NhuGQbNjKeyoQiIXBeNkQW/VqATl6qR5RPFoBAoGACNqV
JQBCd/Y8W0Ry3eM3vgQ5SyqCQMcY5UwYez0Rz3efvJknY72InAhH8o2+VxOlsOjJ
M5iAR3MfHLdeg7Q6J2E5m0gOCJ34ALi3WV8TqXMI+iH1rlnNnjVFU7bbTz4HFXnw
oZSc9w/x53a0KkVtjmOmRg0OGDaI9ILG2MfMmhMCgYB8ZqJtX8qZ2TqKU3XdLZ4z
T2N7xMFuKohWP420r5jKm3Xw85IC+y1SUTB9XGcL79r2eJzmzmdKQ3A3sf3oyUH3
RdYWxtKcZ5PAE8hVRtn1ETZqUgxASGOUn/6w0npkYSOXPU5bc0W6RSLkjES0i+c3
fv3OMNI8qpmQhEjpHHQS1g==
-----END PRIVATE KEY-----

View File

@ -1,35 +0,0 @@
name: Test alerts
on:
push:
branches-ignore:
- 'development/**'
- 'q/*/**'
jobs:
run-alert-tests:
runs-on: ubuntu-latest
strategy:
matrix:
tests:
- name: 1 minute interval tests
file: monitoring/alerts.test.yaml
- name: 10 seconds interval tests
file: monitoring/alerts.10s.test.yaml
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Render and test ${{ matrix.tests.name }}
uses: scality/action-prom-render-test@1.0.3
with:
alert_file_path: monitoring/alerts.yaml
test_file_path: ${{ matrix.tests.file }}
alert_inputs: |
namespace=zenko
service=artesca-data-connector-s3api-metrics
reportJob=artesca-data-ops-report-handler
replicas=3
github_token: ${{ secrets.GITHUB_TOKEN }}

View File

@ -1,25 +0,0 @@
---
name: codeQL
on:
push:
branches: [w/**, q/*]
pull_request:
branches: [development/*, stabilization/*, hotfix/*]
workflow_dispatch:
jobs:
analyze:
name: Static analysis with CodeQL
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: javascript, python, ruby
- name: Build and analyze
uses: github/codeql-action/analyze@v3

View File

@ -1,16 +0,0 @@
---
name: dependency review
on:
pull_request:
branches: [development/*, stabilization/*, hotfix/*]
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@v4
- name: 'Dependency Review'
uses: actions/dependency-review-action@v4

View File

@ -1,80 +0,0 @@
---
name: release
run-name: release ${{ inputs.tag }}
on:
workflow_dispatch:
inputs:
tag:
description: 'Tag to be released'
required: true
env:
PROJECT_NAME: ${{ github.event.repository.name }}
jobs:
build-federation-image:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Build and push image for federation
uses: docker/build-push-action@v5
with:
push: true
context: .
file: images/svc-base/Dockerfile
tags: |
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
cache-from: type=gha,scope=federation
cache-to: type=gha,mode=max,scope=federation
release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildk
uses: docker/setup-buildx-action@v3
- name: Login to Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Push dashboards into the production namespace
run: |
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
dashboard.json:application/grafana-dashboard+json \
alerts.yaml:application/prometheus-alerts+yaml
working-directory: monitoring
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Create Release
uses: softprops/action-gh-release@v2
env:
GITHUB_TOKEN: ${{ github.token }}
with:
name: Release ${{ github.event.inputs.tag }}
tag_name: ${{ github.event.inputs.tag }}
generate_release_notes: true
target_commitish: ${{ github.sha }}

View File

@ -1,533 +0,0 @@
---
name: tests
on:
workflow_dispatch:
push:
branches-ignore:
- 'development/**'
- 'q/*/**'
env:
# Secrets
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurebackend_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
b2backend_B2_STORAGE_ACCESS_KEY: >-
${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
gcpbackendmismatch_GCP_SERVICE_KEY: >-
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
# Configs
ENABLE_LOCAL_CACHE: "true"
REPORT_TOKEN: "report-token-1"
REMOTE_MANAGEMENT_DISABLE: "1"
# https://github.com/git-lfs/git-lfs/issues/5749
GIT_CLONE_PROTECTION_ACTIVE: 'false'
jobs:
linting-coverage:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '16'
cache: yarn
- name: install dependencies
run: yarn install --frozen-lockfile --network-concurrency 1
- uses: actions/setup-python@v5
with:
python-version: '3.9'
- uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
- name: Install python deps
run: pip install flake8
- name: Lint Javascript
run: yarn run --silent lint -- --max-warnings 0
- name: Lint Markdown
run: yarn run --silent lint_md
- name: Lint python
run: flake8 $(git ls-files "*.py")
- name: Lint Yaml
run: yamllint -c yamllint.yml $(git ls-files "*.yml")
- name: Unit Coverage
run: |
set -ex
mkdir -p $CIRCLE_TEST_REPORTS/unit
yarn test
yarn run test_legacy_location
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
CIRCLE_TEST_REPORTS: /tmp
CIRCLE_ARTIFACTS: /tmp
CI_REPORTS: /tmp
- name: Unit Coverage logs
run: find /tmp/unit -exec cat {} \;
- name: preparing junit files for upload
run: |
mkdir -p artifacts/junit
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
if: always()
- name: Upload files to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: artifacts
if: always()
build:
runs-on: ubuntu-20.04
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Build and push cloudserver image
uses: docker/build-push-action@v5
with:
push: true
context: .
provenance: false
tags: |
ghcr.io/${{ github.repository }}:${{ github.sha }}
labels: |
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=cloudserver
cache-to: type=gha,mode=max,scope=cloudserver
- name: Build and push pykmip image
uses: docker/build-push-action@v5
with:
push: true
context: .github/pykmip
tags: |
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
labels: |
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=pykmip
cache-to: type=gha,mode=max,scope=pykmip
- name: Build and push MongoDB
uses: docker/build-push-action@v5
with:
push: true
context: .github/docker/mongodb
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
cache-from: type=gha,scope=mongodb
cache-to: type=gha,mode=max,scope=mongodb
multiple-backend:
runs-on: ubuntu-latest
needs: build
env:
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
S3BACKEND: mem
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
S3DATA: multiple
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Login to Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile sproxyd up -d
working-directory: .github/docker
- name: Run multiple backend test
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 81 40
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
mongo-v0-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
MPU_TESTING: "yes"
S3METADATA: mongodb
S3KMS: file
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
DEFAULT_BUCKET_KEY_FORMAT: v0
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run functional tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
mongo-v1-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
MPU_TESTING: "yes"
S3METADATA: mongodb
S3KMS: file
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
DEFAULT_BUCKET_KEY_FORMAT: v1
METADATA_MAX_CACHED_BUCKETS: 1
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run functional tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
file-ft-tests:
strategy:
matrix:
include:
- job-name: file-ft-tests
name: ${{ matrix.job-name }}
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: file
S3VAULT: mem
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
MPU_TESTING: "yes"
JOB_NAME: ${{ matrix.job-name }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup matrix job artifacts directory
shell: bash
run: |
set -exu
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
- name: Setup CI services
run: docker compose up -d
working-directory: .github/docker
- name: Run file ft tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
utapi-v2-tests:
runs-on: ubuntu-latest
needs: build
env:
ENABLE_UTAPI_V2: t
S3BACKEND: mem
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose up -d
working-directory: .github/docker
- name: Run file utapi v2 tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
quota-tests:
runs-on: ubuntu-latest
needs: build
strategy:
matrix:
inflights:
- name: "With Inflights"
value: "true"
- name: "Without Inflights"
value: "false"
env:
S3METADATA: mongodb
S3BACKEND: mem
S3QUOTA: scuba
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
SCUBA_HOST: localhost
SCUBA_PORT: 8100
SCUBA_HEALTHCHECK_FREQUENCY: 100
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run quota tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
kmip-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: file
S3VAULT: mem
MPU_TESTING: "yes"
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Copy KMIP certs
run: cp -r ./certs /tmp/ssl-kmip
working-directory: .github/pykmip
- name: Setup CI services
run: docker compose --profile pykmip up -d
working-directory: .github/docker
- name: Run file KMIP tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 5696 40
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
ceph-backend-test:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
S3DATA: multiple
S3KMS: file
CI_CEPH: 'true'
MPU_TESTING: "yes"
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- uses: ruby/setup-ruby@v1
with:
ruby-version: '2.5.9'
- name: Install Ruby dependencies
run: |
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
- name: Install Java dependencies
run: |
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
- name: Setup CI services
run: docker compose --profile ceph up -d
working-directory: .github/docker
env:
S3METADATA: mongodb
- name: Run Ceph multiple backend tests
run: |-
set -ex -o pipefail;
bash .github/ceph/wait_for_ceph.sh
bash wait_for_local_port.bash 27018 40
bash wait_for_local_port.bash 8000 40
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
S3METADATA: mem
- name: Run Java tests
run: |-
set -ex -o pipefail;
mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
working-directory: tests/functional/jaws
- name: Run Ruby tests
run: |-
set -ex -o pipefail;
rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
working-directory: tests/functional/fog
- name: Run Javascript AWS SDK tests
run: |-
set -ex -o pipefail;
yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
S3BACKEND: file
S3VAULT: mem
S3METADATA: mongodb
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()

View File

@ -1,60 +1,28 @@
ARG NODE_VERSION=16.20-bullseye-slim FROM node:10-slim
MAINTAINER Giorgio Regni <gr@scality.com>
FROM node:${NODE_VERSION} as builder
WORKDIR /usr/src/app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
curl \
git \
gnupg2 \
jq \
python3 \
ssh \
wget \
libffi-dev \
zlib1g-dev \
&& apt-get clean \
&& mkdir -p /root/ssh \
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
ENV PYTHON=python3
COPY package.json yarn.lock /usr/src/app/
RUN npm install typescript -g
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
################################################################################
FROM node:${NODE_VERSION}
RUN apt-get update && \
apt-get install -y --no-install-recommends \
jq \
&& rm -rf /var/lib/apt/lists/*
ENV NO_PROXY localhost,127.0.0.1
ENV no_proxy localhost,127.0.0.1
EXPOSE 8000
EXPOSE 8002
RUN apt-get update && \
apt-get install -y --no-install-recommends \
jq \
tini \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /usr/src/app WORKDIR /usr/src/app
# Keep the .git directory in order to properly report version # Keep the .git directory in order to properly report version
COPY . /usr/src/app COPY ./package.json .
COPY --from=builder /usr/src/app/node_modules ./node_modules/
RUN apt-get update \
&& apt-get install -y jq python git build-essential --no-install-recommends \
&& npm install --production \
&& apt-get autoremove --purge -y python git build-essential \
&& rm -rf /var/lib/apt/lists/* \
&& npm cache clear --force \
&& rm -rf ~/.node-gyp \
&& rm -rf /tmp/npm-*
COPY ./ ./
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"] VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"] ENV NO_PROXY localhost,127.0.0.1
ENV no_proxy localhost,127.0.0.1
CMD [ "yarn", "start" ] ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
CMD [ "npm", "start" ]
EXPOSE 8000

View File

@ -7,16 +7,16 @@ COPY . /usr/src/app
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y jq python git build-essential --no-install-recommends \ && apt-get install -y jq python git build-essential --no-install-recommends \
&& yarn install --production \ && npm install --production \
&& apt-get autoremove --purge -y python git build-essential \ && apt-get autoremove --purge -y python git build-essential \
&& rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/apt/lists/* \
&& yarn cache clean \ && npm cache clear \
&& rm -rf ~/.node-gyp \ && rm -rf ~/.node-gyp \
&& rm -rf /tmp/yarn-* && rm -rf /tmp/npm-*
ENV S3BACKEND mem ENV S3BACKEND mem
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"] ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
CMD [ "yarn", "start" ] CMD [ "npm", "start" ]
EXPOSE 8000 EXPOSE 8000

View File

@ -1,7 +1,6 @@
# S3 Healthcheck # S3 Healthcheck
Scality S3 exposes a healthcheck route `/live` on the port used Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
for the metrics (defaults to port 8002) which returns a
response with HTTP code response with HTTP code
- 200 OK - 200 OK

166
README.md
View File

@ -1,7 +1,12 @@
# Zenko CloudServer with Vitastor Backend # Zenko CloudServer
![Zenko CloudServer logo](res/scality-cloudserver-logo.png) ![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
[![CircleCI][badgepub]](https://circleci.com/gh/scality/S3)
[![Scality CI][badgepriv]](http://ci.ironmann.io/gh/scality/S3)
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/scality/s3server/)
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
## Overview ## Overview
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
@ -11,71 +16,126 @@ Scalitys Open Source Multi-Cloud Data Controller.
CloudServer provides a single AWS S3 API interface to access multiple CloudServer provides a single AWS S3 API interface to access multiple
backend data storage both on-premise or public in the cloud. backend data storage both on-premise or public in the cloud.
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor) CloudServer is useful for Developers, either to run as part of a
backend support. continous integration test environment to emulate the AWS S3 service locally
or as an abstraction layer to develop object storage enabled
application on the go.
## Quick Start with Vitastor ## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
Vitastor Backend is in experimental status, however you can already try to ## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
it works too 😊.
Installation instructions: ## Docker
### Install Vitastor [Run your Zenko CloudServer with Docker](https://hub.docker.com/r/scality/s3server/)
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md). ## Contributing
### Install Zenko with Vitastor Backend In order to contribute, please follow the
[Contributing Guidelines](
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor` ## Installation
- Install dependencies: `npm install --omit dev` or just `npm install`
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
### Install and Configure MongoDB ### Dependencies
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/). Building and running the Zenko CloudServer requires node.js 6.9.5 and npm v3
. Up-to-date versions can be found at
[Nodesource](https://github.com/nodesource/distributions).
### Setup Zenko ### Clone source code
- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data` ```shell
- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data` git clone https://github.com/scality/S3.git
- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
access keys, but it's not published, so let's use a file for now.
- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
in this file.
Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
### Start Zenko
Start the S3 server with: `node index.js`
If you use default settings, Zenko CloudServer starts on port 8000.
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
Now you can access your S3 with `s3cmd` or `geesefs`:
```
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
``` ```
``` ### Install js dependencies
AWS_ACCESS_KEY_ID=accessKey1 \
AWS_SECRET_ACCESS_KEY=verySecretKey1 \ Go to the ./S3 folder,
geesefs --endpoint http://localhost:8000 testbucket mountdir
```shell
npm install
``` ```
# Author & License If you get an error regarding installation of the diskUsage module,
please install g++.
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0) If you get an error regarding level-down bindings, try clearing your npm cache:
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way) ```shell
npm cache clear
```
## Run it with a file backend
```shell
npm start
```
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
9991 are also open locally for internal transfer of metadata and data,
respectively.
The default access key is accessKey1 with
a secret key of verySecretKey1.
By default the metadata files will be saved in the
localMetadata directory and the data files will be saved
in the localData directory within the ./S3 directory on your
machine. These directories have been pre-created within the
repository. If you would like to save the data or metadata in
different locations of your choice, you must specify them with absolute paths.
So, when starting the server:
```shell
mkdir -m 700 $(pwd)/myFavoriteDataPath
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
npm start
```
## Run it with multiple data backends
```shell
export S3DATA='multiple'
npm start
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
With multiple backends, you have the ability to
choose where each object will be saved by setting
the following header with a locationConstraint on
a PUT request:
```shell
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
```
If no header is sent with a PUT object request, the
location constraint of the bucket will determine
where the data is saved. If the bucket has no location
constraint, the endpoint of the PUT request will be
used to determine location.
See the Configuration section in our documentation
[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
to learn how to set location constraints.
## Run it with an in-memory backend
```shell
npm run mem_backend
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae

View File

@ -1,2 +1,2 @@
--- ---
theme: jekyll-theme-modernist theme: jekyll-theme-minimal

View File

@ -13,26 +13,20 @@ function _performSearch(host,
port, port,
bucketName, bucketName,
query, query,
listVersions,
accessKey, accessKey,
secretKey, secretKey,
sessionToken,
verbose, ssl) { verbose, ssl) {
const escapedSearch = encodeURIComponent(query); const escapedSearch = encodeURIComponent(query);
const options = { const options = {
host, host,
port, port,
method: 'GET', method: 'GET',
path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`, path: `/${bucketName}/?search=${escapedSearch}`,
headers: { headers: {
'Content-Length': 0, 'Content-Length': 0,
}, },
rejectUnauthorized: false, rejectUnauthorized: false,
versions: '',
}; };
if (sessionToken) {
options.headers['x-amz-security-token'] = sessionToken;
}
const transport = ssl ? https : http; const transport = ssl ? https : http;
const request = transport.request(options, response => { const request = transport.request(options, response => {
if (verbose) { if (verbose) {
@ -61,9 +55,9 @@ function _performSearch(host,
// generateV4Headers exepects request object with path that does not // generateV4Headers exepects request object with path that does not
// include query // include query
request.path = `/${bucketName}`; request.path = `/${bucketName}`;
const requestData = listVersions ? { search: query, versions: '' } : { search: query }; auth.client.generateV4Headers(request, { search: query },
auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3'); accessKey, secretKey, 's3');
request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`; request.path = `/${bucketName}?search=${escapedSearch}`;
if (verbose) { if (verbose) {
logger.info('request headers', { headers: request._headers }); logger.info('request headers', { headers: request._headers });
} }
@ -82,17 +76,15 @@ function searchBucket() {
.version('0.0.1') .version('0.0.1')
.option('-a, --access-key <accessKey>', 'Access key id') .option('-a, --access-key <accessKey>', 'Access key id')
.option('-k, --secret-key <secretKey>', 'Secret access key') .option('-k, --secret-key <secretKey>', 'Secret access key')
.option('-t, --session-token <sessionToken>', 'Session token')
.option('-b, --bucket <bucket>', 'Name of the bucket') .option('-b, --bucket <bucket>', 'Name of the bucket')
.option('-q, --query <query>', 'Search query') .option('-q, --query <query>', 'Search query')
.option('-h, --host <host>', 'Host of the server') .option('-h, --host <host>', 'Host of the server')
.option('-p, --port <port>', 'Port of the server') .option('-p, --port <port>', 'Port of the server')
.option('-s', '--ssl', 'Enable ssl') .option('-s', '--ssl', 'Enable ssl')
.option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
'otherwise only list the latest version')
.option('-v, --verbose') .option('-v, --verbose')
.parse(process.argv); .parse(process.argv);
const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
const { host, port, accessKey, secretKey, bucket, query, verbose, ssl } =
commander; commander;
if (!host || !port || !accessKey || !secretKey || !bucket || !query) { if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
@ -101,7 +93,7 @@ function searchBucket() {
process.exit(1); process.exit(1);
} }
_performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose, _performSearch(host, port, bucket, query, accessKey, secretKey, verbose,
ssl); ssl);
} }

View File

@ -1,10 +1,7 @@
{ {
"port": 8000, "port": 8000,
"listenOn": [], "listenOn": [],
"metricsPort": 8002,
"metricsListenOn": [],
"replicationGroupId": "RG001", "replicationGroupId": "RG001",
"workers": 4,
"restEndpoints": { "restEndpoints": {
"localhost": "us-east-1", "localhost": "us-east-1",
"127.0.0.1": "us-east-1", "127.0.0.1": "us-east-1",
@ -42,10 +39,6 @@
"host": "localhost", "host": "localhost",
"port": 8900 "port": 8900
}, },
"workflowEngineOperator": {
"host": "localhost",
"port": 3001
},
"cdmi": { "cdmi": {
"host": "localhost", "host": "localhost",
"port": 81, "port": 81,
@ -75,10 +68,6 @@
"host": "localhost", "host": "localhost",
"port": 9991 "port": 9991
}, },
"pfsClient": {
"host": "localhost",
"port": 9992
},
"metadataDaemon": { "metadataDaemon": {
"bindAddress": "localhost", "bindAddress": "localhost",
"port": 9990 "port": 9990
@ -87,10 +76,6 @@
"bindAddress": "localhost", "bindAddress": "localhost",
"port": 9991 "port": 9991
}, },
"pfsDaemon": {
"bindAddress": "localhost",
"port": 9992
},
"recordLog": { "recordLog": {
"enabled": true, "enabled": true,
"recordLogName": "s3-recordlog" "recordLogName": "s3-recordlog"
@ -102,14 +87,6 @@
"readPreference": "primary", "readPreference": "primary",
"database": "metadata" "database": "metadata"
}, },
"authdata": "authdata.json",
"backends": {
"auth": "file",
"data": "file",
"metadata": "mongodb",
"kms": "file",
"quota": "none"
},
"externalBackends": { "externalBackends": {
"aws_s3": { "aws_s3": {
"httpAgent": { "httpAgent": {
@ -127,17 +104,5 @@
"maxSockets": null "maxSockets": null
} }
} }
},
"requests": {
"viaProxy": false,
"trustedProxyCIDRs": [],
"extractClientIPFromHeader": ""
},
"bucketNotificationDestinations": [
{
"resource": "target1",
"type": "dummy",
"host": "localhost:6000"
} }
]
} }

View File

@ -1,71 +0,0 @@
{
"port": 8000,
"listenOn": [],
"metricsPort": 8002,
"metricsListenOn": [],
"replicationGroupId": "RG001",
"restEndpoints": {
"localhost": "STANDARD",
"127.0.0.1": "STANDARD",
"yourhostname.ru": "STANDARD"
},
"websiteEndpoints": [
"static.yourhostname.ru"
],
"replicationEndpoints": [ {
"site": "zenko",
"servers": ["127.0.0.1:8000"],
"default": true
} ],
"log": {
"logLevel": "info",
"dumpLevel": "error"
},
"healthChecks": {
"allowFrom": ["127.0.0.1/8", "::1"]
},
"backends": {
"metadata": "mongodb"
},
"mongodb": {
"replicaSetHosts": "127.0.0.1:27017",
"writeConcern": "majority",
"replicaSet": "rs0",
"readPreference": "primary",
"database": "s3",
"authCredentials": {
"username": "s3",
"password": ""
}
},
"externalBackends": {
"aws_s3": {
"httpAgent": {
"keepAlive": false,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
},
"gcp": {
"httpAgent": {
"keepAlive": true,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
}
},
"requests": {
"viaProxy": false,
"trustedProxyCIDRs": [],
"extractClientIPFromHeader": ""
},
"bucketNotificationDestinations": [
{
"resource": "target1",
"type": "dummy",
"host": "localhost:6000"
}
]
}

View File

@ -86,51 +86,38 @@ const constants = {
// In testing, AWS seems to allow up to 88 more bytes, so we do the same. // In testing, AWS seems to allow up to 88 more bytes, so we do the same.
maximumMetaHeadersSize: 2136, maximumMetaHeadersSize: 2136,
// Maximum HTTP headers size allowed
maxHttpHeadersSize: 14122,
// hex digest of sha256 hash of empty string: // hex digest of sha256 hash of empty string:
emptyStringHash: crypto.createHash('sha256') emptyStringHash: crypto.createHash('sha256')
.update('', 'binary').digest('hex'), .update('', 'binary').digest('hex'),
// Queries supported by AWS that we do not currently support. // Queries supported by AWS that we do not currently support.
// Non-bucket queries
unsupportedQueries: [ unsupportedQueries: [
'accelerate', 'accelerate',
'analytics', 'analytics',
'inventory', 'inventory',
'logging', 'logging',
'metrics', 'metrics',
'policyStatus', 'notification',
'publicAccessBlock',
'requestPayment', 'requestPayment',
'restore',
'torrent', 'torrent',
], ],
// Headers supported by AWS that we do not currently support. // Headers supported by AWS that we do not currently support.
unsupportedHeaders: [ unsupportedHeaders: [
'x-amz-server-side-encryption',
'x-amz-server-side-encryption-customer-algorithm', 'x-amz-server-side-encryption-customer-algorithm',
'x-amz-server-side-encryption-aws-kms-key-id',
'x-amz-server-side-encryption-context', 'x-amz-server-side-encryption-context',
'x-amz-server-side-encryption-customer-key', 'x-amz-server-side-encryption-customer-key',
'x-amz-server-side-encryption-customer-key-md5', 'x-amz-server-side-encryption-customer-key-md5',
], ],
// user metadata header to set object locationConstraint // user metadata header to set object locationConstraint
objectLocationConstraintHeader: 'x-amz-storage-class', objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
legacyLocations: ['sproxyd', 'legacy'], legacyLocations: ['sproxyd', 'legacy'],
// declare here all existing service accounts and their properties
// (if any, otherwise an empty object)
serviceAccountProperties: {
replication: {},
lifecycle: {},
gc: {},
'md-ingestion': {
canReplicate: true,
},
},
/* eslint-disable camelcase */ /* eslint-disable camelcase */
externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true }, externalBackends: { aws_s3: true, azure: true, gcp: true },
replicationBackends: { aws_s3: true, azure: true, gcp: true },
// some of the available data backends (if called directly rather // some of the available data backends (if called directly rather
// than through the multiple backend gateway) need a key provided // than through the multiple backend gateway) need a key provided
// as a string as first parameter of the get/delete methods. // as a string as first parameter of the get/delete methods.
@ -150,8 +137,6 @@ const constants = {
base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' + base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
'(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'), '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko', productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
// location constraint delimiter
zenkoSeparator: ':',
// user metadata applied on zenko objects // user metadata applied on zenko objects
zenkoIDHeader: 'x-amz-meta-zenko-instance-id', zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
bucketOwnerActions: [ bucketOwnerActions: [
@ -176,73 +161,7 @@ const constants = {
'objectDeleteTagging', 'objectDeleteTagging',
'objectGetTagging', 'objectGetTagging',
'objectPutTagging', 'objectPutTagging',
'objectPutLegalHold',
'objectPutRetention',
], ],
// response header to be sent when there are invalid
// user metadata in the object's metadata
invalidObjectUserMetadataHeader: 'x-amz-missing-meta',
// Bucket specific queries supported by AWS that we do not currently support
// these queries may or may not be supported at object level
unsupportedBucketQueries: [
],
suppressedUtapiEventFields: [
'object',
'location',
'versionId',
],
allowedUtapiEventFilterFields: [
'operationId',
'location',
'account',
'user',
'bucket',
],
arrayOfAllowed: [
'objectPutTagging',
'objectPutLegalHold',
'objectPutRetention',
],
allowedUtapiEventFilterStates: ['allow', 'deny'],
allowedRestoreObjectRequestTierValues: ['Standard'],
lifecycleListing: {
CURRENT_TYPE: 'current',
NON_CURRENT_TYPE: 'noncurrent',
ORPHAN_DM_TYPE: 'orphan',
},
multiObjectDeleteConcurrency: 50,
maxScannedLifecycleListingEntries: 10000,
overheadField: [
'content-length',
'owner-id',
'versionId',
'isNull',
'isDeleteMarker',
],
unsupportedSignatureChecksums: new Set([
'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
]),
supportedSignatureChecksums: new Set([
'UNSIGNED-PAYLOAD',
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
]),
ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
// The AWS assumed Role resource type
assumedRoleArnResourceType: 'assumed-role',
// Session name of the backbeat lifecycle assumed role session.
backbeatLifecycleSessionName: 'backbeat-lifecycle',
actionsToConsiderAsObjectPut: [
'initiateMultipartUpload',
'objectPutPart',
'completeMultipartUpload',
],
// if requester is not bucket owner, bucket policy actions should be denied with
// MethodNotAllowed error
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
}; };
module.exports = constants; module.exports = constants;

View File

@ -71,14 +71,9 @@ fi
if [[ "$LISTEN_ADDR" ]]; then if [[ "$LISTEN_ADDR" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\"" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\"" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
fi fi
if [[ "$REPLICATION_GROUP_ID" ]] ; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
fi
if [[ "$DATA_HOST" ]]; then if [[ "$DATA_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\"" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
fi fi
@ -87,10 +82,6 @@ if [[ "$METADATA_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\"" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
fi fi
if [[ "$PFSD_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
fi
if [[ "$MONGODB_HOSTS" ]]; then if [[ "$MONGODB_HOSTS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\"" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
fi fi
@ -147,14 +138,6 @@ if [[ "$CRR_METRICS_PORT" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
fi fi
if [[ "$WE_OPERATOR_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
fi
if [[ "$WE_OPERATOR_PORT" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
fi
if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
fi fi
@ -195,14 +178,6 @@ if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
fi fi
if [[ -n "$BUCKET_DENY_FILTER" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
fi
if [[ "$TESTING_MODE" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
fi
if [[ $JQ_FILTERS_CONFIG != "." ]]; then if [[ $JQ_FILTERS_CONFIG != "." ]]; then
jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
mv config.json.tmp config.json mv config.json.tmp config.json

View File

@ -66,7 +66,7 @@ The second section, `"Implementation of Bucket Versioning in
API" <#implementation-of-bucket-versioning-in-api>`__, describes the way API" <#implementation-of-bucket-versioning-in-api>`__, describes the way
the metadata options are used in the API within S3 actions to create new the metadata options are used in the API within S3 actions to create new
versions, update their metadata, and delete them. The management of null versions, update their metadata, and delete them. The management of null
versions and creation of delete markers is also described in this versions and creation of delete markers are also described in this
section. section.
Implementation of Bucket Versioning in Metadata Implementation of Bucket Versioning in Metadata
@ -746,7 +746,7 @@ Operation
Startup Startup
~~~~~~~ ~~~~~~~
The simplest deployment is still to launch with yarn start, this will The simplest deployment is still to launch with npm start, this will
start one instance of the Zenko CloudServer connector and will listen on the start one instance of the Zenko CloudServer connector and will listen on the
locally bound dmd ports 9990 and 9991 (by default, see below). locally bound dmd ports 9990 and 9991 (by default, see below).
@ -755,7 +755,7 @@ command in the Zenko CloudServer directory:
:: ::
yarn run start_dmd npm run start_dmd
This will open two ports: This will open two ports:
@ -770,7 +770,7 @@ elsewhere with:
.. code:: sh .. code:: sh
yarn run start_s3server npm run start_s3server
Configuration Configuration
~~~~~~~~~~~~~ ~~~~~~~~~~~~~

View File

@ -295,51 +295,3 @@ Should force path-style requests even though v3 advertises it does by default.
$client->createBucket(array( $client->createBucket(array(
'Bucket' => 'bucketphp', 'Bucket' => 'bucketphp',
)); ));
Go
~~
`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. code:: go
package main
import (
"context"
"fmt"
"log"
"os"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
func main() {
os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
endpoint := "http://localhost:8000"
timeout := time.Duration(10) * time.Second
sess := session.Must(session.NewSession())
// Create a context with a timeout that will abort the upload if it takes
// more than the passed in timeout.
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
svc := s3.New(sess, &aws.Config{
Region: aws.String(endpoints.UsEast1RegionID),
Endpoint: &endpoint,
})
out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
if err != nil {
log.Fatal(err)
} else {
fmt.Println(out)
}
}

View File

@ -14,7 +14,7 @@ Got an idea? Get started!
In order to contribute, please follow the `Contributing In order to contribute, please follow the `Contributing
Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__. Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__.
If anything is unclear to you, reach out to us on If anything is unclear to you, reach out to us on
`forum <https://forum.zenko.io/>`__ or via a GitHub issue. `slack <https://zenko-io.slack.com/>`__ or via a GitHub issue.
Don't write code? There are other ways to help! Don't write code? There are other ways to help!
----------------------------------------------- -----------------------------------------------

View File

@ -1,7 +1,11 @@
Docker Docker
====== ======
.. _environment-variables: - `Environment Variables <#environment-variables>`__
- `Tunables and setup tips <#tunables-and-setup-tips>`__
- `Examples for continuous integration with
Docker <#continuous-integration-with-docker-hosted CloudServer>`__
- `Examples for going in production with Docker <#in-production-with-docker-hosted CloudServer>`__
Environment Variables Environment Variables
--------------------- ---------------------
@ -11,23 +15,21 @@ S3DATA
S3DATA=multiple S3DATA=multiple
^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^
Allows you to run Scality Zenko CloudServer with multiple data backends, defined
This variable enables running CloudServer with multiple data backends, defined
as regions. as regions.
When using multiple data backends, a custom ``locationConfig.json`` file is
mandatory. It will allow you to set custom regions. You will then need to
provide associated rest_endpoints for each custom region in your
``config.json`` file.
`Learn more about multiple backends configuration <../GETTING_STARTED/#location-configuration>`__
For multiple data backends, a custom locationConfig.json file is required. If you are using Scality RING endpoints, please refer to your customer
This file enables you to set custom regions. You must provide associated documentation.
rest_endpoints for each custom region in config.json.
`Learn more about multiple-backend configurations <GETTING_STARTED.html#location-configuration>`__ Running it with an AWS S3 hosted backend
""""""""""""""""""""""""""""""""""""""""
If you are using Scality RING endpoints, refer to your customer documentation. To run CloudServer with an S3 AWS backend, you will have to add a new section
to your ``locationConfig.json`` file with the ``aws_s3`` location type:
Running CloudServer with an AWS S3-Hosted Backend
"""""""""""""""""""""""""""""""""""""""""""""""""
To run CloudServer with an S3 AWS backend, add a new section to the
``locationConfig.json`` file with the ``aws_s3`` location type:
.. code:: json .. code:: json
@ -43,9 +45,10 @@ To run CloudServer with an S3 AWS backend, add a new section to the
} }
(...) (...)
Edit your AWS credentials file to enable your preferred command-line tool. You will also have to edit your AWS credentials file to be able to use your
This file must mention credentials for all backends in use. You can use command line tool of choice. This file should mention credentials for all the
several profiles if multiple profiles are configured. backends you're using. You can use several profiles when using multiple
profiles.
.. code:: json .. code:: json
@ -56,124 +59,110 @@ several profiles if multiple profiles are configured.
aws_access_key_id={{YOUR_ACCESS_KEY}} aws_access_key_id={{YOUR_ACCESS_KEY}}
aws_secret_access_key={{YOUR_SECRET_KEY}} aws_secret_access_key={{YOUR_SECRET_KEY}}
As with locationConfig.json, the AWS credentials file must be mounted at Just as you need to mount your locationConfig.json, you will need to mount your
run time: ``-v ~/.aws/credentials:/root/.aws/credentials`` on Unix-like AWS credentials file at run time:
systems (Linux, OS X, etc.), or ``-v ~/.aws/credentials:/root/.aws/credentials`` on Linux, OS X, or Unix or
``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows ``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows
.. note:: One account cannot copy to another account with a source and NOTE: One account can't copy to another account with a source and
destination on real AWS unless the account associated with the destination on real AWS unless the account associated with the
accessKey/secretKey pairs used for the destination bucket has source access Key/secret Key pairs used for the destination bucket has rights
bucket access privileges. To enable this, update ACLs directly on AWS. to get in the source bucket. ACL's would have to be updated
on AWS directly to enable this.
S3BACKEND S3BACKEND
~~~~~~~~~ ~~~~~~~~~
S3BACKEND=file S3BACKEND=file
^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^
When storing file data, for it to be persistent you must mount docker volumes
For stored file data to persist, you must mount Docker volumes for both data and metadata. See `this section <#using-docker-volumes-in-production>`__
for both data and metadata. See :ref:`In Production with a Docker-Hosted CloudServer <in-production-w-a-Docker-hosted-cloudserver>`
S3BACKEND=mem S3BACKEND=mem
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
This is ideal for testing - no data will remain after container is shutdown.
This is ideal for testing: no data remains after the container is shut down.
ENDPOINT ENDPOINT
~~~~~~~~ ~~~~~~~~
This variable specifies the endpoint. To direct CloudServer requests to This variable specifies your endpoint. If you have a domain such as
new.host.com, for example, specify the endpoint with: new.host.com, by specifying that here, you and your users can direct s3
server requests to new.host.com.
.. code-block:: shell .. code-block:: shell
$ docker run -d --name cloudserver -p 8000:8000 -e ENDPOINT=new.host.com zenko/cloudserver $ docker run -d --name s3server -p 8000:8000 -e ENDPOINT=new.host.com scality/s3server
.. note:: On Unix-like systems (Linux, OS X, etc.) edit /etc/hosts Note: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
to associate 127.0.0.1 with new.host.com. permissions, make sure to associate 127.0.0.1 with ``new.host.com``
REMOTE_MANAGEMENT_DISABLE
~~~~~~~~~~~~~~~~~~~~~~~~~
CloudServer is a part of `Zenko <https://www.zenko.io/>`__. When you run CloudServer standalone it will still try to connect to Orbit by default (browser-based graphical user interface for Zenko).
Setting this variable to true(1) will default to accessKey1 and verySecretKey1 for credentials and disable the automatic Orbit management:
.. code-block:: shell
$ docker run -d --name cloudserver -p 8000:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
These variables specify authentication credentials for an account named These variables specify authentication credentials for an account named
“CustomAccount”. "CustomAccount".
Set account credentials for multiple accounts by editing conf/authdata.json You can set credentials for many accounts by editing
(see below for further details). To specify one set for personal use, set these ``conf/authdata.json`` (see below for further info), but if you just
environment variables: want to specify one set of your own, you can use these environment
variables.
.. code-block:: shell .. code-block:: shell
$ docker run -d --name cloudserver -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey \ docker run -d --name s3server -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey
-e SCALITY_SECRET_ACCESS_KEY=newSecretKey zenko/cloudserver -e SCALITY_SECRET_ACCESS_KEY=newSecretKey scality/s3server
.. note:: This takes precedence over the contents of the authdata.json Note: Anything in the ``authdata.json`` file will be ignored. Note: The
file. The authdata.json file is ignored. old ``ACCESS_KEY`` and ``SECRET_KEY`` environment variables are now
deprecated
.. note:: The ACCESS_KEY and SECRET_KEY environment variables are
deprecated.
LOG\_LEVEL LOG\_LEVEL
~~~~~~~~~~ ~~~~~~~~~~
This variable changes the log level. There are three levels: info, debug, This variable allows you to change the log level: info, debug or trace.
and trace. The default is info. Debug provides more detailed logs, and trace The default is info. Debug will give you more detailed logs and trace
provides the most detailed logs. will give you the most detailed.
.. code-block:: shell .. code-block:: shell
$ docker run -d --name cloudserver -p 8000:8000 -e LOG_LEVEL=trace zenko/cloudserver $ docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
SSL SSL
~~~ ~~~
Set true, this variable runs CloudServer with SSL. This variable set to true allows you to run S3 with SSL:
If SSL is set true: **Note1**: You also need to specify the ENDPOINT environment variable.
**Note2**: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
permissions, make sure to associate 127.0.0.1 with ``<YOUR_ENDPOINT>``
* The ENDPOINT environment variable must also be specified. **Warning**: These certs, being self-signed (and the CA being generated
inside the container) will be untrusted by any clients, and could
disappear on a container upgrade. That's ok as long as it's for quick
testing. Also, best security practice for non-testing would be to use an
extra container to do SSL/TLS termination such as haproxy/nginx/stunnel
to limit what an exploit on either component could expose, as well as
certificates in a mounted volume
* On Unix-like systems (Linux, OS X, etc.), 127.0.0.1 must be associated with .. code-block:: shell
<YOUR_ENDPOINT> in /etc/hosts.
.. Warning:: Self-signed certs with a CA generated within the container are $ docker run -d --name s3server -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT>
suitable for testing purposes only. Clients cannot trust them, and they may scality/s3server
disappear altogether on a container upgrade. The best security practice for
production environments is to use an extra container, such as
haproxy/nginx/stunnel, for SSL/TLS termination and to pull certificates
from a mounted volume, limiting what an exploit on either component
can expose.
.. code:: shell More information about how to use S3 server with SSL
`here <https://s3.scality.com/v1.0/page/scality-with-ssl>`__
$ docker run -d --name cloudserver -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT> \
zenko/cloudserver
For more information about using ClousdServer with SSL, see `Using SSL <GETTING_STARTED.html#Using SSL>`__
LISTEN\_ADDR LISTEN\_ADDR
~~~~~~~~~~~~ ~~~~~~~~~~~~
This variable causes CloudServer and its data and metadata components to This variable instructs the Zenko CloudServer, and its data and metadata
listen on the specified address. This allows starting the data or metadata components to listen on the specified address. This allows starting the data
servers as standalone services, for example. or metadata servers as standalone services, for example.
.. code:: shell .. code-block:: shell
docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0 $ docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0
scality/s3server yarn run start_dataserver scality/s3server npm run start_dataserver
DATA\_HOST and METADATA\_HOST DATA\_HOST and METADATA\_HOST
@ -183,10 +172,10 @@ These variables configure the data and metadata servers to use,
usually when they are running on another host and only starting the stateless usually when they are running on another host and only starting the stateless
Zenko CloudServer. Zenko CloudServer.
.. code:: shell .. code-block:: shell
$ docker run -d --name cloudserver -e DATA_HOST=cloudserver-data \ $ docker run -d --name s3server -e DATA_HOST=s3server-data
-e METADATA_HOST=cloudserver-metadata zenko/cloudserver yarn run start_s3server -e METADATA_HOST=s3server-metadata scality/s3server npm run start_s3server
REDIS\_HOST REDIS\_HOST
~~~~~~~~~~~ ~~~~~~~~~~~
@ -194,23 +183,21 @@ REDIS\_HOST
Use this variable to connect to the redis cache server on another host than Use this variable to connect to the redis cache server on another host than
localhost. localhost.
.. code:: shell .. code-block:: shell
$ docker run -d --name cloudserver -p 8000:8000 \ $ docker run -d --name s3server -p 8000:8000
-e REDIS_HOST=my-redis-server.example.com zenko/cloudserver -e REDIS_HOST=my-redis-server.example.com scality/s3server
REDIS\_PORT REDIS\_PORT
~~~~~~~~~~~ ~~~~~~~~~~~
Use this variable to connect to the Redis cache server on a port other Use this variable to connect to the redis cache server on another port than
than the default 6379. the default 6379.
.. code:: shell .. code-block:: shell
$ docker run -d --name cloudserver -p 8000:8000 \ $ docker run -d --name s3server -p 8000:8000
-e REDIS_PORT=6379 zenko/cloudserver -e REDIS_PORT=6379 scality/s3server
.. _tunables-and-setup-tips:
Tunables and Setup Tips Tunables and Setup Tips
----------------------- -----------------------
@ -218,57 +205,61 @@ Tunables and Setup Tips
Using Docker Volumes Using Docker Volumes
~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~
CloudServer runs with a file backend by default, meaning that data is Zenko CloudServer runs with a file backend by default.
stored inside the CloudServers Docker container.
For data and metadata to persist, data and metadata must be hosted in Docker So, by default, the data is stored inside your Zenko CloudServer Docker
volumes outside the CloudServers Docker container. Otherwise, the data container.
and metadata are destroyed when the container is erased.
However, if you want your data and metadata to persist, you **MUST** use
Docker volumes to host your data and metadata outside your Zenko CloudServer
Docker container. Otherwise, the data and metadata will be destroyed
when you erase the container.
.. code-block:: shell .. code-block:: shell
$ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata \ $ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata
-p 8000:8000 ­-d zenko/cloudserver -p 8000:8000 ­-d scality/s3server
This command mounts the ./data host directory to the container This command mounts the host directory, ``./data``, into the container
at /usr/src/app/localData and the ./metadata host directory to at ``/usr/src/app/localData`` and the host directory, ``./metadata``, into
the container at /usr/src/app/localMetaData. the container at ``/usr/src/app/localMetaData``. It can also be any host
mount point, like ``/mnt/data`` and ``/mnt/metadata``.
.. tip:: These host directories can be mounted to any accessible mount Adding modifying or deleting accounts or users credentials
point, such as /mnt/data and /mnt/metadata, for example. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Adding, Modifying, or Deleting Accounts or Credentials 1. Create locally a customized ``authdata.json`` based on our ``/conf/authdata.json``.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1. Create a customized authdata.json file locally based on /conf/authdata.json. 2. Use `Docker
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__
2. Use `Docker volumes <https://docs.docker.com/storage/volumes/>`__ to override the default ``authdata.json`` through a docker file mapping.
to override the default ``authdata.json`` through a Docker file mapping.
For example: For example:
.. code-block:: shell .. code-block:: shell
$ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d \ $ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
zenko/cloudserver scality/s3server
Specifying a Host Name Specifying your own host name
~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
To specify a host name (for example, s3.domain.name), provide your own To specify a host name (e.g. s3.domain.name), you can provide your own
`config.json <https://github.com/scality/cloudserver/blob/master/config.json>`__ `config.json <https://github.com/scality/S3/blob/master/config.json>`__
file using `Docker volumes <https://docs.docker.com/storage/volumes/>`__. using `Docker
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__.
First, add a new key-value pair to the restEndpoints section of your First add a new key-value pair in the restEndpoints section of your
config.json. Make the key the host name you want, and the value the default config.json. The key in the key-value pair should be the host name you
location\_constraint for this endpoint. would like to add and the value is the default location\_constraint for
this endpoint.
For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one
of the ``location_constraints`` listed in your locationConfig.json file of the ``location_constraints`` listed in your locationConfig.json file
`here <https://github.com/scality/S3/blob/master/locationConfig.json>`__. `here <https://github.com/scality/S3/blob/master/locationConfig.json>`__.
For more information about location configuration, see: More information about location configuration
`GETTING STARTED <GETTING_STARTED.html#location-configuration>`__ `here <https://github.com/scality/S3/blob/master/README.md#location-configuration>`__
.. code:: json .. code:: json
@ -276,31 +267,31 @@ For more information about location configuration, see:
"localhost": "file", "localhost": "file",
"127.0.0.1": "file", "127.0.0.1": "file",
... ...
"cloudserver.example.com": "us-east-1" "s3.example.com": "us-east-1"
}, },
Next, run CloudServer using a `Docker volume Then, run your Scality S3 Server using `Docker
<https://docs.docker.com/engine/tutorials/dockervolumes/>`__: Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
.. code-block:: shell .. code-block:: shell
$ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d zenko/cloudserver $ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
The local ``config.json`` file overrides the default one through a Docker Your local ``config.json`` file will override the default one through a
file mapping. docker file mapping.
Running as an Unprivileged User Running as an unprivileged user
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
CloudServer runs as root by default. Zenko CloudServer runs as root by default.
To change this, modify the dockerfile and specify a user before the You can change that by modifing the dockerfile and specifying a user
entry point. before the entrypoint.
The user must exist within the container, and must own the The user needs to exist within the container, and own the folder
/usr/src/app directory for CloudServer to run. **/usr/src/app** for Scality Zenko CloudServer to run properly.
For example, the following dockerfile lines can be modified: For instance, you can modify these lines in the dockerfile:
.. code-block:: shell .. code-block:: shell
@ -314,58 +305,54 @@ For example, the following dockerfile lines can be modified:
USER scality USER scality
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"] ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
.. _continuous-integration-with-docker-hosted-cloudserver: Continuous integration with Docker hosted CloudServer
-----------------------------------------------------
Continuous Integration with a Docker-Hosted CloudServer When you start the Docker Scality Zenko CloudServer image, you can adjust the
------------------------------------------------------- configuration of the Scality Zenko CloudServer instance by passing one or more
environment variables on the docker run command line.
When you start the Docker CloudServer image, you can adjust the Sample ways to run it for CI are:
configuration of the CloudServer instance by passing one or more
environment variables on the ``docker run`` command line.
- With custom locations (one in-memory, one hosted on AWS), and custom
To run CloudServer for CI with custom locations (one in-memory, credentials mounted:
one hosted on AWS), and custom credentials mounted:
.. code-block:: shell .. code-block:: shell
$ docker run --name CloudServer -p 8000:8000 \ docker run --name CloudServer -p 8000:8000
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \ -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \ -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-v ~/.aws/credentials:/root/.aws/credentials \ -v ~/.aws/credentials:/root/.aws/credentials
-e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver -e S3DATA=multiple -e S3BACKEND=mem scality/s3server
To run CloudServer for CI with custom locations, (one in-memory, one - With custom locations, (one in-memory, one hosted on AWS, one file),
hosted on AWS, and one file), and custom credentials `set as environment and custom credentials set as environment variables
variables <GETTING_STARTED.html#scality-access-key-id-and-scality-secret-access-key>`__): (see `this section <#scality-access-key-id-and-scality-secret-access-key>`__):
.. code-block:: shell .. code-block:: shell
$ docker run --name CloudServer -p 8000:8000 \ docker run --name CloudServer -p 8000:8000
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \ -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-v ~/.aws/credentials:/root/.aws/credentials \ -v ~/.aws/credentials:/root/.aws/credentials
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \ -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-e SCALITY_ACCESS_KEY_ID=accessKey1 \ -e SCALITY_ACCESS_KEY_ID=accessKey1
-e SCALITY_SECRET_ACCESS_KEY=verySecretKey1 \ -e SCALITY_SECRET_ACCESS_KEY=verySecretKey1
-e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver -e S3DATA=multiple -e S3BACKEND=mem scality/s3server
.. _in-production-w-a-Docker-hosted-cloudserver: In production with Docker hosted CloudServer
--------------------------------------------
In Production with a Docker-Hosted CloudServer In production, we expect that data will be persistent, that you will use the
---------------------------------------------- multiple backends capabilities of Zenko CloudServer, and that you will have a
custom endpoint for your local storage, and custom credentials for your local
Because data must persist in production settings, CloudServer offers storage:
multiple-backend capabilities. This requires a custom endpoint
and custom credentials for local storage.
Customize these with:
.. code-block:: shell .. code-block:: shell
$ docker run -d --name CloudServer \ docker run -d --name CloudServer
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \ -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \ -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \ -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple \ -v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple
-e ENDPOINT=custom.endpoint.com \ -e ENDPOINT=custom.endpoint.com
-p 8000:8000 ­-d zenko/cloudserver \ -p 8000:8000 ­-d scality/s3server

View File

@ -4,198 +4,191 @@ Getting Started
.. figure:: ../res/scality-cloudserver-logo.png .. figure:: ../res/scality-cloudserver-logo.png
:alt: Zenko CloudServer logo :alt: Zenko CloudServer logo
|CircleCI| |Scality CI|
Dependencies
------------
Building and running the Scality Zenko CloudServer requires node.js 10.x and
yarn v1.17.x. Up-to-date versions can be found at
`Nodesource <https://github.com/nodesource/distributions>`__.
Installation Installation
------------ ------------
1. Clone the source code Dependencies
~~~~~~~~~~~~
Building and running the Scality Zenko CloudServer requires node.js 6.9.5 and
npm v3 . Up-to-date versions can be found at
`Nodesource <https://github.com/nodesource/distributions>`__.
Clone source code
~~~~~~~~~~~~~~~~~
.. code-block:: shell .. code-block:: shell
$ git clone https://github.com/scality/cloudserver.git git clone https://github.com/scality/S3.git
2. Go to the cloudserver directory and use yarn to install the js dependencies. Install js dependencies
~~~~~~~~~~~~~~~~~~~~~~~
Go to the ./S3 folder,
.. code-block:: shell .. code-block:: shell
$ cd cloudserver npm install
$ yarn install
Running CloudServer with a File Backend Run it with a file backend
--------------------------------------- --------------------------
.. code-block:: shell .. code-block:: shell
$ yarn start npm start
This starts a Zenko CloudServer on port 8000. Two additional ports, 9990 This starts an Zenko CloudServer on port 8000. Two additional ports 9990 and
and 9991, are also open locally for internal transfer of metadata and 9991 are also open locally for internal transfer of metadata and data,
data, respectively. respectively.
The default access key is accessKey1. The secret key is verySecretKey1. The default access key is accessKey1 with a secret key of
verySecretKey1.
By default, metadata files are saved in the localMetadata directory and By default the metadata files will be saved in the localMetadata
data files are saved in the localData directory in the local ./cloudserver directory and the data files will be saved in the localData directory
directory. These directories are pre-created within the repository. To within the ./S3 directory on your machine. These directories have been
save data or metadata in different locations, you must specify them using pre-created within the repository. If you would like to save the data or
absolute paths. Thus, when starting the server: metadata in different locations of your choice, you must specify them
with absolute paths. So, when starting the server:
.. code-block:: shell .. code-block:: shell
$ mkdir -m 700 $(pwd)/myFavoriteDataPath mkdir -m 700 $(pwd)/myFavoriteDataPath
$ mkdir -m 700 $(pwd)/myFavoriteMetadataPath mkdir -m 700 $(pwd)/myFavoriteMetadataPath
$ export S3DATAPATH="$(pwd)/myFavoriteDataPath" export S3DATAPATH="$(pwd)/myFavoriteDataPath"
$ export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath" export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
$ yarn start npm start
Running CloudServer with Multiple Data Backends Run it with multiple data backends
----------------------------------------------- ----------------------------------
.. code-block:: shell .. code-block:: shell
$ export S3DATA='multiple' export S3DATA='multiple'
$ yarn start npm start
This starts a Zenko CloudServer on port 8000. This starts an Zenko CloudServer on port 8000. The default access key is
accessKey1 with a secret key of verySecretKey1.
The default access key is accessKey1. The secret key is verySecretKey1. With multiple backends, you have the ability to choose where each object
will be saved by setting the following header with a locationConstraint
With multiple backends, you can choose where each object is saved by setting on a PUT request:
the following header with a location constraint in a PUT request:
.. code-block:: shell .. code-block:: shell
'x-amz-meta-scal-location-constraint':'myLocationConstraint' 'x-amz-meta-scal-location-constraint':'myLocationConstraint'
If no header is sent with a PUT object request, the buckets location If no header is sent with a PUT object request, the location constraint
constraint determines where the data is saved. If the bucket has no of the bucket will determine where the data is saved. If the bucket has
location constraint, the endpoint of the PUT request determines location. no location constraint, the endpoint of the PUT request will be used to
determine location.
See the Configuration_ section to set location constraints. See the Configuration section below to learn how to set location
constraints.
Run CloudServer with an In-Memory Backend Run it with an in-memory backend
-----------------------------------------
.. code-block:: shell
$ yarn run mem_backend
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1. The secret key is verySecretKey1.
Run CloudServer with Vault User Management
------------------------------------------
.. code:: shell
export S3VAULT=vault
yarn start
Note: Vault is proprietary and must be accessed separately.
This starts a Zenko CloudServer using Vault for user management.
Run CloudServer for Continuous Integration Testing or in Production with Docker
-------------------------------------------------------------------------------
Run Cloudserver with `DOCKER <DOCKER.html>`__
Testing
~~~~~~~
Run unit tests with the command:
.. code-block:: shell
$ yarn test
Run multiple-backend unit tests with:
.. code-block:: shell
$ CI=true S3DATA=multiple yarn start
$ yarn run multiple_backend_test
Run the linter with:
.. code-block:: shell
$ yarn run lint
Running Functional Tests Locally
-------------------------------- --------------------------------
To pass AWS and Azure backend tests locally, modify .. code-block:: shell
tests/locationConfig/locationConfigTests.json so that ``awsbackend``
specifies the bucketname of a bucket you have access to based on your npm run mem_backend
credentials, and modify ``azurebackend`` with details for your Azure account.
This starts an Zenko CloudServer on port 8000. The default access key is
accessKey1 with a secret key of verySecretKey1.
Run it for continuous integration testing or in production with Docker
----------------------------------------------------------------------
`DOCKER <../DOCKER/>`__
Testing
-------
You can run the unit tests with the following command:
.. code-block:: shell
npm test
You can run the multiple backend unit tests with:
.. code-block:: shell
CI=true S3DATA=multiple npm start
npm run multiple_backend_test
You can run the linter with:
.. code-block:: shell
npm run lint
Running functional tests locally:
For the AWS backend and Azure backend tests to pass locally,
you must modify tests/locationConfigTests.json so that awsbackend
specifies a bucketname of a bucket you have access to based on
your credentials profile and modify "azurebackend" with details
for your Azure account.
The test suite requires additional tools, **s3cmd** and **Redis** The test suite requires additional tools, **s3cmd** and **Redis**
installed in the environment the tests are running in. installed in the environment the tests are running in.
1. Install `s3cmd <http://s3tools.org/download>`__ - Install `s3cmd <http://s3tools.org/download>`__
- Install `redis <https://redis.io/download>`__ and start Redis.
- Add localCache section to your ``config.json``:
2. Install `redis <https://redis.io/download>`__ and start Redis. ::
3. Add localCache section to ``config.json``:
.. code:: json
"localCache": { "localCache": {
"host": REDIS_HOST, "host": REDIS_HOST,
"port": REDIS_PORT "port": REDIS_PORT
} }
where ``REDIS_HOST`` is the Redis instance IP address (``"127.0.0.1"`` where ``REDIS_HOST`` is your Redis instance IP address (``"127.0.0.1"``
if Redis is running locally) and ``REDIS_PORT`` is the Redis instance if your Redis is running locally) and ``REDIS_PORT`` is your Redis
port (``6379`` by default) instance port (``6379`` by default)
4. Add the following to the local etc/hosts file: - Add the following to the etc/hosts file on your machine:
.. code-block:: shell .. code-block:: shell
127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com 127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
5. Start Zenko CloudServer in memory and run the functional tests: - Start the Zenko CloudServer in memory and run the functional tests:
.. code-block:: shell .. code-block:: shell
$ CI=true yarn run mem_backend CI=true npm run mem_backend
$ CI=true yarn run ft_test CI=true npm run ft_test
.. _Configuration:
Configuration Configuration
------------- -------------
There are three configuration files for Zenko CloudServer: There are three configuration files for your Scality Zenko CloudServer:
* ``conf/authdata.json``, for authentication. 1. ``conf/authdata.json``, described above for authentication
* ``locationConfig.json``, to configure where data is saved. 2. ``locationConfig.json``, to set up configuration options for
* ``config.json``, for general configuration options. where data will be saved
.. _location-configuration: 3. ``config.json``, for general configuration options
Location Configuration Location Configuration
~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~
You must specify at least one locationConstraint in locationConfig.json You must specify at least one locationConstraint in your
(or leave it as pre-configured). locationConfig.json (or leave as pre-configured).
You must also specify 'us-east-1' as a locationConstraint. If you put a You must also specify 'us-east-1' as a locationConstraint so if you only
bucket to an unknown endpoint and do not specify a locationConstraint in define one locationConstraint, that would be it. If you put a bucket to
the PUT bucket call, us-east-1 is used. an unknown endpoint and do not specify a locationConstraint in the put
bucket call, us-east-1 will be used.
For instance, the following locationConstraint saves data sent to For instance, the following locationConstraint will save data sent to
``myLocationConstraint`` to the file backend: ``myLocationConstraint`` to the file backend:
.. code:: json .. code:: json
@ -206,16 +199,17 @@ For instance, the following locationConstraint saves data sent to
"details": {} "details": {}
}, },
Each locationConstraint must include the ``type``, ``legacyAwsBehavior``, Each locationConstraint must include the ``type``,
and ``details`` keys. ``type`` indicates which backend is used for that ``legacyAwsBehavior``, and ``details`` keys. ``type`` indicates which
region. Supported backends are mem, file, and scality.``legacyAwsBehavior`` backend will be used for that region. Currently, mem, file, and scality
indicates whether the region behaves the same as the AWS S3 'us-east-1' are the supported backends. ``legacyAwsBehavior`` indicates whether the
region. If the locationConstraint type is ``scality``, ``details`` must region will have the same behavior as the AWS S3 'us-east-1' region. If
contain connector information for sproxyd. If the locationConstraint type the locationConstraint type is scality, ``details`` should contain
is ``mem`` or ``file``, ``details`` must be empty. connector information for sproxyd. If the locationConstraint type is mem
or file, ``details`` should be empty.
Once locationConstraints is set in locationConfig.json, specify a default Once you have your locationConstraints in your locationConfig.json, you
locationConstraint for each endpoint. can specify a default locationConstraint for each of your endpoints.
For instance, the following sets the ``localhost`` endpoint to the For instance, the following sets the ``localhost`` endpoint to the
``myLocationConstraint`` data backend defined above: ``myLocationConstraint`` data backend defined above:
@ -226,24 +220,26 @@ For instance, the following sets the ``localhost`` endpoint to the
"localhost": "myLocationConstraint" "localhost": "myLocationConstraint"
}, },
To use an endpoint other than localhost for Zenko CloudServer, the endpoint If you would like to use an endpoint other than localhost for your
must be listed in ``restEndpoints``. Otherwise, if the server is running Scality Zenko CloudServer, that endpoint MUST be listed in your
with a: ``restEndpoints``. Otherwise if your server is running with a:
* **file backend**: The default location constraint is ``file`` - **file backend**: your default location constraint will be ``file``
* **memory backend**: The default location constraint is ``mem``
- **memory backend**: your default location constraint will be ``mem``
Endpoints Endpoints
~~~~~~~~~ ~~~~~~~~~
The Zenko CloudServer supports endpoints that are rendered in either: Note that our Zenko CloudServer supports both:
* path style: http://myhostname.com/mybucket or - path-style: http://myhostname.com/mybucket
* hosted style: http://mybucket.myhostname.com - hosted-style: http://mybucket.myhostname.com
However, if an IP address is specified for the host, hosted-style requests However, hosted-style requests will not hit the server if you are using
cannot reach the server. Use path-style requests in that case. For example, an ip address for your host. So, make sure you are using path-style
if you are using the AWS SDK for JavaScript, instantiate your client like this: requests in that case. For instance, if you are using the AWS SDK for
JavaScript, you would instantiate your client like this:
.. code:: js .. code:: js
@ -252,97 +248,85 @@ if you are using the AWS SDK for JavaScript, instantiate your client like this:
s3ForcePathStyle: true, s3ForcePathStyle: true,
}); });
Setting Your Own Access and Secret Key Pairs Setting your own access key and secret key pairs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Credentials can be set for many accounts by editing ``conf/authdata.json``, You can set credentials for many accounts by editing
but use the ``SCALITY_ACCESS_KEY_ID`` and ``SCALITY_SECRET_ACCESS_KEY`` ``conf/authdata.json`` but if you want to specify one set of your own
environment variables to specify your own credentials. credentials, you can use ``SCALITY_ACCESS_KEY_ID`` and
``SCALITY_SECRET_ACCESS_KEY`` environment variables.
_`scality-access-key-id-and-scality-secret-access-key`
SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
These variables specify authentication credentials for an account named These variables specify authentication credentials for an account named
“CustomAccount”. "CustomAccount".
.. note:: Anything in the ``authdata.json`` file is ignored. Note: Anything in the ``authdata.json`` file will be ignored.
.. code-block:: shell .. code-block:: shell
$ SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey yarn start SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey npm start
.. _Using_SSL:
Using SSL Scality with SSL
~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~
To use https with your local CloudServer, you must set up If you wish to use https with your local Zenko CloudServer, you need to set up
SSL certificates. SSL certificates. Here is a simple guide of how to do it.
1. Deploy CloudServer using `our DockerHub page Deploying Zenko CloudServer
<https://hub.docker.com/r/zenko/cloudserver/>`__ (run it with a file ^^^^^^^^^^^^^^^^^^^^^^^^^^^
backend).
.. Note:: If Docker is not installed locally, follow the First, you need to deploy **Zenko CloudServer**. This can be done very easily
`instructions to install it for your distribution via `our **DockerHub**
<https://docs.docker.com/engine/installation/>`__ page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
with a file backend).
2. Update the CloudServer containers config *Note:* *- If you don't have docker installed on your machine, here
are the `instructions to install it for your
distribution <https://docs.docker.com/engine/installation/>`__*
Add your certificates to your container. To do this, Updating your Zenko CloudServer container's config
#. exec inside the CloudServer container. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#. Run ``$> docker ps`` to find the containers ID (the corresponding You're going to add your certificates to your container. In order to do
image name is ``scality/cloudserver``. so, you need to exec inside your Zenko CloudServer container. Run a
``$> docker ps`` and find your container's id (the corresponding image
#. Copy the corresponding container ID (``894aee038c5e`` in the present name should be ``scality/s3server``. Copy the corresponding container id
example), and run: (here we'll use ``894aee038c5e``, and run:
.. code-block:: shell .. code-block:: shell
$> docker exec -it 894aee038c5e bash $> docker exec -it 894aee038c5e bash
This puts you inside your container, using an interactive terminal. You're now inside your container, using an interactive terminal :)
3. Generate the SSL key and certificates. The paths where the different Generate SSL key and certificates
files are stored are defined after the ``-out`` option in each of the **********************************
following commands.
#. Generate a private key for your certificate signing request (CSR): There are 5 steps to this generation. The paths where the different
files are stored are defined after the ``-out`` option in each command
.. code-block:: shell .. code-block:: shell
# Generate a private key for your CSR
$> openssl genrsa -out ca.key 2048 $> openssl genrsa -out ca.key 2048
# Generate a self signed certificate for your local Certificate Authority
#. Generate a self-signed certificate for your local certificate
authority (CA):
.. code:: shell
$> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test" $> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
#. Generate a key for the CloudServer: # Generate a key for Zenko CloudServer
.. code:: shell
$> openssl genrsa -out test.key 2048 $> openssl genrsa -out test.key 2048
# Generate a Certificate Signing Request for S3 Server
#. Generate a CSR for CloudServer:
.. code:: shell
$> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test" $> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
# Generate a local-CA-signed certificate for S3 Server
#. Generate a certificate for CloudServer signed by the local CA:
.. code:: shell
$> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256 $> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
4. Update Zenko CloudServer ``config.json``. Add a ``certFilePaths`` Update Zenko CloudServer ``config.json``
section to ``./config.json`` with appropriate paths: ****************************************
Add a ``certFilePaths`` section to ``./config.json`` with the
appropriate paths:
.. code:: json .. code:: json
@ -352,36 +336,42 @@ SSL certificates.
"ca": "./ca.crt" "ca": "./ca.crt"
} }
5. Run your container with the new config. Run your container with the new config
**************************************
#. Exit the container by running ``$> exit``. First, you need to exit your container. Simply run ``$> exit``. Then,
you need to restart your container. Normally, a simple
``$> docker restart s3server`` should do the trick.
#. Restart the container with ``$> docker restart cloudserver``. Update your host config
^^^^^^^^^^^^^^^^^^^^^^^
6. Update the host configuration by adding s3.scality.test Associates local IP addresses with hostname
to /etc/hosts: *******************************************
.. code:: bash In your ``/etc/hosts`` file on Linux, OS X, or Unix (with root
permissions), edit the line of localhost so it looks like this:
::
127.0.0.1 localhost s3.scality.test 127.0.0.1 localhost s3.scality.test
7. Copy the local certificate authority (ca.crt in step 4) from your Copy the local certificate authority from your container
container. Choose the path to save this file to (in the present ********************************************************
example, ``/root/ca.crt``), and run:
.. code:: shell In the above commands, it's the file named ``ca.crt``. Choose the path
you want to save this file at (here we chose ``/root/ca.crt``), and run
something like:
.. code-block:: shell
$> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt $> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
.. note:: Your container ID will be different, and your path to Test your config
ca.crt may be different. ^^^^^^^^^^^^^^^^^
Test the Config If you do not have aws-sdk installed, run ``$> npm install aws-sdk``. In
^^^^^^^^^^^^^^^ a ``test.js`` file, paste the following script:
If aws-sdk is not installed, run ``$> yarn install aws-sdk``.
Paste the following script into a file named "test.js":
.. code:: js .. code:: js
@ -421,13 +411,8 @@ Paste the following script into a file named "test.js":
}); });
}); });
Now run this script with: Now run that script with ``$> nodejs test.js``. If all goes well, it
should output ``SSL is cool!``. Enjoy that added security!
.. code::
$> nodejs test.js
On success, the script outputs ``SSL is cool!``.
.. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg .. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg

View File

@ -4,32 +4,38 @@ Integrations
High Availability High Availability
================= =================
`Docker Swarm <https://docs.docker.com/engine/swarm/>`__ is a clustering tool `Docker swarm <https://docs.docker.com/engine/swarm/>`__ is a
developed by Docker for use with its containers. It can be used to start clustering tool developped by Docker and ready to use with its
services, which we define to ensure CloudServer's continuous availability to containers. It allows to start a service, which we define and use as a
end users. A swarm defines a manager and *n* workers among *n* + 1 servers. means to ensure Zenko CloudServer's continuous availability to the end user.
Indeed, a swarm defines a manager and n workers among n+1 servers. We
This tutorial shows how to perform a basic setup with three servers, which will do a basic setup in this tutorial, with just 3 servers, which
provides strong service resiliency, while remaining easy to use and already provides a strong service resiliency, whilst remaining easy to
maintain. We will use NFS through Docker to share data and do as an individual. We will use NFS through docker to share data and
metadata between the different servers. metadata between the different servers.
Sections are labeled **On Server**, **On Clients**, or You will see that the steps of this tutorial are defined as **On
**On All Machines**, referring respectively to NFS server, NFS clients, or Server**, **On Clients**, **On All Machines**. This refers respectively
NFS server and clients. In the present example, the servers IP address is to NFS Server, NFS Clients, or NFS Server and Clients. In our example,
**10.200.15.113** and the client IP addresses are **10.200.15.96** and the IP of the Server will be **10.200.15.113**, while the IPs of the
**10.200.15.97** Clients will be **10.200.15.96 and 10.200.15.97**
1. Install Docker (on All Machines) Installing docker
-----------------
Docker 17.03.0-ce is used for this tutorial. Docker 1.12.6 and later will Any version from docker 1.12.6 onwards should work; we used Docker
likely work, but is not tested. 17.03.0-ce for this tutorial.
* On Ubuntu 14.04 On All Machines
Install Docker CE for Ubuntu as `documented at Docker ~~~~~~~~~~~~~~~
<https://docs.docker.com/install/linux/docker-ce/ubuntu/>`__.
Install the aufs dependency as recommended by Docker. The required On Ubuntu 14.04
commands are: ^^^^^^^^^^^^^^^
The docker website has `solid
documentation <https://docs.docker.com/engine/installation/linux/ubuntu/>`__.
We have chosen to install the aufs dependency, as recommended by Docker.
Here are the required commands:
.. code:: sh .. code:: sh
@ -41,10 +47,12 @@ NFS server and clients. In the present example, the servers IP address is
$> sudo apt-get update $> sudo apt-get update
$> sudo apt-get install docker-ce $> sudo apt-get install docker-ce
* On CentOS 7 On CentOS 7
Install Docker CE as `documented at Docker ^^^^^^^^^^^
<https://docs.docker.com/install/linux/docker-ce/centos/>`__.
The required commands are: The docker website has `solid
documentation <https://docs.docker.com/engine/installation/linux/centos/>`__.
Here are the required commands:
.. code:: sh .. code:: sh
@ -54,22 +62,29 @@ NFS server and clients. In the present example, the servers IP address is
$> sudo yum install docker-ce $> sudo yum install docker-ce
$> sudo systemctl start docker $> sudo systemctl start docker
2. Install NFS on Client(s) Configure NFS
-------------
NFS clients mount Docker volumes over the NFS servers shared folders. On Clients
If the NFS commons are installed, manual mounts are no longer needed. ~~~~~~~~~~
* On Ubuntu 14.04 Your NFS Clients will mount Docker volumes over your NFS Server's shared
folders. Hence, you don't have to mount anything manually, you just have
to install the NFS commons:
Install the NFS commons with apt-get: On Ubuntu 14.04
^^^^^^^^^^^^^^^
Simply install the NFS commons:
.. code:: sh .. code:: sh
$> sudo apt-get install nfs-common $> sudo apt-get install nfs-common
* On CentOS 7 On CentOS 7
^^^^^^^^^^^
Install the NFS utils; then start required services: Install the NFS utils, and then start the required services:
.. code:: sh .. code:: sh
@ -83,22 +98,27 @@ NFS server and clients. In the present example, the servers IP address is
$> sudo systemctl start nfs-lock $> sudo systemctl start nfs-lock
$> sudo systemctl start nfs-idmap $> sudo systemctl start nfs-idmap
3. Install NFS (on Server) On Server
~~~~~~~~~
The NFS server hosts the data and metadata. The package(s) to install on it Your NFS Server will be the machine to physically host the data and
differs from the package installed on the clients. metadata. The package(s) we will install on it is slightly different
from the one we installed on the clients.
* On Ubuntu 14.04 On Ubuntu 14.04
^^^^^^^^^^^^^^^
Install the NFS server-specific package and the NFS commons: Install the NFS server specific package and the NFS commons:
.. code:: sh .. code:: sh
$> sudo apt-get install nfs-kernel-server nfs-common $> sudo apt-get install nfs-kernel-server nfs-common
* On CentOS 7 On CentOS 7
^^^^^^^^^^^
Install the NFS utils and start the required services: Same steps as with the client: install the NFS utils and start the
required services:
.. code:: sh .. code:: sh
@ -112,55 +132,74 @@ NFS server and clients. In the present example, the servers IP address is
$> sudo systemctl start nfs-lock $> sudo systemctl start nfs-lock
$> sudo systemctl start nfs-idmap $> sudo systemctl start nfs-idmap
For both distributions: On Ubuntu 14.04 and CentOS 7
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#. Choose where shared data and metadata from the local Choose where your shared data and metadata from your local `Zenko CloudServer
`CloudServer <http://www.zenko.io/cloudserver/>`__ shall be stored (The <http://www.zenko.io/cloudserver/>`__ will be stored.
present example uses /var/nfs/data and /var/nfs/metadata). Set permissions We chose to go with /var/nfs/data and /var/nfs/metadata. You also need
for these folders for to set proper sharing permissions for these folders as they'll be shared
sharing over NFS: over NFS:
.. code:: sh .. code:: sh
$> mkdir -p /var/nfs/data /var/nfs/metadata $> mkdir -p /var/nfs/data /var/nfs/metadata
$> chmod -R 777 /var/nfs/ $> chmod -R 777 /var/nfs/
#. The /etc/exports file configures network permissions and r-w-x permissions Now you need to update your **/etc/exports** file. This is the file that
for NFS access. Edit /etc/exports, adding the following lines: configures network permissions and rwx permissions for NFS access. By
default, Ubuntu applies the no\_subtree\_check option, so we declared
both folders with the same permissions, even though they're in the same
tree:
.. code:: sh
$> sudo vim /etc/exports
In this file, add the following lines:
.. code:: sh .. code:: sh
/var/nfs/data 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash) /var/nfs/data 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
/var/nfs/metadata 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash) /var/nfs/metadata 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
Ubuntu applies the no\_subtree\_check option by default, so both Export this new NFS table:
folders are declared with the same permissions, even though theyre in
the same tree.
#. Export this new NFS table:
.. code:: sh .. code:: sh
$> sudo exportfs -a $> sudo exportfs -a
#. Edit the ``MountFlags`` option in the Docker config in Eventually, you need to allow for NFS mount from Docker volumes on other
/lib/systemd/system/docker.service to enable NFS mount from Docker volumes machines. You need to change the Docker config in
on other machines: **/lib/systemd/system/docker.service**:
.. code:: sh
$> sudo vim /lib/systemd/system/docker.service
In this file, change the **MountFlags** option:
.. code:: sh .. code:: sh
MountFlags=shared MountFlags=shared
#. Restart the NFS server and Docker daemons to apply these changes. Now you just need to restart the NFS server and docker daemons so your
changes apply.
* On Ubuntu 14.04 On Ubuntu 14.04
^^^^^^^^^^^^^^^
Restart your NFS Server and docker services:
.. code:: sh .. code:: sh
$> sudo service nfs-kernel-server restart $> sudo service nfs-kernel-server restart
$> sudo service docker restart $> sudo service docker restart
* On CentOS 7 On CentOS 7
^^^^^^^^^^^
Restart your NFS Server and docker daemons:
.. code:: sh .. code:: sh
@ -168,30 +207,34 @@ NFS server and clients. In the present example, the servers IP address is
$> sudo systemctl daemon-reload $> sudo systemctl daemon-reload
$> sudo systemctl restart docker $> sudo systemctl restart docker
Set up your Docker Swarm service
--------------------------------
4. Set Up a Docker Swarm On All Machines
~~~~~~~~~~~~~~~
* On all machines and distributions: On Ubuntu 14.04 and CentOS 7
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Set up the Docker volumes to be mounted to the NFS server for CloudServers We will now set up the Docker volumes that will be mounted to the NFS
data and metadata storage. The following commands must be replicated on all Server and serve as data and metadata storage for Zenko CloudServer. These two
machines: commands have to be replicated on all machines:
.. code:: sh .. code:: sh
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata $> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
There is no need to ``docker exec`` these volumes to mount them: the There is no need to ""docker exec" these volumes to mount them: the
Docker Swarm manager does this when the Docker service is started. Docker Swarm manager will do it when the Docker service will be started.
* On a server: On Server
^^^^^^^^^
To start a Docker service on a Docker Swarm cluster, initialize the cluster To start a Docker service on a Docker Swarm cluster, you first have to
(that is, define a manager), prompt workers/nodes to join in, and then start initialize that cluster (i.e.: define a manager), then have the
the service. workers/nodes join in, and then start the service. Initialize the swarm
cluster, and look at the response:
Initialize the swarm cluster, and review its response:
.. code:: sh .. code:: sh
@ -207,10 +250,11 @@ NFS server and clients. In the present example, the servers IP address is
To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions. To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
* On clients: On Clients
^^^^^^^^^^
Copy and paste the command provided by your Docker Swarm init. A successful Simply copy/paste the command provided by your docker swarm init. When
request/response will resemble: all goes well, you'll get something like this:
.. code:: sh .. code:: sh
@ -218,159 +262,182 @@ NFS server and clients. In the present example, the servers IP address is
This node joined a swarm as a worker. This node joined a swarm as a worker.
Set Up Docker Swarm on Clients on a Server On Server
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ^^^^^^^^^
Start the service on the Swarm cluster. Start the service on your swarm cluster!
.. code:: sh .. code:: sh
$> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/cloudserver $> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/s3server
On a successful installation, ``docker service ls`` returns the following If you run a docker service ls, you should have the following output:
output:
.. code:: sh .. code:: sh
$> docker service ls $> docker service ls
ID NAME MODE REPLICAS IMAGE ID NAME MODE REPLICAS IMAGE
ocmggza412ft s3 replicated 1/1 scality/cloudserver:latest ocmggza412ft s3 replicated 1/1 scality/s3server:latest
If the service does not start, consider disabling apparmor/SELinux. If your service won't start, consider disabling apparmor/SELinux.
Testing the High-Availability CloudServer Testing your High Availability S3Server
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ---------------------------------------
On all machines (client/server) and distributions (Ubuntu and CentOS), On All Machines
determine where CloudServer is running using ``docker ps``. CloudServer can
operate on any node of the Swarm cluster, manager or worker. When you find
it, you can kill it with ``docker stop <container id>``. It will respawn
on a different node. Now, if one server falls, or if Docker stops
unexpectedly, the end user will still be able to access your the local CloudServer.
Troubleshooting
~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~
To troubleshoot the service, run: On Ubuntu 14.04 and CentOS 7
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Try to find out where your Scality Zenko CloudServer is actually running using
the **docker ps** command. It can be on any node of the swarm cluster,
manager or worker. When you find it, you can kill it, with **docker stop
<container id>** and you'll see it respawn on a different node of the
swarm cluster. Now you see, if one of your servers falls, or if docker
stops unexpectedly, your end user will still be able to access your
local Zenko CloudServer.
Troubleshooting
---------------
To troubleshoot the service you can run:
.. code:: sh .. code:: sh
$> docker service ps s3docker service ps s3 $> docker service ps s3docker service ps s3
ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR
0ar81cw4lvv8chafm8pw48wbc s3.1 scality/cloudserver localhost.localdomain.localdomain Running Running 7 days ago 0ar81cw4lvv8chafm8pw48wbc s3.1 scality/s3server localhost.localdomain.localdomain Running Running 7 days ago
cvmf3j3bz8w6r4h0lf3pxo6eu \_ s3.1 scality/cloudserver localhost.localdomain.localdomain Shutdown Failed 7 days ago "task: non-zero exit (137)" cvmf3j3bz8w6r4h0lf3pxo6eu \_ s3.1 scality/s3server localhost.localdomain.localdomain Shutdown Failed 7 days ago "task: non-zero exit (137)"
If the error is truncated, view the error in detail by inspecting the If the error is truncated it is possible to have a more detailed view of
Docker task ID: the error by inspecting the docker task ID:
.. code:: sh .. code:: sh
$> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu $> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu
Off you go! Off you go!
~~~~~~~~~~~ -----------
Let us know what you use this functionality for, and if you'd like any
specific developments around it. Or, even better: come and contribute to
our `Github repository <https://github.com/scality/s3/>`__! We look
forward to meeting you!
Let us know how you use this and if you'd like any specific developments
around it. Even better: come and contribute to our `Github repository
<https://github.com/scality/s3/>`__! We look forward to meeting you!
S3FS S3FS
==== ====
Export your buckets as a filesystem with s3fs on top of Zenko CloudServer
You can export buckets as a filesystem with s3fs on CloudServer.
`s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source `s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source
tool, available both on Debian and RedHat distributions, that enables tool that allows you to mount an S3 bucket on a filesystem-like backend.
you to mount an S3 bucket on a filesystem-like backend. This tutorial uses It is available both on Debian and RedHat distributions. For this
an Ubuntu 14.04 host to deploy and use s3fs over CloudServer. tutorial, we used an Ubuntu 14.04 host to deploy and use s3fs over
Scality's Zenko CloudServer.
Deploying Zenko CloudServer with SSL Deploying Zenko CloudServer with SSL
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ------------------------------------
First, deploy CloudServer with a file backend using `our DockerHub page First, you need to deploy **Zenko CloudServer**. This can be done very easily
<https://hub.docker.com/r/zenko/cloudserver>`__. via `our DockerHub
page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
with a file backend).
.. note:: *Note:* *- If you don't have docker installed on your machine, here
are the `instructions to install it for your
distribution <https://docs.docker.com/engine/installation/>`__*
If Docker is not installed on your machine, follow You also necessarily have to set up SSL with Zenko CloudServer to use s3fs. We
`these instructions <https://docs.docker.com/engine/installation/>`__ have a nice
to install it for your distribution. `tutorial <https://s3.scality.com/v1.0/page/scality-with-ssl>`__ to help
you do it.
You must also set up SSL with CloudServer to use s3fs. See `Using SSL s3fs setup
<./GETTING_STARTED#Using_SSL>`__ for instructions. ----------
s3fs Setup
~~~~~~~~~~
Installing s3fs Installing s3fs
--------------- ~~~~~~~~~~~~~~~
Follow the instructions in the s3fs `README s3fs has quite a few dependencies. As explained in their
<https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation-from-pre-built-packages>`__, `README <https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation>`__,
the following commands should install everything for Ubuntu 14.04:
Check that s3fs is properly installed. A version check should return .. code:: sh
a response resembling:
$> sudo apt-get install automake autotools-dev g++ git libcurl4-gnutls-dev
$> sudo apt-get install libfuse-dev libssl-dev libxml2-dev make pkg-config
Now you want to install s3fs per se:
.. code:: sh
$> git clone https://github.com/s3fs-fuse/s3fs-fuse.git
$> cd s3fs-fuse
$> ./autogen.sh
$> ./configure
$> make
$> sudo make install
Check that s3fs is properly installed by checking its version. it should
answer as below:
.. code:: sh .. code:: sh
$> s3fs --version $> s3fs --version
Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL
Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Configuring s3fs Configuring s3fs
---------------- ~~~~~~~~~~~~~~~~
s3fs expects you to provide it with a password file. Our file is s3fs expects you to provide it with a password file. Our file is
``/etc/passwd-s3fs``. The structure for this file is ``/etc/passwd-s3fs``. The structure for this file is
``ACCESSKEYID:SECRETKEYID``, so, for CloudServer, you can run: ``ACCESSKEYID:SECRETKEYID``, so, for S3Server, you can run:
.. code:: sh .. code:: sh
$> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs $> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs
$> chmod 600 /etc/passwd-s3fs $> chmod 600 /etc/passwd-s3fs
Using CloudServer with s3fs Using Zenko CloudServer with s3fs
--------------------------- ---------------------------------
1. Use /mnt/tests3fs as a mount point. First, you're going to need a mountpoint; we chose ``/mnt/tests3fs``:
.. code:: sh .. code:: sh
$> mkdir /mnt/tests3fs $> mkdir /mnt/tests3fs
2. Create a bucket on your local CloudServer. In the present example it is Then, you want to create a bucket on your local Zenko CloudServer; we named it
named “tests3fs”. ``tests3fs``:
.. code:: sh .. code:: sh
$> s3cmd mb s3://tests3fs $> s3cmd mb s3://tests3fs
3. Mount the bucket to your mount point with s3fs: *Note:* *- If you've never used s3cmd with our Zenko CloudServer, our README
provides you with a `recommended
config <https://github.com/scality/S3/blob/master/README.md#s3cmd>`__*
Now you can mount your bucket to your mountpoint with s3fs:
.. code:: sh .. code:: sh
$> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style $> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style
The structure of this command is: *If you're curious, the structure of this command is*
``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``. Of these mandatory ``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``\ *, and the
options: options are mandatory and serve the following purposes:
* ``passwd_file``\ *: specifiy path to password file;
* ``url``\ *: specify the hostname used by your SSL provider;
* ``use_path_request_style``\ *: force path style (by default, s3fs
uses subdomains (DNS style)).*
* ``passwd_file`` specifies the path to the password file. | From now on, you can either add files to your mountpoint, or add
* ``url`` specifies the host name used by your SSL provider. objects to your bucket, and they'll show in the other.
* ``use_path_request_style`` forces the path style (by default, | For example, let's' create two files, and then a directory with a file
s3fs uses DNS-style subdomains). in our mountpoint:
Once the bucket is mounted, files added to the mount point or
objects added to the bucket will appear in both locations.
Example
-------
Create two files, and then a directory with a file in our mount point:
.. code:: sh .. code:: sh
@ -378,7 +445,7 @@ Example
$> mkdir /mnt/tests3fs/dir1 $> mkdir /mnt/tests3fs/dir1
$> touch /mnt/tests3fs/dir1/file3 $> touch /mnt/tests3fs/dir1/file3
Now, use s3cmd to show what is in CloudServer: Now, I can use s3cmd to show me what is actually in S3Server:
.. code:: sh .. code:: sh
@ -389,30 +456,27 @@ Example
2017-02-28 17:28 0 s3://tests3fs/file1 2017-02-28 17:28 0 s3://tests3fs/file1
2017-02-28 17:28 0 s3://tests3fs/file2 2017-02-28 17:28 0 s3://tests3fs/file2
Now you can enjoy a filesystem view on your local CloudServer. Now you can enjoy a filesystem view on your local Zenko CloudServer!
Duplicity Duplicity
========= =========
How to back up your files with CloudServer. How to backup your files with Zenko CloudServer.
Installing Duplicity and its Dependencies Installing
-----------
Installing Duplicity and its dependencies
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
To install `Duplicity <http://duplicity.nongnu.org/>`__, Second, you want to install
go to `this site <https://code.launchpad.net/duplicity/0.7-series>`__. `Duplicity <http://duplicity.nongnu.org/index.html>`__. You have to
Download the latest tarball. Decompress it and follow the instructions download `this
in the README. tarball <https://code.launchpad.net/duplicity/0.7-series/0.7.11/+download/duplicity-0.7.11.tar.gz>`__,
decompress it, and then checkout the README inside, which will give you
.. code:: sh a list of dependencies to install. If you're using Ubuntu 14.04, this is
your lucky day: here is a lazy step by step install.
$> tar zxvf duplicity-0.7.11.tar.gz
$> cd duplicity-0.7.11
$> python setup.py install
You may receive error messages indicating the need to install some or all
of the following dependencies:
.. code:: sh .. code:: sh
@ -420,20 +484,30 @@ of the following dependencies:
$> apt-get install python-dev python-pip python-lockfile $> apt-get install python-dev python-pip python-lockfile
$> pip install -U boto $> pip install -U boto
Testing the Installation Then you want to actually install Duplicity:
------------------------
1. Check that CloudServer is running. Run ``$> docker ps``. You should .. code:: sh
see one container named ``scality/cloudserver``. If you do not, run
``$> docker start cloudserver`` and check again.
$> tar zxvf duplicity-0.7.11.tar.gz
$> cd duplicity-0.7.11
$> python setup.py install
2. Duplicity uses a module called “Boto” to send requests to S3. Boto Using
requires a configuration file located in ``/etc/boto.cfg`` to store ------
your credentials and preferences. A minimal configuration
you can fine tune `following these instructions Testing your installation
<http://boto.cloudhackers.com/en/latest/getting_started.html>`__ is ~~~~~~~~~~~~~~~~~~~~~~~~~~~
shown here:
First, we're just going to quickly check that Zenko CloudServer is actually
running. To do so, simply run ``$> docker ps`` . You should see one
container named ``scality/s3server``. If that is not the case, try
``$> docker start s3server``, and check again.
Secondly, as you probably know, Duplicity uses a module called **Boto**
to send requests to S3. Boto requires a configuration file located in
**``/etc/boto.cfg``** to have your credentials and preferences. Here is
a minimalistic config `that you can finetune following these
instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
:: ::
@ -447,25 +521,26 @@ Testing the Installation
# If using SSL, unmute and provide absolute path to local CA certificate # If using SSL, unmute and provide absolute path to local CA certificate
# ca_certificates_file = /absolute/path/to/ca.crt # ca_certificates_file = /absolute/path/to/ca.crt
.. note:: To set up SSL with CloudServer, check out our `Using SSL *Note:* *If you want to set up SSL with Zenko CloudServer, check out our
<./GETTING_STARTED#Using_SSL>`__ in GETTING STARTED. `tutorial <http://link/to/SSL/tutorial>`__*
3. At this point all requirements to run CloudServer as a backend to Duplicity At this point, we've met all the requirements to start running Zenko CloudServer
have been met. A local folder/file should back up to the local S3. as a backend to Duplicity. So we should be able to back up a local
Try it with the decompressed Duplicity folder: folder/file to local S3. Let's try with the duplicity decompressed
folder:
.. code:: sh .. code:: sh
$> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/" $> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/"
.. note:: Duplicity will prompt for a symmetric encryption passphrase. *Note:* *Duplicity will prompt you for a symmetric encryption
Save it carefully, as you will need it to recover your data. passphrase. Save it somewhere as you will need it to recover your
Alternatively, you can add the ``--no-encryption`` flag data. Alternatively, you can also add the ``--no-encryption`` flag
and the data will be stored plain. and the data will be stored plain.*
If this command is successful, you will receive an output resembling: If this command is succesful, you will get an output looking like this:
.. code:: sh ::
--------------[ Backup Statistics ]-------------- --------------[ Backup Statistics ]--------------
StartTime 1486486547.13 (Tue Feb 7 16:55:47 2017) StartTime 1486486547.13 (Tue Feb 7 16:55:47 2017)
@ -485,13 +560,15 @@ Testing the Installation
Errors 0 Errors 0
------------------------------------------------- -------------------------------------------------
Congratulations! You can now back up to your local S3 through Duplicity. Congratulations! You can now backup to your local S3 through duplicity
:)
Automating Backups Automating backups
------------------ ~~~~~~~~~~~~~~~~~~~
The easiest way to back up files periodically is to write a bash script Now you probably want to back up your files periodically. The easiest
and add it to your crontab. A suggested script follows. way to do this is to write a bash script and add it to your crontab.
Here is my suggestion for such a file:
.. code:: sh .. code:: sh
@ -500,33 +577,33 @@ and add it to your crontab. A suggested script follows.
# Export your passphrase so you don't have to type anything # Export your passphrase so you don't have to type anything
export PASSPHRASE="mypassphrase" export PASSPHRASE="mypassphrase"
# To use a GPG key, put it here and uncomment the line below # If you want to use a GPG Key, put it here and unmute the line below
#GPG_KEY= #GPG_KEY=
# Define your backup bucket, with localhost specified # Define your backup bucket, with localhost specified
DEST="s3://127.0.0.1:8000/testbucketcloudserver/" DEST="s3://127.0.0.1:8000/testbuckets3server/"
# Define the absolute path to the folder to back up # Define the absolute path to the folder you want to backup
SOURCE=/root/testfolder SOURCE=/root/testfolder
# Set to "full" for full backups, and "incremental" for incremental backups # Set to "full" for full backups, and "incremental" for incremental backups
# Warning: you must perform one full backup befor you can perform # Warning: you have to perform one full backup befor you can perform
# incremental ones on top of it # incremental ones on top of it
FULL=incremental FULL=incremental
# How long to keep backups. If you don't want to delete old backups, keep # How long to keep backups for; if you don't want to delete old
# this value empty; otherwise, the syntax is "1Y" for one year, "1M" for # backups, keep empty; otherwise, syntax is "1Y" for one year, "1M"
# one month, "1D" for one day. # for one month, "1D" for one day
OLDER_THAN="1Y" OLDER_THAN="1Y"
# is_running checks whether Duplicity is currently completing a task # is_running checks whether duplicity is currently completing a task
is_running=$(ps -ef | grep duplicity | grep python | wc -l) is_running=$(ps -ef | grep duplicity | grep python | wc -l)
# If Duplicity is already completing a task, this will not run # If duplicity is already completing a task, this will simply not run
if [ $is_running -eq 0 ]; then if [ $is_running -eq 0 ]; then
echo "Backup for ${SOURCE} started" echo "Backup for ${SOURCE} started"
# To delete backups older than a certain time, do it here # If you want to delete backups older than a certain time, we do it here
if [ "$OLDER_THAN" != "" ]; then if [ "$OLDER_THAN" != "" ]; then
echo "Removing backups older than ${OLDER_THAN}" echo "Removing backups older than ${OLDER_THAN}"
duplicity remove-older-than ${OLDER_THAN} ${DEST} duplicity remove-older-than ${OLDER_THAN} ${DEST}
@ -549,17 +626,17 @@ and add it to your crontab. A suggested script follows.
# Forget the passphrase... # Forget the passphrase...
unset PASSPHRASE unset PASSPHRASE
Put this file in ``/usr/local/sbin/backup.sh``. Run ``crontab -e`` and So let's say you put this file in ``/usr/local/sbin/backup.sh.`` Next
paste your configuration into the file that opens. If you're unfamiliar you want to run ``crontab -e`` and paste your configuration in the file
with Cron, here is a good `HowTo that opens. If you're unfamiliar with Cron, here is a good `How
<https://help.ubuntu.com/community/CronHowto>`__. If the folder being To <https://help.ubuntu.com/community/CronHowto>`__. The folder I'm
backed up is a folder to be modified permanently during the work day, backing up is a folder I modify permanently during my workday, so I want
we can set incremental backups every 5 minutes from 8 AM to 9 PM Monday incremental backups every 5mn from 8AM to 9PM monday to friday. Here is
through Friday by pasting the following line into crontab: the line I will paste in my crontab:
.. code:: sh .. code:: sh
*/5 8-20 * * 1-5 /usr/local/sbin/backup.sh */5 8-20 * * 1-5 /usr/local/sbin/backup.sh
Adding or removing files from the folder being backed up will result in Now I can try and add / remove files from the folder I'm backing up, and
incremental backups in the bucket. I will see incremental backups in my bucket.

229
docs/MD_SEARCH.md Normal file
View File

@ -0,0 +1,229 @@
# Metadata Search Documenation
## Description
This feature enables metadata search to be performed on the metadata of objects
stored in Zenko.
## Requirements
+ MongoDB
## Design
The MD Search feature expands on the existing `GET Bucket` S3 API. It allows
users to conduct metadata searches by adding the custom Zenko querystring
parameter, `search`. The `search` parameter is of a pseudo
SQL WHERE clause structure and supports basic SQL operators:
ex. `"A=1 AND B=2 OR C=3"` (more complex queries can also be achieved with the
use of nesting operators, `(` and `)`).
The search process is as follows:
+ Zenko receives a `GET` request.
```
# regular getBucket request
GET /bucketname HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# getBucket versions request
GET /bucketname?versions HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# search getBucket request
GET /bucketname?search=key%3Dsearch-item HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
```
+ If the request does not contain the query param `search`, a normal bucket
listing is performed and a XML result containing the list of objects will be
returned as the response.
+ If the request does contain the query parameter `search`, the search string is
parsed and validated.
+ If the search string is invalid, an `InvalidArgument` error will be
returned as response.
+ If the search string is valid, it will be parsed and an abstract syntax
tree (AST) is generated.
+ The AST is then passed to the MongoDB backend to be used as the query filter
for retrieving objects in a bucket that satisfies the requested search
conditions.
+ The filtered results are then parsed and returned as the response.
The results from MD search is of the same structure as the `GET Bucket`
results:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bucketname</Name>
<Prefix/>
<Marker/>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>objectKey</Key>
<LastModified>2018-04-19T18:31:49.426Z</LastModified>
<ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
<Size>0</Size>
<Owner>
<ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
<DisplayName>Bart</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
...
</Contents>
</ListBucketResult>
```
## Performing MD Search with Zenko
To make a successful request to Zenko, you would need
+ Zenko Credentials
+ Sign request with Auth V4
With requirements, you can peform metadata searches by:
+ using the `search_bucket` tool in the
[Scality/S3](https://github.com/scality/S3) GitHub repository.
+ creating an AuthV4 signed HTTP request to Zenko in the programming language of
choice
### Using the S3 Tool
After cloning the [Scality/S3](https://github.com/scality/S3) GitHub repository
and installing the necessary dependencies, you can run the following command
in the S3 project root directory to access the search tool.
```
node bin/search_bucket
```
This will generate the following output
```
Usage: search_bucket [options]
Options:
-V, --version output the version number
-a, --access-key <accessKey> Access key id
-k, --secret-key <secretKey> Secret access key
-b, --bucket <bucket> Name of the bucket
-q, --query <query> Search query
-h, --host <host> Host of the server
-p, --port <port> Port of the server
-s --ssl
-v, --verbose
-h, --help output usage information
```
In the following examples, our Zenko Server is accessible on endpoint
`http://127.0.0.1:8000` and contains the bucket `zenkobucket`.
```
# search for objects with metadata "blue"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
# search for objects tagged with "type=color"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "tags.type=color" -h 127.0.0.1 -p 8000
```
### Coding Examples
Search requests can be also performed by making HTTP requests authenticated
with the `AWS Signature version 4` scheme.\
See the following urls for more information about the V4 authentication scheme.
+ http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
+ http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
You can also view examples for making requests with Auth V4 in various
languages [here](../exmaples).
### Specifying Metadata Fields
To search common metadata headers:
```
{metadata-key}{supported SQL op}{search value}
# example
key = blueObject
size > 0
key LIKE "blue.*"
```
To search custom user metadata:
```
# metadata must be prefixed with "x-amz-meta-"
x-amz-meta-{usermetadata-key}{supported SQL op}{search value}
# example
x-amz-meta-color = blue
x-amz-meta-color != red
x-amz-meta-color LIKE "b.*"
```
To search tags:
```
# tag searches must be prefixed with "tags."
tags.{tag-key}{supported SQL op}{search value}
# example
tags.type = color
```
### Differences from SQL
The MD search queries are similar to the `WHERE` clauses of SQL queries, but
they differ in that:
+ MD search queries follow the `PCRE` format
+ Search queries do not require values with hyphens to be enclosed in
backticks, ``(`)``
```
# SQL query
`x-amz-meta-search-item` = `ice-cream-cone`
# MD Search query
x-amz-meta-search-item = ice-cream-cone
```
+ The search queries do not support all of the SQL operators.
+ Supported SQL Operators: `=`, `<`, `>`, `<=`, `>=`, `!=`,
`AND`, `OR`, `LIKE`, `<>`
+ Unsupported SQL Operators: `NOT`, `BETWEEN`, `IN`, `IS`, `+`,
`-`, `%`, `^`, `/`, `*`, `!`
#### Using Regular Expressions in MD Search
+ Regular expressions used in MD search differs from SQL in that wildcards are
represented with `.*` instead of `%`.
+ Regex patterns must be wrapped in quotes as not doing so can lead to
misinterpretation of patterns.
+ Regex patterns can be written in form of the `/pattern/` syntax or
just the pattern if one does not require regex options, similar to `PCRE`.
Example regular expressions:
```
# search for strings containing word substring "helloworld"
".*helloworld.*"
"/.*helloworld.*/"
"/.*helloworld.*/i"
```

View File

@ -1,263 +0,0 @@
Metadata Search Documentation
=============================
Description
-----------
This feature enables metadata search to be performed on the metadata of objects
stored in Zenko.
Requirements
------------
* MongoDB
Design
------
The Metadata Search feature expands on the existing :code:`GET Bucket` S3 API by
enabling users to conduct metadata searches by adding the custom Zenko query
string parameter, :code:`search`. The :code:`search` parameter is structured as a pseudo
SQL WHERE clause, and supports basic SQL operators. For example:
:code:`"A=1 AND B=2 OR C=3"` (complex queries can be built using nesting
operators, :code:`(` and :code:`)`).
The search process is as follows:
* Zenko receives a :code:`GET` request.
.. code::
# regular getBucket request
GET /bucketname HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# getBucket versions request
GET /bucketname?versions HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# search getBucket request
GET /bucketname?search=key%3Dsearch-item HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
* If the request does *not* contain the :code:`search` query parameter, Zenko performs
a normal bucket listing and returns an XML result containing the list of
objects.
* If the request *does* contain the :code:`search` query parameter, Zenko parses and
validates the search string.
- If the search string is invalid, Zenko returns an :code:`InvalidArgument` error.
.. code::
<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<Error>
<Code>InvalidArgument</Code>
<Message>Invalid sql where clause sent as search query</Message>
<Resource></Resource>
<RequestId>d1d6afc64345a8e1198e</RequestId>
</Error>
- If the search string is valid, Zenko parses it and generates an abstract
syntax tree (AST). The AST is then passed to the MongoDB backend to be
used as the query filter for retrieving objects from a bucket that
satisfies the requested search conditions. Zenko parses the filtered
results and returns them as the response.
Metadata search results have the same structure as a :code:`GET Bucket` response:
.. code:: xml
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bucketname</Name>
<Prefix/>
<Marker/>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>objectKey</Key>
<LastModified>2018-04-19T18:31:49.426Z</LastModified>
<ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
<Size>0</Size>
<Owner>
<ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
<DisplayName>Bart</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
...
</Contents>
</ListBucketResult>
Performing Metadata Searches with Zenko
---------------------------------------
You can perform metadata searches by:
+ Using the :code:`search_bucket` tool in the
`Scality/S3 <https://github.com/scality/S3>`_ GitHub repository.
+ Creating a signed HTTP request to Zenko in your preferred programming
language.
Using the S3 Tool
+++++++++++++++++
After cloning the `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository
and installing the necessary dependencies, run the following command in the S3
projects root directory to access the search tool:
.. code::
node bin/search_bucket
This generates the following output:
.. code::
Usage: search_bucket [options]
Options:
-V, --version output the version number
-a, --access-key <accessKey> Access key id
-k, --secret-key <secretKey> Secret access key
-b, --bucket <bucket> Name of the bucket
-q, --query <query> Search query
-h, --host <host> Host of the server
-p, --port <port> Port of the server
-s --ssl
-v, --verbose
-h, --help output usage information
In the following examples, Zenko Server is accessible on endpoint
:code:`http://127.0.0.1:8000` and contains the bucket :code:`zenkobucket`.
.. code::
# search for objects with metadata "blue"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
# search for objects tagged with "type=color"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "tags.type=color" -h 127.0.0.1 -p 8000
Coding Examples
+++++++++++++++
Search requests can be also performed by making HTTP requests authenticated
with one of the AWS Signature schemes: version 2 or version 4. \
For more about authentication scheme, see:
* https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
* http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
* http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
You can also view examples for making requests with Auth V4 in various
languages `here <../../../examples>`__.
Specifying Metadata Fields
~~~~~~~~~~~~~~~~~~~~~~~~~~
To search system metadata headers:
.. code::
{system-metadata-key}{supported SQL op}{search value}
# example
key = blueObject
size > 0
key LIKE "blue.*"
To search custom user metadata:
.. code::
# metadata must be prefixed with "x-amz-meta-"
x-amz-meta-{user-metadata-key}{supported SQL op}{search value}
# example
x-amz-meta-color = blue
x-amz-meta-color != red
x-amz-meta-color LIKE "b.*"
To search tags:
.. code::
# tag searches must be prefixed with "tags."
tags.{tag-key}{supported SQL op}{search value}
# example
tags.type = color
Examples queries:
.. code::
# searching for objects with custom metadata "color"=blue" and are tagged
# "type"="color"
tags.type="color" AND x-amz-meta-color="blue"
# searching for objects with the object key containing the substring "blue"
# or (custom metadata "color"=blue" and are tagged "type"="color")
key LIKE '.*blue.*' OR (x-amz-meta-color="blue" AND tags.type="color")
Differences from SQL
++++++++++++++++++++
Zenko metadata search queries are similar to SQL-query :code:`WHERE` clauses, but
differ in that:
* They follow the :code:`PCRE` format
* They do not require values with hyphens to be enclosed in
backticks, :code:``(`)``
.. code::
# SQL query
`x-amz-meta-search-item` = `ice-cream-cone`
# MD Search query
x-amz-meta-search-item = ice-cream-cone
* Search queries do not support all SQL operators.
.. code::
# Supported SQL operators:
=, <, >, <=, >=, !=, AND, OR, LIKE, <>
# Unsupported SQL operators:
NOT, BETWEEN, IN, IS, +, -, %, ^, /, *, !
Using Regular Expressions in Metadata Search
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Regular expressions in Zenko metadata search differ from SQL in the following
ways:
+ Wildcards are represented with :code:`.*` instead of :code:`%`.
+ Regex patterns must be wrapped in quotes. Failure to do this can lead to
misinterpretation of patterns.
+ As with :code:`PCRE`, regular expressions can be entered in either the
:code:`/pattern/` syntax or as the pattern itself if regex options are
not required.
Example regular expressions:
.. code::
# search for strings containing word substring "helloworld"
".*helloworld.*"
"/.*helloworld.*/"
"/.*helloworld.*/i"

View File

@ -1,161 +0,0 @@
# Object Lock Feature Test Plan
## Feature Component Description
Implementing Object Lock will introduce six new APIs:
- putObjectLockConfiguration
- getObjectLockConfiguration
- putObjectRetention
- getObjectRetention
- putObjectLegalHold
- getObjectLegalHold
Along with these APIs, putBucket, putObject, deleteObject, and multiObjectDelete
be affected. In Arsenal, both the BucketInfo and ObjectMD models will be
updated. Bucket policy and IAM policy permissions will be updated to include
the new API actions.
## Functional Tests
### putBucket tests
- passing option to enable object lock updates bucket metadata and enables
bucket versioning
### putBucketVersioning tests
- suspending versioning on bucket with object lock enabled returns error
### putObject tests
- putting retention configuration on object should be allowed
- putting invalid retention configuration returns error
### getObject tests
- getting object with retention information should include retention information
### copyObject tests
- copying object with retention information should include retention information
### initiateMultipartUpload tests
- mpu object initiated with retention information should include retention
information
### putObjectLockConfiguration tests
- putting configuration as non-bucket-owner user returns AccessDenied error
- disabling object lock on bucket created with object lock returns error
- enabling object lock on bucket created without object lock returns
InvalidBucketState error
- enabling object lock with token on bucket created without object lock succeeds
- putting valid object lock configuration when bucket does not have object
lock enabled returns error (InvalidRequest?)
- putting valid object lock configuration updates bucket metadata
- putting invalid object lock configuration returns error
- ObjectLockEnabled !== "Enabled"
- Rule object doesn't contain DefaultRetention key
- Mode !== "GOVERNANCE" or "COMPLIANCE"
- Days are not an integer
- Years are not an integer
### getObjectLockConfiguration tests
- getting configuration as non-bucket-owner user returns AccessDenied error
- getting configuration when none is set returns
ObjectLockConfigurationNotFoundError error
- getting configuration returns correct object lock configuration for bucket
### putObjectRetention
- putting retention as non-bucket-owner user returns AccessDenied error
- putting retention on object in bucket without object lock enabled returns
InvalidRequest error
- putting valid retention period updates object metadata
### getObjectRetention
- getting retention as non-bucket-owner user returns AccessDenied error
- getting retention when none is set returns NoSuchObjectLockConfiguration
error
- getting retention returns correct object retention period
### putObjectLegalHold
- putting legal hold as non-bucket-owner user returns AccessDenied error
- putting legal hold on object in bucket without object lock enabled returns
InvalidRequest error
- putting valid legal hold updates object metadata
### getObjectLegalHold
- getting legal hold as non-bucket-owner user returns AccessDenied error
- getting legal hold when none is set returns NoSuchObjectLockConfiguration
error
- getting legal hold returns correct object legal hold
## End to End Tests
### Scenarios
- Create bucket with object lock enabled. Put object. Put object lock
configuration. Put another object.
- Ensure object put before configuration does not have retention period set
- Ensure object put after configuration does have retention period set
- Create bucket without object lock. Put object. Enable object lock with token
and put object lock configuration. Put another object.
- Ensure object put before configuration does not have retention period set
- Ensure object put after configuration does have retention period set
- Create bucket with object lock enabled and put configuration with COMPLIANCE
mode. Put object.
- Ensure object cannot be deleted (returns AccessDenied error).
- Ensure object cannot be overwritten.
- Create bucket with object lock enabled and put configuration with GOVERNANCE
mode. Put object.
- Ensure user without permission cannot delete object
- Ensure user without permission cannot overwrite object
- Ensure user with permission can delete object
- Ensure user with permission can overwrite object
- Ensure user with permission can lengthen retention period
- Ensure user with permission cannot shorten retention period
- Create bucket with object lock enabled and put configuration. Edit bucket
metadata so retention period is expired. Put object.
- Ensure object can be deleted.
- Ensure object can be overwritten.
- Create bucket with object lock enabled and put configuration. Edit bucket
metadata so retention period is expired. Put object. Put new retention
period on object.
- Ensure object cannot be deleted.
- Ensure object cannot be overwritten.
- Create bucket with object locked enabled and put configuration. Put object.
Edit object metadata so retention period is past expiration.
- Ensure object can be deleted.
- Ensure object can be overwritten.
- Create bucket with object lock enabled and put configuration. Edit bucket
metadata so retention period is expired. Put object. Put legal hold
on object.
- Ensure object cannot be deleted.
- Ensure object cannot be overwritten.
- Create bucket with object lock enabled and put configuration. Put object.
Check object retention. Change bucket object lock configuration.
- Ensure object retention period has not changed with bucket configuration.
- Create bucket with object lock enabled. Put object with legal hold.
- Ensure object cannot be deleted.
- Ensure object cannot be overwritten.
- Create bucket with object lock enabled. Put object with legal hold. Remove
legal hold.
- Ensure object can be deleted.
- Ensure object can be overwritten.

View File

@ -1,73 +0,0 @@
# Cloudserver Release Plan
## Docker Image Generation
Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
CloudServer has a few images there:
* Cloudserver container image: ghcr.io/scality/cloudserver
* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
With every CI build, the CI will push images, tagging the
content with the developer branch's short SHA-1 commit hash.
This allows those images to be used by developers, CI builds,
build chain and so on.
Tagged versions of cloudserver will be stored in the production namespace.
## How to Pull Docker Images
```sh
docker pull ghcr.io/scality/cloudserver:<commit hash>
docker pull ghcr.io/scality/cloudserver:<tag>
```
## Release Process
To release a production image:
* Create a PR to bump the package version
Update Cloudserver's `package.json` by bumping it to the relevant next
version in a new PR. Per example if the last released version was
`8.4.7`, the next version would be `8.4.8`.
```js
{
"name": "cloudserver",
"version": "8.4.8", <--- Here
[...]
}
```
* Review & merge the PR
* Create the release on GitHub
* Go the Release tab (https://github.com/scality/cloudserver/releases);
* Click on the `Draft new release button`;
* In the `tag` field, type the name of the release (`8.4.8`), and confirm
to create the tag on publish;
* Click on `Generate release notes` button to fill the fields;
* Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
* Click to `Publish the release` to create the GitHub release and git tag
Notes:
* the Git tag will be created automatically.
* this should be done as soon as the PR is merged, so that the tag
is put on the "version bump" commit.
* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
* Branch Name: The one used for the tag earlier. In this example `development/8.4`
* Override Stage: 'release'
* Extra properties:
* name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
* Release the release version on Jira
* Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
* Create a next version
* Name: `[next version]`, in this example `8.4.9`
* Click `...` and select `Release` on the recently released version (`8.4.8`)
* Fill in the field to move incomplete version to the next one

View File

@ -6,7 +6,7 @@ Using Public Clouds as data backends
Introduction Introduction
------------ ------------
As stated in our `GETTING STARTED guide <GETTING_STARTED.html#location-configuration>`__, As stated in our `GETTING STARTED guide <../GETTING_STARTED/#location-configuration>`__,
new data backends can be added by creating a region (also called location new data backends can be added by creating a region (also called location
constraint) with the right endpoint and credentials. constraint) with the right endpoint and credentials.
This section of the documentation shows you how to set up our currently This section of the documentation shows you how to set up our currently
@ -139,7 +139,7 @@ to start the server and start writing data to AWS S3 through CloudServer.
.. code:: shell .. code:: shell
# Start the server locally # Start the server locally
$> S3DATA=multiple yarn start $> S3DATA=multiple npm start
Run the server as a docker container with the ability to write to AWS S3 Run the server as a docker container with the ability to write to AWS S3
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -162,8 +162,8 @@ CloudServer.
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \ -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
-v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \ -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
-v ~/.aws/credentials:/root/.aws/credentials \ -v ~/.aws/credentials:/root/.aws/credentials \
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 \ -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-d scality/cloudserver -d scality/s3server
Testing: put an object to AWS S3 using CloudServer Testing: put an object to AWS S3 using CloudServer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -306,7 +306,7 @@ to start the server and start writing data to MS Azure through CloudServer.
.. code:: shell .. code:: shell
# Start the server locally # Start the server locally
$> S3DATA=multiple yarn start $> S3DATA=multiple npm start
Run the server as a docker container with the ability to write to MS Azure Run the server as a docker container with the ability to write to MS Azure
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -324,7 +324,7 @@ CloudServer.
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \ -v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
-v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \ -v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 -e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-d scality/cloudserver -d scality/s3server
Testing: put an object to MS Azure using CloudServer Testing: put an object to MS Azure using CloudServer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

View File

@ -1,8 +1,8 @@
============================================ ============================================
Add New Backend Storage To Zenko CloudServer Add New Backend Storage To Zenko Cloudserver
============================================ ============================================
This set of documents aims at bootstrapping developers with Zenko's CloudServer This set of documents aims at bootstrapping developers with Zenko's Cloudserver
module, so they can then go on and contribute features. module, so they can then go on and contribute features.
.. toctree:: .. toctree::
@ -43,7 +43,7 @@ Openstack Swift x
and filling out the "Feature Request" section of our and filling out the "Feature Request" section of our
template. template.
To add support for a new backend support to CloudServer official To add support for a new backend support to Cloudserver official
repository, please follow these steps: repository, please follow these steps:
- familiarize yourself with our `Contributing Guidelines`_ - familiarize yourself with our `Contributing Guidelines`_

View File

@ -2,28 +2,28 @@
Add A New Backend Add A New Backend
================= =================
Supporting all possible public cloud storage APIs is CloudServer's Supporting all possible public cloud storage APIs is Cloudserver's
ultimate goal. As an open source project, contributions are welcome. ultimate goal. As an open source project, contributions are welcome.
The first step is to get familiar with building a custom Docker image The first step is to get familiar with building a custom Docker image
for CloudServer. for Cloudserver.
Build a Custom Docker Image Build a Custom Docker Image
~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~
Clone Zenko's CloudServer, install all dependencies and start the Clone Zenko's Cloudserver, install all dependencies and start the
service: service:
.. code-block:: shell .. code-block:: shell
$ git clone https://github.com/scality/cloudserver $ git clone https://github.com/scality/cloudserver
$ cd cloudserver $ cd cloudserver
$ yarn install $ npm install
$ yarn start $ npm start
.. tip:: .. tip::
Some optional dependencies may fail, resulting in you seeing `yarn Some optional dependencies may fail, resulting in you seeing `NPM
WARN` messages; these can safely be ignored. Refer to the User WARN` messages; these can safely be ignored. Refer to the User
documentation for all available options. documentation for all available options.

View File

@ -6,7 +6,7 @@ These backends abstract the complexity of multiple APIs to let users
work on a single common namespace across multiple clouds. work on a single common namespace across multiple clouds.
This documents aims at introducing you to the right files in This documents aims at introducing you to the right files in
CloudServer (the Zenko stack's subcomponent in charge of API Cloudserver (the Zenko stack's subcomponent in charge of API
translation, among other things) to add support to your own backend of translation, among other things) to add support to your own backend of
choice. choice.

View File

@ -1,10 +1,10 @@
====================== ======================
S3-Compatible Backends S3 compatible backends
====================== ======================
Adding Support in CloudServer Adding support in Zenkos Cloudserver
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This is the easiest case for backend support integration: there is nothing to do This is the easiest case for backend support integration: there is nothing to do
but configuration! Follow the steps described in our but configuration! Follow the steps described in our
@ -33,7 +33,7 @@ definition for that backend will look something like:
} }
}, },
Adding Support in Zenko Orbit Adding support in Zenko Orbit
############################# #############################
This can only be done by our core developpers' team. If thats what youre This can only be done by our core developpers' team. If thats what youre

13
eve/get_product_version.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/sh
LOCAL_BRANCH=$(git branch | grep \* | cut -d ' ' -f2)
BRANCHES=(development q stabilization)
for branch in ${BRANCHES[@]}; do
if echo "${LOCAL_BRANCH}\/" | grep -q ^${branch} ; then
cat .git/HEAD | sed 's/.*\///'
exit 0
fi
done
echo 0.0.0

363
eve/main.yml Normal file
View File

@ -0,0 +1,363 @@
---
version: 0.2
branches:
feature/*, improvement/*, bugfix/*, w/*, q/*, hotfix/*:
stage: pre-merge
development/*:
stage: post-merge
models:
- env: &global-env
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurebackend_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key_2)s
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name_2)s
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint_2)s
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azuretest_AZURE_BLOB_ENDPOINT: "%(secret:azure_storage_endpoint)s"
b2backend_B2_ACCOUNT_ID: "%(secret:b2backend_b2_account_id)s"
b2backend_B2_STORAGE_ACCESS_KEY: >-
%(secret:b2backend_b2_storage_access_key)s
GOOGLE_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
GOOGLE_SERVICE_KEY: "%(secret:gcp_service_key)s"
AWS_S3_BACKEND_ACCESS_KEY: "%(secret:aws_s3_backend_access_key)s"
AWS_S3_BACKEND_SECRET_KEY: "%(secret:aws_s3_backend_secret_key)s"
AWS_S3_BACKEND_ACCESS_KEY_2: "%(secret:aws_s3_backend_access_key_2)s"
AWS_S3_BACKEND_SECRET_KEY_2: "%(secret:aws_s3_backend_secret_key_2)s"
AWS_GCP_BACKEND_ACCESS_KEY: "%(secret:aws_gcp_backend_access_key)s"
AWS_GCP_BACKEND_SECRET_KEY: "%(secret:aws_gcp_backend_secret_key)s"
AWS_GCP_BACKEND_ACCESS_KEY_2: "%(secret:aws_gcp_backend_access_key_2)s"
AWS_GCP_BACKEND_SECRET_KEY_2: "%(secret:aws_gcp_backend_secret_key_2)s"
b2backend_B2_STORAGE_ENDPOINT: "%(secret:b2backend_b2_storage_endpoint)s"
gcpbackend2_GCP_SERVICE_EMAIL: "%(secret:gcp2_service_email)s"
gcpbackend2_GCP_SERVICE_KEY: "%(secret:gcp2_service_key)s"
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackend_GCP_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
gcpbackend_GCP_SERVICE_KEY: "%(secret:gcp_service_key)s"
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
%(secret:gcpbackendmismatch_gcp_service_email)s
gcpbackendmismatch_GCP_SERVICE_KEY: >-
%(secret:gcpbackendmismatch_gcp_service_key)s
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
- env: &mongo-vars
S3BACKEND: "mem"
MPU_TESTING: "yes"
S3METADATA: mongodb
- env: &multiple-backend-vars
S3BACKEND: "mem"
S3DATA: "multiple"
- env: &file-mem-mpu
S3BACKEND: "file"
S3VAULT: "mem"
MPU_TESTING: "yes"
- Git: &clone
name: Pull repo
repourl: '%(prop:git_reference)s'
shallow: True
retryFetch: True
haltOnFailure: True
- ShellCommand: &credentials
name: Setup Credentials
command: bash eve/workers/build/credentials.bash
haltOnFailure: True
env: *global-env
- ShellCommand: &npm-install
name: install modules
command: npm install
haltOnFailure: True
- Upload: &upload-artifacts
source: /artifacts
urls:
- "*"
- ShellCommand: &follow-s3-log
logfiles:
s3:
filename: /artifacts/s3.log
follow: true
- ShellCommand: &add-hostname
name: add hostname
command: |
echo "127.0.0.1 testrequestbucket.localhost" >> /etc/hosts
echo \
"127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" \
>> /etc/hosts
haltOnFailure: True
- ShellCommand: &setup-junit-upload
name: preparing junit files for upload
command: |
mkdir -p artifacts/junit
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
alwaysRun: true
- Upload: &upload-junits
source: artifacts
urls:
- "*"
alwaysRun: true
stages:
pre-merge:
worker:
type: local
steps:
- TriggerStages:
name: Launch all workers
stage_names:
- linting-coverage
- file-ft-tests
- multiple-backend-test
- mongo-ft-tests
- kmip-ft-tests
waitForFinish: True
haltOnFailure: True
linting-coverage:
worker:
type: docker
path: eve/workers/build
volumes: &default_volumes
- '/home/eve/workspace'
steps:
- Git: *clone
- ShellCommand: *npm-install
- ShellCommand: *add-hostname
- ShellCommand: *credentials
- ShellCommand:
name: Linting
command: |
set -ex
npm run --silent lint -- --max-warnings 0
npm run --silent lint_md
flake8 $(git ls-files "*.py")
yamllint $(git ls-files "*.yml")
- ShellCommand:
name: Unit Coverage
command: |
set -ex
unset HTTP_PROXY HTTPS_PROXY NO_PROXY
unset http_proxy https_proxy no_proxy
mkdir -p $CIRCLE_TEST_REPORTS/unit
npm test
npm run test_legacy_location
env: &shared-vars
<<: *global-env
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
CIRCLE_TEST_REPORTS: /tmp
CIRCLE_ARTIFACTS: /tmp
CI_REPORTS: /tmp
- ShellCommand:
name: Unit Coverage logs
command: find /tmp/unit -exec cat {} \;
- ShellCommand: *setup-junit-upload
- Upload: *upload-junits
multiple-backend-test:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMem: "2Gi"
s3Mem: "2Gi"
env:
<<: *multiple-backend-vars
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *npm-install
- ShellCommand:
command: |
bash -c "
source /root/.aws/exports &> /dev/null
set -ex
bash wait_for_local_port.bash 8000 40
npm run multiple_backend_test"
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
<<: *global-env
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- ShellCommand:
command: mvn test
workdir: build/tests/functional/jaws
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
- ShellCommand:
command: rspec tests.rb
workdir: build/tests/functional/fog
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
mongo-ft-tests:
worker: &s3-pod
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMem: "2Gi"
s3Mem: "1664Mi"
redis: enabled
env:
<<: *mongo-vars
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *npm-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
npm run ft_test
<<: *follow-s3-log
env:
<<: *mongo-vars
<<: *global-env
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
file-ft-tests:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMem: "1920Mi"
s3Mem: "2Gi"
redis: enabled
env:
<<: *file-mem-mpu
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *npm-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
npm run ft_test
<<: *follow-s3-log
env:
<<: *file-mem-mpu
<<: *global-env
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
kmip-ft-tests:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
pykmip: eve/workers/pykmip
vars:
aggressorMem: "2Gi"
s3Mem: "1664Mi"
redis: enabled
pykmip: enabled
env:
<<: *mongo-vars
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *npm-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 5696 40
npm run ft_kmip
logfiles:
pykmip:
filename: /artifacts/pykmip.log
follow: true
s3:
filename: /artifacts/s3.log
follow: true
env:
<<: *mongo-vars
<<: *global-env
- ShellCommand: *setup-junit-upload
- Upload: *upload-artifacts
- Upload: *upload-junits
post-merge:
worker:
type: local
steps:
- Git: *clone
- ShellCommand: &docker_login
name: Private Registry Login
command: >
docker login
-u '%(secret:private_registry_username)s'
-p '%(secret:private_registry_password)s'
'%(secret:private_registry_url)s'
- ShellCommand:
name: Dockerhub Login
command: >
docker login
-u '%(secret:dockerhub_ro_user)s'
-p '%(secret:dockerhub_ro_password)s'
- SetProperty: &docker_image_name
name: Set docker image name property
property: docker_image_name
value:
"%(secret:private_registry_url)s/zenko/cloudserver:\
%(prop:commit_short_revision)s"
- ShellCommand:
name: Build docker image
command: >-
docker build
--no-cache
-t %(prop:docker_image_name)s
.
- ShellCommand:
name: Tag images
command: |
docker tag %(prop:docker_image_name)s zenko/cloudserver:$TAG
env:
TAG: "latest-%(prop:product_version)s"
- ShellCommand:
name: Push image
command: |
docker push %(prop:docker_image_name)s
docker push zenko/cloudserver:latest-%(prop:product_version)s

View File

@ -0,0 +1,57 @@
FROM buildpack-deps:xenial-curl
#
# Install packages needed by the buildchain
#
ENV LANG C.UTF-8
COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
&& apt-get update \
&& cat /tmp/*packages.list | xargs apt-get install -y \
&& git clone https://github.com/tj/n.git \
&& make -C ./n \
&& n 10 latest \
&& pip install pip==9.0.1 \
&& rm -rf ./n \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /tmp/packages.list
#
# Add user eve
#
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
&& adduser eve sudo \
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
#
# Install Dependencies
#
# Install RVM and gems
ENV RUBY_VERSION="2.4.1"
COPY ./gems.list /tmp/
RUN cat /tmp/gems.list | xargs gem install
#RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
# && curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
# && usermod -a -G rvm eve
#RUN /bin/bash -l -c "\
# source /usr/local/rvm/scripts/rvm \
# && cat /tmp/gems.list | xargs gem install \
# && rm /tmp/gems.list"
# Install Pip packages
COPY ./pip_packages.list /tmp/
RUN cat /tmp/pip_packages.list | xargs pip install \
&& rm -f /tmp/pip_packages.list \
&& mkdir /home/eve/.aws \
&& chown eve /home/eve/.aws
#
# Run buildbot-worker on startup
#
ARG BUILDBOT_VERSION
RUN pip install buildbot-worker==$BUILDBOT_VERSION
CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS && buildbot-worker start --nodaemon"]

View File

@ -0,0 +1,13 @@
ca-certificates
git
gnupg
libffi-dev
libssl-dev
python-pip
python2.7
python2.7-dev
software-properties-common
sudo
tcl
wget
procps

View File

@ -2,9 +2,9 @@
set -x #echo on set -x #echo on
set -e #exit at the first error set -e #exit at the first error
mkdir -p $HOME/.aws mkdir -p ~/.aws
cat >>$HOME/.aws/credentials <<EOF cat >>/root/.aws/credentials <<EOF
[default] [default]
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY

View File

@ -0,0 +1,4 @@
fog-aws:1.3.0
json
mime-types:3.1
rspec:3.5

View File

@ -0,0 +1,3 @@
flake8
s3cmd==1.6.1
yamllint

View File

@ -0,0 +1,11 @@
build-essential
curl
default-jdk
libdigest-hmac-perl
lsof
maven
netcat
redis-server
ruby-full
yarn=1.7.0-1
zlib1g-dev

213
eve/workers/pod.yaml Normal file
View File

@ -0,0 +1,213 @@
---
apiVersion: v1
kind: Pod
metadata:
name: "proxy-ci-test-pod"
spec:
restartPolicy: Never
terminationGracePeriodSeconds: 10
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com"
- "testrequestbucket.localhost"
- "pykmip.local"
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
initContainers:
- name: kmip-certs-installer
image: {{ images.pykmip }}
command: [ 'sh', '-c', 'cp /ssl/* /ssl-kmip/']
volumeMounts:
- name: kmip-certs
readOnly: false
mountPath: /ssl-kmip
{%- endif %}
containers:
{% if vars.env.S3METADATA is defined and vars.env.S3METADATA == "mongodb" -%}
- name: mongo
image: scality/ci-mongo:3.4
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: 500m
memory: 1Gi
{%- endif %}
- name: aggressor
image: {{ images.aggressor }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: "1"
memory: {{ vars.aggressorMem }}
limits:
cpu: "1"
memory: {{ vars.aggressorMem }}
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
- name: artifacts
readOnly: true
mountPath: /artifacts
command:
- bash
- -lc
- |
buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS
buildbot-worker start --nodaemon
env:
- name: CI
value: "true"
- name: ENABLE_LOCAL_CACHE
value: "true"
- name: REPORT_TOKEN
value: "report-token-1"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
{% for key, value in vars.env.items() %}
- name: {{ key }}
value: "{{ value }}"
{% endfor %}
- name: s3
image: {{ images.s3 }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: "2"
memory: {{ vars.s3Mem }}
limits:
cpu: "2"
memory: {{ vars.s3Mem }}
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
- name: certs
readOnly: true
mountPath: /tmp
- name: artifacts
readOnly: false
mountPath: /artifacts
- name: kmip-certs
readOnly: false
mountPath: /ssl-kmip
command:
- bash
- -ec
- |
sleep 10 # wait for mongo
/usr/src/app/docker-entrypoint.sh npm start | tee -a /artifacts/s3.log
env:
{% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" -%}
- name: S3_LOCATION_FILE
value: "/usr/src/app/tests/locationConfig/locationConfigTests.json"
{%- endif %}
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
- name: S3KMS
value: kmip
- name: S3KMIP_PORT
value: "5696"
- name: S3KMIP_HOSTS
value: "pykmip.local"
- name: S3KMIP_COMPOUND_CREATE
value: "false"
- name: S3KMIP_BUCKET_ATTRIBUTE_NAME
value: ''
- name: S3KMIP_PIPELINE_DEPTH
value: "8"
- name: S3KMIP_KEY
value: /ssl-kmip/kmip-client-key.pem
- name: S3KMIP_CERT
value: /ssl-kmip/kmip-client-cert.pem
- name: S3KMIP_CA
value: /ssl-kmip/kmip-ca.pem
{%- endif %}
- name: CI
value: "true"
- name: ENABLE_LOCAL_CACHE
value: "true"
- name: MONGODB_HOSTS
value: "localhost:27018"
- name: MONGODB_RS
value: "rs0"
- name: REDIS_HOST
value: "localhost"
- name: REDIS_PORT
value: "6379"
- name: REPORT_TOKEN
value: "report-token-1"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
- name: HEALTHCHECKS_ALLOWFROM
value: "0.0.0.0/0"
{% for key, value in vars.env.items() %}
- name: {{ key }}
value: "{{ value }}"
{% endfor %}
{% if vars.redis is defined and vars.redis == "enabled" -%}
- name: redis
image: redis:alpine
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 200m
memory: 128Mi
limits:
cpu: 200m
memory: 128Mi
{%- endif %}
{% if vars.env.CI_PROXY is defined and vars.env.CI_PROXY == "true" -%}
- name: squid
image: scality/ci-squid
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 250m
memory: 128Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: certs
readOnly: false
mountPath: /ssl
command:
- sh
- -exc
- |
mkdir -p /ssl
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-keyout /ssl/myca.pem -out /ssl/myca.pem
cp /ssl/myca.pem /ssl/CA.pem
squid -f /etc/squid/squid.conf -N -z
squid -f /etc/squid/squid.conf -NYCd 1
{%- endif %}
{% if vars.pykmip is defined and vars.pykmip == 'enabled' -%}
- name: pykmip
image: {{ images.pykmip }}
imagePullPolicy: IfNotPresent
volumeMounts:
- name: artifacts
readOnly: false
mountPath: /artifacts
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 100m
memory: 128Mi
{%- endif %}
volumes:
- name: creds
emptyDir: {}
- name: certs
emptyDir: {}
- name: artifacts
emptyDir: {}
- name: kmip-certs
emptyDir: {}

View File

@ -1,4 +1,5 @@
FROM python:3.10-alpine FROM python:3-alpine
RUN apk add --no-cache \ RUN apk add --no-cache \
libressl && \ libressl && \
@ -7,14 +8,8 @@ RUN apk add --no-cache \
libffi-dev \ libffi-dev \
libressl-dev \ libressl-dev \
sqlite-dev \ sqlite-dev \
build-base \ build-base && \
curl pip install pykmip requests && \
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN pip3 install -U pip && \
pip3 install pykmip requests && \
apk del .build-deps && \ apk del .build-deps && \
mkdir /pykmip mkdir /pykmip

View File

@ -30,9 +30,7 @@ def create_rsa_private_key(key_size=2048, public_exponent=65537):
return private_key return private_key
def create_self_signed_certificate(subject_name, def create_self_signed_certificate(subject_name, private_key, days_valid=365):
private_key,
days_valid=36500):
subject = x509.Name([ subject = x509.Name([
x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"), x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),
x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name) x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name)
@ -61,7 +59,7 @@ def create_certificate(subject_name,
private_key, private_key,
signing_certificate, signing_certificate,
signing_key, signing_key,
days_valid=36500, days_valid=365,
client_auth=False): client_auth=False):
subject = x509.Name([ subject = x509.Name([
x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"), x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),

View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC6TCCAdGgAwIBAgIUO54wXmqIJGCKxQAH4jhGQXa6ZHIwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAeFw0xOTA0
MDIxODE2MjBaFw0yMDA0MDExODE2MjBaMCQxEDAOBgNVBAoMB1NjYWxpdHkxEDAO
BgNVBAMMB1Jvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDL
T3oapzN3ZEWh5cfe5PlOTgewZ55j7Xjz5ZEWNjPWYmBJfh0+5dntK+c1HvtEL/oa
6vnliPbb3kcl01eoOgmWX7ZgRwWsSb05otBSXJ040eJ8IFKw5Pp8OiWS3wXNusBs
HI/exrGdDTukqarhTBuscbBVtVd/IdQNQZRxB14ci1DjD+i3zBv/oRfrDUbXoBDJ
/ucyCICMthqWzFI509FU6DD1554xvDOoryhCOTHfQFcEWgSln8HaiELlJk9D7164
9qRse2R0s0STTrDgclbQpvt8gJfsWpTuRhjEFe0MKmWWhuYfA+o8eHNvCqQKdRwH
QLx9q1fCwi6Czz7aO8lTAgMBAAGjEzARMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
hvcNAQELBQADggEBAITJfBlmABEwO/d7320ukoQyV/i2dhC/y2C/mKxuzPXPU8hR
FsBC6LA2oJbatSTSuG3YmHcf8/0mj3o4fCgz2+B7rJLH9WXd8lZdz8CRMwsyVmFY
aI3NvtMc0tV+4W1pxmxBs5IDITLAYIxuTm6kowH9jy85bAnGDYjGK9Hr84keWJIg
a2z7DGhL1HEd6tqJvhEFJFLL4VqDB9vEdvILnav3D3EkKU6lQ0Bvi2XO4t8rOclm
lEfYU7taNgCAYc9y/KcQ13jAokjZxmT8Bhep+Xq4BAHDakzqgD7USUnSUZ0inu8e
2bZItCXIJq/wD5ysOyeT490qVJ6F/8LKS1HYsUE=
-----END CERTIFICATE-----

View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC2TCCAcGgAwIBAgIUG83im4Ny72RE536mxXOtJxzjOgAwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAeFw0xOTA0
MDIxODE2MjBaFw0yMDA0MDExODE2MjBaMCkxEDAOBgNVBAoMB1NjYWxpdHkxFTAT
BgNVBAMMDHB5a21pcC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
ggEBALCSoV6WSaIS0FHmJk1+b1/DEBolk4hru4gkk7qqS+vSlHMog2pu37Dvd6Ll
2G2EUVxkrf7bJP1pz5qxLYoZeMDd2MVCzGpdVHrLx7yRB+A7UwSrVpMcQkpxaBrK
upBZuJZoiYgsqAdxxD/NLUAUSyTm3RQ/xJmLRSs3w8F8AfjoGbFZwsAgfnO4kxNR
tdeVf4a4yGp2tmF7QMtMQr2ov0ktGiFwvmosvhkwJjCgvq5IfL4kc4iLhW6vSKPh
e/51Mhq2ntqX+4WxXuhGV4O9rc8tMZ/8zQY8KuabETDgIeBgpE9u+uEqvjnOZmNx
/bZW7MW2tldBmiPFQ+HMFTNICMsCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAhcJt
1605IDKAlCisolU49574gjJv6OdMSOuMF2ZDibK0mIqta65+t9eoRSmwSzysTyWe
6dDt0BRwxzEwuGN8B4DHe5jV8+9NHq8wh/ImyfohX75xSvgnCW2aRuA0qZhi4qCa
pZjy+rZu7Xa10hAmQ8lllYmLrNyRld8dL0eyL3sYzxb5SWX/60kJ4Fo/OkDAOzBo
7P5PKNlCthHL0ND/jV1jqrr+822xWGzGeN4vvqMYMvR71J+dBTCkJj5lRW3MailR
R2zu+W7idIxbs4Gh2JZ4LwPrWG63KhA7Hc+4sMSdrw3Mcp5IMpwIjllBJlllNbKE
x8ARszTHSgP6WtzvaQ==
-----END CERTIFICATE-----

View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC8TCCAdmgAwIBAgIUaGl4Kplsv0uUwIgJKaZByGk6JMwwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAeFw0xOTA0
MDIxODE2MjBaFw0yMDA0MDExODE2MjBaMCUxEDAOBgNVBAoMB1NjYWxpdHkxETAP
BgNVBAMMCEpvaG4gRG9lMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
xrms0paQYVeXg/Gra21DQNW9t6lm7/CzsYZRKcqWnALVhg1Ol6NzZYxhx4ezYlga
ztHnKwFUReOqNYWl2Cgivxwav9lpZp4W7LvHemfOcSmy+7fX8ttEs9df/1p9uMu+
AtnE0yiqWrDYQAu2gSQMWv+SjdnTOrOuz88//6CgxKzE6ccC4EKkJuxIr5Vqy+Up
OgF8wZSnx6JuW+o20XhodCLBac/diwvltODif8FHIhkzR948PfGKAofIaV0s2eri
Kfsc7bRDLnUjYLGeEk/PxjydpCy4e+U+fctTvrRxi0KVqnbxz3+B0nx2Cp0IW/DM
RP09Lfx7uQvieiKaqLJElQIDAQABoxowGDAWBgNVHSUBAf8EDDAKBggrBgEFBQcD
AjANBgkqhkiG9w0BAQsFAAOCAQEAk5Xh1EpxMsWhlorkrBadYtkTqsiC4UIBEJvT
hqU2eb3Fom39gSoKTFSJEO2mopMCq34TRG6klkxfMQRxzQxWAmAHAu0BLeWcJ0Rx
FCxZZ5CexZYAH2yJBQDvvTfFCXZ6VmpHfDa+7Z9DBNYm3WPuDROWgnwiTqrtVmu7
3HeBi4KG/DE6tC6QxI+A9Ofj3wcfv25et1NdBQnNMoMmjyWIGlEmhNShgtasNScV
mp+9LxZUFxeVDx3Qnw+wo/bwbyjYGh+osI+7RHpwXobmSQxxC8Vs+hZlKWGIh47b
DU2ONZAO4565Kppp48mTcgmq96IQFLoIK9XY3CkLsowS7IWczQ==
-----END CERTIFICATE-----

View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGuazSlpBhV5eD
8atrbUNA1b23qWbv8LOxhlEpypacAtWGDU6Xo3NljGHHh7NiWBrO0ecrAVRF46o1
haXYKCK/HBq/2Wlmnhbsu8d6Z85xKbL7t9fy20Sz11//Wn24y74C2cTTKKpasNhA
C7aBJAxa/5KN2dM6s67Pzz//oKDErMTpxwLgQqQm7EivlWrL5Sk6AXzBlKfHom5b
6jbReGh0IsFpz92LC+W04OJ/wUciGTNH3jw98YoCh8hpXSzZ6uIp+xzttEMudSNg
sZ4ST8/GPJ2kLLh75T59y1O+tHGLQpWqdvHPf4HSfHYKnQhb8MxE/T0t/Hu5C+J6
IpqoskSVAgMBAAECggEAJJ4aBkPQHt+w/5MMbyMW/V9lMe55FUVZFyEU24qE7gJn
refNz4tCvDd93PJYT4rEhc+PtRDtomMs/ee+g3IB1Q0ssKUzEsGWn9CKFTgDhj6U
yGU72Xgl1K2e9sKJ3/9K5+OQrQgVO9jSQBroaitmS25EZvb4QRzS3V/m/wduGE8a
V08HuJX7K6XDptaj2wuYxJmb+Zx4RMVc7D9R38zEoPu5yDVIFQSmZtbGSzo2ElyL
93zEiCno9PVIvNCvEayoYA1Mm4hvp7/gmeF8K9aYGHBeMmHJQpixrBRca0zm1en5
TxirD07P7mqENTHUKq9GXciHAmflzBAVYXB7S0zDkQKBgQDrRUNZEN5d9YuteN18
RYdc7bttE5lY1CLtdWera1d1SZ912selsdKBBYl0ukAvVHvjC/XdA/FGfykfK1h1
UV8bWSH3lxaW9wlwn4mtNS4itWY5CaLLlhmUMM3wehSw+iZgTUiNDhhmt5Anj3/H
uSVwumO54+JWC+XjTheIOikAPwKBgQDYPBnbrbEFHp4ZNVArNApyfpefgrnj/ypV
Bs/rJxGtbSe9x92POYmJwHIH44LuU14XQAwSIIKoGwXX66auJ+JPaFf8tAoPQS6j
MxP4YfLesS1sQweEKoYgZ9O+AdyBgOWdubyi063XlM7pUzeCrmxdC5koJ/PQoGOc
8haBAsJGKwKBgQC8qbxKDfbjjeZGY6fo4bCc2p7z50WPL/4aQY2yrs9hZHqU/a4f
tytA/3msuzaBPdRiy9KLO4Adshb9wbqbyXbk7WMJsoUQ5mURhT3YQc8PUjv4/Tso
2uMELObYMm2pRc/EZfUJ+AWlSQo2TyJ+vH/DmBQkmxODQONGlfbU7R637QKBgQC7
aFuA6ajipwafEnXI+/GSCeWfec1irWQjDSRmyhWoGVK4SODdoSBzIzexXp27sMV7
oSbVDxguWj1WRgbQKgEakXSwr9mIHxYsm7hTLZExMJ4NloqNIc3diB8cLsDN/MkF
SlUTSiMBFRe/YUBbIpEIk2TKSNYnmtq6y5Z1ec6mwQKBgDY9sv5xUkON9CjNn4KO
lIs/Ef4id2awMOb1fsTSFTxPjo47UdDQsEF/anxjwc2tYto3WuVpurWxU8Jqknkz
uDtCr8aNVar3d5y3NNFg8oWY1tk9Ha7U05t/PraJRmeduOzy/nWMuZAi+mW9kdRK
WYX5TjM6GzAOrlUr4QdHAfP/
-----END PRIVATE KEY-----

View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwkqFelkmiEtBR
5iZNfm9fwxAaJZOIa7uIJJO6qkvr0pRzKINqbt+w73ei5dhthFFcZK3+2yT9ac+a
sS2KGXjA3djFQsxqXVR6y8e8kQfgO1MEq1aTHEJKcWgayrqQWbiWaImILKgHccQ/
zS1AFEsk5t0UP8SZi0UrN8PBfAH46BmxWcLAIH5zuJMTUbXXlX+GuMhqdrZhe0DL
TEK9qL9JLRohcL5qLL4ZMCYwoL6uSHy+JHOIi4Vur0ij4Xv+dTIatp7al/uFsV7o
RleDva3PLTGf/M0GPCrmmxEw4CHgYKRPbvrhKr45zmZjcf22VuzFtrZXQZojxUPh
zBUzSAjLAgMBAAECggEAPXHpQdcerI3LfJSQg6sZ6sMgmVi2LGUBD3FbDzwvy1Ku
YhyZDrKimRncEg3V8NZ40aQfgG6WQrFNk1FQvZv7j3Ij+xExvVnZChpb2VzG1tsO
GrPdHrhYwTsRtTETFySBvaWHJqITnvOSDXnC42esdpz4FhHSwnPakB2Ju40ByrG2
M/ljiLpWZC9lMdk+BK7MdjoDOvawCKghovKqxHkxjLVA1CG1sovmm3mTCqO0iOst
b6go/ETb/oSLB+aHV9++bK1Zc3fO26vjz/BDkOqbz66U/74KVqIyOeJy2WarwiDa
sqaRi5c3izwZynhl/eiIDcJJDKD6AyoCHWUbuIdTgQKBgQDW6/Hfb7iO/M+lx4ey
uTxgS5MXOn2ozQFxLbfE5aTdRjPHlq2OhPOK+rmfk0nupEQTPDe7YFwvKaI5kiQQ
R6eJMAcpO4iLGGqjQw1jCBHLo73Ri58v2KEkjOAA+S1jVcWi/nqCPrlpjZqj2640
SKksEUMOfXsPVVbpfEAvqOL0IQKBgQDSUkkdPAALrbaZuOovUurbJoKY6GFHPVJ5
kvrpqNNBYZ12FnEWqJFCTWeF9g3WEjy8XKltxkf8ARxOwXC9P7FTBHX2hXmxRNuE
N7tZ/WML9/9GDlwb4TOQgiIxBOBT5NN7DAS+1api8ik93ZFExD2ufpcBdjVfLxn+
RIIpxVwfawKBgQCirlAUJ9XUbeqjeqftkabw4OPC9mQ9jIfl6owqvwUO9N+m2Rgg
Q+SxM12kO3H/8FkTEkbBT4wXqvT/jO49YG+hOTiCbmzJlL7LO6r7ZhVKRnQdFAl3
xwsaxoOcWQCRK1CBMwz6X44rJqOCGnv/WWysTZirdDHdBmTWMVXIfZbk4QKBgC9M
FekfHxuBOzkinnd5/BrAdEoSqB2vKqbwaMC3GJrxasmtjkz8J35zjb5QcRgdDc+G
PwvStUl0rnr/gWztr+DtdeG0boNw6rS3G8jG9MkyQhPtEsWqRUBQI4RGhnQXV3+q
Wj7YKfMKZj/lXc/LGdvt1+OaQ7JeE0hc+7CNE4R1AoGAGXbXJLj5kGLuq0r2Sj2p
JQPgzp4ZWjQwbkOcTLmGYpn61YjZrlbDDB/3gQ3sQ7q8LYQhgKEpH+cuX0vZixul
V7fj/D8RUxI72+2WlE6rCxYXYmSXEOS2DIx61KdzsL/oSw28dujKxyPRdt8H4uxg
Z7zoolTN1y/WZUxYD2Pxfb4=
-----END PRIVATE KEY-----

View File

@ -4,7 +4,7 @@ port=5696
certificate_path=/ssl/kmip-cert.pem certificate_path=/ssl/kmip-cert.pem
key_path=/ssl/kmip-key.pem key_path=/ssl/kmip-key.pem
ca_path=/ssl/kmip-ca.pem ca_path=/ssl/kmip-ca.pem
auth_suite=TLS1.2 auth_suite=Basic
policy_path=/etc/pykmip/policies policy_path=/etc/pykmip/policies
enable_tls_client_auth=True enable_tls_client_auth=True
database_path=/pykmip/pykmip.db database_path=/pykmip/pykmip.db

View File

@ -48,7 +48,7 @@ signed_headers = 'host;x-amz-content-sha256;x-amz-date'
canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \ canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
.format(method, canonical_uri, canonical_querystring, canonical_headers, .format(method, canonical_uri, canonical_querystring, canonical_headers,
signed_headers, payload_hash) signed_headers, payload_hash)
print(canonical_request) print canonical_request
credential_scope = '{0}/{1}/{2}/aws4_request' \ credential_scope = '{0}/{1}/{2}/aws4_request' \
.format(date_stamp, region, service) .format(date_stamp, region, service)

View File

@ -1,28 +0,0 @@
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
COPY . ${HOME_DIR}/s3
RUN chown -R ${USER} ${HOME_DIR}
RUN pip3 install redis===3.5.3 requests==2.27.1 && \
apt-get install -y git-lfs
USER ${USER}
WORKDIR ${HOME_DIR}/s3
RUN rm -f ~/.gitconfig && \
git config --global --add safe.directory . && \
git lfs install && \
GIT_LFS_SKIP_SMUDGE=1 && \
yarn global add typescript && \
yarn install --frozen-lockfile --production --network-concurrency 1 && \
yarn cache clean --all && \
yarn global remove typescript
# run symlinking separately to avoid yarn installation errors
# we might have to check if the symlinking is really needed!
RUN ln -sf /scality-kms node_modules
EXPOSE 8000
CMD bash -c "source ${CONF_DIR}/env && export && supervisord -c ${CONF_DIR}/supervisord.conf"

View File

@ -1,10 +1,3 @@
'use strict'; // eslint-disable-line strict 'use strict'; // eslint-disable-line strict
require('werelogs').stderrUtils.catchAndTimestampStderr(
undefined,
// Do not exit as workers have their own listener that will exit
// But primary don't have another listener
require('cluster').isPrimary ? 1 : null,
);
require('./lib/server.js')(); require('./lib/server.js')();

File diff suppressed because it is too large Load Diff

View File

@ -1,41 +1,27 @@
const { auth, errors, policies } = require('arsenal'); const { auth, errors } = require('arsenal');
const async = require('async');
const bucketDelete = require('./bucketDelete'); const bucketDelete = require('./bucketDelete');
const bucketDeleteCors = require('./bucketDeleteCors'); const bucketDeleteCors = require('./bucketDeleteCors');
const bucketDeleteEncryption = require('./bucketDeleteEncryption');
const bucketDeleteWebsite = require('./bucketDeleteWebsite'); const bucketDeleteWebsite = require('./bucketDeleteWebsite');
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle'); const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
const bucketDeletePolicy = require('./bucketDeletePolicy'); const bucketDeletePolicy = require('./bucketDeletePolicy');
const bucketDeleteQuota = require('./bucketDeleteQuota'); const bucketGet = require('./bucketGet');
const { bucketGet } = require('./bucketGet');
const bucketGetACL = require('./bucketGetACL'); const bucketGetACL = require('./bucketGetACL');
const bucketGetCors = require('./bucketGetCors'); const bucketGetCors = require('./bucketGetCors');
const bucketGetVersioning = require('./bucketGetVersioning'); const bucketGetVersioning = require('./bucketGetVersioning');
const bucketGetWebsite = require('./bucketGetWebsite'); const bucketGetWebsite = require('./bucketGetWebsite');
const bucketGetLocation = require('./bucketGetLocation'); const bucketGetLocation = require('./bucketGetLocation');
const bucketGetLifecycle = require('./bucketGetLifecycle'); const bucketGetLifecycle = require('./bucketGetLifecycle');
const bucketGetNotification = require('./bucketGetNotification');
const bucketGetObjectLock = require('./bucketGetObjectLock');
const bucketGetPolicy = require('./bucketGetPolicy'); const bucketGetPolicy = require('./bucketGetPolicy');
const bucketGetQuota = require('./bucketGetQuota');
const bucketGetEncryption = require('./bucketGetEncryption');
const bucketHead = require('./bucketHead'); const bucketHead = require('./bucketHead');
const { bucketPut } = require('./bucketPut'); const { bucketPut } = require('./bucketPut');
const bucketPutACL = require('./bucketPutACL'); const bucketPutACL = require('./bucketPutACL');
const bucketPutCors = require('./bucketPutCors'); const bucketPutCors = require('./bucketPutCors');
const bucketPutVersioning = require('./bucketPutVersioning'); const bucketPutVersioning = require('./bucketPutVersioning');
const bucketPutTagging = require('./bucketPutTagging');
const bucketDeleteTagging = require('./bucketDeleteTagging');
const bucketGetTagging = require('./bucketGetTagging');
const bucketPutWebsite = require('./bucketPutWebsite'); const bucketPutWebsite = require('./bucketPutWebsite');
const bucketPutReplication = require('./bucketPutReplication'); const bucketPutReplication = require('./bucketPutReplication');
const bucketPutLifecycle = require('./bucketPutLifecycle'); const bucketPutLifecycle = require('./bucketPutLifecycle');
const bucketPutNotification = require('./bucketPutNotification');
const bucketPutEncryption = require('./bucketPutEncryption');
const bucketPutPolicy = require('./bucketPutPolicy'); const bucketPutPolicy = require('./bucketPutPolicy');
const bucketPutObjectLock = require('./bucketPutObjectLock');
const bucketUpdateQuota = require('./bucketUpdateQuota');
const bucketGetReplication = require('./bucketGetReplication'); const bucketGetReplication = require('./bucketGetReplication');
const bucketDeleteReplication = require('./bucketDeleteReplication'); const bucketDeleteReplication = require('./bucketDeleteReplication');
const corsPreflight = require('./corsPreflight'); const corsPreflight = require('./corsPreflight');
@ -43,75 +29,40 @@ const completeMultipartUpload = require('./completeMultipartUpload');
const initiateMultipartUpload = require('./initiateMultipartUpload'); const initiateMultipartUpload = require('./initiateMultipartUpload');
const listMultipartUploads = require('./listMultipartUploads'); const listMultipartUploads = require('./listMultipartUploads');
const listParts = require('./listParts'); const listParts = require('./listParts');
const metadataSearch = require('./metadataSearch');
const { multiObjectDelete } = require('./multiObjectDelete'); const { multiObjectDelete } = require('./multiObjectDelete');
const multipartDelete = require('./multipartDelete'); const multipartDelete = require('./multipartDelete');
const objectCopy = require('./objectCopy'); const objectCopy = require('./objectCopy');
const { objectDelete } = require('./objectDelete'); const objectDelete = require('./objectDelete');
const objectDeleteTagging = require('./objectDeleteTagging'); const objectDeleteTagging = require('./objectDeleteTagging');
const objectGet = require('./objectGet'); const objectGet = require('./objectGet');
const objectGetACL = require('./objectGetACL'); const objectGetACL = require('./objectGetACL');
const objectGetLegalHold = require('./objectGetLegalHold');
const objectGetRetention = require('./objectGetRetention');
const objectGetTagging = require('./objectGetTagging'); const objectGetTagging = require('./objectGetTagging');
const objectHead = require('./objectHead'); const objectHead = require('./objectHead');
const objectPut = require('./objectPut'); const objectPut = require('./objectPut');
const objectPutACL = require('./objectPutACL'); const objectPutACL = require('./objectPutACL');
const objectPutLegalHold = require('./objectPutLegalHold');
const objectPutTagging = require('./objectPutTagging'); const objectPutTagging = require('./objectPutTagging');
const objectPutPart = require('./objectPutPart'); const objectPutPart = require('./objectPutPart');
const objectPutCopyPart = require('./objectPutCopyPart'); const objectPutCopyPart = require('./objectPutCopyPart');
const objectPutRetention = require('./objectPutRetention');
const objectRestore = require('./objectRestore');
const prepareRequestContexts const prepareRequestContexts
= require('./apiUtils/authorization/prepareRequestContexts'); = require('./apiUtils/authorization/prepareRequestContexts');
const serviceGet = require('./serviceGet'); const serviceGet = require('./serviceGet');
const vault = require('../auth/vault'); const vault = require('../auth/vault');
const website = require('./website'); const websiteGet = require('./websiteGet');
const websiteHead = require('./websiteHead');
const writeContinue = require('../utilities/writeContinue'); const writeContinue = require('../utilities/writeContinue');
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders'); const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
const parseCopySource = require('./apiUtils/object/parseCopySource'); const parseCopySource = require('./apiUtils/object/parseCopySource');
const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
auth.setHandler(vault); auth.setHandler(vault);
/* eslint-disable no-param-reassign */ /* eslint-disable no-param-reassign */
const api = { const api = {
callApiMethod(apiMethod, request, response, log, callback) { callApiMethod(apiMethod, request, response, log, callback) {
// Attach the apiMethod method to the request, so it can used by monitoring in the server
// eslint-disable-next-line no-param-reassign
request.apiMethod = apiMethod;
// Array of end of API callbacks, used to perform some logic
// at the end of an API.
// eslint-disable-next-line no-param-reassign
request.finalizerHooks = [];
const actionLog = monitoringMap[apiMethod];
if (!actionLog &&
apiMethod !== 'websiteGet' &&
apiMethod !== 'websiteHead' &&
apiMethod !== 'corsPreflight') {
log.error('callApiMethod(): No actionLog for this api method', {
apiMethod,
});
}
log.addDefaultFields({
service: 's3',
action: actionLog,
bucketName: request.bucketName,
});
if (request.objectKey) {
log.addDefaultFields({
objectKey: request.objectKey,
});
}
let returnTagCount = true; let returnTagCount = true;
const validationRes = validateQueryAndHeaders(request, log); const validationRes =
validateQueryAndHeaders(request.method, request.query,
request.headers, log);
if (validationRes.error) { if (validationRes.error) {
log.debug('request query / header validation failed', { log.debug('request query / header validation failed', {
error: validationRes.error, error: validationRes.error,
@ -123,7 +74,6 @@ const api = {
// no need to check auth on website or cors preflight requests // no need to check auth on website or cors preflight requests
if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' || if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
apiMethod === 'corsPreflight') { apiMethod === 'corsPreflight') {
request.actionImplicitDenies = false;
return this[apiMethod](request, log, callback); return this[apiMethod](request, log, callback);
} }
@ -136,92 +86,43 @@ const api = {
return process.nextTick(callback, parsingError); return process.nextTick(callback, parsingError);
} }
const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers);
if (httpHeadersSizeError) {
log.debug('http header size limit exceeded', {
error: httpHeadersSizeError,
});
return process.nextTick(callback, httpHeadersSizeError);
}
const requestContexts = prepareRequestContexts(apiMethod, request, const requestContexts = prepareRequestContexts(apiMethod, request,
sourceBucket, sourceObject, sourceVersionId); sourceBucket, sourceObject, sourceVersionId);
// Extract all the _apiMethods and store them in an array return auth.server.doAuth(request, log, (err, userInfo,
const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : []; authorizationResults, streamingV4Params) => {
// Attach the names to the current request if (err) {
// eslint-disable-next-line no-param-reassign log.trace('authentication error', { error: err });
request.apiMethods = apiMethods; return callback(err);
}
function checkAuthResults(authResults) { if (authorizationResults) {
let returnTagCount = true;
const isImplicitDeny = {};
let isOnlyImplicitDeny = true;
if (apiMethod === 'objectGet') { if (apiMethod === 'objectGet') {
// first item checks s3:GetObject(Version) action // first item checks s3:GetObject(Version) action
if (!authResults[0].isAllowed && !authResults[0].isImplicit) { if (!authorizationResults[0].isAllowed) {
log.trace('get object authorization denial from Vault'); log.trace('get object authorization denial from Vault');
return errors.AccessDenied; return callback(errors.AccessDenied);
} }
// TODO add support for returnTagCount in the bucket policy
// checks
isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
// second item checks s3:GetObject(Version)Tagging action // second item checks s3:GetObject(Version)Tagging action
if (!authResults[1].isAllowed) { if (!authorizationResults[1].isAllowed) {
log.trace('get tagging authorization denial ' + log.trace('get tagging authorization denial ' +
'from Vault'); 'from Vault');
returnTagCount = false; returnTagCount = false;
} }
} else { } else {
for (let i = 0; i < authResults.length; i++) { for (let i = 0; i < authorizationResults.length; i++) {
isImplicitDeny[authResults[i].action] = true; if (!authorizationResults[i].isAllowed) {
if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
// Any explicit deny rejects the current API call
log.trace('authorization denial from Vault'); log.trace('authorization denial from Vault');
return errors.AccessDenied; return callback(errors.AccessDenied);
}
if (authResults[i].isAllowed) {
// If the action is allowed, the result is not implicit
// Deny.
isImplicitDeny[authResults[i].action] = false;
isOnlyImplicitDeny = false;
} }
} }
} }
// These two APIs cannot use ACLs or Bucket Policies, hence, any
// implicit deny from vault must be treated as an explicit deny.
if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
return errors.AccessDenied;
}
return { returnTagCount, isImplicitDeny };
}
return async.waterfall([
next => auth.server.doAuth(
request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
if (err) {
// VaultClient returns standard errors, but the route requires
// Arsenal errors
const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
log.trace('authentication error', { error: err });
return next(arsenalError);
}
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
}, 's3', requestContexts),
(userInfo, authorizationResults, streamingV4Params, infos, next) => {
const authNames = { accountName: userInfo.getAccountDisplayName() };
if (userInfo.isRequesterAnIAMUser()) {
authNames.userName = userInfo.getIAMdisplayName();
}
if (isRequesterASessionUser(userInfo)) {
authNames.sessionName = userInfo.getShortid().split(':')[1];
}
log.addDefaultFields(authNames);
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
} }
// issue 100 Continue to the client // issue 100 Continue to the client
writeContinue(request, response); writeContinue(request, response);
const MAX_POST_LENGTH = request.method === 'POST' ? if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
return this[apiMethod](userInfo, request, streamingV4Params,
log, callback);
}
const MAX_POST_LENGTH = request.method.toUpperCase() === 'POST' ?
1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB 1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
const post = []; const post = [];
let postLength = 0; let postLength = 0;
@ -231,147 +132,85 @@ const api = {
if (postLength <= MAX_POST_LENGTH) { if (postLength <= MAX_POST_LENGTH) {
post.push(chunk); post.push(chunk);
} }
return undefined;
}); });
request.on('error', err => { request.on('error', err => {
log.trace('error receiving request', { log.trace('error receiving request', {
error: err, error: err,
}); });
return next(errors.InternalError); return callback(errors.InternalError);
}); });
request.on('end', () => { request.on('end', () => {
if (postLength > MAX_POST_LENGTH) { if (postLength > MAX_POST_LENGTH) {
log.error('body length is too long for request type', log.error('body length is too long for request type',
{ postLength }); { postLength });
return next(errors.InvalidRequest); return callback(errors.InvalidRequest);
} }
// Convert array of post buffers into one string // Convert array of post buffers into one string
request.post = Buffer.concat(post, postLength).toString(); request.post = Buffer.concat(post, postLength).toString();
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
});
return undefined;
},
// Tag condition keys require information from CloudServer for evaluation
(userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
authorizationResults,
request,
requestContexts,
apiMethod,
log,
(err, authResultsWithTags) => {
if (err) {
log.trace('tag authentication error', { error: err });
return next(err);
}
return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
},
),
], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
if (err) {
return callback(err);
}
request.accountQuotas = infos?.accountQuota;
if (authorizationResults) {
const checkedResults = checkAuthResults(authorizationResults);
if (checkedResults instanceof Error) {
return callback(checkedResults);
}
returnTagCount = checkedResults.returnTagCount;
request.actionImplicitDenies = checkedResults.isImplicitDeny;
} else {
// create an object of keys apiMethods with all values to false:
// for backward compatibility, all apiMethods are allowed by default
// thus it is explicitly allowed, so implicit deny is false
request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
acc[curr] = false;
return acc;
}, {});
}
const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
(hook, done) => hook(err, done),
() => callback(err, ...results));
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') { if (apiMethod === 'objectCopy' ||
request._response = response; apiMethod === 'objectPutCopyPart') {
return this[apiMethod](userInfo, request, streamingV4Params,
log, methodCallback, authorizationResults);
}
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
return this[apiMethod](userInfo, request, sourceBucket, return this[apiMethod](userInfo, request, sourceBucket,
sourceObject, sourceVersionId, log, methodCallback); sourceObject, sourceVersionId, log, callback);
} }
if (apiMethod === 'objectGet') { if (apiMethod === 'objectGet') {
return this[apiMethod](userInfo, request, returnTagCount, log, callback); return this[apiMethod](userInfo, request,
returnTagCount, log, callback);
} }
return this[apiMethod](userInfo, request, log, methodCallback); return this[apiMethod](userInfo, request, log, callback);
}); });
return undefined;
}, 's3', requestContexts);
}, },
bucketDelete, bucketDelete,
bucketDeleteCors, bucketDeleteCors,
bucketDeleteEncryption,
bucketDeleteWebsite, bucketDeleteWebsite,
bucketGet, bucketGet,
bucketGetACL, bucketGetACL,
bucketGetCors, bucketGetCors,
bucketGetObjectLock,
bucketGetVersioning, bucketGetVersioning,
bucketGetWebsite, bucketGetWebsite,
bucketGetLocation, bucketGetLocation,
bucketGetEncryption,
bucketHead, bucketHead,
bucketPut, bucketPut,
bucketPutACL, bucketPutACL,
bucketPutCors, bucketPutCors,
bucketPutVersioning, bucketPutVersioning,
bucketPutTagging,
bucketDeleteTagging,
bucketGetTagging,
bucketPutWebsite, bucketPutWebsite,
bucketPutReplication, bucketPutReplication,
bucketGetReplication, bucketGetReplication,
bucketDeleteReplication, bucketDeleteReplication,
bucketDeleteQuota,
bucketPutLifecycle, bucketPutLifecycle,
bucketUpdateQuota,
bucketGetLifecycle, bucketGetLifecycle,
bucketDeleteLifecycle, bucketDeleteLifecycle,
bucketPutPolicy, bucketPutPolicy,
bucketGetPolicy, bucketGetPolicy,
bucketGetQuota,
bucketDeletePolicy, bucketDeletePolicy,
bucketPutObjectLock,
bucketPutNotification,
bucketGetNotification,
bucketPutEncryption,
corsPreflight, corsPreflight,
completeMultipartUpload, completeMultipartUpload,
initiateMultipartUpload, initiateMultipartUpload,
listMultipartUploads, listMultipartUploads,
listParts, listParts,
metadataSearch,
multiObjectDelete, multiObjectDelete,
multipartDelete, multipartDelete,
objectDelete, objectDelete,
objectDeleteTagging, objectDeleteTagging,
objectGet, objectGet,
objectGetACL, objectGetACL,
objectGetLegalHold,
objectGetRetention,
objectGetTagging, objectGetTagging,
objectCopy, objectCopy,
objectHead, objectHead,
objectPut, objectPut,
objectPutACL, objectPutACL,
objectPutLegalHold,
objectPutTagging, objectPutTagging,
objectPutPart, objectPutPart,
objectPutCopyPart, objectPutCopyPart,
objectPutRetention,
objectRestore,
serviceGet, serviceGet,
websiteGet: website, websiteGet,
websiteHead: website, websiteHead,
}; };
module.exports = api; module.exports = api;

View File

@ -1,29 +0,0 @@
const { errors } = require('arsenal');
const vault = require('../../../auth/vault');
function checkExpectedBucketOwner(headers, bucket, log, cb) {
const expectedOwner = headers['x-amz-expected-bucket-owner'];
if (expectedOwner === undefined) {
return cb();
}
const bucketOwner = bucket.getOwner();
return vault.getAccountIds([bucketOwner], log, (error, res) => {
if (error) {
log.error('error fetch accountId from vault', {
method: 'checkExpectedBucketOwner',
error,
});
}
if (error || res[bucketOwner] !== expectedOwner) {
return cb(errors.AccessDenied);
}
return cb();
});
}
module.exports = {
checkExpectedBucketOwner,
};

View File

@ -1,137 +1,83 @@
const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
const { errors } = require('arsenal');
const { parseCIDR, isValid } = require('ipaddr.js');
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { config } = require('../../../Config');
const { const actionMap = {
allAuthedUsersId, 's3:AbortMultipartUpload': 'multipartDelete',
bucketOwnerActions, 's3:CreateBucket': 'bucketPut',
logId, 's3:DeleteBucket': 'bucketDelete',
publicId, 's3:DeleteBucketPolicy': 'bucketDeletePolicy',
arrayOfAllowed, 's3:DeleteBucketWebsite': 'bucketDeleteWebsite',
assumedRoleArnResourceType, 's3:DeleteObject': 'objectDelete',
backbeatLifecycleSessionName, 's3:DeleteObjectTagging': 'objectDeleteTagging',
actionsToConsiderAsObjectPut, 's3:GetBucketAcl': 'bucketGetACL',
} = constants; 's3:GetBucketCORS': 'bucketGetCors',
's3:GetBucketLocation': 'bucketGetLocation',
's3:GetBucketPolicy': 'bucketGetPolicy',
's3:GetBucketVersioning': 'bucketGetVersioning',
's3:GetBucketWebsite': 'bucketGetWebsite',
's3:GetLifecycleConfiguration': 'bucketGetLifecycle',
's3:GetObject': 'objectGet',
's3:GetObjectAcl': 'objectGetACL',
's3:GetObjectTagging': 'objectGetTagging',
's3:GetReplicationConfiguration': 'bucketGetReplication',
's3:HeadBucket': 'bucketHead',
's3:ListBucket': 'bucketGet',
's3:ListBucketMultipartUploads': 'listMultipartUploads',
's3:ListMultipartUploadParts': 'listParts',
's3:PutBucketAcl': 'bucketPutACL',
's3:PutBucketCORS': 'bucketPutCors',
's3:PutBucketPolicy': 'bucketPutPolicy',
's3:PutBucketVersioning': 'bucketPutVersioning',
's3:PutBucketWebsite': 'bucketPutWebsite',
's3:PutLifecycleConfiguration': 'bucketPutLifecycle',
's3:PutObject': 'objectPut',
's3:PutObjectAcl': 'objectPutACL',
's3:PutObjectTagging': 'objectPutTagging',
's3:PutReplicationConfiguration': 'bucketPutReplication',
's3:ReplicateObject': 'objectCopy',
};
// whitelist buckets to allow public read on objects // whitelist buckets to allow public read on objects
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ?
? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : []; process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
function getServiceAccountProperties(canonicalID) {
const canonicalIDArray = canonicalID.split('/');
const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
return constants.serviceAccountProperties[serviceName];
}
function isServiceAccount(canonicalID) {
return getServiceAccountProperties(canonicalID) !== undefined;
}
function isRequesterASessionUser(authInfo) {
const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
return regexpAssumedRoleArn.test(authInfo.getArn());
}
function isRequesterNonAccountUser(authInfo) {
return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
}
/**
* Checks the access control for a given bucket based on the request type and user's canonical ID.
*
* @param {Bucket} bucket - The bucket to check access control for.
* @param {string} requestType - The list of s3 actions to check within the API call.
* @param {string} canonicalID - The canonical ID of the user making the request.
* @param {string} mainApiCall - The main API call (first item of the requestType).
*
* @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
*/
function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
// Same logic applies on the Versioned APIs, so let's simplify it.
let requestTypeParsed = requestType.endsWith('Version') ?
requestType.slice(0, 'Version'.length * -1) : requestType;
requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
'objectPut' : requestTypeParsed;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
'objectPut' : mainApiCall;
if (bucket.getOwner() === canonicalID) {
return true;
}
if (parsedMainApiCall === 'objectGet') {
if (requestTypeParsed === 'objectGetTagging') {
return true;
}
}
if (parsedMainApiCall === 'objectPut') {
if (arrayOfAllowed.includes(requestTypeParsed)) {
return true;
}
}
function checkBucketAcls(bucket, requestType, canonicalID) {
const bucketAcl = bucket.getAcl(); const bucketAcl = bucket.getAcl();
if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') { if (requestType === 'bucketGet' || requestType === 'bucketHead') {
if (bucketAcl.Canned === 'public-read' if (bucketAcl.Canned === 'public-read'
|| bucketAcl.Canned === 'public-read-write' || bucketAcl.Canned === 'public-read-write'
|| (bucketAcl.Canned === 'authenticated-read' || (bucketAcl.Canned === 'authenticated-read'
&& canonicalID !== publicId)) { && canonicalID !== constants.publicId)) {
return true; return true;
} else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ.indexOf(canonicalID) > -1) { || bucketAcl.READ.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.READ.indexOf(publicId) > -1
|| (bucketAcl.READ.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'bucketGetACL') { if (requestType === 'bucketGetACL') {
if ((bucketAcl.Canned === 'log-delivery-write' if ((bucketAcl.Canned === 'log-delivery-write'
&& canonicalID === logId) && canonicalID === constants.logId)
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ_ACP.indexOf(canonicalID) > -1) { || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.READ_ACP.indexOf(publicId) > -1
|| (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'bucketPutACL') { if (requestType === 'bucketPutACL') {
if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) { || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1
|| (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') { if (requestType === 'bucketDelete' && bucket.getOwner() === canonicalID) {
return true;
}
if (requestType === 'objectDelete' || requestType === 'objectPut') {
if (bucketAcl.Canned === 'public-read-write' if (bucketAcl.Canned === 'public-read-write'
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE.indexOf(canonicalID) > -1) { || bucketAcl.WRITE.indexOf(canonicalID) > -1) {
return true; return true;
} else if (bucketAcl.WRITE.indexOf(publicId) > -1
|| (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
// Note that an account can have the ability to do objectPutACL, // Note that an account can have the ability to do objectPutACL,
@ -140,43 +86,21 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
// objectPutACL, objectGetACL, objectHead or objectGet, the bucket // objectPutACL, objectGetACL, objectHead or objectGet, the bucket
// authorization check should just return true so can move on to check // authorization check should just return true so can move on to check
// rights at the object level. // rights at the object level.
return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL' return (requestType === 'objectPutACL' || requestType === 'objectGetACL' ||
|| requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); requestType === 'objectGet' || requestType === 'objectHead');
}
function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
isUserUnauthenticated, mainApiCall) {
const bucketOwner = bucket.getOwner();
const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
'objectPut' : requestType;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
'objectPut' : mainApiCall;
// acls don't distinguish between users and accounts, so both should be allowed
if (bucketOwnerActions.includes(requestTypeParsed)
&& (bucketOwner === canonicalID)) {
return true;
}
if (objectMD['owner-id'] === canonicalID) {
return true;
}
// Backward compatibility
if (parsedMainApiCall === 'objectGet') {
if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
&& requestTypeParsed === 'objectGetTagging') {
return true;
}
} }
function checkObjectAcls(bucket, objectMD, requestType, canonicalID) {
if (!objectMD.acl) { if (!objectMD.acl) {
return false; return false;
} }
const bucketOwner = bucket.getOwner();
if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') { if (requestType === 'objectGet' || requestType === 'objectHead') {
if (objectMD.acl.Canned === 'public-read' if (objectMD.acl.Canned === 'public-read'
|| objectMD.acl.Canned === 'public-read-write' || objectMD.acl.Canned === 'public-read-write'
|| (objectMD.acl.Canned === 'authenticated-read' || (objectMD.acl.Canned === 'authenticated-read'
&& canonicalID !== publicId)) { && canonicalID !== constants.publicId)) {
return true; return true;
} else if (objectMD.acl.Canned === 'bucket-owner-read' } else if (objectMD.acl.Canned === 'bucket-owner-read'
&& bucketOwner === canonicalID) { && bucketOwner === canonicalID) {
@ -186,306 +110,153 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ.indexOf(canonicalID) > -1) { || objectMD.acl.READ.indexOf(canonicalID) > -1) {
return true; return true;
} else if (objectMD.acl.READ.indexOf(publicId) > -1
|| (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
// User is already authorized on the bucket for FULL_CONTROL or WRITE or // User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write // bucket has canned ACL public-read-write
if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') { if (requestType === 'objectPut' || requestType === 'objectDelete') {
return true; return true;
} }
if (requestTypeParsed === 'objectPutACL') { if (requestType === 'objectPutACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control' if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID) && bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) { || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1
|| (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
if (requestTypeParsed === 'objectGetACL') { if (requestType === 'objectGetACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control' if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID) && bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) { || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
return true; return true;
} else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1
|| (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
} }
} }
// allow public reads on buckets that are whitelisted for anonymous reads // allow public reads on buckets that are whitelisted for anonymous reads
// TODO: remove this after bucket policies are implemented // TODO: remove this after bucket policies are implemented
const bucketAcl = bucket.getAcl(); const bucketAcl = bucket.getAcl();
const allowPublicReads = publicReadBuckets.includes(bucket.getName()) const allowPublicReads = publicReadBuckets.includes(bucket.getName()) &&
&& bucketAcl.Canned === 'public-read' bucketAcl.Canned === 'public-read' &&
&& (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); (requestType === 'objectGet' || requestType === 'objectHead');
if (allowPublicReads) { if (allowPublicReads) {
return true; return true;
} }
return false; return false;
} }
function _checkBucketPolicyActions(requestType, actions, log) { function _checkAction(requestType, action) {
const mappedAction = actionMaps.actionMapBP[requestType]; // if requestType isn't in list of controlled actions
// Deny any action that isn't in list of controlled actions if (!Object.values(actionMap).includes(requestType)) {
if (!mappedAction) { return true;
}
if (action === 's3:*') {
return true;
}
if (requestType === actionMap[action]) {
return true;
}
if (Array.isArray(action)
&& action.some(a => requestType === actionMap[a])) {
return true;
}
return false; return false;
} }
return evaluators.isActionApplicable(mappedAction, actions, log);
}
function _checkBucketPolicyResources(request, resource, log) { function _checkPrincipal(canonicalID, arn, principal) {
if (!request || (Array.isArray(resource) && resource.length === 0)) {
return true;
}
// build request context from the request!
const requestContext = new RequestContext(request.headers, request.query,
request.bucketName, request.objectKey, null,
request.connection.encrypted, request.resourceType, 's3');
return evaluators.isResourceApplicable(requestContext, resource, log);
}
function _checkBucketPolicyConditions(request, conditions, log) {
const ip = request ? requestUtils.getClientIp(request, config) : undefined;
if (!conditions) {
return true;
}
// build request context from the request!
const requestContext = new RequestContext(request.headers, request.query,
request.bucketName, request.objectKey, ip,
request.connection.encrypted, request.resourceType, 's3', null, null,
null, null, null, null, null, null, null, null, null,
request.objectLockRetentionDays);
return evaluators.meetConditions(requestContext, conditions, log);
}
function _getAccountId(arn) {
// account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
return arn.substr(13, 12);
}
function _isAccountId(principal) {
return (principal.length === 12 && /^\d+$/.test(principal));
}
function _checkPrincipal(requester, principal) {
if (principal === '*') { if (principal === '*') {
return true; return true;
} }
if (principal === requester) { if (principal.CanonicalUser && principal.CanonicalUser === canonicalID) {
return true; return true;
} }
if (_isAccountId(principal)) { if (Array.isArray(principal.CanonicalUser)
return _getAccountId(requester) === principal; && principal.CanonicalUser.includes(canonicalID)) {
return true;
} }
if (principal.endsWith('root')) { if (principal.AWS && (principal.AWS === '*' || principal.AWS === arn)) {
return _getAccountId(requester) === _getAccountId(principal); return true;
}
if (Array.isArray(principal.AWS) && principal.AWS.includes(arn)) {
return true;
} }
return false; return false;
} }
function _checkPrincipals(canonicalID, arn, principal) { function checkBucketPolicy(policy, requestType, canonicalID, arn) {
if (principal === '*') {
return true;
}
if (principal.CanonicalUser) {
if (Array.isArray(principal.CanonicalUser)) {
return principal.CanonicalUser.some(p => _checkPrincipal(canonicalID, p));
}
return _checkPrincipal(canonicalID, principal.CanonicalUser);
}
if (principal.AWS) {
if (Array.isArray(principal.AWS)) {
return principal.AWS.some(p => _checkPrincipal(arn, p));
}
return _checkPrincipal(arn, principal.AWS);
}
return false;
}
function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) {
let permission = 'defaultDeny'; let permission = 'defaultDeny';
// if requester is user within bucket owner account, actions should be while (policy.Statement.length > 0) {
// allowed unless explicitly denied (assumes allowed by IAM policy) const s = policy.Statement[0];
if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) { const principalMatch = _checkPrincipal(canonicalID, arn, s.Principal);
permission = 'allow'; const actionMatch = _checkAction(requestType, s.Action);
}
let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
while (copiedStatement.length > 0) {
const s = copiedStatement[0];
const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') { if (principalMatch && actionMatch && s.Effect === 'Deny') {
// explicit deny trumps any allows, so return immediately // explicit deny trumps any allows, so return immediately
return 'explicitDeny'; return 'explicitDeny';
} }
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') { if (principalMatch && actionMatch && s.Effect === 'Allow') {
permission = 'allow'; permission = 'allow';
} }
copiedStatement = copiedStatement.splice(1); // eslint-disable-next-line no-param-reassign
policy.Statement = policy.Statement.splice(1);
} }
return permission; return permission;
} }
function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log, function isBucketAuthorized(bucket, requestType, canonicalID, arn) {
request, aclPermission, results, actionImplicitDenies) {
const bucketPolicy = bucket.getBucketPolicy();
let processedResult = results[requestType];
if (!bucketPolicy) {
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
} else {
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
bucketOwner, log, request, actionImplicitDenies);
if (bucketPolicyPermission === 'explicitDeny') {
processedResult = false;
} else if (bucketPolicyPermission === 'allow') {
processedResult = true;
} else {
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
}
}
return processedResult;
}
function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
actionImplicitDeniesInput = {}, isWebsite = false) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const mainApiCall = requestTypes[0];
const results = {};
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
// Check to see if user is authorized to perform a // Check to see if user is authorized to perform a
// particular action on bucket based on ACLs. // particular action on bucket based on ACLs.
// TODO: Add IAM checks // TODO: Add IAM checks
let requesterIsNotUser = true; if (bucket.getOwner() === canonicalID) {
let arn = null; return true;
if (authInfo) { } else if (constants.bucketOwnerActions.includes(requestType)) {
requesterIsNotUser = !isRequesterNonAccountUser(authInfo); // only bucket owner can modify or retrieve this property of a bucket
arn = authInfo.getArn(); return false;
} }
// if the bucket owner is an account, users should not have default access const aclPermission = checkBucketAcls(bucket, requestType, canonicalID);
if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) { const bucketPolicy = bucket.getBucketPolicy();
results[_requestType] = actionImplicitDenies[_requestType] === false; if (!bucketPolicy) {
return results[_requestType]; return aclPermission;
} }
const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall); const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
// In case of error bucket access is checked with bucketGet canonicalID, arn);
// For website, bucket policy only uses objectGet and ignores bucketGet if (bucketPolicyPermission === 'explicitDeny') {
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html return false;
// bucketGet should be used to check acl but switched to objectGet for bucket policy
if (isWebsite && _requestType === 'bucketGet') {
// eslint-disable-next-line no-param-reassign
_requestType = 'objectGet';
actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
} }
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log, return (aclPermission || (bucketPolicyPermission === 'allow'));
request, aclPermission, results, actionImplicitDenies);
});
} }
function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {}, function isObjAuthorized(bucket, objectMD, requestType, canonicalID, arn) {
log, request) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const results = {};
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
let arn = null;
if (authInfo) {
arn = authInfo.getArn();
}
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
request, true, results, actionImplicitDenies);
});
}
function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
actionImplicitDeniesInput = {}, isWebsite = false) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const results = {};
const mainApiCall = requestTypes[0];
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
const parsedMethodName = _requestType.endsWith('Version')
? _requestType.slice(0, -7) : _requestType;
const bucketOwner = bucket.getOwner(); const bucketOwner = bucket.getOwner();
if (!objectMD) { if (!objectMD) {
// check bucket has read access return false;
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
let permission = 'bucketGet';
if (actionsToConsiderAsObjectPut.includes(_requestType)) {
permission = 'objectPut';
} }
results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request, if (objectMD['owner-id'] === canonicalID) {
actionImplicitDenies, isWebsite); return true;
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write
if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
&& results[_requestType] === false) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
}
return results[_requestType];
}
let requesterIsNotUser = true;
let arn = null;
let isUserUnauthenticated = false;
if (authInfo) {
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
arn = authInfo.getArn();
isUserUnauthenticated = arn === undefined;
}
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
return results[_requestType];
} }
// account is authorized if: // account is authorized if:
// - requesttype is included in bucketOwnerActions and // - requesttype is included in bucketOwnerActions and
// - account is the bucket owner // - account is the bucket owner
// - requester is account, not user if (constants.bucketOwnerActions.includes(requestType)
if (bucketOwnerActions.includes(parsedMethodName) && bucketOwner === canonicalID) {
&& (bucketOwner === canonicalID) return true;
&& requesterIsNotUser) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
return results[_requestType];
} }
const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName, const aclPermission = checkObjectAcls(bucket, objectMD, requestType,
canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall); canonicalID);
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner, const bucketPolicy = bucket.getBucketPolicy();
log, request, aclPermission, results, actionImplicitDenies); if (!bucketPolicy) {
}); return aclPermission;
}
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType,
canonicalID, arn);
if (bucketPolicyPermission === 'explicitDeny') {
return false;
}
return (aclPermission || (bucketPolicyPermission === 'allow'));
} }
function _checkResource(resource, bucketArn) { function _checkResource(resource, bucketArn) {
@ -514,128 +285,10 @@ function validatePolicyResource(bucketName, policy) {
}); });
} }
function checkIp(value) {
const errString = 'Invalid IP address in Conditions';
const values = Array.isArray(value) ? value : [value];
for (let i = 0; i < values.length; i++) {
// these preliminary checks are validating the provided
// ip address against ipaddr.js, the library we use when
// evaluating IP condition keys. It ensures compatibility,
// but additional checks are required to enforce the right
// notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
// we would accept different ip formats, which is not
// standard in an AWS use case.
try {
try {
parseCIDR(values[i]);
} catch (err) {
isValid(values[i]);
}
} catch (err) {
return errString;
}
// Apply the existing IP validation logic to each element
const validateIpRegex = ip => {
if (constants.ipv4Regex.test(ip)) {
return ip.split('.').every(part => parseInt(part, 10) <= 255);
}
if (constants.ipv6Regex.test(ip)) {
return ip.split(':').every(part => part.length <= 4);
}
return false;
};
if (validateIpRegex(values[i]) !== true) {
return errString;
}
}
// If the function hasn't returned by now, all elements are valid
return null;
}
// This function checks all bucket policy conditions if the values provided
// are valid for the condition type. If not it returns a relevant Malformed policy error string
function validatePolicyConditions(policy) {
const validConditions = [
{ conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
{ conditionKey: 's3:object-lock-remaining-retention-days' },
];
// keys where value type does not seem to be checked by AWS:
// - s3:object-lock-remaining-retention-days
if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
return null;
}
// there can be multiple statements in the policy, each with a Condition enclosure
for (let i = 0; i < policy.Statement.length; i++) {
const s = policy.Statement[i];
if (s.Condition) {
const conditionOperators = Object.keys(s.Condition);
// there can be multiple condition operations in the Condition enclosure
// eslint-disable-next-line no-restricted-syntax
for (const conditionOperator of conditionOperators) {
const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
const conditionValue = s.Condition[conditionOperator][conditionKey];
const validCondition = validConditions.find(validCondition =>
validCondition.conditionKey === conditionKey
);
// AWS returns does not return an error if the condition starts with 'aws:'
// so we reproduce this behaviour
if (!validCondition && !conditionKey.startsWith('aws:')) {
return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
}
if (validCondition && validCondition.conditionValueTypeChecker) {
const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
if (conditionValueTypeError) {
return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
}
}
}
}
}
return null;
}
/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
* @param {string} arn - Amazon resource name - example:
* arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
* @return {boolean} true if Lifecycle assumed role session arn, false if not.
*/
function isLifecycleSession(arn) {
if (!arn) {
return false;
}
const arnSplits = arn.split(':');
const service = arnSplits[2];
const resourceNames = arnSplits[arnSplits.length - 1].split('/');
const resourceType = resourceNames[0];
const sessionName = resourceNames[resourceNames.length - 1];
return (service === 'sts'
&& resourceType === assumedRoleArnResourceType
&& sessionName === backbeatLifecycleSessionName);
}
module.exports = { module.exports = {
isBucketAuthorized, isBucketAuthorized,
isObjAuthorized, isObjAuthorized,
getServiceAccountProperties,
isServiceAccount,
isRequesterASessionUser,
isRequesterNonAccountUser,
checkBucketAcls, checkBucketAcls,
checkObjectAcls, checkObjectAcls,
validatePolicyResource, validatePolicyResource,
validatePolicyConditions,
isLifecycleSession,
evaluateBucketPolicyWithIAM,
}; };

View File

@ -1,20 +1,10 @@
const { policies } = require('arsenal'); const { policies } = require('arsenal');
const { config } = require('../../../Config');
const { RequestContext, requestUtils } = policies; const RequestContext = policies.RequestContext;
let apiMethodAfterVersionCheck; let apiMethodAfterVersionCheck;
const apiMethodWithVersion = { const apiMethodWithVersion = { objectGetACL: true, objectPutACL: true,
objectGetACL: true, objectGet: true, objectDelete: true, objectPutTagging: true,
objectPutACL: true, objectGetTagging: true, objectDeleteTagging: true };
objectGet: true,
objectDelete: true,
objectPutTagging: true,
objectGetTagging: true,
objectDeleteTagging: true,
objectGetLegalHold: true,
objectPutLegalHold: true,
objectPutRetention: true,
};
function isHeaderAcl(headers) { function isHeaderAcl(headers) {
return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] || return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] ||
@ -43,7 +33,8 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
// null as the requestContext to Vault so it will only do an authentication // null as the requestContext to Vault so it will only do an authentication
// check. // check.
const ip = requestUtils.getClientIp(request, config); const ip = request.headers['x-forwarded-for'] ||
request.socket.remoteAddress;
function generateRequestContext(apiMethod) { function generateRequestContext(apiMethod) {
return new RequestContext(request.headers, return new RequestContext(request.headers,
@ -52,7 +43,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
apiMethod, 's3'); apiMethod, 's3');
} }
if (apiMethod === 'bucketPut') { if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
return null; return null;
} }
@ -65,17 +56,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
const requestContexts = []; const requestContexts = [];
if (apiMethod === 'multiObjectDelete') { if (apiMethodAfterVersionCheck === 'objectCopy'
// MultiObjectDelete does not require any authorization when evaluating
// the API. Instead, we authorize each object passed.
// But in order to get any relevant information from the authorization service
// for example, the account quota, we must send a request context object
// with no `specificResource`. We expect the result to be an implicit deny.
// In the API, we then ignore these authorization results, and we can use
// any information returned, e.g., the quota.
const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
requestContexts.push(requestContextMultiObjectDelete);
} else if (apiMethodAfterVersionCheck === 'objectCopy'
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') { || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
const objectGetAction = sourceVersionId ? 'objectGetVersion' : const objectGetAction = sourceVersionId ? 'objectGetVersion' :
'objectGet'; 'objectGet';
@ -107,63 +88,12 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
const objectGetTaggingAction = (request.query && const objectGetTaggingAction = (request.query &&
request.query.versionId) ? 'objectGetTaggingVersion' : request.query.versionId) ? 'objectGetTaggingVersion' :
'objectGetTagging'; 'objectGetTagging';
if (request.headers['x-amz-version-id']) {
const objectGetVersionAction = 'objectGetVersion';
const getVersionResourceVersion =
generateRequestContext(objectGetVersionAction);
requestContexts.push(getVersionResourceVersion);
}
const getRequestContext = const getRequestContext =
generateRequestContext(apiMethodAfterVersionCheck); generateRequestContext(apiMethodAfterVersionCheck);
const getTaggingRequestContext = const getTaggingRequestContext =
generateRequestContext(objectGetTaggingAction); generateRequestContext(objectGetTaggingAction);
requestContexts.push(getRequestContext, getTaggingRequestContext); requestContexts.push(getRequestContext, getTaggingRequestContext);
} else if (apiMethodAfterVersionCheck === 'objectGetTagging') {
const objectGetTaggingAction = 'objectGetTagging';
const getTaggingResourceVersion =
generateRequestContext(objectGetTaggingAction);
requestContexts.push(getTaggingResourceVersion);
if (request.headers['x-amz-version-id']) {
const objectGetTaggingVersionAction = 'objectGetTaggingVersion';
const getTaggingVersionResourceVersion =
generateRequestContext(objectGetTaggingVersionAction);
requestContexts.push(getTaggingVersionResourceVersion);
}
} else if (apiMethodAfterVersionCheck === 'objectHead') {
const objectHeadAction = 'objectHead';
const headObjectAction =
generateRequestContext(objectHeadAction);
requestContexts.push(headObjectAction);
if (request.headers['x-amz-version-id']) {
const objectHeadVersionAction = 'objectGetVersion';
const headObjectVersion =
generateRequestContext(objectHeadVersionAction);
requestContexts.push(headObjectVersion);
}
} else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
const putObjectTaggingRequestContext =
generateRequestContext('objectPutTagging');
requestContexts.push(putObjectTaggingRequestContext);
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
} else if (apiMethodAfterVersionCheck === 'objectPutCopyPart') {
const putObjectRequestContext =
generateRequestContext('objectPut');
requestContexts.push(putObjectRequestContext);
const getObjectRequestContext =
generateRequestContext('objectGet');
requestContexts.push(getObjectRequestContext);
} else if (apiMethodAfterVersionCheck === 'objectPut') { } else if (apiMethodAfterVersionCheck === 'objectPut') {
// if put object with version
if (request.headers['x-scal-s3-version-id'] ||
request.headers['x-scal-s3-version-id'] === '') {
const putVersionRequestContext =
generateRequestContext('objectPutVersion');
requestContexts.push(putVersionRequestContext);
} else {
const putRequestContext = const putRequestContext =
generateRequestContext(apiMethodAfterVersionCheck); generateRequestContext(apiMethodAfterVersionCheck);
requestContexts.push(putRequestContext); requestContexts.push(putRequestContext);
@ -173,60 +103,12 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
generateRequestContext('objectPutTagging'); generateRequestContext('objectPutTagging');
requestContexts.push(putTaggingRequestContext); requestContexts.push(putTaggingRequestContext);
} }
if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
const putLegalHoldStatusAction =
generateRequestContext('objectPutLegalHold');
requestContexts.push(putLegalHoldStatusAction);
}
// if put object (versioning) with ACL // if put object (versioning) with ACL
if (isHeaderAcl(request.headers)) { if (isHeaderAcl(request.headers)) {
const putAclRequestContext = const putAclRequestContext =
generateRequestContext('objectPutACL'); generateRequestContext('objectPutACL');
requestContexts.push(putAclRequestContext); requestContexts.push(putAclRequestContext);
} }
if (request.headers['x-amz-object-lock-mode']) {
const putObjectLockRequestContext =
generateRequestContext('objectPutRetention');
requestContexts.push(putObjectLockRequestContext);
}
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
}
} else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
apiMethodAfterVersionCheck === 'objectPutPart' ||
apiMethodAfterVersionCheck === 'completeMultipartUpload'
) {
if (request.headers['x-scal-s3-version-id'] ||
request.headers['x-scal-s3-version-id'] === '') {
const putVersionRequestContext =
generateRequestContext('objectPutVersion');
requestContexts.push(putVersionRequestContext);
} else {
const putRequestContext =
generateRequestContext(apiMethodAfterVersionCheck);
requestContexts.push(putRequestContext);
}
// if put object (versioning) with ACL
if (isHeaderAcl(request.headers)) {
const putAclRequestContext =
generateRequestContext('objectPutACL');
requestContexts.push(putAclRequestContext);
}
if (request.headers['x-amz-object-lock-mode']) {
const putObjectLockRequestContext =
generateRequestContext('objectPutRetention');
requestContexts.push(putObjectLockRequestContext);
}
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
} else { } else {
const requestContext = const requestContext =
generateRequestContext(apiMethodAfterVersionCheck); generateRequestContext(apiMethodAfterVersionCheck);

View File

@ -1,99 +0,0 @@
const async = require('async');
const { auth, s3middleware } = require('arsenal');
const metadata = require('../../../metadata/wrapper');
const { decodeVersionId } = require('../object/versioning');
const { parseTagXml } = s3middleware.tagging;
function makeTagQuery(tags) {
return Object.entries(tags)
.map(i => i.join('='))
.join('&');
}
function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) {
async.waterfall([
next => {
if (request.headers['x-amz-tagging']) {
return next(null, request.headers['x-amz-tagging']);
}
if (request.post && apiMethod === 'objectPutTagging') {
return parseTagXml(request.post, log, (err, tags) => {
if (err) {
log.trace('error parsing request tags');
return next(err);
}
return next(null, makeTagQuery(tags));
});
}
return next(null, null);
},
(requestTagsQuery, next) => {
const objectKey = request.objectKey;
const bucketName = request.bucketName;
const decodedVidResult = decodeVersionId(request.query);
if (decodedVidResult instanceof Error) {
log.trace('invalid versionId query', {
versionId: request.query.versionId,
error: decodedVidResult,
});
return next(decodedVidResult);
}
const reqVersionId = decodedVidResult;
return metadata.getObjectMD(
bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => {
if (err) {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err.NoSuchKey) {
return next(null, requestTagsQuery, null);
}
log.trace('error getting request object tags');
return next(err);
}
const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags);
return next(null, requestTagsQuery, existingTagsQuery);
});
},
], (err, requestTagsQuery, existingTagsQuery) => {
if (err) {
log.trace('error processing tag condition key evaluation');
return cb(err);
}
// FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
// eslint-disable-next-line no-restricted-syntax
for (const rc of requestContexts) {
rc.setNeedTagEval(true);
if (requestTagsQuery) {
rc.setRequestObjTags(requestTagsQuery);
}
if (existingTagsQuery) {
rc.setExistingObjTag(existingTagsQuery);
}
}
return cb();
});
}
function tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log, cb) {
if (!authorizationResults) {
return cb();
}
if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
return cb(null, authorizationResults);
}
return updateRequestContextsWithTags(request, requestContexts, apiMethod, log, err => {
if (err) {
return cb(err);
}
return auth.server.doAuth(request, log,
(err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts);
});
}
module.exports = {
tagConditionKeyAuth,
updateRequestContextsWithTags,
makeTagQuery,
};

View File

@ -6,34 +6,29 @@ const acl = require('../../../metadata/acl');
const BucketInfo = require('arsenal').models.BucketInfo; const BucketInfo = require('arsenal').models.BucketInfo;
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const createKeyForUserBucket = require('./createKeyForUserBucket'); const createKeyForUserBucket = require('./createKeyForUserBucket');
const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
const metadata = require('../../../metadata/wrapper'); const metadata = require('../../../metadata/wrapper');
const kms = require('../../../kms/wrapper'); const kms = require('../../../kms/wrapper');
const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior'); const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
const { isServiceAccount } = require('../authorization/permissionChecks'); const { isBackbeatUser } = require('../authorization/aclChecks');
const usersBucket = constants.usersBucket; const usersBucket = constants.usersBucket;
const oldUsersBucket = constants.oldUsersBucket; const oldUsersBucket = constants.oldUsersBucket;
const zenkoSeparator = constants.zenkoSeparator;
const userBucketOwner = 'admin'; const userBucketOwner = 'admin';
function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) { function addToUsersBucket(canonicalID, bucketName, log, cb) {
// BACKWARD: Simplify once do not have to deal with old // BACKWARD: Simplify once do not have to deal with old
// usersbucket name and old splitter // usersbucket name and old splitter
// Get new format usersBucket to see if it exists // Get new format usersBucket to see if it exists
return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => { return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => {
if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) { if (err && !err.NoSuchBucket && !err.BucketAlreadyExists) {
return cb(err); return cb(err);
} }
const splitter = usersBucketAttrs ? const splitter = usersBucketAttrs ?
constants.splitter : constants.oldSplitter; constants.splitter : constants.oldSplitter;
let key = createKeyForUserBucket(canonicalID, splitter, bucketName); let key = createKeyForUserBucket(canonicalID, splitter, bucketName);
const omVal = { const omVal = { creationDate: new Date().toJSON() };
creationDate: new Date().toJSON(),
ingestion: bucketMD.getIngestion(),
};
// If the new format usersbucket does not exist, try to put the // If the new format usersbucket does not exist, try to put the
// key in the old usersBucket using the old splitter. // key in the old usersBucket using the old splitter.
// Otherwise put the key in the new format usersBucket // Otherwise put the key in the new format usersBucket
@ -41,7 +36,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
usersBucket : oldUsersBucket; usersBucket : oldUsersBucket;
return metadata.putObjectMD(usersBucketBeingCalled, key, return metadata.putObjectMD(usersBucketBeingCalled, key,
omVal, {}, log, err => { omVal, {}, log, err => {
if (err?.is?.NoSuchBucket) { if (err && err.NoSuchBucket) {
// There must be no usersBucket so createBucket // There must be no usersBucket so createBucket
// one using the new format // one using the new format
log.trace('users bucket does not exist, ' + log.trace('users bucket does not exist, ' +
@ -61,8 +56,9 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
// from getting a BucketAlreadyExists // from getting a BucketAlreadyExists
// error with respect // error with respect
// to the usersBucket. // to the usersBucket.
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err &&
if (err && !err.BucketAlreadyExists) { err !==
errors.BucketAlreadyExists) {
log.error('error from metadata', { log.error('error from metadata', {
error: err, error: err,
}); });
@ -96,7 +92,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
return callback(err); return callback(err);
} }
log.trace('created bucket in metadata'); log.trace('created bucket in metadata');
return addToUsersBucket(canonicalID, bucketName, bucket, log, err => { return addToUsersBucket(canonicalID, bucketName, log, err => {
if (err) { if (err) {
return callback(err); return callback(err);
} }
@ -119,7 +115,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
*/ */
function cleanUpBucket(bucketMD, canonicalID, log, callback) { function cleanUpBucket(bucketMD, canonicalID, log, callback) {
const bucketName = bucketMD.getName(); const bucketName = bucketMD.getName();
return addToUsersBucket(canonicalID, bucketName, bucketMD, log, err => { return addToUsersBucket(canonicalID, bucketName, log, err => {
if (err) { if (err) {
return callback(err); return callback(err);
} }
@ -171,36 +167,14 @@ function createBucket(authInfo, bucketName, headers,
authInfo.getAccountDisplayName(); authInfo.getAccountDisplayName();
const creationDate = new Date().toJSON(); const creationDate = new Date().toJSON();
const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true'; const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
const headerObjectLock = headers['x-amz-bucket-object-lock-enabled']; const bucket = new BucketInfo(bucketName,
const objectLockEnabled canonicalID, ownerDisplayName, creationDate,
= headerObjectLock && headerObjectLock.toLowerCase() === 'true'; BucketInfo.currentModelVersion(), null, null, null,
const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName, null, null, null, null, null, null, null, null,
creationDate, BucketInfo.currentModelVersion(), null, null, null, null, null, isNFSEnabled);
null, null, null, null, null, null, null, null, null, isNFSEnabled,
null, null, objectLockEnabled);
let locationConstraintVal = null;
if (locationConstraint) { if (locationConstraint !== undefined) {
const [locationConstraintStr, ingestion] = bucket.setLocationConstraint(locationConstraint);
locationConstraint.split(zenkoSeparator);
if (locationConstraintStr) {
locationConstraintVal = locationConstraintStr;
bucket.setLocationConstraint(locationConstraintStr);
}
if (ingestion === 'ingest') {
bucket.enableIngestion();
//automatically enable versioning for ingestion buckets
bucket.setVersioningConfiguration({ Status: 'Enabled' });
}
}
if (objectLockEnabled) {
// default versioning configuration AWS sets
// when a bucket is created with object lock
const versioningConfiguration = {
Status: 'Enabled',
MfaDelete: 'Disabled',
};
bucket.setVersioningConfiguration(versioningConfiguration);
} }
const parseAclParams = { const parseAclParams = {
headers, headers,
@ -223,7 +197,6 @@ function createBucket(authInfo, bucketName, headers,
}, },
getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) { getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
metadata.getBucket(bucketName, log, (err, data) => { metadata.getBucket(bucketName, log, (err, data) => {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err && err.NoSuchBucket) { if (err && err.NoSuchBucket) {
return callback(null, 'NoBucketYet'); return callback(null, 'NoBucketYet');
} }
@ -242,15 +215,14 @@ function createBucket(authInfo, bucketName, headers,
const existingBucketMD = results.getAnyExistingBucketInfo; const existingBucketMD = results.getAnyExistingBucketInfo;
if (existingBucketMD instanceof BucketInfo && if (existingBucketMD instanceof BucketInfo &&
existingBucketMD.getOwner() !== canonicalID && existingBucketMD.getOwner() !== canonicalID &&
!isServiceAccount(canonicalID)) { !isBackbeatUser(canonicalID)) {
// return existingBucketMD to collect cors headers // return existingBucketMD to collect cors headers
return cb(errors.BucketAlreadyExists, existingBucketMD); return cb(errors.BucketAlreadyExists, existingBucketMD);
} }
const newBucketMD = results.prepareNewBucketMD; const newBucketMD = results.prepareNewBucketMD;
if (existingBucketMD === 'NoBucketYet') { if (existingBucketMD === 'NoBucketYet') {
const sseConfig = parseBucketEncryptionHeaders(headers);
return bucketLevelServerSideEncryption( return bucketLevelServerSideEncryption(
bucketName, sseConfig, log, bucketName, headers, log,
(err, sseInfo) => { (err, sseInfo) => {
if (err) { if (err) {
return cb(err); return cb(err);
@ -273,7 +245,7 @@ function createBucket(authInfo, bucketName, headers,
// error unless old AWS behavior (us-east-1) // error unless old AWS behavior (us-east-1)
// Existing locationConstraint must have legacyAwsBehavior === true // Existing locationConstraint must have legacyAwsBehavior === true
// New locationConstraint should have legacyAwsBehavior === true // New locationConstraint should have legacyAwsBehavior === true
if (isLegacyAWSBehavior(locationConstraintVal) && if (isLegacyAWSBehavior(locationConstraint) &&
isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) { isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) {
log.trace('returning 200 instead of 409 to mirror us-east-1'); log.trace('returning 200 instead of 409 to mirror us-east-1');
return cb(null, existingBucketMD); return cb(null, existingBucketMD);

View File

@ -3,7 +3,6 @@ const async = require('async');
const { errors } = require('arsenal'); const { errors } = require('arsenal');
const abortMultipartUpload = require('../object/abortMultipartUpload'); const abortMultipartUpload = require('../object/abortMultipartUpload');
const { pushMetric } = require('../../../utapi/utilities');
const { splitter, oldSplitter, mpuBucketPrefix } = const { splitter, oldSplitter, mpuBucketPrefix } =
require('../../../../constants'); require('../../../../constants');
@ -16,7 +15,6 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
`${mpuBucketPrefix}${destinationBucketName}`; `${mpuBucketPrefix}${destinationBucketName}`;
return metadata.deleteBucket(mpuBucketName, log, err => { return metadata.deleteBucket(mpuBucketName, log, err => {
// If the mpu bucket does not exist, just move on // If the mpu bucket does not exist, just move on
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err && err.NoSuchBucket) { if (err && err.NoSuchBucket) {
return cb(); return cb();
} }
@ -24,23 +22,14 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
}); });
} }
function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) { function _deleteOngoingMPUs(authInfo, bucketName, mpus, log, cb) {
async.mapLimit(mpus, 1, (mpu, next) => { async.mapLimit(mpus, 1, (mpu, next) => {
const splitterChar = mpu.key.includes(oldSplitter) ? const splitterChar = mpu.key.includes(oldSplitter) ?
oldSplitter : splitter; oldSplitter : splitter;
// `overview${splitter}${objectKey}${splitter}${uploadId} // `overview${splitter}${objectKey}${splitter}${uploadId}
const [, objectKey, uploadId] = mpu.key.split(splitterChar); const [, objectKey, uploadId] = mpu.key.split(splitterChar);
abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log, abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
(err, destBucket, partSizeSum) => { next);
pushMetric('abortMultipartUpload', log, {
authInfo,
canonicalID: bucketMD.getOwner(),
bucket: bucketName,
keys: [objectKey],
byteLength: partSizeSum,
});
next(err);
}, request);
}, cb); }, cb);
} }
/** /**
@ -49,13 +38,11 @@ function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log,
* @param {object} bucketMD - bucket attributes/metadata * @param {object} bucketMD - bucket attributes/metadata
* @param {string} bucketName - bucket in which objectMetadata is stored * @param {string} bucketName - bucket in which objectMetadata is stored
* @param {string} canonicalID - account canonicalID of requester * @param {string} canonicalID - account canonicalID of requester
* @param {object} request - request object given by router
* including normalized headers
* @param {object} log - Werelogs logger * @param {object} log - Werelogs logger
* @param {function} cb - callback from async.waterfall in bucketDelete * @param {function} cb - callback from async.waterfall in bucketDelete
* @return {undefined} * @return {undefined}
*/ */
function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, cb) { function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, log, cb) {
log.trace('deleting bucket from metadata'); log.trace('deleting bucket from metadata');
assert.strictEqual(typeof bucketName, 'string'); assert.strictEqual(typeof bucketName, 'string');
assert.strictEqual(typeof canonicalID, 'string'); assert.strictEqual(typeof canonicalID, 'string');
@ -93,7 +80,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
log, (err, objectsListRes) => { log, (err, objectsListRes) => {
// If no shadow bucket ever created, no ongoing MPU's, so // If no shadow bucket ever created, no ongoing MPU's, so
// continue with deletion // continue with deletion
if (err?.is.NoSuchBucket) { if (err && err.NoSuchBucket) {
return next(); return next();
} }
if (err) { if (err) {
@ -102,7 +89,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
} }
if (objectsListRes.Contents.length) { if (objectsListRes.Contents.length) {
return _deleteOngoingMPUs(authInfo, bucketName, return _deleteOngoingMPUs(authInfo, bucketName,
bucketMD, objectsListRes.Contents, request, log, err => { objectsListRes.Contents, log, err => {
if (err) { if (err) {
return next(err); return next(err);
} }

View File

@ -1,255 +0,0 @@
const { errors } = require('arsenal');
const metadata = require('../../../metadata/wrapper');
const kms = require('../../../kms/wrapper');
const { parseString } = require('xml2js');
/**
* ServerSideEncryptionInfo - user configuration for server side encryption
* @typedef {Object} ServerSideEncryptionInfo
* @property {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
* @property {string} masterKeyId - Key id for the kms key used to encrypt data keys.
* @property {string} configuredMasterKeyId - User configured master key id.
* @property {boolean} mandatory - Whether a default encryption policy has been enabled.
*/
/**
* @callback ServerSideEncryptionInfo~callback
* @param {Object} error - Instance of Arsenal error
* @param {ServerSideEncryptionInfo} - SSE configuration
*/
/**
* parseEncryptionXml - Parses and validates a ServerSideEncryptionConfiguration xml document
* @param {object} xml - ServerSideEncryptionConfiguration doc
* @param {object} log - logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function parseEncryptionXml(xml, log, cb) {
return parseString(xml, (err, parsed) => {
if (err) {
log.trace('xml parsing failed', {
error: err,
method: 'parseEncryptionXml',
});
log.debug('invalid xml', { xml });
return cb(errors.MalformedXML);
}
if (!parsed
|| !parsed.ServerSideEncryptionConfiguration
|| !parsed.ServerSideEncryptionConfiguration.Rule) {
log.trace('error in sse config, invalid ServerSideEncryptionConfiguration section', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const { Rule } = parsed.ServerSideEncryptionConfiguration;
if (!Array.isArray(Rule)
|| Rule.length > 1
|| !Rule[0]
|| !Rule[0].ApplyServerSideEncryptionByDefault
|| !Rule[0].ApplyServerSideEncryptionByDefault[0]) {
log.trace('error in sse config, invalid ApplyServerSideEncryptionByDefault section', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const [encConfig] = Rule[0].ApplyServerSideEncryptionByDefault;
if (!encConfig.SSEAlgorithm || !encConfig.SSEAlgorithm[0]) {
log.trace('error in sse config, no SSEAlgorithm provided', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const [algorithm] = encConfig.SSEAlgorithm;
if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
log.trace('error in sse config, unknown SSEAlgorithm', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const result = { algorithm, mandatory: true };
if (encConfig.KMSMasterKeyID) {
if (algorithm === 'AES256') {
log.trace('error in sse config, can not specify KMSMasterKeyID when using AES256', {
method: 'parseEncryptionXml',
});
return cb(errors.InvalidArgument.customizeDescription(
'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'));
}
if (!encConfig.KMSMasterKeyID[0] || typeof encConfig.KMSMasterKeyID[0] !== 'string') {
log.trace('error in sse config, invalid KMSMasterKeyID', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
result.configuredMasterKeyId = encConfig.KMSMasterKeyID[0];
}
return cb(null, result);
});
}
/**
* hydrateEncryptionConfig - Constructs a ServerSideEncryptionInfo object from arguments
* ensuring no invalid or undefined keys are added
*
* @param {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
* @param {string} configuredMasterKeyId - User configured master key id.
* @param {boolean} [mandatory] - Whether a default encryption policy has been enabled.
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function hydrateEncryptionConfig(algorithm, configuredMasterKeyId, mandatory = null) {
if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
return {
algorithm: null,
};
}
const sseConfig = { algorithm, mandatory };
if (algorithm === 'aws:kms' && configuredMasterKeyId) {
sseConfig.configuredMasterKeyId = configuredMasterKeyId;
}
if (mandatory !== null) {
sseConfig.mandatory = mandatory;
}
return sseConfig;
}
/**
* parseBucketEncryptionHeaders - retrieves bucket level sse configuration from request headers
* @param {object} headers - Request headers
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function parseBucketEncryptionHeaders(headers) {
const sseAlgorithm = headers['x-amz-scal-server-side-encryption'];
const configuredMasterKeyId = headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] || null;
return hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId, true);
}
/**
* parseObjectEncryptionHeaders - retrieves bucket level sse configuration from request headers
* @param {object} headers - Request headers
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function parseObjectEncryptionHeaders(headers) {
const sseAlgorithm = headers['x-amz-server-side-encryption'];
const configuredMasterKeyId = headers['x-amz-server-side-encryption-aws-kms-key-id'] || null;
if (sseAlgorithm && sseAlgorithm !== 'AES256' && sseAlgorithm !== 'aws:kms') {
return {
error: errors.InvalidArgument.customizeDescription('The encryption method specified is not supported'),
};
}
if (sseAlgorithm !== 'aws:kms' && configuredMasterKeyId) {
return {
error: errors.InvalidArgument.customizeDescription(
'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'),
};
}
return { objectSSE: hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId) };
}
/**
* createDefaultBucketEncryptionMetadata - Creates master key and sets up default server side encryption configuration
* @param {BucketInfo} bucket - bucket metadata
* @param {object} log - werelogs logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function createDefaultBucketEncryptionMetadata(bucket, log, cb) {
return kms.bucketLevelEncryption(
bucket.getName(),
{ algorithm: 'AES256', mandatory: false },
log,
(error, sseConfig) => {
if (error) {
return cb(error);
}
bucket.setServerSideEncryption(sseConfig);
return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig));
});
}
/**
*
* @param {object} headers - request headers
* @param {BucketInfo} bucket - BucketInfo model
* @param {*} log - werelogs logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function getObjectSSEConfiguration(headers, bucket, log, cb) {
const bucketSSE = bucket.getServerSideEncryption();
const { error, objectSSE } = parseObjectEncryptionHeaders(headers);
if (error) {
return cb(error);
}
// If a per object sse algo has been passed through
// x-amz-server-side-encryption
if (objectSSE.algorithm) {
// If aws:kms and a custom key id
// pass it through without updating the bucket md
if (objectSSE.algorithm === 'aws:kms' && objectSSE.configuredMasterKeyId) {
return cb(null, objectSSE);
}
// If the client has not specified a key id,
// and we have a default config, then we reuse
// it and pass it through
if (!objectSSE.configuredMasterKeyId && bucketSSE) {
// The default configs algo is overridden with the one passed in the
// request headers. Our implementations of AES256 and aws:kms are the
// same underneath so this is only cosmetic change.
const sseConfig = Object.assign({}, bucketSSE, { algorithm: objectSSE.algorithm });
return cb(null, sseConfig);
}
// If the client has not specified a key id, and we
// don't have a default config, generate it
if (!objectSSE.configuredMasterKeyId && !bucketSSE) {
return createDefaultBucketEncryptionMetadata(bucket, log, (error, sseConfig) => {
if (error) {
return cb(error);
}
// Override the algorithm, for the same reasons as above.
Object.assign(sseConfig, { algorithm: objectSSE.algorithm });
return cb(null, sseConfig);
});
}
}
// If the bucket has a default encryption config, and it is mandatory
// (created with putBucketEncryption or legacy headers)
// pass it through
if (bucketSSE && bucketSSE.mandatory) {
return cb(null, bucketSSE);
}
// No encryption config
return cb(null, null);
}
module.exports = {
createDefaultBucketEncryptionMetadata,
getObjectSSEConfiguration,
hydrateEncryptionConfig,
parseEncryptionXml,
parseBucketEncryptionHeaders,
parseObjectEncryptionHeaders,
};

View File

@ -30,9 +30,6 @@ function bucketShield(bucket, requestType) {
// Otherwise return an error to the client // Otherwise return an error to the client
if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) && if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
(requestType !== 'objectPut' && (requestType !== 'objectPut' &&
requestType !== 'initiateMultipartUpload' &&
requestType !== 'objectPutPart' &&
requestType !== 'completeMultipartUpload' &&
requestType !== 'bucketPutACL' && requestType !== 'bucketPutACL' &&
requestType !== 'bucketDelete')) { requestType !== 'bucketDelete')) {
return true; return true;

View File

@ -11,16 +11,15 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => { metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
// If the object representing the bucket is not in the // If the object representing the bucket is not in the
// users bucket just continue // users bucket just continue
if (error?.is.NoSuchKey) { if (error && error.NoSuchKey) {
return cb(null); return cb(null);
// BACKWARDS COMPATIBILITY: Remove this once no longer // BACKWARDS COMPATIBILITY: Remove this once no longer
// have old user bucket format // have old user bucket format
} else if (error?.is.NoSuchBucket) { } else if (error && error.NoSuchBucket) {
const keyForUserBucket2 = createKeyForUserBucket(canonicalID, const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
oldSplitter, bucketName); oldSplitter, bucketName);
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2, return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
{}, log, error => { {}, log, error => {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (error && !error.NoSuchKey) { if (error && !error.NoSuchKey) {
log.error('from metadata while deleting user bucket', log.error('from metadata while deleting user bucket',
{ error }); { error });

View File

@ -1,37 +0,0 @@
const { errors, models } = require('arsenal');
const { NotificationConfiguration } = models;
const { config } = require('../../../Config');
function getNotificationConfiguration(parsedXml) {
const notifConfig = new NotificationConfiguration(parsedXml).getValidatedNotificationConfiguration();
// if notifConfig is empty object, effectively delete notification configuration
if (notifConfig.error || Object.keys(notifConfig).length === 0) {
return notifConfig;
}
if (!config.bucketNotificationDestinations) {
return { error: errors.InvalidArgument.customizeDescription(
'Unable to validate the following destination configurations') };
}
const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource));
const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]);
// getting invalid targets
const invalidTargets = [];
notifConfigTargets.forEach((t, i) => {
if (!targets.has(t)) {
invalidTargets.push({
ArgumentName: notifConfig.queueConfig[i].queueArn,
ArgumentValue: 'The destination queue does not exist',
});
}
});
if (invalidTargets.length > 0) {
const errDesc = 'Unable to validate the following destination configurations';
let error = errors.InvalidArgument.customizeDescription(errDesc);
error = error.addMetadataEntry('invalidArguments', invalidTargets);
return { error };
}
return notifConfig;
}
module.exports = getNotificationConfiguration;

View File

@ -0,0 +1,233 @@
const { config } = require('../../../Config');
const { legacyLocations } = require('../../../../constants.js');
const { locationConstraints } = config;
const escapeForXml = require('arsenal').s3middleware.escapeForXml;
class BackendInfo {
/**
* Represents the info necessary to evaluate which data backend to use
* on a data put call.
* @constructor
* @param {string | undefined} objectLocationConstraint - location constraint
* for object based on user meta header
* @param {string | undefined } bucketLocationConstraint - location
* constraint for bucket based on bucket metadata
* @param {string} requestEndpoint - endpoint to which request was made
* @param {string | undefined } legacyLocationConstraint - legacy location
* constraint
*/
constructor(objectLocationConstraint, bucketLocationConstraint,
requestEndpoint, legacyLocationConstraint) {
this._objectLocationConstraint = objectLocationConstraint;
this._bucketLocationConstraint = bucketLocationConstraint;
this._requestEndpoint = requestEndpoint;
this._legacyLocationConstraint = legacyLocationConstraint;
return this;
}
/**
* validate proposed location constraint against config
* @param {string | undefined} locationConstraint - value of user
* metadata location constraint header or bucket location constraint
* @param {object} log - werelogs logger
* @return {boolean} - true if valid, false if not
*/
static isValidLocationConstraint(locationConstraint, log) {
if (Object.keys(config.locationConstraints).
indexOf(locationConstraint) < 0) {
log.trace('proposed locationConstraint is invalid',
{ locationConstraint });
return false;
}
return true;
}
/**
* validate that request endpoint is listed in the restEndpoint config
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if present, false if not
*/
static isRequestEndpointPresent(requestEndpoint, log) {
if (Object.keys(config.restEndpoints).indexOf(requestEndpoint) < 0) {
log.trace('requestEndpoint does not match config restEndpoints',
{ requestEndpoint });
return false;
}
return true;
}
/**
* validate that locationConstraint for request Endpoint matches
* one config locationConstraint
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if matches, false if not
*/
static isRequestEndpointValueValid(requestEndpoint, log) {
if (Object.keys(config.locationConstraints).indexOf(config
.restEndpoints[requestEndpoint]) < 0) {
log.trace('the default locationConstraint for request' +
'Endpoint does not match any config locationConstraint',
{ requestEndpoint });
return false;
}
return true;
}
/**
* validate that s3 server is running with a file or memory backend
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if running with file/mem backend, false if not
*/
static isMemOrFileBackend(requestEndpoint, log) {
if (config.backends.data === 'mem' ||
config.backends.data === 'file') {
log.trace('use data backend for the location', {
dataBackend: config.backends.data,
method: 'isMemOrFileBackend',
});
return true;
}
return false;
}
/**
* validate requestEndpoint against config or mem/file data backend
* - if there is no match for the request endpoint in the config
* restEndpoints and data backend is set to mem or file we will use this
* data backend for the location.
* - if locationConstraint for request Endpoint does not match
* any config locationConstraint, we will return an error
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if valid, false if not
*/
static isValidRequestEndpointOrBackend(requestEndpoint, log) {
if (!BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
return BackendInfo.isMemOrFileBackend(requestEndpoint, log);
}
return BackendInfo.isRequestEndpointValueValid(requestEndpoint, log);
}
/**
* validate controlling BackendInfo Parameter
* @param {string | undefined} objectLocationConstraint - value of user
* metadata location constraint header
* @param {string | null} bucketLocationConstraint - location
* constraint from bucket metadata
* @param {string} requestEndpoint - endpoint of request
* @param {object} log - werelogs logger
* @return {object} - location constraint validity
*/
static controllingBackendParam(objectLocationConstraint,
bucketLocationConstraint, requestEndpoint, log) {
if (objectLocationConstraint) {
if (BackendInfo.isValidLocationConstraint(objectLocationConstraint,
log)) {
log.trace('objectLocationConstraint is valid');
return { isValid: true };
}
log.trace('objectLocationConstraint is invalid');
return { isValid: false, description: 'Object Location Error - ' +
`Your object location "${escapeForXml(objectLocationConstraint)}"` +
'is not in your location config - Please update.' };
}
if (bucketLocationConstraint) {
if (BackendInfo.isValidLocationConstraint(bucketLocationConstraint,
log)) {
log.trace('bucketLocationConstraint is valid');
return { isValid: true };
}
log.trace('bucketLocationConstraint is invalid');
return { isValid: false, description: 'Bucket Location Error - ' +
`Your bucket location "${escapeForXml(bucketLocationConstraint)}"` +
' is not in your location config - Please update.' };
}
const legacyLocationConstraint =
BackendInfo.getLegacyLocationConstraint();
if (legacyLocationConstraint) {
log.trace('legacy location is valid');
return { isValid: true, legacyLocationConstraint };
}
if (!BackendInfo.isValidRequestEndpointOrBackend(requestEndpoint,
log)) {
return { isValid: false, description: 'Endpoint Location Error - ' +
`Your endpoint "${requestEndpoint}" is not in restEndpoints ` +
'in your config OR the default location constraint for request ' +
`endpoint "${escapeForXml(requestEndpoint)}" does not ` +
'match any config locationConstraint - Please update.' };
}
if (BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
return { isValid: true };
}
return { isValid: true, defaultedToDataBackend: true };
}
/**
* Return legacyLocationConstraint
* @return {string | undefined} legacyLocationConstraint;
*/
static getLegacyLocationConstraint() {
return legacyLocations.find(ll => locationConstraints[ll]);
}
/**
* Return objectLocationConstraint
* @return {string | undefined} objectLocationConstraint;
*/
getObjectLocationConstraint() {
return this._objectLocationConstraint;
}
/**
* Return bucketLocationConstraint
* @return {string | undefined} bucketLocationConstraint;
*/
getBucketLocationConstraint() {
return this._bucketLocationConstraint;
}
/**
* Return requestEndpoint
* @return {string} requestEndpoint;
*/
getRequestEndpoint() {
return this._requestEndpoint;
}
/**
* Return locationConstraint that should be used with put request
* Order of priority is:
* (1) objectLocationConstraint,
* (2) bucketLocationConstraint,
* (3) legacyLocationConstraint,
* (4) default locationConstraint for requestEndpoint if requestEndpoint
* is listed in restEndpoints in config.json
* (5) default data backend
* @return {string} locationConstraint;
*/
getControllingLocationConstraint() {
const objectLC = this.getObjectLocationConstraint();
const bucketLC = this.getBucketLocationConstraint();
const reqEndpoint = this.getRequestEndpoint();
if (objectLC) {
return objectLC;
}
if (bucketLC) {
return bucketLC;
}
if (this._legacyLocationConstraint) {
return this._legacyLocationConstraint;
}
if (config.restEndpoints[reqEndpoint]) {
return config.restEndpoints[reqEndpoint];
}
return config.backends.data;
}
}
module.exports = {
BackendInfo,
};

View File

@ -1,10 +1,12 @@
const async = require('async'); const async = require('async');
const { config } = require('../../../Config');
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { data } = require('../../../data/wrapper'); const data = require('../../../data/wrapper');
const locationConstraintCheck = require('../object/locationConstraintCheck'); const locationConstraintCheck = require('../object/locationConstraintCheck');
const { standardMetadataValidateBucketAndObj } = const { metadataValidateBucketAndObj } =
require('../../../metadata/metadataUtils'); require('../../../metadata/metadataUtils');
const multipleBackendGateway = require('../../../data/multipleBackendGateway');
const services = require('../../../services'); const services = require('../../../services');
function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log, function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
@ -14,19 +16,17 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
bucketName, bucketName,
objectKey, objectKey,
uploadId, uploadId,
preciseRequestType: request.apiMethods || 'multipartDelete', requestType: 'deleteMPU',
request,
}; };
// For validating the request at the destinationBucket level // For validating the request at the destinationBucket level
// params are the same as validating at the MPU level // params are the same as validating at the MPU level
// but the requestType is the more general 'objectDelete' // but the requestType is the more general 'objectDelete'
const metadataValParams = Object.assign({}, metadataValMPUparams); const metadataValParams = Object.assign({}, metadataValMPUparams);
metadataValParams.requestType = 'objectPut'; metadataValParams.requestType = 'objectPut';
const authzIdentityResult = request ? request.actionImplicitDenies : false;
async.waterfall([ async.waterfall([
function checkDestBucketVal(next) { function checkDestBucketVal(next) {
standardMetadataValidateBucketAndObj(metadataValParams, authzIdentityResult, log, metadataValidateBucketAndObj(metadataValParams, log,
(err, destinationBucket) => { (err, destinationBucket) => {
if (err) { if (err) {
return next(err, destinationBucket); return next(err, destinationBucket);
@ -54,24 +54,35 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
return next(err, mpuBucket, mpuOverviewObj, destBucket); return next(err, mpuBucket, mpuOverviewObj, destBucket);
}); });
}, },
function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket, function ifMultipleBackend(mpuBucket, mpuOverviewObj, destBucket,
next) { next) {
const location = mpuOverviewObj.controllingLocationConstraint; if (config.backends.data === 'multiple') {
const originalIdentityAuthzResults = request.actionImplicitDenies; let location;
// eslint-disable-next-line no-param-reassign // if controlling location constraint is not stored in object
delete request.actionImplicitDenies; // metadata, mpu was initiated in legacy S3C, so need to
return data.abortMPU(objectKey, uploadId, location, bucketName, // determine correct location constraint
request, destBucket, locationConstraintCheck, log, if (!mpuOverviewObj.controllingLocationConstraint) {
(err, skipDataDelete) => { const backendInfoObj = locationConstraintCheck(request,
// eslint-disable-next-line no-param-reassign null, destBucket, log);
request.actionImplicitDenies = originalIdentityAuthzResults; if (backendInfoObj.err) {
return process.nextTick(() => {
next(backendInfoObj.err, destBucket);
});
}
location = backendInfoObj.controllingLC;
} else {
location = mpuOverviewObj.controllingLocationConstraint;
}
return multipleBackendGateway.abortMPU(objectKey, uploadId,
location, bucketName, log, (err, skipDataDelete) => {
if (err) { if (err) {
return next(err, destBucket); return next(err, destBucket);
} }
// for Azure and GCP we do not need to delete data return next(null, mpuBucket, destBucket,
// for all other backends, skipDataDelete will be set to false skipDataDelete);
return next(null, mpuBucket, destBucket, skipDataDelete);
}); });
}
return next(null, mpuBucket, destBucket, false);
}, },
function getPartLocations(mpuBucket, destBucket, skipDataDelete, function getPartLocations(mpuBucket, destBucket, skipDataDelete,
next) { next) {
@ -87,6 +98,7 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
}, },
function deleteData(mpuBucket, storedParts, destBucket, function deleteData(mpuBucket, storedParts, destBucket,
skipDataDelete, next) { skipDataDelete, next) {
// for Azure we do not need to delete data
if (skipDataDelete) { if (skipDataDelete) {
return next(null, mpuBucket, storedParts, destBucket); return next(null, mpuBucket, storedParts, destBucket);
} }

View File

@ -1,19 +0,0 @@
const { zenkoIDHeader } = require('arsenal').constants;
const _config = require('../../../Config').config;
/**
* applyZenkoUserMD - if request is within a Zenko deployment, apply user
* metadata called "zenko-source" to the object
* @param {Object} metaHeaders - user metadata object
* @return {undefined}
*/
function applyZenkoUserMD(metaHeaders) {
if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' &&
!metaHeaders[zenkoIDHeader]) {
// eslint-disable-next-line no-param-reassign
metaHeaders[zenkoIDHeader] = _config.getPublicInstanceId();
}
}
module.exports = applyZenkoUserMD;

View File

@ -1,25 +0,0 @@
const { errors } = require('arsenal');
const { maxHttpHeadersSize } = require('../../../../constants');
/**
* Checks the size of the HTTP headers
* @param {object} requestHeaders - HTTP request headers
* @return {object} object with error or null
*/
function checkHttpHeadersSize(requestHeaders) {
let httpHeadersSize = 0;
Object.keys(requestHeaders).forEach(header => {
httpHeadersSize += Buffer.byteLength(header, 'utf8') +
Buffer.byteLength(requestHeaders[header], 'utf8');
});
if (httpHeadersSize > maxHttpHeadersSize) {
return {
httpHeadersSizeError: errors.HttpHeadersTooLarge,
};
}
return {};
}
module.exports = checkHttpHeadersSize;

View File

@ -1,38 +0,0 @@
const { maximumMetaHeadersSize,
invalidObjectUserMetadataHeader } = require('../../../../constants');
/**
* Checks the size of the user metadata in the object metadata and removes
* them from the response if the size of the user metadata is larger than
* the maximum size allowed. A custome metadata key is added to the response
* with the number of user metadata keys not returned as its value
* @param {object} responseMetadata - response metadata
* @return {object} responseMetaHeaders headers with object metadata to include
* in response to client
*/
function checkUserMetadataSize(responseMetadata) {
let userMetadataSize = 0;
// collect the user metadata keys from the object metadata
const userMetadataHeaders = Object.keys(responseMetadata)
.filter(key => key.startsWith('x-amz-meta-'));
// compute the size of all user metadata key and its value
userMetadataHeaders.forEach(header => {
userMetadataSize += header.length + responseMetadata[header].length;
});
// check the size computed against the maximum allowed
// if the computed size is greater, then remove all the
// user metadata from the response object
if (userMetadataSize > maximumMetaHeadersSize) {
const md = Object.assign({}, responseMetadata);
userMetadataHeaders.forEach(header => {
delete md[header];
});
// add the prescribed/custom metadata with number of user metadata
// as its value
md[invalidObjectUserMetadataHeader] = userMetadataHeaders.length;
return md;
}
return responseMetadata;
}
module.exports = checkUserMetadataSize;

View File

@ -1,247 +0,0 @@
/*
* Code based on Yutaka Oishi (Fujifilm) contributions
* Date: 11 Sep 2020
*/
const { ObjectMDArchive } = require('arsenal').models;
const errors = require('arsenal').errors;
const { config } = require('../../../Config');
const { locationConstraints } = config;
const { scaledMsPerDay } = config.getTimeOptions();
/**
* Get response header "x-amz-restore"
* Be called by objectHead.js
* @param {object} objMD - object's metadata
* @returns {string|undefined} x-amz-restore
*/
function getAmzRestoreResHeader(objMD) {
if (objMD.archive &&
objMD.archive.restoreRequestedAt &&
!objMD.archive.restoreCompletedAt) {
// Avoid race condition by relying on the `archive` MD of the object
// and return the right header after a RESTORE request.
// eslint-disable-next-line
return `ongoing-request="true"`;
}
if (objMD['x-amz-restore']) {
if (objMD['x-amz-restore']['expiry-date']) {
const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString();
// eslint-disable-next-line
return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
}
}
return undefined;
}
/**
* Check if restore can be done.
*
* @param {ObjectMD} objectMD - object metadata
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
*/
function _validateStartRestore(objectMD, log) {
if (objectMD.archive?.restoreCompletedAt) {
if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
// return InvalidObjectState error if the restored object is expired
// but restore info md of this object has not yet been cleared
log.debug('The restored object already expired.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
}
// If object is already restored, no further check is needed
// Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have
// been reset.
return undefined;
}
const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
if (!isLocationCold) {
// return InvalidObjectState error if the object is not in cold storage,
// not in cold storage means either location cold flag not exists or cold flag is explicit false
log.debug('The bucket of the object is not in a cold storage location.',
{
isLocationCold,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
}
if (objectMD.archive?.restoreRequestedAt) {
// return RestoreAlreadyInProgress error if the object is currently being restored
// check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
log.debug('The object is currently being restored.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.RestoreAlreadyInProgress;
}
return undefined;
}
/**
* Check if "put version id" is allowed
*
* @param {ObjectMD} objMD - object metadata
* @param {string} versionId - object's version id
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} - undefined if "put version id" is allowed
*/
function validatePutVersionId(objMD, versionId, log) {
if (!objMD) {
const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
return err;
}
if (objMD.isDeleteMarker) {
log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
return errors.MethodNotAllowed;
}
const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
if (!isLocationCold) {
log.error('The object data is not stored in a cold storage location.',
{
isLocationCold,
dataStoreName: objMD.dataStoreName,
method: 'validatePutVersionId',
});
return errors.InvalidObjectState;
}
// make sure object archive restoration is in progress
// NOTE: we do not use putObjectVersion to update the restoration period.
if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
|| objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
log.error('object archive restoration is not in progress',
{ method: 'validatePutVersionId', versionId });
return errors.InvalidObjectState;
}
return undefined;
}
/**
* Check if the object is already restored, and update the expiration date accordingly:
* > After restoring an archived object, you can update the restoration period by reissuing the
* > request with a new period. Amazon S3 updates the restoration period relative to the current
* > time.
*
* @param {ObjectMD} objectMD - object metadata
* @param {object} log - werelogs logger
* @return {boolean} - true if the object is already restored
*/
function _updateObjectExpirationDate(objectMD, log) {
// Check if restoreCompletedAt field exists
// Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
// checked earlier in the process, so checking again here would create weird states
const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
log.debug('The restore status of the object.', {
isObjectAlreadyRestored,
method: 'isObjectAlreadyRestored'
});
if (isObjectAlreadyRestored) {
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
/* eslint-disable no-param-reassign */
objectMD.archive.restoreWillExpireAt = expiryDate;
objectMD['x-amz-restore'] = {
'ongoing-request': false,
'expiry-date': expiryDate,
};
/* eslint-enable no-param-reassign */
}
return isObjectAlreadyRestored;
}
/**
* update restore expiration date.
*
* @param {ObjectMD} objectMD - objectMD instance
* @param {object} restoreParam - restore param
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} internal error if object MD is not valid
*
*/
function _updateRestoreInfo(objectMD, restoreParam, log) {
if (!objectMD.archive) {
log.debug('objectMD.archive doesn\'t exits', {
objectMD,
method: '_updateRestoreInfo'
});
return errors.InternalError.customizeDescription('Archive metadata is missing.');
}
/* eslint-disable no-param-reassign */
objectMD.archive.restoreRequestedAt = new Date();
objectMD.archive.restoreRequestedDays = restoreParam.days;
objectMD.originOp = 's3:ObjectRestore:Post';
/* eslint-enable no-param-reassign */
if (!ObjectMDArchive.isValid(objectMD.archive)) {
log.debug('archive is not valid', {
archive: objectMD.archive,
method: '_updateRestoreInfo'
});
return errors.InternalError.customizeDescription('Invalid archive metadata.');
}
return undefined;
}
/**
* start to restore object.
* If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
* calculate restore expiry-date and add it to objectMD.
* Be called by objectRestore.js
*
* @param {ObjectMD} objectMD - objectMd instance
* @param {object} restoreParam - bucket name
* @param {object} log - werelogs logger
* @param {function} cb - bucket name
* @return {undefined}
*
*/
function startRestore(objectMD, restoreParam, log, cb) {
log.info('Validating if restore can be done or not.');
const checkResultError = _validateStartRestore(objectMD, log);
if (checkResultError) {
return cb(checkResultError);
}
log.info('Updating restore information.');
const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
if (updateResultError) {
return cb(updateResultError);
}
const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
return cb(null, isObjectAlreadyRestored);
}
/**
* checks if object data is available or if it's in cold storage
* @param {ObjectMD} objMD Object metadata
* @returns {ArsenalError|null} error if object data is not available
*/
function verifyColdObjectAvailable(objMD) {
// return error when object is cold
if (objMD.archive &&
// Object is in cold backend
(!objMD.archive.restoreRequestedAt ||
// Object is being restored
(objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
const err = errors.InvalidObjectState
.customizeDescription('The operation is not valid for the object\'s storage class');
return err;
}
return null;
}
module.exports = {
startRestore,
getAmzRestoreResHeader,
validatePutVersionId,
verifyColdObjectAvailable,
};

View File

@ -3,24 +3,51 @@ const { errors, s3middleware } = require('arsenal');
const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders; const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { data } = require('../../../data/wrapper'); const data = require('../../../data/wrapper');
const services = require('../../../services'); const services = require('../../../services');
const logger = require('../../../utilities/logger');
const { dataStore } = require('./storeObject'); const { dataStore } = require('./storeObject');
const locationConstraintCheck = require('./locationConstraintCheck'); const locationConstraintCheck = require('./locationConstraintCheck');
const { versioningPreprocessing, overwritingVersioning } = require('./versioning'); const { versioningPreprocessing } = require('./versioning');
const removeAWSChunked = require('./removeAWSChunked'); const removeAWSChunked = require('./removeAWSChunked');
const getReplicationInfo = require('./getReplicationInfo'); const getReplicationInfo = require('./getReplicationInfo');
const { config } = require('../../../Config'); const { config } = require('../../../Config');
const validateWebsiteHeader = require('./websiteServing') const validateWebsiteHeader = require('./websiteServing')
.validateWebsiteHeader; .validateWebsiteHeader;
const applyZenkoUserMD = require('./applyZenkoUserMD'); const {
const { externalBackends, versioningNotImplBackends } = constants; externalBackends, versioningNotImplBackends, zenkoIDHeader,
} = constants;
const externalVersioningErrorMessage = 'We do not currently support putting ' + const externalVersioningErrorMessage = 'We do not currently support putting ' +
'a versioned object to a location-constraint of type Azure or GCP.'; 'a versioned object to a location-constraint of type Azure or GCP.';
/**
* Retro-propagation is where S3C ingestion will re-ingest an object whose
* request originated from Zenko.
* To avoid this, Zenko requests which create objects/versions will be tagged
* with a user-metadata header defined in constants.zenkoIDHeader. When
* ingesting objects into Zenko, we can determine if this object has already
* been created in Zenko.
* Delete marker requests cannot specify user-metadata fields, so we instead
* rely on checking the "user-agent" to see the origin of a request.
* If delete marker, and user-agent came from a Zenko client, we add the
* user-metadata field to the object metadata.
* @param {Object} metaHeaders - user metadata object
* @param {http.ClientRequest} request - client request with user-agent header
* @param {Boolean} isDeleteMarker - delete marker indicator
* @return {undefined}
*/
function _checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker) {
const userAgent = request.headers['user-agent'];
if (isDeleteMarker && userAgent && userAgent.includes('Zenko')) {
// eslint-disable-next-line no-param-reassign
metaHeaders[zenkoIDHeader] = 'zenko';
}
}
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
metadataStoreParams, dataToDelete, log, requestMethod, callback) { metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
services.metadataStoreObject(bucketName, dataGetInfo, services.metadataStoreObject(bucketName, dataGetInfo,
cipherBundle, metadataStoreParams, (err, result) => { cipherBundle, metadataStoreParams, (err, result) => {
if (err) { if (err) {
@ -30,7 +57,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
const newDataStoreName = Array.isArray(dataGetInfo) ? const newDataStoreName = Array.isArray(dataGetInfo) ?
dataGetInfo[0].dataStoreName : null; dataGetInfo[0].dataStoreName : null;
return data.batchDelete(dataToDelete, requestMethod, return data.batchDelete(dataToDelete, requestMethod,
newDataStoreName, log, err => callback(err, result)); newDataStoreName, deleteLog, err => callback(err, result));
} }
return callback(null, result); return callback(null, result);
}); });
@ -50,9 +77,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
* @param {(object|null)} streamingV4Params - if v4 auth, object containing * @param {(object|null)} streamingV4Params - if v4 auth, object containing
* accessKey, signatureFromRequest, region, scopeDate, timestamp, and * accessKey, signatureFromRequest, region, scopeDate, timestamp, and
* credentialScope (to be used for streaming v4 auth if applicable) * credentialScope (to be used for streaming v4 auth if applicable)
* @param {(object|null)} overheadField - fields to be included in metadata overhead
* @param {RequestLogger} log - logger instance * @param {RequestLogger} log - logger instance
* @param {string} originOp - Origin operation
* @param {function} callback - callback function * @param {function} callback - callback function
* @return {undefined} and call callback with (err, result) - * @return {undefined} and call callback with (err, result) -
* result.contentMD5 - content md5 of new object or version * result.contentMD5 - content md5 of new object or version
@ -60,10 +85,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
*/ */
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params, canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
overheadField, log, originOp, callback) { log, callback) {
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
const size = isDeleteMarker ? 0 : request.parsedContentLength; const size = isDeleteMarker ? 0 : request.parsedContentLength;
// although the request method may actually be 'DELETE' if creating a // although the request method may actually be 'DELETE' if creating a
// delete marker, for our purposes we consider this to be a 'PUT' // delete marker, for our purposes we consider this to be a 'PUT'
@ -86,9 +108,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
}); });
return process.nextTick(() => callback(metaHeaders)); return process.nextTick(() => callback(metaHeaders));
} }
// if the request occurs within a Zenko deployment, we place a user-metadata // if receiving a request from Zenko for a delete marker, we place a
// field on the object // user-metadata field on the object
applyZenkoUserMD(metaHeaders); _checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker);
log.trace('meta headers', { metaHeaders, method: 'objectPut' }); log.trace('meta headers', { metaHeaders, method: 'objectPut' });
const objectKeyContext = { const objectKeyContext = {
@ -116,24 +138,8 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
isDeleteMarker, isDeleteMarker,
replicationInfo: getReplicationInfo( replicationInfo: getReplicationInfo(
objectKey, bucketMD, false, size, null, null, authInfo), objectKey, bucketMD, false, size, null, null, authInfo),
overheadField,
log, log,
}; };
// For Azure BlobStorage API compatability
// If an object already exists copy/repair creation-time
// creation-time must remain static after an object is created
// --> EVEN FOR VERSIONS <--
if (objMD) {
if (objMD['creation-time']) {
metadataStoreParams.creationTime = objMD['creation-time'];
} else {
// If creation-time is not set (for old objects)
// fall back to the last modified and store it back to the db
metadataStoreParams.creationTime = objMD['last-modified'];
}
}
if (!isDeleteMarker) { if (!isDeleteMarker) {
metadataStoreParams.contentType = request.headers['content-type']; metadataStoreParams.contentType = request.headers['content-type'];
metadataStoreParams.cacheControl = request.headers['cache-control']; metadataStoreParams.cacheControl = request.headers['cache-control'];
@ -143,13 +149,6 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
removeAWSChunked(request.headers['content-encoding']); removeAWSChunked(request.headers['content-encoding']);
metadataStoreParams.expires = request.headers.expires; metadataStoreParams.expires = request.headers.expires;
metadataStoreParams.tagging = request.headers['x-amz-tagging']; metadataStoreParams.tagging = request.headers['x-amz-tagging'];
metadataStoreParams.originOp = originOp;
const defaultObjectLockConfiguration
= bucketMD.getObjectLockConfiguration();
if (defaultObjectLockConfiguration) {
metadataStoreParams.defaultRetention
= defaultObjectLockConfiguration;
}
} }
// if creating new delete marker and there is an existing object, copy // if creating new delete marker and there is an existing object, copy
@ -158,7 +157,6 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
// eslint-disable-next-line no-param-reassign // eslint-disable-next-line no-param-reassign
request.headers[constants.objectLocationConstraintHeader] = request.headers[constants.objectLocationConstraintHeader] =
objMD[constants.objectLocationConstraintHeader]; objMD[constants.objectLocationConstraintHeader];
metadataStoreParams.originOp = originOp;
} }
const backendInfoObj = const backendInfoObj =
@ -189,50 +187,18 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
} }
} }
if (objMD && objMD.uploadId) {
metadataStoreParams.oldReplayId = objMD.uploadId;
}
/* eslint-disable camelcase */ /* eslint-disable camelcase */
const dontSkipBackend = externalBackends; const dontSkipBackend = externalBackends;
/* eslint-enable camelcase */ /* eslint-enable camelcase */
const mdOnlyHeader = request.headers['x-amz-meta-mdonly']; const requestLogger =
const mdOnlySize = request.headers['x-amz-meta-size']; logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
return async.waterfall([ return async.waterfall([
function storeData(next) { function storeData(next) {
if (size === 0) { if (size === 0 && !dontSkipBackend[locationType]) {
if (!dontSkipBackend[locationType]) {
metadataStoreParams.contentMD5 = constants.emptyFileMd5; metadataStoreParams.contentMD5 = constants.emptyFileMd5;
return next(null, null, null); return next(null, null, null);
} }
// Handle mdOnlyHeader as a metadata only operation. If
// the object in question is actually 0 byte or has a body size
// then handle normally.
if (mdOnlyHeader === 'true' && mdOnlySize > 0) {
log.debug('metadata only operation x-amz-meta-mdonly');
const md5 = request.headers['x-amz-meta-md5chksum']
? new Buffer(request.headers['x-amz-meta-md5chksum'],
'base64').toString('hex') : null;
const numParts = request.headers['x-amz-meta-md5numparts'];
let _md5;
if (numParts === undefined) {
_md5 = md5;
} else {
_md5 = `${md5}-${numParts}`;
}
const versionId = request.headers['x-amz-meta-version-id'];
const dataGetInfo = {
key: objectKey,
dataStoreName: location,
dataStoreType: locationType,
dataStoreVersionId: versionId,
dataStoreMD5: _md5,
};
return next(null, dataGetInfo, _md5);
}
}
return dataStore(objectKeyContext, cipherBundle, request, size, return dataStore(objectKeyContext, cipherBundle, request, size,
streamingV4Params, backendInfo, log, next); streamingV4Params, backendInfo, log, next);
}, },
@ -256,25 +222,16 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
dataGetInfoArr[0].cipheredDataKey = dataGetInfoArr[0].cipheredDataKey =
cipherBundle.cipheredDataKey; cipherBundle.cipheredDataKey;
} }
if (mdOnlyHeader === 'true') {
metadataStoreParams.size = mdOnlySize;
dataGetInfoArr[0].size = mdOnlySize;
}
metadataStoreParams.contentMD5 = calculatedHash; metadataStoreParams.contentMD5 = calculatedHash;
return next(null, dataGetInfoArr); return next(null, dataGetInfoArr);
}, },
function getVersioningInfo(infoArr, next) { function getVersioningInfo(infoArr, next) {
// if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
if (isPutVersion) {
const options = overwritingVersioning(objMD, metadataStoreParams);
return process.nextTick(() => next(null, options, infoArr));
}
return versioningPreprocessing(bucketName, bucketMD, return versioningPreprocessing(bucketName, bucketMD,
metadataStoreParams.objectKey, objMD, log, (err, options) => { metadataStoreParams.objectKey, objMD, log, (err, options) => {
if (err) { if (err) {
// TODO: check AWS error when user requested a specific // TODO: check AWS error when user requested a specific
// version before any versions have been put // version before any versions have been put
const logLvl = err.is.BadRequest ? const logLvl = err === errors.BadRequest ?
'debug' : 'error'; 'debug' : 'error';
log[logLvl]('error getting versioning info', { log[logLvl]('error getting versioning info', {
error: err, error: err,
@ -288,13 +245,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
metadataStoreParams.versionId = options.versionId; metadataStoreParams.versionId = options.versionId;
metadataStoreParams.versioning = options.versioning; metadataStoreParams.versioning = options.versioning;
metadataStoreParams.isNull = options.isNull; metadataStoreParams.isNull = options.isNull;
metadataStoreParams.deleteNullKey = options.deleteNullKey; metadataStoreParams.nullVersionId = options.nullVersionId;
if (options.extraMD) {
Object.assign(metadataStoreParams, options.extraMD);
}
return _storeInMDandDeleteData(bucketName, infoArr, return _storeInMDandDeleteData(bucketName, infoArr,
cipherBundle, metadataStoreParams, cipherBundle, metadataStoreParams,
options.dataToDelete, log, requestMethod, next); options.dataToDelete, requestLogger, requestMethod, next);
}, },
], callback); ], callback);
} }

View File

@ -1,18 +1,16 @@
/** const data = require('../../../data/wrapper');
* _bucketRequiresOplogUpdate - DELETE an object from a bucket
* @param {BucketInfo} bucket - bucket object function dataDelete(objectGetInfo, log, cb) {
* @return {boolean} whether objects require oplog updates on deletion, or not data.delete(objectGetInfo, log, err => {
*/ if (err) {
function _bucketRequiresOplogUpdate(bucket) { log.error('error deleting object data', {
// Default behavior is to require an oplog update error: err,
if (!bucket || !bucket.getLifecycleConfiguration || !bucket.getNotificationConfiguration) { method: 'dataDelete',
return true; });
return cb(err);
} }
// If the bucket has lifecycle configuration or notification configuration return cb();
// set, we also require an oplog update });
return bucket.getLifecycleConfiguration() || bucket.getNotificationConfiguration();
} }
module.exports = { module.exports = { dataDelete };
_bucketRequiresOplogUpdate,
};

View File

@ -1,140 +0,0 @@
const { supportedLifecycleRules } = require('arsenal').constants;
const { LifecycleConfiguration } = require('arsenal').models;
const {
LifecycleDateTime,
LifecycleUtils,
} = require('arsenal').s3middleware.lifecycleHelpers;
const { config } = require('../../../Config');
const {
expireOneDayEarlier,
transitionOneDayEarlier,
timeProgressionFactor,
scaledMsPerDay,
} = config.getTimeOptions();
const lifecycleDateTime = new LifecycleDateTime({
transitionOneDayEarlier,
expireOneDayEarlier,
timeProgressionFactor,
});
const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor);
function calculateDate(objDate, expDays, datetime) {
return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay));
}
function formatExpirationHeader(date, id) {
return `expiry-date="${date}", rule-id="${encodeURIComponent(id)}"`;
}
// format: x-amz-expiration: expiry-date="Fri, 21 Dec 2012 00:00:00 GMT", rule-id="id"
const AMZ_EXP_HEADER = 'x-amz-expiration';
// format: x-amz-abort-date: "Fri, 21 Dec 2012 00:00:00 GMT"
const AMZ_ABORT_DATE_HEADER = 'x-amz-abort-date';
// format: x-amz-abort-rule-id: "rule id"
const AMZ_ABORT_ID_HEADER = 'x-amz-abort-rule-id';
function _generateExpHeadersObjects(rules, params, datetime) {
const tags = {
TagSet: params.tags
? Object.keys(params.tags)
.map(key => ({ Key: key, Value: params.tags[key] }))
: [],
};
const objectInfo = { Key: params.key };
const filteredRules = lifecycleUtils.filterRules(rules, objectInfo, tags);
const applicable = lifecycleUtils.getApplicableRules(filteredRules, objectInfo, datetime);
if (applicable.Expiration) {
const rule = applicable.Expiration;
if (rule.Days === undefined && rule.Date === undefined) {
return {};
}
if (rule.Date) {
return {
[AMZ_EXP_HEADER]: formatExpirationHeader(rule.Date, rule.ID),
};
}
const date = calculateDate(params.date, rule.Days, datetime);
return {
[AMZ_EXP_HEADER]: formatExpirationHeader(date.toUTCString(), rule.ID),
};
}
return {};
}
function _generateExpHeadresMPU(rules, params, datetime) {
const noTags = { TagSet: [] };
const objectInfo = { Key: params.key };
const filteredRules = lifecycleUtils.filterRules(rules, objectInfo, noTags);
const applicable = lifecycleUtils.getApplicableRules(filteredRules, {}, datetime);
if (applicable.AbortIncompleteMultipartUpload) {
const rule = applicable.AbortIncompleteMultipartUpload;
const date = calculateDate(
params.date,
rule.DaysAfterInitiation,
datetime
);
return {
[AMZ_ABORT_ID_HEADER]: encodeURIComponent(rule.ID),
[AMZ_ABORT_DATE_HEADER]: date.toUTCString(),
};
}
return {};
}
/**
* generate response expiration headers
* @param {object} params - params
* @param {LifecycleDateTime} datetime - lifecycle datetime object
* @returns {object} - expiration response headers
*/
function generateExpirationHeaders(params, datetime) {
const { lifecycleConfig, objectParams, mpuParams, isVersionedReq } = params;
if (!lifecycleConfig || isVersionedReq) {
return {};
}
const lcfg = LifecycleConfiguration.getConfigJson(lifecycleConfig);
if (objectParams) {
return _generateExpHeadersObjects(lcfg.Rules, objectParams, datetime);
}
if (mpuParams) {
return _generateExpHeadresMPU(lcfg.Rules, mpuParams, datetime);
}
return {};
}
/**
* set response expiration headers to target header object
* @param {object} headers - target header object
* @param {object} params - params
* @returns {undefined}
*/
function setExpirationHeaders(headers, params) {
const expHeaders = generateExpirationHeaders(params, lifecycleDateTime);
Object.assign(headers, expHeaders);
}
module.exports = {
lifecycleDateTime,
generateExpirationHeaders,
setExpirationHeaders,
};

View File

@ -1,7 +1,6 @@
const s3config = require('../../../Config').config; const s3config = require('../../../Config').config;
const { isServiceAccount, getServiceAccountProperties } = const constants = require('../../../../constants');
require('../authorization/permissionChecks'); const { isBackbeatUser } = require('../authorization/aclChecks');
const { replicationBackends } = require('arsenal').constants;
function _getBackend(objectMD, site) { function _getBackend(objectMD, site) {
const backends = objectMD ? objectMD.replicationInfo.backends : []; const backends = objectMD ? objectMD.replicationInfo.backends : [];
@ -23,12 +22,12 @@ function _getStorageClasses(rule) {
} }
const { replicationEndpoints } = s3config; const { replicationEndpoints } = s3config;
// If no storage class, use the given default endpoint or the sole endpoint // If no storage class, use the given default endpoint or the sole endpoint
if (replicationEndpoints.length > 0) { if (replicationEndpoints.length > 1) {
const endPoint = const endPoint =
replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0]; replicationEndpoints.find(endpoint => endpoint.default);
return [endPoint.site]; return [endPoint.site];
} }
return undefined; return [replicationEndpoints[0].site];
} }
function _getReplicationInfo(rule, replicationConfig, content, operationType, function _getReplicationInfo(rule, replicationConfig, content, operationType,
@ -36,15 +35,12 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
const storageTypes = []; const storageTypes = [];
const backends = []; const backends = [];
const storageClasses = _getStorageClasses(rule); const storageClasses = _getStorageClasses(rule);
if (!storageClasses) {
return undefined;
}
storageClasses.forEach(storageClass => { storageClasses.forEach(storageClass => {
const storageClassName = const storageClassName =
storageClass.endsWith(':preferred_read') ? storageClass.endsWith(':preferred_read') ?
storageClass.split(':')[0] : storageClass; storageClass.split(':')[0] : storageClass;
const location = s3config.locationConstraints[storageClassName]; const location = s3config.locationConstraints[storageClassName];
if (location && replicationBackends[location.type]) { if (location && constants.replicationBackends[location.type]) {
storageTypes.push(location.type); storageTypes.push(location.type);
} }
backends.push(_getBackend(objectMD, storageClassName)); backends.push(_getBackend(objectMD, storageClassName));
@ -91,28 +87,16 @@ function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
// - replication configuration applies to the object (i.e. a rule matches // - replication configuration applies to the object (i.e. a rule matches
// object prefix) but the status is disabled // object prefix) but the status is disabled
// //
// - object owner is an internal service account like Lifecycle, // - object owner is an internal service account like Lifecycle
// unless the account properties explicitly allow it to // (because we do not want to replicate objects created from
// replicate like MD ingestion (because we do not want to // actions triggered by internal services, by design)
// replicate objects created from actions triggered by internal
// services, by design)
if (config) { if (config && (!authInfo || !isBackbeatUser(authInfo.getCanonicalID()))) {
let doReplicate = false; const rule = config.rules.find(rule =>
if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) { (objKey.startsWith(rule.prefix) && rule.enabled));
doReplicate = true;
} else {
const serviceAccountProps = getServiceAccountProperties(
authInfo.getCanonicalID());
doReplicate = serviceAccountProps.canReplicate;
}
if (doReplicate) {
const rule = config.rules.find(
rule => (objKey.startsWith(rule.prefix) && rule.enabled));
if (rule) { if (rule) {
return _getReplicationInfo( return _getReplicationInfo(rule, config, content, operationType,
rule, config, content, operationType, objectMD, bucketMD); objectMD, bucketMD);
}
} }
} }
return undefined; return undefined;

Some files were not shown because too many files have changed in this diff Show More