Compare commits
6 Commits
developmen
...
feature/CL
Author | SHA1 | Date |
---|---|---|
Will Toozs | 6ff75dbe88 | |
Will Toozs | b740e13d15 | |
Will Toozs | 9d9c4ae66d | |
Will Toozs | 41b839c014 | |
Will Toozs | 6f82826984 | |
Will Toozs | efc79b293c |
|
@ -1,9 +1,3 @@
|
|||
node_modules
|
||||
localData/*
|
||||
localMetadata/*
|
||||
# Keep the .git/HEAD file in order to properly report version
|
||||
.git/objects
|
||||
.git
|
||||
.github
|
||||
.tox
|
||||
coverage
|
||||
.DS_Store
|
||||
node_modules
|
54
.eslintrc
54
.eslintrc
|
@ -1,54 +1,10 @@
|
|||
{
|
||||
"extends": "scality",
|
||||
"plugins": [
|
||||
"mocha"
|
||||
],
|
||||
"rules": {
|
||||
"import/extensions": "off",
|
||||
"lines-around-directive": "off",
|
||||
"no-underscore-dangle": "off",
|
||||
"indent": "off",
|
||||
"object-curly-newline": "off",
|
||||
"operator-linebreak": "off",
|
||||
"function-paren-newline": "off",
|
||||
"import/newline-after-import": "off",
|
||||
"prefer-destructuring": "off",
|
||||
"implicit-arrow-linebreak": "off",
|
||||
"no-bitwise": "off",
|
||||
"dot-location": "off",
|
||||
"comma-dangle": "off",
|
||||
"no-undef-init": "off",
|
||||
"global-require": "off",
|
||||
"import/no-dynamic-require": "off",
|
||||
"class-methods-use-this": "off",
|
||||
"no-plusplus": "off",
|
||||
"no-else-return": "off",
|
||||
"object-property-newline": "off",
|
||||
"import/order": "off",
|
||||
"no-continue": "off",
|
||||
"no-tabs": "off",
|
||||
"lines-between-class-members": "off",
|
||||
"prefer-spread": "off",
|
||||
"no-lonely-if": "off",
|
||||
"no-useless-escape": "off",
|
||||
"no-restricted-globals": "off",
|
||||
"no-buffer-constructor": "off",
|
||||
"import/no-extraneous-dependencies": "off",
|
||||
"space-unary-ops": "off",
|
||||
"no-useless-return": "off",
|
||||
"no-unexpected-multiline": "off",
|
||||
"no-mixed-operators": "off",
|
||||
"newline-per-chained-call": "off",
|
||||
"operator-assignment": "off",
|
||||
"spaced-comment": "off",
|
||||
"comma-style": "off",
|
||||
"no-restricted-properties": "off",
|
||||
"new-parens": "off",
|
||||
"no-multi-spaces": "off",
|
||||
"quote-props": "off",
|
||||
"mocha/no-exclusive-tests": "error",
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2020
|
||||
"ecmaVersion": 2018,
|
||||
"sourceType": "module",
|
||||
"ecmaFeatures": {
|
||||
"jsx": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,32 +1,19 @@
|
|||
# General support information
|
||||
# Issue template
|
||||
|
||||
GitHub Issues are **reserved** for actionable bug reports (including
|
||||
documentation inaccuracies), and feature requests.
|
||||
**All questions** (regarding configuration, use cases, performance, community,
|
||||
events, setup and usage recommendations, among other things) should be asked on
|
||||
the **[Zenko Forum](http://forum.zenko.io/)**.
|
||||
If you are reporting a new issue, make sure that we do not have any
|
||||
duplicates already open. You can ensure this by searching the issue list for
|
||||
this repository. If there is a duplicate, please close your issue and add a
|
||||
comment to the existing issue instead.
|
||||
|
||||
> Questions opened as GitHub issues will systematically be closed, and moved to
|
||||
> the [Zenko Forum](http://forum.zenko.io/).
|
||||
## General support information
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
## Avoiding duplicates
|
||||
|
||||
When reporting a new issue/requesting a feature, make sure that we do not have
|
||||
any duplicates already open:
|
||||
|
||||
- search the issue list for this repository (use the search bar, select
|
||||
"Issues" on the left pane after searching);
|
||||
- if there is a duplicate, please do not open your issue, and add a comment
|
||||
to the existing issue instead.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
GitHub Issues are reserved for actionable bug reports and feature requests.
|
||||
General questions should be sent to the
|
||||
[S3 scality server Forum](http://forum.scality.com/).
|
||||
|
||||
## Bug report information
|
||||
|
||||
(delete this section (everything between the lines) if you're not reporting a bug
|
||||
but requesting a feature)
|
||||
(delete this section if not applicable)
|
||||
|
||||
### Description
|
||||
|
||||
|
@ -42,22 +29,13 @@ Describe the results you received
|
|||
|
||||
### Expected result
|
||||
|
||||
Describe the results you expected
|
||||
Describe the results you expecteds
|
||||
|
||||
### Additional information
|
||||
|
||||
- Node.js version,
|
||||
- Docker version,
|
||||
- yarn version,
|
||||
- distribution/OS,
|
||||
- optional: anything else you deem helpful to us.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
### Additional information: (Node.js version, Docker version, etc)
|
||||
|
||||
## Feature Request
|
||||
|
||||
(delete this section (everything between the lines) if you're not requesting
|
||||
a feature but reporting a bug)
|
||||
(delete this section if not applicable)
|
||||
|
||||
### Proposal
|
||||
|
||||
|
@ -74,14 +52,3 @@ What you would like to happen
|
|||
### Use case
|
||||
|
||||
Please provide use cases for changing the current behavior
|
||||
|
||||
### Additional information
|
||||
|
||||
- Is this request for your company? Y/N
|
||||
- If Y: Company name:
|
||||
- Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N
|
||||
- Are you willing to contribute this feature yourself?
|
||||
- Position/Title:
|
||||
- How did you hear about us?
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
|
|
@ -15,7 +15,7 @@ runs:
|
|||
shell: bash
|
||||
run: |-
|
||||
set -exu;
|
||||
mkdir -p /tmp/artifacts/${JOB_NAME}/;
|
||||
mkdir -p /tmp/artifacts/${{ github.job }}/;
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '16'
|
||||
|
@ -23,21 +23,3 @@ runs:
|
|||
- name: install dependencies
|
||||
shell: bash
|
||||
run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup python2 test environment
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install -y libdigest-hmac-perl
|
||||
pip install 's3cmd==2.3.0'
|
||||
- name: fix sproxyd.conf permissions
|
||||
shell: bash
|
||||
run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
|
||||
- name: ensure fuse kernel module is loaded (for sproxyd)
|
||||
shell: bash
|
||||
run: sudo modprobe fuse
|
||||
|
|
|
@ -1,25 +0,0 @@
|
|||
FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
|
||||
|
||||
ENV CEPH_DAEMON demo
|
||||
ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
|
||||
|
||||
ENV CEPH_DEMO_UID zenko
|
||||
ENV CEPH_DEMO_ACCESS_KEY accessKey1
|
||||
ENV CEPH_DEMO_SECRET_KEY verySecretKey1
|
||||
ENV CEPH_DEMO_BUCKET zenkobucket
|
||||
|
||||
ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
|
||||
ENV MON_IP 0.0.0.0
|
||||
ENV NETWORK_AUTO_DETECT 4
|
||||
ENV RGW_CIVETWEB_PORT 8001
|
||||
|
||||
RUN rm /etc/yum.repos.d/tcmu-runner.repo
|
||||
|
||||
ADD ./entrypoint-wrapper.sh /
|
||||
RUN chmod +x /entrypoint-wrapper.sh && \
|
||||
yum install -y python-pip && \
|
||||
yum clean all && \
|
||||
pip install awscli && \
|
||||
rm -rf /root/.cache/pip
|
||||
|
||||
ENTRYPOINT [ "/entrypoint-wrapper.sh" ]
|
|
@ -1,37 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
touch /artifacts/ceph.log
|
||||
mkfifo /tmp/entrypoint_output
|
||||
# We run this in the background so that we can tail the RGW log after init,
|
||||
# because entrypoint.sh never returns
|
||||
|
||||
# The next line will be needed when ceph builds 3.2.2 so I'll leave it here
|
||||
# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
|
||||
|
||||
bash /entrypoint.sh > /tmp/entrypoint_output &
|
||||
entrypoint_pid="$!"
|
||||
while read -r line; do
|
||||
echo $line
|
||||
# When we find this line server has started
|
||||
if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
|
||||
break
|
||||
fi
|
||||
done < /tmp/entrypoint_output
|
||||
|
||||
# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
|
||||
s3cmd mb s3://cephbucket s3://cephbucket2
|
||||
|
||||
mkdir /root/.aws
|
||||
cat > /root/.aws/credentials <<EOF
|
||||
[default]
|
||||
aws_access_key_id = accessKey1
|
||||
aws_secret_access_key = verySecretKey1
|
||||
EOF
|
||||
|
||||
# Enable versioning on them
|
||||
for bucket in cephbucket cephbucket2; do
|
||||
echo "Enabling versiong for $bucket"
|
||||
aws --endpoint http://127.0.0.1:8001 s3api put-bucket-versioning --bucket $bucket --versioning Status=Enabled
|
||||
done
|
||||
tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
|
||||
wait $entrypoint_pid
|
|
@ -1,11 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# This script is needed because RADOS Gateway
|
||||
# will open the port before beginning to serve traffic
|
||||
# causing wait_for_local_port.bash to exit immediately
|
||||
|
||||
echo 'Waiting for ceph'
|
||||
while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
|
||||
sleep 1
|
||||
echo -n "."
|
||||
done
|
|
@ -34,3 +34,4 @@ gcpbackendmismatch_GCP_SERVICE_KEY
|
|||
gcpbackend_GCP_SERVICE_KEYFILE
|
||||
gcpbackendmismatch_GCP_SERVICE_KEYFILE
|
||||
gcpbackendnoproxy_GCP_SERVICE_KEYFILE
|
||||
gcpbackendproxy_GCP_SERVICE_KEYFILE
|
||||
|
|
|
@ -20,7 +20,6 @@ services:
|
|||
- METADATA_HOST=0.0.0.0
|
||||
- S3BACKEND
|
||||
- S3DATA
|
||||
- S3METADATA
|
||||
- MPU_TESTING
|
||||
- S3VAULT
|
||||
- S3_LOCATION_FILE
|
||||
|
@ -35,16 +34,7 @@ services:
|
|||
- S3KMIP_KEY
|
||||
- S3KMIP_CERT
|
||||
- S3KMIP_CA
|
||||
- MONGODB_HOSTS=0.0.0.0:27018
|
||||
- MONGODB_RS=rs0
|
||||
- DEFAULT_BUCKET_KEY_FORMAT
|
||||
- METADATA_MAX_CACHED_BUCKETS
|
||||
- ENABLE_NULL_VERSION_COMPAT_MODE
|
||||
- SCUBA_HOST
|
||||
- SCUBA_PORT
|
||||
- SCUBA_HEALTHCHECK_FREQUENCY
|
||||
- S3QUOTA
|
||||
- QUOTA_ENABLE_INFLIGHTS
|
||||
env_file:
|
||||
- creds.env
|
||||
depends_on:
|
||||
|
@ -75,18 +65,3 @@ services:
|
|||
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
|
||||
volumes:
|
||||
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
||||
mongo:
|
||||
network_mode: "host"
|
||||
profiles: ['mongo', 'ceph']
|
||||
image: ${MONGODB_IMAGE}
|
||||
ceph:
|
||||
network_mode: "host"
|
||||
profiles: ['ceph']
|
||||
image: ghcr.io/scality/cloudserver/ci-ceph
|
||||
sproxyd:
|
||||
network_mode: "host"
|
||||
profiles: ['sproxyd']
|
||||
image: sproxyd-standalone
|
||||
build: ./sproxyd
|
||||
user: 0:0
|
||||
privileged: yes
|
||||
|
|
|
@ -1,28 +0,0 @@
|
|||
FROM mongo:5.0.21
|
||||
|
||||
ENV USER=scality \
|
||||
HOME_DIR=/home/scality \
|
||||
CONF_DIR=/conf \
|
||||
DATA_DIR=/data
|
||||
|
||||
# Set up directories and permissions
|
||||
RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
|
||||
mkdir /logs; \
|
||||
adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
|
||||
|
||||
# Set up environment variables and directories for scality user
|
||||
RUN mkdir ${CONF_DIR} && \
|
||||
chown -R ${USER} ${CONF_DIR} && \
|
||||
chown -R ${USER} ${DATA_DIR}
|
||||
|
||||
# copy the mongo config file
|
||||
COPY /conf/mongod.conf /conf/mongod.conf
|
||||
COPY /conf/mongo-run.sh /conf/mongo-run.sh
|
||||
COPY /conf/initReplicaSet /conf/initReplicaSet.js
|
||||
|
||||
EXPOSE 27017/tcp
|
||||
EXPOSE 27018
|
||||
|
||||
# Set up CMD
|
||||
ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
|
||||
CMD ["bash", "/conf/mongo-run.sh"]
|
|
@ -1,4 +0,0 @@
|
|||
rs.initiate({
|
||||
_id: "rs0",
|
||||
members: [{ _id: 0, host: "127.0.0.1:27018" }]
|
||||
});
|
|
@ -1,10 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -exo pipefail
|
||||
|
||||
init_RS() {
|
||||
sleep 5
|
||||
mongo --port 27018 /conf/initReplicaSet.js
|
||||
}
|
||||
init_RS &
|
||||
|
||||
mongod --bind_ip_all --config=/conf/mongod.conf
|
|
@ -1,15 +0,0 @@
|
|||
storage:
|
||||
journal:
|
||||
enabled: true
|
||||
engine: wiredTiger
|
||||
dbPath: "/data/db"
|
||||
processManagement:
|
||||
fork: false
|
||||
net:
|
||||
port: 27018
|
||||
bindIp: 0.0.0.0
|
||||
replication:
|
||||
replSetName: "rs0"
|
||||
enableMajorityReadConcern: true
|
||||
security:
|
||||
authorization: disabled
|
|
@ -1,3 +0,0 @@
|
|||
FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
|
||||
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
|
||||
RUN chown root:root /conf/sproxyd0.conf
|
|
@ -1,26 +0,0 @@
|
|||
fastcgi_param QUERY_STRING $query_string;
|
||||
fastcgi_param REQUEST_METHOD $request_method;
|
||||
fastcgi_param CONTENT_TYPE $content_type;
|
||||
fastcgi_param CONTENT_LENGTH $content_length;
|
||||
|
||||
#fastcgi_param SCRIPT_NAME $fastcgi_script_name;
|
||||
fastcgi_param SCRIPT_NAME /var/www;
|
||||
fastcgi_param PATH_INFO $document_uri;
|
||||
|
||||
fastcgi_param REQUEST_URI $request_uri;
|
||||
fastcgi_param DOCUMENT_URI $document_uri;
|
||||
fastcgi_param DOCUMENT_ROOT $document_root;
|
||||
fastcgi_param SERVER_PROTOCOL $server_protocol;
|
||||
fastcgi_param HTTPS $https if_not_empty;
|
||||
|
||||
fastcgi_param GATEWAY_INTERFACE CGI/1.1;
|
||||
fastcgi_param SERVER_SOFTWARE nginx/$nginx_version;
|
||||
|
||||
fastcgi_param REMOTE_ADDR $remote_addr;
|
||||
fastcgi_param REMOTE_PORT $remote_port;
|
||||
fastcgi_param SERVER_ADDR $server_addr;
|
||||
fastcgi_param SERVER_PORT $server_port;
|
||||
fastcgi_param SERVER_NAME $server_name;
|
||||
|
||||
# PHP only, required if PHP was built with --enable-force-cgi-redirect
|
||||
fastcgi_param REDIRECT_STATUS 200;
|
|
@ -1,88 +0,0 @@
|
|||
worker_processes 1;
|
||||
error_log /logs/error.log;
|
||||
user root root;
|
||||
events {
|
||||
worker_connections 1000;
|
||||
reuse_port on;
|
||||
multi_accept on;
|
||||
}
|
||||
worker_rlimit_nofile 20000;
|
||||
http {
|
||||
root /var/www/;
|
||||
upstream sproxyds {
|
||||
least_conn;
|
||||
keepalive 40;
|
||||
server 127.0.0.1:20000;
|
||||
}
|
||||
server {
|
||||
client_max_body_size 0;
|
||||
client_body_timeout 150;
|
||||
client_header_timeout 150;
|
||||
postpone_output 0;
|
||||
client_body_postpone_size 0;
|
||||
keepalive_requests 1100;
|
||||
keepalive_timeout 300s;
|
||||
server_tokens off;
|
||||
default_type application/octet-stream;
|
||||
gzip off;
|
||||
tcp_nodelay on;
|
||||
tcp_nopush on;
|
||||
sendfile on;
|
||||
listen 81;
|
||||
server_name localhost;
|
||||
rewrite ^/arc/(.*)$ /dc1/$1 permanent;
|
||||
location ~* ^/proxy/(.*)$ {
|
||||
rewrite ^/proxy/(.*)$ /$1 last;
|
||||
}
|
||||
allow 127.0.0.1;
|
||||
|
||||
deny all;
|
||||
set $usermd '-';
|
||||
set $sentusermd '-';
|
||||
set $elapsed_ms '-';
|
||||
set $now '-';
|
||||
log_by_lua '
|
||||
if not(ngx.var.http_x_scal_usermd == nil) and string.len(ngx.var.http_x_scal_usermd) > 2 then
|
||||
ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
|
||||
end
|
||||
if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
|
||||
ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
|
||||
end
|
||||
local elapsed_ms = tonumber(ngx.var.request_time)
|
||||
if not ( elapsed_ms == nil) then
|
||||
elapsed_ms = elapsed_ms * 1000
|
||||
ngx.var.elapsed_ms = tostring(elapsed_ms)
|
||||
end
|
||||
local time = tonumber(ngx.var.msec) * 1000
|
||||
ngx.var.now = time
|
||||
';
|
||||
log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
|
||||
'"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
|
||||
'"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
|
||||
'"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
|
||||
'"contentType":"$content_type","s3Address":"$remote_addr",'
|
||||
'"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
|
||||
'"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
|
||||
'"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
|
||||
'"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
|
||||
'"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
|
||||
'"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
|
||||
access_log /dev/stdout irm;
|
||||
error_log /dev/stdout error;
|
||||
location / {
|
||||
proxy_request_buffering off;
|
||||
fastcgi_request_buffering off;
|
||||
fastcgi_no_cache 1;
|
||||
fastcgi_cache_bypass 1;
|
||||
fastcgi_buffering off;
|
||||
fastcgi_ignore_client_abort on;
|
||||
fastcgi_keep_conn on;
|
||||
include fastcgi_params;
|
||||
fastcgi_pass sproxyds;
|
||||
fastcgi_next_upstream error timeout;
|
||||
fastcgi_send_timeout 285s;
|
||||
fastcgi_read_timeout 285s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
{
|
||||
"general": {
|
||||
"ring": "DATA",
|
||||
"port": 20000,
|
||||
"syslog_facility": "local0"
|
||||
},
|
||||
"ring_driver:0": {
|
||||
"alias": "dc1",
|
||||
"type": "local",
|
||||
"queue_path": "/tmp/ring-objs"
|
||||
},
|
||||
}
|
|
@ -1,43 +0,0 @@
|
|||
[supervisord]
|
||||
nodaemon = true
|
||||
loglevel = info
|
||||
logfile = %(ENV_LOG_DIR)s/supervisord.log
|
||||
pidfile = %(ENV_SUP_RUN_DIR)s/supervisord.pid
|
||||
logfile_maxbytes = 20MB
|
||||
logfile_backups = 2
|
||||
|
||||
[unix_http_server]
|
||||
file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
[supervisorctl]
|
||||
serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
|
||||
|
||||
[program:nginx]
|
||||
directory=%(ENV_SUP_RUN_DIR)s
|
||||
command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
|
||||
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
|
||||
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
|
||||
stdout_logfile_maxbytes=100MB
|
||||
stdout_logfile_backups=7
|
||||
stderr_logfile_maxbytes=100MB
|
||||
stderr_logfile_backups=7
|
||||
autorestart=true
|
||||
autostart=true
|
||||
user=root
|
||||
|
||||
[program:sproxyd]
|
||||
directory=%(ENV_SUP_RUN_DIR)s
|
||||
process_name=%(program_name)s-%(process_num)s
|
||||
numprocs=1
|
||||
numprocs_start=0
|
||||
command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
|
||||
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
|
||||
stdout_logfile_maxbytes=100MB
|
||||
stdout_logfile_backups=7
|
||||
redirect_stderr=true
|
||||
autorestart=true
|
||||
autostart=true
|
||||
user=root
|
|
@ -1,35 +0,0 @@
|
|||
name: Test alerts
|
||||
|
||||
on:
|
||||
push:
|
||||
branches-ignore:
|
||||
- 'development/**'
|
||||
- 'q/*/**'
|
||||
|
||||
jobs:
|
||||
run-alert-tests:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
tests:
|
||||
- name: 1 minute interval tests
|
||||
file: monitoring/alerts.test.yaml
|
||||
|
||||
- name: 10 seconds interval tests
|
||||
file: monitoring/alerts.10s.test.yaml
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Render and test ${{ matrix.tests.name }}
|
||||
uses: scality/action-prom-render-test@1.0.3
|
||||
with:
|
||||
alert_file_path: monitoring/alerts.yaml
|
||||
test_file_path: ${{ matrix.tests.file }}
|
||||
alert_inputs: |
|
||||
namespace=zenko
|
||||
service=artesca-data-connector-s3api-metrics
|
||||
reportJob=artesca-data-ops-report-handler
|
||||
replicas=3
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
@ -9,9 +9,6 @@ on:
|
|||
description: 'Tag to be released'
|
||||
required: true
|
||||
|
||||
env:
|
||||
PROJECT_NAME: ${{ github.event.repository.name }}
|
||||
|
||||
jobs:
|
||||
build-federation-image:
|
||||
runs-on: ubuntu-20.04
|
||||
|
@ -37,38 +34,32 @@ jobs:
|
|||
cache-from: type=gha,scope=federation
|
||||
cache-to: type=gha,mode=max,scope=federation
|
||||
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
build-image:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildk
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Registry
|
||||
- name: Login to GitHub Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Push dashboards into the production namespace
|
||||
run: |
|
||||
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
|
||||
dashboard.json:application/grafana-dashboard+json \
|
||||
alerts.yaml:application/prometheus-alerts+yaml
|
||||
working-directory: monitoring
|
||||
|
||||
- name: Build and push
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
|
||||
context: .
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
github-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
env:
|
||||
|
|
|
@ -2,8 +2,6 @@
|
|||
name: tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
push:
|
||||
branches-ignore:
|
||||
- 'development/**'
|
||||
|
@ -128,9 +126,6 @@ jobs:
|
|||
|
||||
build:
|
||||
runs-on: ubuntu-20.04
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
@ -150,9 +145,6 @@ jobs:
|
|||
provenance: false
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
labels: |
|
||||
git.repository=${{ github.repository }}
|
||||
git.commit-sha=${{ github.sha }}
|
||||
cache-from: type=gha,scope=cloudserver
|
||||
cache-to: type=gha,mode=max,scope=cloudserver
|
||||
- name: Build and push pykmip image
|
||||
|
@ -162,26 +154,38 @@ jobs:
|
|||
context: .github/pykmip
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
||||
labels: |
|
||||
git.repository=${{ github.repository }}
|
||||
git.commit-sha=${{ github.sha }}
|
||||
cache-from: type=gha,scope=pykmip
|
||||
cache-to: type=gha,mode=max,scope=pykmip
|
||||
- name: Build and push MongoDB
|
||||
|
||||
build-federation-image:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to GitHub Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ github.token }}
|
||||
- name: Build and push image for federation
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
context: .github/docker/mongodb
|
||||
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
cache-from: type=gha,scope=mongodb
|
||||
cache-to: type=gha,mode=max,scope=mongodb
|
||||
context: .
|
||||
file: images/svc-base/Dockerfile
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}:${{ github.sha }}-svc-base
|
||||
cache-from: type=gha,scope=federation
|
||||
cache-to: type=gha,mode=max,scope=federation
|
||||
|
||||
multiple-backend:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
env:
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
S3BACKEND: mem
|
||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||
S3DATA: multiple
|
||||
|
@ -189,22 +193,18 @@ jobs:
|
|||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Login to Registry
|
||||
uses: docker/login-action@v3
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ github.token }}
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Setup CI services
|
||||
run: docker compose --profile sproxyd up -d
|
||||
run: docker compose up -d
|
||||
working-directory: .github/docker
|
||||
- name: Run multiple backend test
|
||||
run: |-
|
||||
set -o pipefail;
|
||||
bash wait_for_local_port.bash 8000 40
|
||||
bash wait_for_local_port.bash 81 40
|
||||
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||
env:
|
||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||
|
@ -218,89 +218,14 @@ jobs:
|
|||
source: /tmp/artifacts
|
||||
if: always()
|
||||
|
||||
mongo-v0-ft-tests:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
env:
|
||||
S3BACKEND: mem
|
||||
MPU_TESTING: "yes"
|
||||
S3METADATA: mongodb
|
||||
S3KMS: file
|
||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||
DEFAULT_BUCKET_KEY_FORMAT: v0
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Setup CI services
|
||||
run: docker compose --profile mongo up -d
|
||||
working-directory: .github/docker
|
||||
- name: Run functional tests
|
||||
run: |-
|
||||
set -o pipefail;
|
||||
bash wait_for_local_port.bash 8000 40
|
||||
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||
env:
|
||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
user: ${{ secrets.ARTIFACTS_USER }}
|
||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||
source: /tmp/artifacts
|
||||
if: always()
|
||||
|
||||
mongo-v1-ft-tests:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
env:
|
||||
S3BACKEND: mem
|
||||
MPU_TESTING: "yes"
|
||||
S3METADATA: mongodb
|
||||
S3KMS: file
|
||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||
DEFAULT_BUCKET_KEY_FORMAT: v1
|
||||
METADATA_MAX_CACHED_BUCKETS: 1
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Setup CI services
|
||||
run: docker compose --profile mongo up -d
|
||||
working-directory: .github/docker
|
||||
- name: Run functional tests
|
||||
run: |-
|
||||
set -o pipefail;
|
||||
bash wait_for_local_port.bash 8000 40
|
||||
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||
yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
|
||||
env:
|
||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
user: ${{ secrets.ARTIFACTS_USER }}
|
||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||
source: /tmp/artifacts
|
||||
if: always()
|
||||
|
||||
file-ft-tests:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- job-name: file-ft-tests
|
||||
- enable-null-compat: ''
|
||||
job-name: file-ft-tests
|
||||
- enable-null-compat: 'true'
|
||||
job-name: file-ft-tests-null-compat
|
||||
name: ${{ matrix.job-name }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
|
@ -308,12 +233,15 @@ jobs:
|
|||
S3BACKEND: file
|
||||
S3VAULT: mem
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
MPU_TESTING: "yes"
|
||||
ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
|
||||
JOB_NAME: ${{ matrix.job-name }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Setup matrix job artifacts directory
|
||||
|
@ -321,6 +249,10 @@ jobs:
|
|||
run: |
|
||||
set -exu
|
||||
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
|
||||
- name: Setup python test environment
|
||||
run: |
|
||||
sudo apt-get install -y libdigest-hmac-perl
|
||||
pip install 's3cmd==2.3.0'
|
||||
- name: Setup CI services
|
||||
run: docker compose up -d
|
||||
working-directory: .github/docker
|
||||
|
@ -347,11 +279,13 @@ jobs:
|
|||
S3BACKEND: mem
|
||||
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Setup CI services
|
||||
|
@ -372,64 +306,22 @@ jobs:
|
|||
source: /tmp/artifacts
|
||||
if: always()
|
||||
|
||||
quota-tests:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
strategy:
|
||||
matrix:
|
||||
inflights:
|
||||
- name: "With Inflights"
|
||||
value: "true"
|
||||
- name: "Without Inflights"
|
||||
value: "false"
|
||||
env:
|
||||
S3METADATA: mongodb
|
||||
S3BACKEND: mem
|
||||
S3QUOTA: scuba
|
||||
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
|
||||
SCUBA_HOST: localhost
|
||||
SCUBA_PORT: 8100
|
||||
SCUBA_HEALTHCHECK_FREQUENCY: 100
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Setup CI services
|
||||
run: docker compose --profile mongo up -d
|
||||
working-directory: .github/docker
|
||||
- name: Run quota tests
|
||||
run: |-
|
||||
set -ex -o pipefail;
|
||||
bash wait_for_local_port.bash 8000 40
|
||||
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
user: ${{ secrets.ARTIFACTS_USER }}
|
||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||
source: /tmp/artifacts
|
||||
if: always()
|
||||
|
||||
kmip-ft-tests:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
env:
|
||||
S3BACKEND: file
|
||||
S3VAULT: mem
|
||||
MPU_TESTING: "yes"
|
||||
MPU_TESTING: true
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Copy KMIP certs
|
||||
|
@ -453,81 +345,3 @@ jobs:
|
|||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||
source: /tmp/artifacts
|
||||
if: always()
|
||||
|
||||
ceph-backend-test:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
env:
|
||||
S3BACKEND: mem
|
||||
S3DATA: multiple
|
||||
S3KMS: file
|
||||
CI_CEPH: 'true'
|
||||
MPU_TESTING: "yes"
|
||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
|
||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Login to GitHub Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ github.token }}
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: '2.5.9'
|
||||
- name: Install Ruby dependencies
|
||||
run: |
|
||||
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
|
||||
- name: Install Java dependencies
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
|
||||
- name: Setup CI services
|
||||
run: docker compose --profile ceph up -d
|
||||
working-directory: .github/docker
|
||||
env:
|
||||
S3METADATA: mongodb
|
||||
- name: Run Ceph multiple backend tests
|
||||
run: |-
|
||||
set -ex -o pipefail;
|
||||
bash .github/ceph/wait_for_ceph.sh
|
||||
bash wait_for_local_port.bash 27018 40
|
||||
bash wait_for_local_port.bash 8000 40
|
||||
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
|
||||
env:
|
||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||
S3METADATA: mem
|
||||
- name: Run Java tests
|
||||
run: |-
|
||||
set -ex -o pipefail;
|
||||
mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
|
||||
working-directory: tests/functional/jaws
|
||||
- name: Run Ruby tests
|
||||
run: |-
|
||||
set -ex -o pipefail;
|
||||
rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
|
||||
working-directory: tests/functional/fog
|
||||
- name: Run Javascript AWS SDK tests
|
||||
run: |-
|
||||
set -ex -o pipefail;
|
||||
yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
|
||||
yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
|
||||
env:
|
||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
|
||||
S3BACKEND: file
|
||||
S3VAULT: mem
|
||||
S3METADATA: mongodb
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
user: ${{ secrets.ARTIFACTS_USER }}
|
||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||
source: /tmp/artifacts
|
||||
if: always()
|
||||
|
|
|
@ -22,14 +22,9 @@ coverage
|
|||
# Compiled binary addons (http://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Sphinx build dir
|
||||
_build
|
||||
|
||||
# Dependency directory
|
||||
# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
|
||||
node_modules
|
||||
yarn.lock
|
||||
.tox
|
||||
|
||||
# Junit directory
|
||||
junit
|
||||
|
|
71
Dockerfile
71
Dockerfile
|
@ -1,19 +1,22 @@
|
|||
ARG NODE_VERSION=16.20-bullseye-slim
|
||||
|
||||
FROM node:${NODE_VERSION} as builder
|
||||
FROM node:16.13.2-slim
|
||||
MAINTAINER Giorgio Regni <gr@scality.com>
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Keep the .git directory in order to properly report version
|
||||
COPY ./package.json yarn.lock ./
|
||||
|
||||
ENV PYTHON=python3.9
|
||||
ENV PY_VERSION=3.9.7
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
gnupg2 \
|
||||
jq \
|
||||
python3 \
|
||||
python \
|
||||
git \
|
||||
build-essential \
|
||||
ssh \
|
||||
ca-certificates \
|
||||
wget \
|
||||
libffi-dev \
|
||||
zlib1g-dev \
|
||||
|
@ -21,40 +24,28 @@ RUN apt-get update \
|
|||
&& mkdir -p /root/ssh \
|
||||
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
|
||||
|
||||
ENV PYTHON=python3
|
||||
COPY package.json yarn.lock /usr/src/app/
|
||||
RUN npm install typescript -g
|
||||
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
|
||||
RUN cd /tmp \
|
||||
&& wget https://www.python.org/ftp/python/$PY_VERSION/Python-$PY_VERSION.tgz \
|
||||
&& tar -C /usr/local/bin -xzvf Python-$PY_VERSION.tgz \
|
||||
&& cd /usr/local/bin/Python-$PY_VERSION \
|
||||
&& ./configure --enable-optimizations \
|
||||
&& make \
|
||||
&& make altinstall \
|
||||
&& rm -rf /tmp/Python-$PY_VERSION.tgz
|
||||
|
||||
################################################################################
|
||||
FROM node:${NODE_VERSION}
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
jq \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV NO_PROXY localhost,127.0.0.1
|
||||
ENV no_proxy localhost,127.0.0.1
|
||||
|
||||
EXPOSE 8000
|
||||
EXPOSE 8002
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
jq \
|
||||
tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Keep the .git directory in order to properly report version
|
||||
COPY . /usr/src/app
|
||||
COPY --from=builder /usr/src/app/node_modules ./node_modules/
|
||||
RUN yarn cache clean \
|
||||
&& yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
|
||||
&& apt-get autoremove --purge -y python git build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& yarn cache clean \
|
||||
&& rm -rf ~/.node-gyp \
|
||||
&& rm -rf /tmp/yarn-*
|
||||
|
||||
COPY ./ ./
|
||||
|
||||
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
|
||||
|
||||
ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
|
||||
|
||||
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
|
||||
CMD [ "yarn", "start" ]
|
||||
|
||||
EXPOSE 8000 8002
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
# S3 Healthcheck
|
||||
|
||||
Scality S3 exposes a healthcheck route `/live` on the port used
|
||||
for the metrics (defaults to port 8002) which returns a
|
||||
Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
|
||||
response with HTTP code
|
||||
|
||||
- 200 OK
|
||||
|
|
165
README.md
165
README.md
|
@ -1,7 +1,12 @@
|
|||
# Zenko CloudServer with Vitastor Backend
|
||||
# Zenko CloudServer
|
||||
|
||||
![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
|
||||
|
||||
[![CircleCI][badgepub]](https://circleci.com/gh/scality/S3)
|
||||
[![Scality CI][badgepriv]](http://ci.ironmann.io/gh/scality/S3)
|
||||
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/scality/s3server/)
|
||||
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
|
||||
|
||||
## Overview
|
||||
|
||||
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
|
||||
|
@ -11,71 +16,125 @@ Scality’s Open Source Multi-Cloud Data Controller.
|
|||
CloudServer provides a single AWS S3 API interface to access multiple
|
||||
backend data storage both on-premise or public in the cloud.
|
||||
|
||||
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
|
||||
backend support.
|
||||
CloudServer is useful for Developers, either to run as part of a
|
||||
continous integration test environment to emulate the AWS S3 service locally
|
||||
or as an abstraction layer to develop object storage enabled
|
||||
application on the go.
|
||||
|
||||
## Quick Start with Vitastor
|
||||
## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
|
||||
|
||||
Vitastor Backend is in experimental status, however you can already try to
|
||||
run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
|
||||
it works too 😊.
|
||||
## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
|
||||
|
||||
Installation instructions:
|
||||
## Docker
|
||||
|
||||
### Install Vitastor
|
||||
[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/scality/s3server/)
|
||||
|
||||
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
|
||||
## Contributing
|
||||
|
||||
### Install Zenko with Vitastor Backend
|
||||
In order to contribute, please follow the
|
||||
[Contributing Guidelines](
|
||||
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
|
||||
|
||||
- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
|
||||
- Install dependencies: `npm install --omit dev` or just `npm install`
|
||||
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
|
||||
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
|
||||
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
|
||||
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
|
||||
## Installation
|
||||
|
||||
### Install and Configure MongoDB
|
||||
### Dependencies
|
||||
|
||||
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
|
||||
Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
|
||||
. Up-to-date versions can be found at
|
||||
[Nodesource](https://github.com/nodesource/distributions).
|
||||
|
||||
### Setup Zenko
|
||||
### Clone source code
|
||||
|
||||
- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
|
||||
- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
|
||||
- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
|
||||
- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
|
||||
- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
|
||||
and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
|
||||
access keys, but it's not published, so let's use a file for now.
|
||||
- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
|
||||
You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
|
||||
in this file.
|
||||
|
||||
Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
|
||||
instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
|
||||
|
||||
### Start Zenko
|
||||
|
||||
Start the S3 server with: `node index.js`
|
||||
|
||||
If you use default settings, Zenko CloudServer starts on port 8000.
|
||||
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
|
||||
|
||||
Now you can access your S3 with `s3cmd` or `geesefs`:
|
||||
|
||||
```
|
||||
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
|
||||
```shell
|
||||
git clone https://github.com/scality/S3.git
|
||||
```
|
||||
|
||||
```
|
||||
AWS_ACCESS_KEY_ID=accessKey1 \
|
||||
AWS_SECRET_ACCESS_KEY=verySecretKey1 \
|
||||
geesefs --endpoint http://localhost:8000 testbucket mountdir
|
||||
### Install js dependencies
|
||||
|
||||
Go to the ./S3 folder,
|
||||
|
||||
```shell
|
||||
yarn install --frozen-lockfile
|
||||
```
|
||||
|
||||
# Author & License
|
||||
If you get an error regarding installation of the diskUsage module,
|
||||
please install g++.
|
||||
|
||||
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
||||
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
|
||||
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way)
|
||||
If you get an error regarding level-down bindings, try clearing your yarn cache:
|
||||
|
||||
```shell
|
||||
yarn cache clean
|
||||
```
|
||||
|
||||
## Run it with a file backend
|
||||
|
||||
```shell
|
||||
yarn start
|
||||
```
|
||||
|
||||
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
|
||||
9991 are also open locally for internal transfer of metadata and data,
|
||||
respectively.
|
||||
|
||||
The default access key is accessKey1 with
|
||||
a secret key of verySecretKey1.
|
||||
|
||||
By default the metadata files will be saved in the
|
||||
localMetadata directory and the data files will be saved
|
||||
in the localData directory within the ./S3 directory on your
|
||||
machine. These directories have been pre-created within the
|
||||
repository. If you would like to save the data or metadata in
|
||||
different locations of your choice, you must specify them with absolute paths.
|
||||
So, when starting the server:
|
||||
|
||||
```shell
|
||||
mkdir -m 700 $(pwd)/myFavoriteDataPath
|
||||
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
|
||||
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
|
||||
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
|
||||
yarn start
|
||||
```
|
||||
|
||||
## Run it with multiple data backends
|
||||
|
||||
```shell
|
||||
export S3DATA='multiple'
|
||||
yarn start
|
||||
```
|
||||
|
||||
This starts a Zenko CloudServer on port 8000.
|
||||
The default access key is accessKey1 with
|
||||
a secret key of verySecretKey1.
|
||||
|
||||
With multiple backends, you have the ability to
|
||||
choose where each object will be saved by setting
|
||||
the following header with a locationConstraint on
|
||||
a PUT request:
|
||||
|
||||
```shell
|
||||
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
|
||||
```
|
||||
|
||||
If no header is sent with a PUT object request, the
|
||||
location constraint of the bucket will determine
|
||||
where the data is saved. If the bucket has no location
|
||||
constraint, the endpoint of the PUT request will be
|
||||
used to determine location.
|
||||
|
||||
See the Configuration section below to learn how to set
|
||||
location constraints.
|
||||
|
||||
## Run it with an in-memory backend
|
||||
|
||||
```shell
|
||||
yarn run mem_backend
|
||||
```
|
||||
|
||||
This starts a Zenko CloudServer on port 8000.
|
||||
The default access key is accessKey1 with
|
||||
a secret key of verySecretKey1.
|
||||
|
||||
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
|
||||
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
|
||||
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
|
||||
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
|
||||
|
|
|
@ -1,2 +1,2 @@
|
|||
---
|
||||
theme: jekyll-theme-modernist
|
||||
theme: jekyll-theme-minimal
|
||||
|
|
|
@ -1,56 +0,0 @@
|
|||
{
|
||||
"accounts": [{
|
||||
"name": "Bart",
|
||||
"email": "sampleaccount1@sampling.com",
|
||||
"arn": "arn:aws:iam::123456789012:root",
|
||||
"canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
|
||||
"shortid": "123456789012",
|
||||
"keys": [{
|
||||
"access": "accessKey1",
|
||||
"secret": "verySecretKey1"
|
||||
}]
|
||||
}, {
|
||||
"name": "Lisa",
|
||||
"email": "sampleaccount2@sampling.com",
|
||||
"arn": "arn:aws:iam::123456789013:root",
|
||||
"canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
|
||||
"shortid": "123456789013",
|
||||
"keys": [{
|
||||
"access": "accessKey2",
|
||||
"secret": "verySecretKey2"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"name": "Clueso",
|
||||
"email": "inspector@clueso.info",
|
||||
"arn": "arn:aws:iam::123456789014:root",
|
||||
"canonicalID": "http://acs.zenko.io/accounts/service/clueso",
|
||||
"shortid": "123456789014",
|
||||
"keys": [{
|
||||
"access": "cluesoKey1",
|
||||
"secret": "cluesoSecretKey1"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"name": "Replication",
|
||||
"email": "inspector@replication.info",
|
||||
"arn": "arn:aws:iam::123456789015:root",
|
||||
"canonicalID": "http://acs.zenko.io/accounts/service/replication",
|
||||
"shortid": "123456789015",
|
||||
"keys": [{
|
||||
"access": "replicationKey1",
|
||||
"secret": "replicationSecretKey1"
|
||||
}]
|
||||
},
|
||||
{
|
||||
"name": "Lifecycle",
|
||||
"email": "inspector@lifecycle.info",
|
||||
"arn": "arn:aws:iam::123456789016:root",
|
||||
"canonicalID": "http://acs.zenko.io/accounts/service/lifecycle",
|
||||
"shortid": "123456789016",
|
||||
"keys": [{
|
||||
"access": "lifecycleKey1",
|
||||
"secret": "lifecycleSecretKey1"
|
||||
}]
|
||||
}]
|
||||
}
|
|
@ -1,4 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
require('../lib/nfs/utilities.js').createBucketWithNFSEnabled();
|
|
@ -1,108 +0,0 @@
|
|||
#!/bin/sh
|
||||
// 2>/dev/null ; exec "$(which nodejs 2>/dev/null || which node)" "$0" "$@"
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const { auth } = require('arsenal');
|
||||
const commander = require('commander');
|
||||
|
||||
const http = require('http');
|
||||
const https = require('https');
|
||||
const logger = require('../lib/utilities/logger');
|
||||
|
||||
function _performSearch(host,
|
||||
port,
|
||||
bucketName,
|
||||
query,
|
||||
listVersions,
|
||||
accessKey,
|
||||
secretKey,
|
||||
sessionToken,
|
||||
verbose, ssl) {
|
||||
const escapedSearch = encodeURIComponent(query);
|
||||
const options = {
|
||||
host,
|
||||
port,
|
||||
method: 'GET',
|
||||
path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`,
|
||||
headers: {
|
||||
'Content-Length': 0,
|
||||
},
|
||||
rejectUnauthorized: false,
|
||||
versions: '',
|
||||
};
|
||||
if (sessionToken) {
|
||||
options.headers['x-amz-security-token'] = sessionToken;
|
||||
}
|
||||
const transport = ssl ? https : http;
|
||||
const request = transport.request(options, response => {
|
||||
if (verbose) {
|
||||
logger.info('response status code', {
|
||||
statusCode: response.statusCode,
|
||||
});
|
||||
logger.info('response headers', { headers: response.headers });
|
||||
}
|
||||
const body = [];
|
||||
response.setEncoding('utf8');
|
||||
response.on('data', chunk => body.push(chunk));
|
||||
response.on('end', () => {
|
||||
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||
logger.info('Success');
|
||||
process.stdout.write(body.join(''));
|
||||
process.exit(0);
|
||||
} else {
|
||||
logger.error('request failed with HTTP Status ', {
|
||||
statusCode: response.statusCode,
|
||||
body: body.join(''),
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
// generateV4Headers exepects request object with path that does not
|
||||
// include query
|
||||
request.path = `/${bucketName}`;
|
||||
const requestData = listVersions ? { search: query, versions: '' } : { search: query };
|
||||
auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3');
|
||||
request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`;
|
||||
if (verbose) {
|
||||
logger.info('request headers', { headers: request._headers });
|
||||
}
|
||||
request.end();
|
||||
}
|
||||
|
||||
/**
|
||||
* This function is used as a binary to send a request to S3 to perform a
|
||||
* search on the objects in a bucket
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
function searchBucket() {
|
||||
// TODO: Include other bucket listing possible query params?
|
||||
commander
|
||||
.version('0.0.1')
|
||||
.option('-a, --access-key <accessKey>', 'Access key id')
|
||||
.option('-k, --secret-key <secretKey>', 'Secret access key')
|
||||
.option('-t, --session-token <sessionToken>', 'Session token')
|
||||
.option('-b, --bucket <bucket>', 'Name of the bucket')
|
||||
.option('-q, --query <query>', 'Search query')
|
||||
.option('-h, --host <host>', 'Host of the server')
|
||||
.option('-p, --port <port>', 'Port of the server')
|
||||
.option('-s', '--ssl', 'Enable ssl')
|
||||
.option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
|
||||
'otherwise only list the latest version')
|
||||
.option('-v, --verbose')
|
||||
.parse(process.argv);
|
||||
const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
|
||||
commander;
|
||||
|
||||
if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
|
||||
logger.error('missing parameter');
|
||||
commander.outputHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
_performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose,
|
||||
ssl);
|
||||
}
|
||||
|
||||
searchBucket();
|
|
@ -0,0 +1,45 @@
|
|||
---
|
||||
general:
|
||||
branches:
|
||||
ignore:
|
||||
- /^ultron\/.*/ # Ignore ultron/* branches
|
||||
artifacts:
|
||||
- coverage/
|
||||
|
||||
machine:
|
||||
node:
|
||||
version: 6.13.1
|
||||
services:
|
||||
- redis
|
||||
- docker
|
||||
ruby:
|
||||
version: "2.4.1"
|
||||
environment:
|
||||
CXX: g++-4.9
|
||||
ENABLE_LOCAL_CACHE: true
|
||||
REPORT_TOKEN: report-token-1
|
||||
hosts:
|
||||
bucketwebsitetester.s3-website-us-east-1.amazonaws.com: 127.0.0.1
|
||||
|
||||
dependencies:
|
||||
override:
|
||||
- rm -rf node_modules
|
||||
- yarn install --frozen-lockfile
|
||||
post:
|
||||
- sudo pip install flake8 yamllint
|
||||
- sudo pip install s3cmd==1.6.1
|
||||
# fog and ruby testing dependencies
|
||||
- gem install fog-aws -v 1.3.0
|
||||
- gem install mime-types -v 3.1
|
||||
- gem install rspec -v 3.5
|
||||
- gem install json
|
||||
# java sdk dependencies
|
||||
- sudo apt-get install -y -q default-jdk
|
||||
|
||||
|
||||
test:
|
||||
override:
|
||||
- docker run --name squid-proxy -d --net=host
|
||||
--publish 3128:3128 sameersbn/squid:3.3.8-23
|
||||
- bash tests.bash:
|
||||
parallel: true
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"accounts": [{
|
||||
"name": "Bart",
|
||||
"email": "sampleaccount1@sampling.com",
|
||||
"arn": "arn:aws:iam::123456789012:root",
|
||||
"canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
|
||||
"shortid": "123456789012",
|
||||
"keys": [{
|
||||
"access": "accessKey1",
|
||||
"secret": "verySecretKey1"
|
||||
}]
|
||||
}, {
|
||||
"name": "Lisa",
|
||||
"email": "sampleaccount2@sampling.com",
|
||||
"arn": "arn:aws:iam::123456789013:root",
|
||||
"canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf",
|
||||
"shortid": "123456789013",
|
||||
"keys": [{
|
||||
"access": "accessKey2",
|
||||
"secret": "verySecretKey2"
|
||||
}]
|
||||
}]
|
||||
}
|
|
@ -4,16 +4,13 @@
|
|||
"metricsPort": 8002,
|
||||
"metricsListenOn": [],
|
||||
"replicationGroupId": "RG001",
|
||||
"workers": 4,
|
||||
"restEndpoints": {
|
||||
"localhost": "us-east-1",
|
||||
"127.0.0.1": "us-east-1",
|
||||
"cloudserver-front": "us-east-1",
|
||||
"s3.docker.test": "us-east-1",
|
||||
"127.0.0.2": "us-east-1",
|
||||
"s3.amazonaws.com": "us-east-1",
|
||||
"zenko-cloudserver-replicator": "us-east-1",
|
||||
"lb": "us-east-1"
|
||||
"s3.amazonaws.com": "us-east-1"
|
||||
},
|
||||
"websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
|
||||
"s3-website.us-east-2.amazonaws.com",
|
||||
|
@ -28,8 +25,7 @@
|
|||
"s3-website-eu-west-1.amazonaws.com",
|
||||
"s3-website-sa-east-1.amazonaws.com",
|
||||
"s3-website.localhost",
|
||||
"s3-website.scality.test",
|
||||
"zenkoazuretest.blob.core.windows.net"],
|
||||
"s3-website.scality.test"],
|
||||
"replicationEndpoints": [{
|
||||
"site": "zenko",
|
||||
"servers": ["127.0.0.1:8000"],
|
||||
|
@ -38,14 +34,6 @@
|
|||
"site": "us-east-2",
|
||||
"type": "aws_s3"
|
||||
}],
|
||||
"backbeat": {
|
||||
"host": "localhost",
|
||||
"port": 8900
|
||||
},
|
||||
"workflowEngineOperator": {
|
||||
"host": "localhost",
|
||||
"port": 3001
|
||||
},
|
||||
"cdmi": {
|
||||
"host": "localhost",
|
||||
"port": 81,
|
||||
|
@ -59,7 +47,7 @@
|
|||
"host": "localhost",
|
||||
"port": 8500
|
||||
},
|
||||
"clusters": 1,
|
||||
"clusters": 10,
|
||||
"log": {
|
||||
"logLevel": "info",
|
||||
"dumpLevel": "error"
|
||||
|
@ -75,10 +63,6 @@
|
|||
"host": "localhost",
|
||||
"port": 9991
|
||||
},
|
||||
"pfsClient": {
|
||||
"host": "localhost",
|
||||
"port": 9992
|
||||
},
|
||||
"metadataDaemon": {
|
||||
"bindAddress": "localhost",
|
||||
"port": 9990
|
||||
|
@ -87,47 +71,10 @@
|
|||
"bindAddress": "localhost",
|
||||
"port": 9991
|
||||
},
|
||||
"pfsDaemon": {
|
||||
"bindAddress": "localhost",
|
||||
"port": 9992
|
||||
},
|
||||
"recordLog": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"recordLogName": "s3-recordlog"
|
||||
},
|
||||
"mongodb": {
|
||||
"replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
|
||||
"writeConcern": "majority",
|
||||
"replicaSet": "rs0",
|
||||
"readPreference": "primary",
|
||||
"database": "metadata"
|
||||
},
|
||||
"authdata": "authdata.json",
|
||||
"backends": {
|
||||
"auth": "file",
|
||||
"data": "file",
|
||||
"metadata": "mongodb",
|
||||
"kms": "file",
|
||||
"quota": "none"
|
||||
},
|
||||
"externalBackends": {
|
||||
"aws_s3": {
|
||||
"httpAgent": {
|
||||
"keepAlive": false,
|
||||
"keepAliveMsecs": 1000,
|
||||
"maxFreeSockets": 256,
|
||||
"maxSockets": null
|
||||
}
|
||||
},
|
||||
"gcp": {
|
||||
"httpAgent": {
|
||||
"keepAlive": true,
|
||||
"keepAliveMsecs": 1000,
|
||||
"maxFreeSockets": 256,
|
||||
"maxSockets": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"requests": {
|
||||
"viaProxy": false,
|
||||
"trustedProxyCIDRs": [],
|
|
@ -1,71 +0,0 @@
|
|||
{
|
||||
"port": 8000,
|
||||
"listenOn": [],
|
||||
"metricsPort": 8002,
|
||||
"metricsListenOn": [],
|
||||
"replicationGroupId": "RG001",
|
||||
"restEndpoints": {
|
||||
"localhost": "STANDARD",
|
||||
"127.0.0.1": "STANDARD",
|
||||
"yourhostname.ru": "STANDARD"
|
||||
},
|
||||
"websiteEndpoints": [
|
||||
"static.yourhostname.ru"
|
||||
],
|
||||
"replicationEndpoints": [ {
|
||||
"site": "zenko",
|
||||
"servers": ["127.0.0.1:8000"],
|
||||
"default": true
|
||||
} ],
|
||||
"log": {
|
||||
"logLevel": "info",
|
||||
"dumpLevel": "error"
|
||||
},
|
||||
"healthChecks": {
|
||||
"allowFrom": ["127.0.0.1/8", "::1"]
|
||||
},
|
||||
"backends": {
|
||||
"metadata": "mongodb"
|
||||
},
|
||||
"mongodb": {
|
||||
"replicaSetHosts": "127.0.0.1:27017",
|
||||
"writeConcern": "majority",
|
||||
"replicaSet": "rs0",
|
||||
"readPreference": "primary",
|
||||
"database": "s3",
|
||||
"authCredentials": {
|
||||
"username": "s3",
|
||||
"password": ""
|
||||
}
|
||||
},
|
||||
"externalBackends": {
|
||||
"aws_s3": {
|
||||
"httpAgent": {
|
||||
"keepAlive": false,
|
||||
"keepAliveMsecs": 1000,
|
||||
"maxFreeSockets": 256,
|
||||
"maxSockets": null
|
||||
}
|
||||
},
|
||||
"gcp": {
|
||||
"httpAgent": {
|
||||
"keepAlive": true,
|
||||
"keepAliveMsecs": 1000,
|
||||
"maxFreeSockets": 256,
|
||||
"maxSockets": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"requests": {
|
||||
"viaProxy": false,
|
||||
"trustedProxyCIDRs": [],
|
||||
"extractClientIPFromHeader": ""
|
||||
},
|
||||
"bucketNotificationDestinations": [
|
||||
{
|
||||
"resource": "target1",
|
||||
"type": "dummy",
|
||||
"host": "localhost:6000"
|
||||
}
|
||||
]
|
||||
}
|
54
constants.js
54
constants.js
|
@ -39,8 +39,6 @@ const constants = {
|
|||
// once the multipart upload is complete.
|
||||
mpuBucketPrefix: 'mpuShadowBucket',
|
||||
blacklistedPrefixes: { bucket: [], object: [] },
|
||||
// GCP Object Tagging Prefix
|
||||
gcpTaggingPrefix: 'aws-tag-',
|
||||
// PublicId is used as the canonicalID for a request that contains
|
||||
// no authentication information. Requestor can access
|
||||
// only public resources
|
||||
|
@ -66,20 +64,14 @@ const constants = {
|
|||
// http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
|
||||
minimumAllowedPartSize: 5242880,
|
||||
|
||||
// AWS sets a maximum total parts limit
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
|
||||
maximumAllowedPartCount: 10000,
|
||||
|
||||
gcpMaximumAllowedPartCount: 1024,
|
||||
|
||||
// Max size on put part or copy part is 5GB. For functional
|
||||
// testing use 110 MB as max
|
||||
maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :
|
||||
5368709120,
|
||||
|
||||
// Max size allowed in a single put object request is 5GB
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
|
||||
maximumAllowedUploadSize: 5368709120,
|
||||
// AWS sets a maximum total parts limit
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
|
||||
maximumAllowedPartCount: 10000,
|
||||
|
||||
// AWS states max size for user-defined metadata (x-amz-meta- headers) is
|
||||
// 2 KB: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
|
||||
|
@ -104,6 +96,7 @@ const constants = {
|
|||
'policyStatus',
|
||||
'publicAccessBlock',
|
||||
'requestPayment',
|
||||
'restore',
|
||||
'torrent',
|
||||
],
|
||||
|
||||
|
@ -116,21 +109,11 @@ const constants = {
|
|||
],
|
||||
|
||||
// user metadata header to set object locationConstraint
|
||||
objectLocationConstraintHeader: 'x-amz-storage-class',
|
||||
objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
|
||||
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
|
||||
legacyLocations: ['sproxyd', 'legacy'],
|
||||
// declare here all existing service accounts and their properties
|
||||
// (if any, otherwise an empty object)
|
||||
serviceAccountProperties: {
|
||||
replication: {},
|
||||
lifecycle: {},
|
||||
gc: {},
|
||||
'md-ingestion': {
|
||||
canReplicate: true,
|
||||
},
|
||||
},
|
||||
/* eslint-disable camelcase */
|
||||
externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
|
||||
externalBackends: { aws_s3: true, azure: true, gcp: true },
|
||||
// some of the available data backends (if called directly rather
|
||||
// than through the multiple backend gateway) need a key provided
|
||||
// as a string as first parameter of the get/delete methods.
|
||||
|
@ -139,19 +122,13 @@ const constants = {
|
|||
// for external backends, don't call unless at least 1 minute
|
||||
// (60,000 milliseconds) since last call
|
||||
externalBackendHealthCheckInterval: 60000,
|
||||
versioningNotImplBackends: { azure: true, gcp: true },
|
||||
mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
|
||||
skipBatchDeleteBackends: { azure: true, gcp: true },
|
||||
s3HandledBackends: { azure: true, gcp: true },
|
||||
hasCopyPartBackends: { aws_s3: true, gcp: true },
|
||||
versioningNotImplBackends: { azure: true },
|
||||
mpuMDStoredExternallyBackend: { aws_s3: true },
|
||||
/* eslint-enable camelcase */
|
||||
mpuMDStoredOnS3Backend: { azure: true },
|
||||
azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
|
||||
base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
|
||||
'(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
|
||||
productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
|
||||
// location constraint delimiter
|
||||
zenkoSeparator: ':',
|
||||
// user metadata applied on zenko objects
|
||||
zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
|
||||
bucketOwnerActions: [
|
||||
|
@ -204,13 +181,16 @@ const constants = {
|
|||
'objectPutRetention',
|
||||
],
|
||||
allowedUtapiEventFilterStates: ['allow', 'deny'],
|
||||
allowedRestoreObjectRequestTierValues: ['Standard'],
|
||||
// The AWS assumed Role resource type
|
||||
assumedRoleArnResourceType: 'assumed-role',
|
||||
// Session name of the backbeat lifecycle assumed role session.
|
||||
backbeatLifecycleSessionName: 'backbeat-lifecycle',
|
||||
multiObjectDeleteConcurrency: 50,
|
||||
lifecycleListing: {
|
||||
CURRENT_TYPE: 'current',
|
||||
NON_CURRENT_TYPE: 'noncurrent',
|
||||
ORPHAN_DM_TYPE: 'orphan',
|
||||
},
|
||||
multiObjectDeleteConcurrency: 50,
|
||||
maxScannedLifecycleListingEntries: 10000,
|
||||
overheadField: [
|
||||
'content-length',
|
||||
|
@ -231,18 +211,12 @@ const constants = {
|
|||
]),
|
||||
ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
|
||||
ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
|
||||
// The AWS assumed Role resource type
|
||||
assumedRoleArnResourceType: 'assumed-role',
|
||||
// Session name of the backbeat lifecycle assumed role session.
|
||||
backbeatLifecycleSessionName: 'backbeat-lifecycle',
|
||||
actionsToConsiderAsObjectPut: [
|
||||
'initiateMultipartUpload',
|
||||
'objectPutPart',
|
||||
'completeMultipartUpload',
|
||||
'objectPost',
|
||||
],
|
||||
// if requester is not bucket owner, bucket policy actions should be denied with
|
||||
// MethodNotAllowed error
|
||||
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
|
||||
};
|
||||
|
||||
module.exports = constants;
|
||||
|
|
|
@ -17,17 +17,13 @@ process.on('uncaughtException', err => {
|
|||
if (config.backends.data === 'file' ||
|
||||
(config.backends.data === 'multiple' &&
|
||||
config.backends.metadata !== 'scality')) {
|
||||
const dataServer = new arsenal.network.rest.RESTServer({
|
||||
bindAddress: config.dataDaemon.bindAddress,
|
||||
const dataServer = new arsenal.network.rest.RESTServer(
|
||||
{ bindAddress: config.dataDaemon.bindAddress,
|
||||
port: config.dataDaemon.port,
|
||||
dataStore: new arsenal.storage.data.file.DataFileStore({
|
||||
dataPath: config.dataDaemon.dataPath,
|
||||
log: config.log,
|
||||
noSync: config.dataDaemon.noSync,
|
||||
noCache: config.dataDaemon.noCache,
|
||||
}),
|
||||
log: config.log,
|
||||
});
|
||||
dataStore: new arsenal.storage.data.file.DataFileStore(
|
||||
{ dataPath: config.dataDaemon.dataPath,
|
||||
log: config.log }),
|
||||
log: config.log });
|
||||
dataServer.setup(err => {
|
||||
if (err) {
|
||||
logger.error('Error initializing REST data server',
|
||||
|
|
|
@ -6,15 +6,14 @@ set -e
|
|||
# modifying config.json
|
||||
JQ_FILTERS_CONFIG="."
|
||||
|
||||
# ENDPOINT var can accept comma separated values
|
||||
# for multiple endpoint locations
|
||||
if [[ "$ENDPOINT" ]]; then
|
||||
IFS="," read -ra HOST_NAMES <<< "$ENDPOINT"
|
||||
for host in "${HOST_NAMES[@]}"; do
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$host\"]=\"us-east-1\""
|
||||
done
|
||||
echo "Host name has been modified to ${HOST_NAMES[@]}"
|
||||
echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with ${HOST_NAMES[@]}"
|
||||
HOST_NAME="$ENDPOINT"
|
||||
fi
|
||||
|
||||
if [[ "$HOST_NAME" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$HOST_NAME\"]=\"us-east-1\""
|
||||
echo "Host name has been modified to $HOST_NAME"
|
||||
echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with $HOST_NAME"
|
||||
fi
|
||||
|
||||
if [[ "$LOG_LEVEL" ]]; then
|
||||
|
@ -26,7 +25,7 @@ if [[ "$LOG_LEVEL" ]]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
if [[ "$SSL" && "$HOST_NAMES" ]]; then
|
||||
if [[ "$SSL" && "$HOST_NAME" ]]; then
|
||||
# This condition makes sure that the certificates are not generated twice. (for docker restart)
|
||||
if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
|
||||
# Compute config for utapi tests
|
||||
|
@ -37,15 +36,15 @@ prompt = no
|
|||
req_extensions = s3_req
|
||||
|
||||
[req_distinguished_name]
|
||||
CN = ${HOST_NAMES[0]}
|
||||
CN = ${HOST_NAME}
|
||||
|
||||
[s3_req]
|
||||
subjectAltName = @alt_names
|
||||
extendedKeyUsage = serverAuth, clientAuth
|
||||
|
||||
[alt_names]
|
||||
DNS.1 = *.${HOST_NAMES[0]}
|
||||
DNS.2 = ${HOST_NAMES[0]}
|
||||
DNS.1 = *.${HOST_NAME}
|
||||
DNS.2 = ${HOST_NAME}
|
||||
|
||||
EOF
|
||||
|
||||
|
@ -71,14 +70,9 @@ fi
|
|||
if [[ "$LISTEN_ADDR" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
|
||||
fi
|
||||
|
||||
if [[ "$REPLICATION_GROUP_ID" ]] ; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
|
||||
fi
|
||||
|
||||
if [[ "$DATA_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
|
||||
fi
|
||||
|
@ -87,22 +81,6 @@ if [[ "$METADATA_HOST" ]]; then
|
|||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$PFSD_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$MONGODB_HOSTS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
|
||||
fi
|
||||
|
||||
if [[ "$MONGODB_RS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSet=\"$MONGODB_RS\""
|
||||
fi
|
||||
|
||||
if [[ "$MONGODB_DATABASE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.database=\"$MONGODB_DATABASE\""
|
||||
fi
|
||||
|
||||
if [ -z "$REDIS_HA_NAME" ]; then
|
||||
REDIS_HA_NAME='mymaster'
|
||||
fi
|
||||
|
@ -135,70 +113,14 @@ if [[ "$RECORDLOG_ENABLED" ]]; then
|
|||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true"
|
||||
fi
|
||||
|
||||
if [[ "$STORAGE_LIMIT_ENABLED" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.metrics[.utapi.metrics | length]=\"location\""
|
||||
fi
|
||||
|
||||
if [[ "$CRR_METRICS_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$CRR_METRICS_PORT" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
|
||||
fi
|
||||
|
||||
if [[ "$WE_OPERATOR_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$WE_OPERATOR_PORT" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
|
||||
fi
|
||||
|
||||
if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
|
||||
fi
|
||||
|
||||
# external backends http(s) agent config
|
||||
|
||||
# AWS
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
|
||||
fi
|
||||
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
|
||||
fi
|
||||
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
|
||||
fi
|
||||
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
|
||||
fi
|
||||
|
||||
#GCP
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
|
||||
fi
|
||||
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
|
||||
fi
|
||||
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
|
||||
fi
|
||||
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
|
||||
fi
|
||||
|
||||
if [[ -n "$BUCKET_DENY_FILTER" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
|
||||
fi
|
||||
|
||||
if [[ "$BUCKETD_BOOTSTRAP" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .bucketd.bootstrap=[\"$BUCKETD_BOOTSTRAP\""]
|
||||
fi
|
||||
|
||||
if [[ "$TESTING_MODE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
|
||||
fi
|
||||
|
@ -208,10 +130,6 @@ if [[ $JQ_FILTERS_CONFIG != "." ]]; then
|
|||
mv config.json.tmp config.json
|
||||
fi
|
||||
|
||||
if test -v INITIAL_INSTANCE_ID && test -v S3METADATAPATH && ! test -f ${S3METADATAPATH}/uuid ; then
|
||||
echo -n ${INITIAL_INSTANCE_ID} > ${S3METADATAPATH}/uuid
|
||||
fi
|
||||
|
||||
# s3 secret credentials for Zenko
|
||||
if [ -r /run/secrets/s3-credentials ] ; then
|
||||
. /run/secrets/s3-credentials
|
||||
|
|
|
@ -27,7 +27,7 @@ including null versions and delete markers, described in the above
|
|||
links.
|
||||
|
||||
Implementation of Bucket Versioning in Zenko CloudServer
|
||||
--------------------------------------------------------
|
||||
-----------------------------------------
|
||||
|
||||
Overview of Metadata and API Component Roles
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
@ -66,7 +66,7 @@ The second section, `"Implementation of Bucket Versioning in
|
|||
API" <#implementation-of-bucket-versioning-in-api>`__, describes the way
|
||||
the metadata options are used in the API within S3 actions to create new
|
||||
versions, update their metadata, and delete them. The management of null
|
||||
versions and creation of delete markers is also described in this
|
||||
versions and creation of delete markers are also described in this
|
||||
section.
|
||||
|
||||
Implementation of Bucket Versioning in Metadata
|
||||
|
@ -179,13 +179,12 @@ PUT
|
|||
the master version with this version.
|
||||
- ``versionId: <versionId>`` create or update a specific version (for updating
|
||||
version's ACL or tags, or remote updates in geo-replication)
|
||||
|
||||
* if the version identified by ``versionId`` happens to be the latest
|
||||
- if the version identified by ``versionId`` happens to be the latest
|
||||
version, the master version will be updated as well
|
||||
* if the master version is not as recent as the version identified by
|
||||
- if the master version is not as recent as the version identified by
|
||||
``versionId``, as may happen with cross-region replication, the master
|
||||
will be updated as well
|
||||
* note that with ``versionId`` set to an empty string ``''``, it will
|
||||
- note that with ``versionId`` set to an empty string ``''``, it will
|
||||
overwrite the master version only (same as no options, but the master
|
||||
version will have a ``versionId`` property set in its metadata like
|
||||
any other version). The ``versionId`` will never be exposed to an
|
||||
|
@ -209,13 +208,10 @@ A deletion targeting the latest version of an object has to:
|
|||
- delete the specified version identified by ``versionId``
|
||||
- replace the master version with a version that is a placeholder for
|
||||
deletion
|
||||
|
||||
- this version contains a special keyword, 'isPHD', to indicate the
|
||||
master version was deleted and needs to be updated
|
||||
|
||||
- initiate a repair operation to update the value of the master
|
||||
version:
|
||||
|
||||
- involves listing the versions of the object and get the latest
|
||||
version to replace the placeholder delete version
|
||||
- if no more versions exist, metadata deletes the master version,
|
||||
|
@ -768,7 +764,7 @@ This will open two ports:
|
|||
Then, one or more instances of Zenko CloudServer without the dmd can be started
|
||||
elsewhere with:
|
||||
|
||||
.. code:: sh
|
||||
::
|
||||
|
||||
yarn run start_s3server
|
||||
|
||||
|
|
|
@ -178,7 +178,7 @@ Ruby
|
|||
~~~~
|
||||
|
||||
`AWS SDK for Ruby - Version 2 <http://docs.aws.amazon.com/sdkforruby/api/>`__
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code:: ruby
|
||||
|
||||
|
@ -239,7 +239,6 @@ Python
|
|||
Client integration
|
||||
|
||||
.. code:: python
|
||||
|
||||
import boto3
|
||||
|
||||
client = boto3.client(
|
||||
|
@ -254,7 +253,6 @@ Client integration
|
|||
Full integration (with object mapping)
|
||||
|
||||
.. code:: python
|
||||
|
||||
import os
|
||||
|
||||
from botocore.utils import fix_s3_host
|
||||
|
@ -295,51 +293,3 @@ Should force path-style requests even though v3 advertises it does by default.
|
|||
$client->createBucket(array(
|
||||
'Bucket' => 'bucketphp',
|
||||
));
|
||||
|
||||
Go
|
||||
~~
|
||||
|
||||
`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code:: go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/endpoints"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
|
||||
os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
|
||||
endpoint := "http://localhost:8000"
|
||||
timeout := time.Duration(10) * time.Second
|
||||
sess := session.Must(session.NewSession())
|
||||
|
||||
// Create a context with a timeout that will abort the upload if it takes
|
||||
// more than the passed in timeout.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
svc := s3.New(sess, &aws.Config{
|
||||
Region: aws.String(endpoints.UsEast1RegionID),
|
||||
Endpoint: &endpoint,
|
||||
})
|
||||
|
||||
out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
} else {
|
||||
fmt.Println(out)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ Got an idea? Get started!
|
|||
In order to contribute, please follow the `Contributing
|
||||
Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__.
|
||||
If anything is unclear to you, reach out to us on
|
||||
`forum <https://forum.zenko.io/>`__ or via a GitHub issue.
|
||||
`slack <https://zenko-io.slack.com/>`__ or via a GitHub issue.
|
||||
|
||||
Don't write code? There are other ways to help!
|
||||
-----------------------------------------------
|
||||
|
|
376
docs/DOCKER.rst
376
docs/DOCKER.rst
|
@ -1,7 +1,11 @@
|
|||
Docker
|
||||
======
|
||||
|
||||
.. _environment-variables:
|
||||
- `Environment Variables <#environment-variables>`__
|
||||
- `Tunables and setup tips <#tunables-and-setup-tips>`__
|
||||
- `Examples for continuous integration with
|
||||
Docker <#continuous-integration-with-docker-hosted CloudServer>`__
|
||||
- `Examples for going in production with Docker <#in-production-with-docker-hosted CloudServer>`__
|
||||
|
||||
Environment Variables
|
||||
---------------------
|
||||
|
@ -11,27 +15,25 @@ S3DATA
|
|||
|
||||
S3DATA=multiple
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
This variable enables running CloudServer with multiple data backends, defined
|
||||
Allows you to run Scality Zenko CloudServer with multiple data backends, defined
|
||||
as regions.
|
||||
When using multiple data backends, a custom ``locationConfig.json`` file is
|
||||
mandatory. It will allow you to set custom regions. You will then need to
|
||||
provide associated rest_endpoints for each custom region in your
|
||||
``config.json`` file.
|
||||
`Learn more about multiple backends configuration <../GETTING_STARTED/#location-configuration>`__
|
||||
|
||||
For multiple data backends, a custom locationConfig.json file is required.
|
||||
This file enables you to set custom regions. You must provide associated
|
||||
rest_endpoints for each custom region in config.json.
|
||||
If you are using Scality RING endpoints, please refer to your customer
|
||||
documentation.
|
||||
|
||||
`Learn more about multiple-backend configurations <GETTING_STARTED.html#location-configuration>`__
|
||||
|
||||
If you are using Scality RING endpoints, refer to your customer documentation.
|
||||
|
||||
Running CloudServer with an AWS S3-Hosted Backend
|
||||
"""""""""""""""""""""""""""""""""""""""""""""""""
|
||||
|
||||
To run CloudServer with an S3 AWS backend, add a new section to the
|
||||
``locationConfig.json`` file with the ``aws_s3`` location type:
|
||||
Running it with an AWS S3 hosted backend
|
||||
""""""""""""""""""""""""""""""""""""""""
|
||||
To run CloudServer with an S3 AWS backend, you will have to add a new section
|
||||
to your ``locationConfig.json`` file with the ``aws_s3`` location type:
|
||||
|
||||
.. code:: json
|
||||
|
||||
(...)
|
||||
(...)
|
||||
"awsbackend": {
|
||||
"type": "aws_s3",
|
||||
"details": {
|
||||
|
@ -41,134 +43,121 @@ To run CloudServer with an S3 AWS backend, add a new section to the
|
|||
"credentialsProfile": "aws_hosted_profile"
|
||||
}
|
||||
}
|
||||
(...)
|
||||
(...)
|
||||
|
||||
Edit your AWS credentials file to enable your preferred command-line tool.
|
||||
This file must mention credentials for all backends in use. You can use
|
||||
several profiles if multiple profiles are configured.
|
||||
You will also have to edit your AWS credentials file to be able to use your
|
||||
command line tool of choice. This file should mention credentials for all the
|
||||
backends you're using. You can use several profiles when using multiple
|
||||
profiles.
|
||||
|
||||
.. code:: json
|
||||
|
||||
[default]
|
||||
aws_access_key_id=accessKey1
|
||||
aws_secret_access_key=verySecretKey1
|
||||
[aws_hosted_profile]
|
||||
aws_access_key_id={{YOUR_ACCESS_KEY}}
|
||||
aws_secret_access_key={{YOUR_SECRET_KEY}}
|
||||
[default]
|
||||
aws_access_key_id=accessKey1
|
||||
aws_secret_access_key=verySecretKey1
|
||||
[aws_hosted_profile]
|
||||
aws_access_key_id={{YOUR_ACCESS_KEY}}
|
||||
aws_secret_access_key={{YOUR_SECRET_KEY}}
|
||||
|
||||
As with locationConfig.json, the AWS credentials file must be mounted at
|
||||
run time: ``-v ~/.aws/credentials:/root/.aws/credentials`` on Unix-like
|
||||
systems (Linux, OS X, etc.), or
|
||||
Just as you need to mount your locationConfig.json, you will need to mount your
|
||||
AWS credentials file at run time:
|
||||
``-v ~/.aws/credentials:/root/.aws/credentials`` on Linux, OS X, or Unix or
|
||||
``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows
|
||||
|
||||
.. note:: One account cannot copy to another account with a source and
|
||||
destination on real AWS unless the account associated with the
|
||||
accessKey/secretKey pairs used for the destination bucket has source
|
||||
bucket access privileges. To enable this, update ACLs directly on AWS.
|
||||
NOTE: One account can't copy to another account with a source and
|
||||
destination on real AWS unless the account associated with the
|
||||
access Key/secret Key pairs used for the destination bucket has rights
|
||||
to get in the source bucket. ACL's would have to be updated
|
||||
on AWS directly to enable this.
|
||||
|
||||
S3BACKEND
|
||||
~~~~~~~~~
|
||||
~~~~~~
|
||||
|
||||
S3BACKEND=file
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
For stored file data to persist, you must mount Docker volumes
|
||||
for both data and metadata. See :ref:`In Production with a Docker-Hosted CloudServer <in-production-w-a-Docker-hosted-cloudserver>`
|
||||
^^^^^^^^^^^
|
||||
When storing file data, for it to be persistent you must mount docker volumes
|
||||
for both data and metadata. See `this section <#using-docker-volumes-in-production>`__
|
||||
|
||||
S3BACKEND=mem
|
||||
^^^^^^^^^^^^^
|
||||
|
||||
This is ideal for testing: no data remains after the container is shut down.
|
||||
^^^^^^^^^^
|
||||
This is ideal for testing - no data will remain after container is shutdown.
|
||||
|
||||
ENDPOINT
|
||||
~~~~~~~~
|
||||
|
||||
This variable specifies the endpoint. To direct CloudServer requests to
|
||||
new.host.com, for example, specify the endpoint with:
|
||||
This variable specifies your endpoint. If you have a domain such as
|
||||
new.host.com, by specifying that here, you and your users can direct s3
|
||||
server requests to new.host.com.
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -p 8000:8000 -e ENDPOINT=new.host.com zenko/cloudserver
|
||||
docker run -d --name s3server -p 8000:8000 -e ENDPOINT=new.host.com scality/s3server
|
||||
|
||||
.. note:: On Unix-like systems (Linux, OS X, etc.) edit /etc/hosts
|
||||
to associate 127.0.0.1 with new.host.com.
|
||||
|
||||
REMOTE_MANAGEMENT_DISABLE
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
CloudServer is a part of `Zenko <https://www.zenko.io/>`__. When you run CloudServer standalone it will still try to connect to Orbit by default (browser-based graphical user interface for Zenko).
|
||||
|
||||
Setting this variable to true(1) will default to accessKey1 and verySecretKey1 for credentials and disable the automatic Orbit management:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -p 8000:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
|
||||
Note: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
|
||||
permissions, make sure to associate 127.0.0.1 with ``new.host.com``
|
||||
|
||||
SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
These variables specify authentication credentials for an account named
|
||||
“CustomAccount”.
|
||||
"CustomAccount".
|
||||
|
||||
Set account credentials for multiple accounts by editing conf/authdata.json
|
||||
(see below for further details). To specify one set for personal use, set these
|
||||
environment variables:
|
||||
You can set credentials for many accounts by editing
|
||||
``conf/authdata.json`` (see below for further info), but if you just
|
||||
want to specify one set of your own, you can use these environment
|
||||
variables.
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey \
|
||||
-e SCALITY_SECRET_ACCESS_KEY=newSecretKey zenko/cloudserver
|
||||
docker run -d --name s3server -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey
|
||||
-e SCALITY_SECRET_ACCESS_KEY=newSecretKey scality/s3server
|
||||
|
||||
.. note:: This takes precedence over the contents of the authdata.json
|
||||
file. The authdata.json file is ignored.
|
||||
|
||||
.. note:: The ACCESS_KEY and SECRET_KEY environment variables are
|
||||
deprecated.
|
||||
Note: Anything in the ``authdata.json`` file will be ignored. Note: The
|
||||
old ``ACCESS_KEY`` and ``SECRET_KEY`` environment variables are now
|
||||
deprecated
|
||||
|
||||
LOG\_LEVEL
|
||||
~~~~~~~~~~
|
||||
|
||||
This variable changes the log level. There are three levels: info, debug,
|
||||
and trace. The default is info. Debug provides more detailed logs, and trace
|
||||
provides the most detailed logs.
|
||||
This variable allows you to change the log level: info, debug or trace.
|
||||
The default is info. Debug will give you more detailed logs and trace
|
||||
will give you the most detailed.
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -p 8000:8000 -e LOG_LEVEL=trace zenko/cloudserver
|
||||
docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
|
||||
|
||||
SSL
|
||||
~~~
|
||||
|
||||
Set true, this variable runs CloudServer with SSL.
|
||||
This variable set to true allows you to run S3 with SSL:
|
||||
|
||||
If SSL is set true:
|
||||
**Note1**: You also need to specify the ENDPOINT environment variable.
|
||||
**Note2**: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
|
||||
permissions, make sure to associate 127.0.0.1 with ``<YOUR_ENDPOINT>``
|
||||
|
||||
* The ENDPOINT environment variable must also be specified.
|
||||
|
||||
* On Unix-like systems (Linux, OS X, etc.), 127.0.0.1 must be associated with
|
||||
<YOUR_ENDPOINT> in /etc/hosts.
|
||||
|
||||
.. Warning:: Self-signed certs with a CA generated within the container are
|
||||
suitable for testing purposes only. Clients cannot trust them, and they may
|
||||
disappear altogether on a container upgrade. The best security practice for
|
||||
production environments is to use an extra container, such as
|
||||
haproxy/nginx/stunnel, for SSL/TLS termination and to pull certificates
|
||||
from a mounted volume, limiting what an exploit on either component
|
||||
can expose.
|
||||
**Warning**: These certs, being self-signed (and the CA being generated
|
||||
inside the container) will be untrusted by any clients, and could
|
||||
disappear on a container upgrade. That's ok as long as it's for quick
|
||||
testing. Also, best security practice for non-testing would be to use an
|
||||
extra container to do SSL/TLS termination such as haproxy/nginx/stunnel
|
||||
to limit what an exploit on either component could expose, as well as
|
||||
certificates in a mounted volume
|
||||
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT> \
|
||||
zenko/cloudserver
|
||||
docker run -d --name s3server -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT>
|
||||
scality/s3server
|
||||
|
||||
For more information about using ClousdServer with SSL, see `Using SSL <GETTING_STARTED.html#Using SSL>`__
|
||||
More information about how to use S3 server with SSL
|
||||
`here <https://s3.scality.com/v1.0/page/scality-with-ssl>`__
|
||||
|
||||
LISTEN\_ADDR
|
||||
~~~~~~~~~~~~
|
||||
|
||||
This variable causes CloudServer and its data and metadata components to
|
||||
listen on the specified address. This allows starting the data or metadata
|
||||
servers as standalone services, for example.
|
||||
This variable instructs the Zenko CloudServer, and its data and metadata
|
||||
components to listen on the specified address. This allows starting the data
|
||||
or metadata servers as standalone services, for example.
|
||||
|
||||
.. code:: shell
|
||||
|
||||
|
@ -185,8 +174,8 @@ Zenko CloudServer.
|
|||
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -e DATA_HOST=cloudserver-data \
|
||||
-e METADATA_HOST=cloudserver-metadata zenko/cloudserver yarn run start_s3server
|
||||
docker run -d --name s3server -e DATA_HOST=s3server-data
|
||||
-e METADATA_HOST=s3server-metadata scality/s3server npm run start_s3server
|
||||
|
||||
REDIS\_HOST
|
||||
~~~~~~~~~~~
|
||||
|
@ -196,21 +185,19 @@ localhost.
|
|||
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -p 8000:8000 \
|
||||
-e REDIS_HOST=my-redis-server.example.com zenko/cloudserver
|
||||
docker run -d --name s3server -p 8000:8000
|
||||
-e REDIS_HOST=my-redis-server.example.com scality/s3server
|
||||
|
||||
REDIS\_PORT
|
||||
~~~~~~~~~~~
|
||||
|
||||
Use this variable to connect to the Redis cache server on a port other
|
||||
than the default 6379.
|
||||
Use this variable to connect to the redis cache server on another port than
|
||||
the default 6379.
|
||||
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -d --name cloudserver -p 8000:8000 \
|
||||
-e REDIS_PORT=6379 zenko/cloudserver
|
||||
|
||||
.. _tunables-and-setup-tips:
|
||||
docker run -d --name s3server -p 8000:8000
|
||||
-e REDIS_PORT=6379 scality/s3server
|
||||
|
||||
Tunables and Setup Tips
|
||||
-----------------------
|
||||
|
@ -218,57 +205,60 @@ Tunables and Setup Tips
|
|||
Using Docker Volumes
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
CloudServer runs with a file backend by default, meaning that data is
|
||||
stored inside the CloudServer’s Docker container.
|
||||
Zenko CloudServer runs with a file backend by default.
|
||||
|
||||
For data and metadata to persist, data and metadata must be hosted in Docker
|
||||
volumes outside the CloudServer’s Docker container. Otherwise, the data
|
||||
and metadata are destroyed when the container is erased.
|
||||
So, by default, the data is stored inside your Zenko CloudServer Docker
|
||||
container.
|
||||
|
||||
.. code-block:: shell
|
||||
However, if you want your data and metadata to persist, you **MUST** use
|
||||
Docker volumes to host your data and metadata outside your Zenko CloudServer
|
||||
Docker container. Otherwise, the data and metadata will be destroyed
|
||||
when you erase the container.
|
||||
|
||||
$ docker run -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
|
||||
-p 8000:8000 -d zenko/cloudserver
|
||||
.. code:: shell
|
||||
|
||||
This command mounts the ./data host directory to the container
|
||||
at /usr/src/app/localData and the ./metadata host directory to
|
||||
the container at /usr/src/app/localMetaData.
|
||||
docker run -v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
|
||||
-p 8000:8000 -d scality/s3server
|
||||
|
||||
.. tip:: These host directories can be mounted to any accessible mount
|
||||
point, such as /mnt/data and /mnt/metadata, for example.
|
||||
This command mounts the host directory, ``./data``, into the container
|
||||
at ``/usr/src/app/localData`` and the host directory, ``./metadata``, into
|
||||
the container at ``/usr/src/app/localMetaData``. It can also be any host
|
||||
mount point, like ``/mnt/data`` and ``/mnt/metadata``.
|
||||
|
||||
Adding, Modifying, or Deleting Accounts or Credentials
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Adding modifying or deleting accounts or users credentials
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
1. Create a customized authdata.json file locally based on /conf/authdata.json.
|
||||
|
||||
2. Use `Docker volumes <https://docs.docker.com/storage/volumes/>`__
|
||||
to override the default ``authdata.json`` through a Docker file mapping.
|
||||
1. Create locally a customized ``authdata.json`` based on our ``/conf/authdata.json``.
|
||||
|
||||
2. Use `Docker
|
||||
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__
|
||||
to override the default ``authdata.json`` through a docker file mapping.
|
||||
For example:
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d \
|
||||
zenko/cloudserver
|
||||
docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
|
||||
scality/s3server
|
||||
|
||||
Specifying a Host Name
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
Specifying your own host name
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To specify a host name (for example, s3.domain.name), provide your own
|
||||
`config.json <https://github.com/scality/cloudserver/blob/master/config.json>`__
|
||||
file using `Docker volumes <https://docs.docker.com/storage/volumes/>`__.
|
||||
To specify a host name (e.g. s3.domain.name), you can provide your own
|
||||
`config.json <https://github.com/scality/S3/blob/master/config.json>`__
|
||||
using `Docker
|
||||
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__.
|
||||
|
||||
First, add a new key-value pair to the restEndpoints section of your
|
||||
config.json. Make the key the host name you want, and the value the default
|
||||
location\_constraint for this endpoint.
|
||||
First add a new key-value pair in the restEndpoints section of your
|
||||
config.json. The key in the key-value pair should be the host name you
|
||||
would like to add and the value is the default location\_constraint for
|
||||
this endpoint.
|
||||
|
||||
For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one
|
||||
of the ``location_constraints`` listed in your locationConfig.json file
|
||||
`here <https://github.com/scality/S3/blob/master/locationConfig.json>`__.
|
||||
|
||||
For more information about location configuration, see:
|
||||
`GETTING STARTED <GETTING_STARTED.html#location-configuration>`__
|
||||
More information about location configuration
|
||||
`here <https://github.com/scality/S3/blob/master/README.md#location-configuration>`__
|
||||
|
||||
.. code:: json
|
||||
|
||||
|
@ -276,33 +266,33 @@ For more information about location configuration, see:
|
|||
"localhost": "file",
|
||||
"127.0.0.1": "file",
|
||||
...
|
||||
"cloudserver.example.com": "us-east-1"
|
||||
"s3.example.com": "us-east-1"
|
||||
},
|
||||
|
||||
Next, run CloudServer using a `Docker volume
|
||||
<https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
|
||||
Then, run your Scality S3 Server using `Docker
|
||||
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d zenko/cloudserver
|
||||
docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
|
||||
|
||||
The local ``config.json`` file overrides the default one through a Docker
|
||||
file mapping.
|
||||
Your local ``config.json`` file will override the default one through a
|
||||
docker file mapping.
|
||||
|
||||
Running as an Unprivileged User
|
||||
Running as an unprivileged user
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
CloudServer runs as root by default.
|
||||
Zenko CloudServer runs as root by default.
|
||||
|
||||
To change this, modify the dockerfile and specify a user before the
|
||||
entry point.
|
||||
You can change that by modifing the dockerfile and specifying a user
|
||||
before the entrypoint.
|
||||
|
||||
The user must exist within the container, and must own the
|
||||
/usr/src/app directory for CloudServer to run.
|
||||
The user needs to exist within the container, and own the folder
|
||||
**/usr/src/app** for Scality Zenko CloudServer to run properly.
|
||||
|
||||
For example, the following dockerfile lines can be modified:
|
||||
For instance, you can modify these lines in the dockerfile:
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
...
|
||||
&& groupadd -r -g 1001 scality \
|
||||
|
@ -314,58 +304,54 @@ For example, the following dockerfile lines can be modified:
|
|||
USER scality
|
||||
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
|
||||
|
||||
.. _continuous-integration-with-docker-hosted-cloudserver:
|
||||
Continuous integration with Docker hosted CloudServer
|
||||
-----------------------------------------------------
|
||||
|
||||
Continuous Integration with a Docker-Hosted CloudServer
|
||||
-------------------------------------------------------
|
||||
When you start the Docker Scality Zenko CloudServer image, you can adjust the
|
||||
configuration of the Scality Zenko CloudServer instance by passing one or more
|
||||
environment variables on the docker run command line.
|
||||
|
||||
When you start the Docker CloudServer image, you can adjust the
|
||||
configuration of the CloudServer instance by passing one or more
|
||||
environment variables on the ``docker run`` command line.
|
||||
Sample ways to run it for CI are:
|
||||
|
||||
- With custom locations (one in-memory, one hosted on AWS), and custom
|
||||
credentials mounted:
|
||||
|
||||
To run CloudServer for CI with custom locations (one in-memory,
|
||||
one hosted on AWS), and custom credentials mounted:
|
||||
.. code:: shell
|
||||
|
||||
.. code-block:: shell
|
||||
docker run --name CloudServer -p 8000:8000
|
||||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
|
||||
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
|
||||
-v ~/.aws/credentials:/root/.aws/credentials
|
||||
-e S3DATA=multiple -e S3BACKEND=mem scality/s3server
|
||||
|
||||
$ docker run --name CloudServer -p 8000:8000 \
|
||||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
|
||||
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
|
||||
-v ~/.aws/credentials:/root/.aws/credentials \
|
||||
-e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
|
||||
- With custom locations, (one in-memory, one hosted on AWS, one file),
|
||||
and custom credentials set as environment variables
|
||||
(see `this section <#scality-access-key-id-and-scality-secret-access-key>`__):
|
||||
|
||||
To run CloudServer for CI with custom locations, (one in-memory, one
|
||||
hosted on AWS, and one file), and custom credentials `set as environment
|
||||
variables <GETTING_STARTED.html#scality-access-key-id-and-scality-secret-access-key>`__):
|
||||
.. code:: shell
|
||||
|
||||
.. code-block:: shell
|
||||
docker run --name CloudServer -p 8000:8000
|
||||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
|
||||
-v ~/.aws/credentials:/root/.aws/credentials
|
||||
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
|
||||
-e SCALITY_ACCESS_KEY_ID=accessKey1
|
||||
-e SCALITY_SECRET_ACCESS_KEY=verySecretKey1
|
||||
-e S3DATA=multiple -e S3BACKEND=mem scality/s3server
|
||||
|
||||
$ docker run --name CloudServer -p 8000:8000 \
|
||||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
|
||||
-v ~/.aws/credentials:/root/.aws/credentials \
|
||||
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
|
||||
-e SCALITY_ACCESS_KEY_ID=accessKey1 \
|
||||
-e SCALITY_SECRET_ACCESS_KEY=verySecretKey1 \
|
||||
-e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
|
||||
In production with Docker hosted CloudServer
|
||||
--------------------------------------------
|
||||
|
||||
.. _in-production-w-a-Docker-hosted-cloudserver:
|
||||
In production, we expect that data will be persistent, that you will use the
|
||||
multiple backends capabilities of Zenko CloudServer, and that you will have a
|
||||
custom endpoint for your local storage, and custom credentials for your local
|
||||
storage:
|
||||
|
||||
In Production with a Docker-Hosted CloudServer
|
||||
----------------------------------------------
|
||||
.. code:: shell
|
||||
|
||||
Because data must persist in production settings, CloudServer offers
|
||||
multiple-backend capabilities. This requires a custom endpoint
|
||||
and custom credentials for local storage.
|
||||
|
||||
Customize these with:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ docker run -d --name CloudServer \
|
||||
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
|
||||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
|
||||
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
|
||||
-v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple \
|
||||
-e ENDPOINT=custom.endpoint.com \
|
||||
-p 8000:8000 -d zenko/cloudserver \
|
||||
docker run -d --name CloudServer
|
||||
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
|
||||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
|
||||
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
|
||||
-v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple
|
||||
-e ENDPOINT=custom.endpoint.com
|
||||
-p 8000:8000 -d scality/s3server
|
||||
|
|
|
@ -1,201 +1,193 @@
|
|||
Getting Started
|
||||
===============
|
||||
=================
|
||||
|
||||
.. figure:: ../res/scality-cloudserver-logo.png
|
||||
:alt: Zenko CloudServer logo
|
||||
|
||||
|
||||
Dependencies
|
||||
------------
|
||||
|
||||
Building and running the Scality Zenko CloudServer requires node.js 10.x and
|
||||
yarn v1.17.x. Up-to-date versions can be found at
|
||||
`Nodesource <https://github.com/nodesource/distributions>`__.
|
||||
|CircleCI| |Scality CI|
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
1. Clone the source code
|
||||
Dependencies
|
||||
~~~~~~~~~~~~
|
||||
|
||||
.. code-block:: shell
|
||||
Building and running the Scality Zenko CloudServer requires node.js 6.9.5 and
|
||||
npm v3 . Up-to-date versions can be found at
|
||||
`Nodesource <https://github.com/nodesource/distributions>`__.
|
||||
|
||||
$ git clone https://github.com/scality/cloudserver.git
|
||||
|
||||
2. Go to the cloudserver directory and use yarn to install the js dependencies.
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ cd cloudserver
|
||||
$ yarn install
|
||||
|
||||
Running CloudServer with a File Backend
|
||||
---------------------------------------
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ yarn start
|
||||
|
||||
This starts a Zenko CloudServer on port 8000. Two additional ports, 9990
|
||||
and 9991, are also open locally for internal transfer of metadata and
|
||||
data, respectively.
|
||||
|
||||
The default access key is accessKey1. The secret key is verySecretKey1.
|
||||
|
||||
By default, metadata files are saved in the localMetadata directory and
|
||||
data files are saved in the localData directory in the local ./cloudserver
|
||||
directory. These directories are pre-created within the repository. To
|
||||
save data or metadata in different locations, you must specify them using
|
||||
absolute paths. Thus, when starting the server:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ mkdir -m 700 $(pwd)/myFavoriteDataPath
|
||||
$ mkdir -m 700 $(pwd)/myFavoriteMetadataPath
|
||||
$ export S3DATAPATH="$(pwd)/myFavoriteDataPath"
|
||||
$ export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
|
||||
$ yarn start
|
||||
|
||||
Running CloudServer with Multiple Data Backends
|
||||
-----------------------------------------------
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ export S3DATA='multiple'
|
||||
$ yarn start
|
||||
|
||||
This starts a Zenko CloudServer on port 8000.
|
||||
|
||||
The default access key is accessKey1. The secret key is verySecretKey1.
|
||||
|
||||
With multiple backends, you can choose where each object is saved by setting
|
||||
the following header with a location constraint in a PUT request:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
|
||||
|
||||
If no header is sent with a PUT object request, the bucket’s location
|
||||
constraint determines where the data is saved. If the bucket has no
|
||||
location constraint, the endpoint of the PUT request determines location.
|
||||
|
||||
See the Configuration_ section to set location constraints.
|
||||
|
||||
Run CloudServer with an In-Memory Backend
|
||||
-----------------------------------------
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ yarn run mem_backend
|
||||
|
||||
This starts a Zenko CloudServer on port 8000.
|
||||
|
||||
The default access key is accessKey1. The secret key is verySecretKey1.
|
||||
|
||||
Run CloudServer with Vault User Management
|
||||
------------------------------------------
|
||||
Clone source code
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: shell
|
||||
|
||||
export S3VAULT=vault
|
||||
yarn start
|
||||
git clone https://github.com/scality/S3.git
|
||||
|
||||
Note: Vault is proprietary and must be accessed separately.
|
||||
This starts a Zenko CloudServer using Vault for user management.
|
||||
Install js dependencies
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Run CloudServer for Continuous Integration Testing or in Production with Docker
|
||||
-------------------------------------------------------------------------------
|
||||
Go to the ./S3 folder,
|
||||
|
||||
Run Cloudserver with `DOCKER <DOCKER.html>`__
|
||||
.. code:: shell
|
||||
|
||||
Testing
|
||||
~~~~~~~
|
||||
npm install
|
||||
|
||||
Run unit tests with the command:
|
||||
Run it with a file backend
|
||||
--------------------------
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ yarn test
|
||||
npm start
|
||||
|
||||
Run multiple-backend unit tests with:
|
||||
This starts an Zenko CloudServer on port 8000. Two additional ports 9990 and
|
||||
9991 are also open locally for internal transfer of metadata and data,
|
||||
respectively.
|
||||
|
||||
.. code-block:: shell
|
||||
The default access key is accessKey1 with a secret key of
|
||||
verySecretKey1.
|
||||
|
||||
$ CI=true S3DATA=multiple yarn start
|
||||
$ yarn run multiple_backend_test
|
||||
By default the metadata files will be saved in the localMetadata
|
||||
directory and the data files will be saved in the localData directory
|
||||
within the ./S3 directory on your machine. These directories have been
|
||||
pre-created within the repository. If you would like to save the data or
|
||||
metadata in different locations of your choice, you must specify them
|
||||
with absolute paths. So, when starting the server:
|
||||
|
||||
Run the linter with:
|
||||
.. code:: shell
|
||||
|
||||
.. code-block:: shell
|
||||
mkdir -m 700 $(pwd)/myFavoriteDataPath
|
||||
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
|
||||
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
|
||||
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
|
||||
npm start
|
||||
|
||||
$ yarn run lint
|
||||
Run it with multiple data backends
|
||||
----------------------------------
|
||||
|
||||
Running Functional Tests Locally
|
||||
.. code:: shell
|
||||
|
||||
export S3DATA='multiple'
|
||||
npm start
|
||||
|
||||
This starts an Zenko CloudServer on port 8000. The default access key is
|
||||
accessKey1 with a secret key of verySecretKey1.
|
||||
|
||||
With multiple backends, you have the ability to choose where each object
|
||||
will be saved by setting the following header with a locationConstraint
|
||||
on a PUT request:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
|
||||
|
||||
If no header is sent with a PUT object request, the location constraint
|
||||
of the bucket will determine where the data is saved. If the bucket has
|
||||
no location constraint, the endpoint of the PUT request will be used to
|
||||
determine location.
|
||||
|
||||
See the Configuration section below to learn how to set location
|
||||
constraints.
|
||||
|
||||
Run it with an in-memory backend
|
||||
--------------------------------
|
||||
|
||||
To pass AWS and Azure backend tests locally, modify
|
||||
tests/locationConfig/locationConfigTests.json so that ``awsbackend``
|
||||
specifies the bucketname of a bucket you have access to based on your
|
||||
credentials, and modify ``azurebackend`` with details for your Azure account.
|
||||
.. code:: shell
|
||||
|
||||
npm run mem_backend
|
||||
|
||||
This starts an Zenko CloudServer on port 8000. The default access key is
|
||||
accessKey1 with a secret key of verySecretKey1.
|
||||
|
||||
Run it for continuous integration testing or in production with Docker
|
||||
----------------------------------------------------------------------
|
||||
|
||||
`DOCKER <../DOCKER/>`__
|
||||
|
||||
Testing
|
||||
-------
|
||||
|
||||
You can run the unit tests with the following command:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
npm test
|
||||
|
||||
You can run the multiple backend unit tests with:
|
||||
|
||||
.. code:: shell
|
||||
CI=true S3DATA=multiple npm start
|
||||
npm run multiple_backend_test
|
||||
|
||||
You can run the linter with:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
npm run lint
|
||||
|
||||
Running functional tests locally:
|
||||
|
||||
For the AWS backend and Azure backend tests to pass locally,
|
||||
you must modify tests/locationConfigTests.json so that awsbackend
|
||||
specifies a bucketname of a bucket you have access to based on
|
||||
your credentials profile and modify "azurebackend" with details
|
||||
for your Azure account.
|
||||
|
||||
The test suite requires additional tools, **s3cmd** and **Redis**
|
||||
installed in the environment the tests are running in.
|
||||
|
||||
1. Install `s3cmd <http://s3tools.org/download>`__
|
||||
- Install `s3cmd <http://s3tools.org/download>`__
|
||||
- Install `redis <https://redis.io/download>`__ and start Redis.
|
||||
- Add localCache section to your ``config.json``:
|
||||
|
||||
2. Install `redis <https://redis.io/download>`__ and start Redis.
|
||||
|
||||
3. Add localCache section to ``config.json``:
|
||||
|
||||
.. code:: json
|
||||
::
|
||||
|
||||
"localCache": {
|
||||
"host": REDIS_HOST,
|
||||
"port": REDIS_PORT
|
||||
}
|
||||
|
||||
where ``REDIS_HOST`` is the Redis instance IP address (``"127.0.0.1"``
|
||||
if Redis is running locally) and ``REDIS_PORT`` is the Redis instance
|
||||
port (``6379`` by default)
|
||||
where ``REDIS_HOST`` is your Redis instance IP address (``"127.0.0.1"``
|
||||
if your Redis is running locally) and ``REDIS_PORT`` is your Redis
|
||||
instance port (``6379`` by default)
|
||||
|
||||
4. Add the following to the local etc/hosts file:
|
||||
- Add the following to the etc/hosts file on your machine:
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
|
||||
|
||||
5. Start Zenko CloudServer in memory and run the functional tests:
|
||||
- Start the Zenko CloudServer in memory and run the functional tests:
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ CI=true yarn run mem_backend
|
||||
$ CI=true yarn run ft_test
|
||||
|
||||
.. _Configuration:
|
||||
CI=true npm run mem_backend
|
||||
CI=true npm run ft_test
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
|
||||
There are three configuration files for Zenko CloudServer:
|
||||
There are three configuration files for your Scality Zenko CloudServer:
|
||||
|
||||
* ``conf/authdata.json``, for authentication.
|
||||
1. ``conf/authdata.json``, described above for authentication
|
||||
|
||||
* ``locationConfig.json``, to configure where data is saved.
|
||||
2. ``locationConfig.json``, to set up configuration options for
|
||||
|
||||
* ``config.json``, for general configuration options.
|
||||
where data will be saved
|
||||
|
||||
.. _location-configuration:
|
||||
3. ``config.json``, for general configuration options
|
||||
|
||||
Location Configuration
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
You must specify at least one locationConstraint in locationConfig.json
|
||||
(or leave it as pre-configured).
|
||||
You must specify at least one locationConstraint in your
|
||||
locationConfig.json (or leave as pre-configured).
|
||||
|
||||
You must also specify 'us-east-1' as a locationConstraint. If you put a
|
||||
bucket to an unknown endpoint and do not specify a locationConstraint in
|
||||
the PUT bucket call, us-east-1 is used.
|
||||
You must also specify 'us-east-1' as a locationConstraint so if you only
|
||||
define one locationConstraint, that would be it. If you put a bucket to
|
||||
an unknown endpoint and do not specify a locationConstraint in the put
|
||||
bucket call, us-east-1 will be used.
|
||||
|
||||
For instance, the following locationConstraint saves data sent to
|
||||
For instance, the following locationConstraint will save data sent to
|
||||
``myLocationConstraint`` to the file backend:
|
||||
|
||||
.. code:: json
|
||||
|
@ -206,16 +198,17 @@ For instance, the following locationConstraint saves data sent to
|
|||
"details": {}
|
||||
},
|
||||
|
||||
Each locationConstraint must include the ``type``, ``legacyAwsBehavior``,
|
||||
and ``details`` keys. ``type`` indicates which backend is used for that
|
||||
region. Supported backends are mem, file, and scality.``legacyAwsBehavior``
|
||||
indicates whether the region behaves the same as the AWS S3 'us-east-1'
|
||||
region. If the locationConstraint type is ``scality``, ``details`` must
|
||||
contain connector information for sproxyd. If the locationConstraint type
|
||||
is ``mem`` or ``file``, ``details`` must be empty.
|
||||
Each locationConstraint must include the ``type``,
|
||||
``legacyAwsBehavior``, and ``details`` keys. ``type`` indicates which
|
||||
backend will be used for that region. Currently, mem, file, and scality
|
||||
are the supported backends. ``legacyAwsBehavior`` indicates whether the
|
||||
region will have the same behavior as the AWS S3 'us-east-1' region. If
|
||||
the locationConstraint type is scality, ``details`` should contain
|
||||
connector information for sproxyd. If the locationConstraint type is mem
|
||||
or file, ``details`` should be empty.
|
||||
|
||||
Once locationConstraints is set in locationConfig.json, specify a default
|
||||
locationConstraint for each endpoint.
|
||||
Once you have your locationConstraints in your locationConfig.json, you
|
||||
can specify a default locationConstraint for each of your endpoints.
|
||||
|
||||
For instance, the following sets the ``localhost`` endpoint to the
|
||||
``myLocationConstraint`` data backend defined above:
|
||||
|
@ -226,24 +219,26 @@ For instance, the following sets the ``localhost`` endpoint to the
|
|||
"localhost": "myLocationConstraint"
|
||||
},
|
||||
|
||||
To use an endpoint other than localhost for Zenko CloudServer, the endpoint
|
||||
must be listed in ``restEndpoints``. Otherwise, if the server is running
|
||||
with a:
|
||||
If you would like to use an endpoint other than localhost for your
|
||||
Scality Zenko CloudServer, that endpoint MUST be listed in your
|
||||
``restEndpoints``. Otherwise if your server is running with a:
|
||||
|
||||
* **file backend**: The default location constraint is ``file``
|
||||
* **memory backend**: The default location constraint is ``mem``
|
||||
- **file backend**: your default location constraint will be ``file``
|
||||
|
||||
- **memory backend**: your default location constraint will be ``mem``
|
||||
|
||||
Endpoints
|
||||
~~~~~~~~~
|
||||
|
||||
The Zenko CloudServer supports endpoints that are rendered in either:
|
||||
Note that our Zenko CloudServer supports both:
|
||||
|
||||
* path style: http://myhostname.com/mybucket or
|
||||
* hosted style: http://mybucket.myhostname.com
|
||||
- path-style: http://myhostname.com/mybucket
|
||||
- hosted-style: http://mybucket.myhostname.com
|
||||
|
||||
However, if an IP address is specified for the host, hosted-style requests
|
||||
cannot reach the server. Use path-style requests in that case. For example,
|
||||
if you are using the AWS SDK for JavaScript, instantiate your client like this:
|
||||
However, hosted-style requests will not hit the server if you are using
|
||||
an ip address for your host. So, make sure you are using path-style
|
||||
requests in that case. For instance, if you are using the AWS SDK for
|
||||
JavaScript, you would instantiate your client like this:
|
||||
|
||||
.. code:: js
|
||||
|
||||
|
@ -252,99 +247,87 @@ if you are using the AWS SDK for JavaScript, instantiate your client like this:
|
|||
s3ForcePathStyle: true,
|
||||
});
|
||||
|
||||
Setting Your Own Access and Secret Key Pairs
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Setting your own access key and secret key pairs
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Credentials can be set for many accounts by editing ``conf/authdata.json``,
|
||||
but use the ``SCALITY_ACCESS_KEY_ID`` and ``SCALITY_SECRET_ACCESS_KEY``
|
||||
environment variables to specify your own credentials.
|
||||
|
||||
_`scality-access-key-id-and-scality-secret-access-key`
|
||||
You can set credentials for many accounts by editing
|
||||
``conf/authdata.json`` but if you want to specify one set of your own
|
||||
credentials, you can use ``SCALITY_ACCESS_KEY_ID`` and
|
||||
``SCALITY_SECRET_ACCESS_KEY`` environment variables.
|
||||
|
||||
SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
These variables specify authentication credentials for an account named
|
||||
“CustomAccount”.
|
||||
"CustomAccount".
|
||||
|
||||
.. note:: Anything in the ``authdata.json`` file is ignored.
|
||||
Note: Anything in the ``authdata.json`` file will be ignored.
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: shell
|
||||
|
||||
$ SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey yarn start
|
||||
SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey npm start
|
||||
|
||||
.. _Using_SSL:
|
||||
|
||||
Using SSL
|
||||
~~~~~~~~~
|
||||
Scality with SSL
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To use https with your local CloudServer, you must set up
|
||||
SSL certificates.
|
||||
If you wish to use https with your local Zenko CloudServer, you need to set up
|
||||
SSL certificates. Here is a simple guide of how to do it.
|
||||
|
||||
1. Deploy CloudServer using `our DockerHub page
|
||||
<https://hub.docker.com/r/zenko/cloudserver/>`__ (run it with a file
|
||||
backend).
|
||||
Deploying Zenko CloudServer
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. Note:: If Docker is not installed locally, follow the
|
||||
`instructions to install it for your distribution
|
||||
<https://docs.docker.com/engine/installation/>`__
|
||||
First, you need to deploy **Zenko CloudServer**. This can be done very easily
|
||||
via `our **DockerHub**
|
||||
page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
|
||||
with a file backend).
|
||||
|
||||
2. Update the CloudServer container’s config
|
||||
*Note:* *- If you don't have docker installed on your machine, here
|
||||
are the `instructions to install it for your
|
||||
distribution <https://docs.docker.com/engine/installation/>`__*
|
||||
|
||||
Add your certificates to your container. To do this,
|
||||
#. exec inside the CloudServer container.
|
||||
Updating your Zenko CloudServer container's config
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
#. Run ``$> docker ps`` to find the container’s ID (the corresponding
|
||||
image name is ``scality/cloudserver``.
|
||||
You're going to add your certificates to your container. In order to do
|
||||
so, you need to exec inside your Zenko CloudServer container. Run a
|
||||
``$> docker ps`` and find your container's id (the corresponding image
|
||||
name should be ``scality/s3server``. Copy the corresponding container id
|
||||
(here we'll use ``894aee038c5e``, and run:
|
||||
|
||||
#. Copy the corresponding container ID (``894aee038c5e`` in the present
|
||||
example), and run:
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: sh
|
||||
|
||||
$> docker exec -it 894aee038c5e bash
|
||||
|
||||
This puts you inside your container, using an interactive terminal.
|
||||
You're now inside your container, using an interactive terminal :)
|
||||
|
||||
3. Generate the SSL key and certificates. The paths where the different
|
||||
files are stored are defined after the ``-out`` option in each of the
|
||||
following commands.
|
||||
Generate SSL key and certificates
|
||||
**********************************
|
||||
|
||||
#. Generate a private key for your certificate signing request (CSR):
|
||||
There are 5 steps to this generation. The paths where the different
|
||||
files are stored are defined after the ``-out`` option in each command
|
||||
|
||||
.. code-block:: shell
|
||||
.. code:: sh
|
||||
|
||||
# Generate a private key for your CSR
|
||||
$> openssl genrsa -out ca.key 2048
|
||||
|
||||
#. Generate a self-signed certificate for your local certificate
|
||||
authority (CA):
|
||||
|
||||
.. code:: shell
|
||||
|
||||
# Generate a self signed certificate for your local Certificate Authority
|
||||
$> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
|
||||
|
||||
#. Generate a key for the CloudServer:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
# Generate a key for Zenko CloudServer
|
||||
$> openssl genrsa -out test.key 2048
|
||||
|
||||
#. Generate a CSR for CloudServer:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
# Generate a Certificate Signing Request for S3 Server
|
||||
$> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
|
||||
|
||||
#. Generate a certificate for CloudServer signed by the local CA:
|
||||
|
||||
.. code:: shell
|
||||
|
||||
# Generate a local-CA-signed certificate for S3 Server
|
||||
$> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
|
||||
|
||||
4. Update Zenko CloudServer ``config.json``. Add a ``certFilePaths``
|
||||
section to ``./config.json`` with appropriate paths:
|
||||
Update Zenko CloudServer ``config.json``
|
||||
**********************************
|
||||
|
||||
.. code:: json
|
||||
Add a ``certFilePaths`` section to ``./config.json`` with the
|
||||
appropriate paths:
|
||||
|
||||
.. code:: json
|
||||
|
||||
"certFilePaths": {
|
||||
"key": "./test.key",
|
||||
|
@ -352,36 +335,42 @@ SSL certificates.
|
|||
"ca": "./ca.crt"
|
||||
}
|
||||
|
||||
5. Run your container with the new config.
|
||||
Run your container with the new config
|
||||
****************************************
|
||||
|
||||
#. Exit the container by running ``$> exit``.
|
||||
First, you need to exit your container. Simply run ``$> exit``. Then,
|
||||
you need to restart your container. Normally, a simple
|
||||
``$> docker restart s3server`` should do the trick.
|
||||
|
||||
#. Restart the container with ``$> docker restart cloudserver``.
|
||||
Update your host config
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
6. Update the host configuration by adding s3.scality.test
|
||||
to /etc/hosts:
|
||||
Associates local IP addresses with hostname
|
||||
*******************************************
|
||||
|
||||
.. code:: bash
|
||||
In your ``/etc/hosts`` file on Linux, OS X, or Unix (with root
|
||||
permissions), edit the line of localhost so it looks like this:
|
||||
|
||||
::
|
||||
|
||||
127.0.0.1 localhost s3.scality.test
|
||||
|
||||
7. Copy the local certificate authority (ca.crt in step 4) from your
|
||||
container. Choose the path to save this file to (in the present
|
||||
example, ``/root/ca.crt``), and run:
|
||||
Copy the local certificate authority from your container
|
||||
*********************************************************
|
||||
|
||||
.. code:: shell
|
||||
In the above commands, it's the file named ``ca.crt``. Choose the path
|
||||
you want to save this file at (here we chose ``/root/ca.crt``), and run
|
||||
something like:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
|
||||
|
||||
.. note:: Your container ID will be different, and your path to
|
||||
ca.crt may be different.
|
||||
Test your config
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
Test the Config
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
If aws-sdk is not installed, run ``$> yarn install aws-sdk``.
|
||||
|
||||
Paste the following script into a file named "test.js":
|
||||
If you do not have aws-sdk installed, run ``$> npm install aws-sdk``. In
|
||||
a ``test.js`` file, paste the following script:
|
||||
|
||||
.. code:: js
|
||||
|
||||
|
@ -421,13 +410,8 @@ Paste the following script into a file named "test.js":
|
|||
});
|
||||
});
|
||||
|
||||
Now run this script with:
|
||||
|
||||
.. code::
|
||||
|
||||
$> nodejs test.js
|
||||
|
||||
On success, the script outputs ``SSL is cool!``.
|
||||
Now run that script with ``$> nodejs test.js``. If all goes well, it
|
||||
should output ``SSL is cool!``. Enjoy that added security!
|
||||
|
||||
|
||||
.. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg
|
||||
|
|
|
@ -4,34 +4,40 @@ Integrations
|
|||
High Availability
|
||||
=================
|
||||
|
||||
`Docker Swarm <https://docs.docker.com/engine/swarm/>`__ is a clustering tool
|
||||
developed by Docker for use with its containers. It can be used to start
|
||||
services, which we define to ensure CloudServer's continuous availability to
|
||||
end users. A swarm defines a manager and *n* workers among *n* + 1 servers.
|
||||
|
||||
This tutorial shows how to perform a basic setup with three servers, which
|
||||
provides strong service resiliency, while remaining easy to use and
|
||||
maintain. We will use NFS through Docker to share data and
|
||||
`Docker swarm <https://docs.docker.com/engine/swarm/>`__ is a
|
||||
clustering tool developped by Docker and ready to use with its
|
||||
containers. It allows to start a service, which we define and use as a
|
||||
means to ensure Zenko CloudServer's continuous availability to the end user.
|
||||
Indeed, a swarm defines a manager and n workers among n+1 servers. We
|
||||
will do a basic setup in this tutorial, with just 3 servers, which
|
||||
already provides a strong service resiliency, whilst remaining easy to
|
||||
do as an individual. We will use NFS through docker to share data and
|
||||
metadata between the different servers.
|
||||
|
||||
Sections are labeled **On Server**, **On Clients**, or
|
||||
**On All Machines**, referring respectively to NFS server, NFS clients, or
|
||||
NFS server and clients. In the present example, the server’s IP address is
|
||||
**10.200.15.113** and the client IP addresses are **10.200.15.96** and
|
||||
**10.200.15.97**
|
||||
You will see that the steps of this tutorial are defined as **On
|
||||
Server**, **On Clients**, **On All Machines**. This refers respectively
|
||||
to NFS Server, NFS Clients, or NFS Server and Clients. In our example,
|
||||
the IP of the Server will be **10.200.15.113**, while the IPs of the
|
||||
Clients will be **10.200.15.96 and 10.200.15.97**
|
||||
|
||||
1. Install Docker (on All Machines)
|
||||
Installing docker
|
||||
-----------------
|
||||
|
||||
Docker 17.03.0-ce is used for this tutorial. Docker 1.12.6 and later will
|
||||
likely work, but is not tested.
|
||||
Any version from docker 1.12.6 onwards should work; we used Docker
|
||||
17.03.0-ce for this tutorial.
|
||||
|
||||
* On Ubuntu 14.04
|
||||
Install Docker CE for Ubuntu as `documented at Docker
|
||||
<https://docs.docker.com/install/linux/docker-ce/ubuntu/>`__.
|
||||
Install the aufs dependency as recommended by Docker. The required
|
||||
commands are:
|
||||
On All Machines
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
On Ubuntu 14.04
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
The docker website has `solid
|
||||
documentation <https://docs.docker.com/engine/installation/linux/ubuntu/>`__.
|
||||
We have chosen to install the aufs dependency, as recommended by Docker.
|
||||
Here are the required commands:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> sudo apt-get update
|
||||
$> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
|
||||
|
@ -41,12 +47,14 @@ NFS server and clients. In the present example, the server’s IP address is
|
|||
$> sudo apt-get update
|
||||
$> sudo apt-get install docker-ce
|
||||
|
||||
* On CentOS 7
|
||||
Install Docker CE as `documented at Docker
|
||||
<https://docs.docker.com/install/linux/docker-ce/centos/>`__.
|
||||
The required commands are:
|
||||
On CentOS 7
|
||||
^^^^^^^^^^^
|
||||
|
||||
.. code:: sh
|
||||
The docker website has `solid
|
||||
documentation <https://docs.docker.com/engine/installation/linux/centos/>`__.
|
||||
Here are the required commands:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> sudo yum install -y yum-utils
|
||||
$> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
|
||||
|
@ -54,24 +62,31 @@ NFS server and clients. In the present example, the server’s IP address is
|
|||
$> sudo yum install docker-ce
|
||||
$> sudo systemctl start docker
|
||||
|
||||
2. Install NFS on Client(s)
|
||||
Configure NFS
|
||||
-------------
|
||||
|
||||
NFS clients mount Docker volumes over the NFS server’s shared folders.
|
||||
If the NFS commons are installed, manual mounts are no longer needed.
|
||||
On Clients
|
||||
~~~~~~~~~~
|
||||
|
||||
* On Ubuntu 14.04
|
||||
Your NFS Clients will mount Docker volumes over your NFS Server's shared
|
||||
folders. Hence, you don't have to mount anything manually, you just have
|
||||
to install the NFS commons:
|
||||
|
||||
Install the NFS commons with apt-get:
|
||||
On Ubuntu 14.04
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
.. code:: sh
|
||||
Simply install the NFS commons:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> sudo apt-get install nfs-common
|
||||
|
||||
* On CentOS 7
|
||||
On CentOS 7
|
||||
^^^^^^^^^^^
|
||||
|
||||
Install the NFS utils; then start required services:
|
||||
Install the NFS utils, and then start the required services:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> yum install nfs-utils
|
||||
$> sudo systemctl enable rpcbind
|
||||
|
@ -83,24 +98,29 @@ NFS server and clients. In the present example, the server’s IP address is
|
|||
$> sudo systemctl start nfs-lock
|
||||
$> sudo systemctl start nfs-idmap
|
||||
|
||||
3. Install NFS (on Server)
|
||||
On Server
|
||||
~~~~~~~~~
|
||||
|
||||
The NFS server hosts the data and metadata. The package(s) to install on it
|
||||
differs from the package installed on the clients.
|
||||
Your NFS Server will be the machine to physically host the data and
|
||||
metadata. The package(s) we will install on it is slightly different
|
||||
from the one we installed on the clients.
|
||||
|
||||
* On Ubuntu 14.04
|
||||
On Ubuntu 14.04
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
Install the NFS server-specific package and the NFS commons:
|
||||
Install the NFS server specific package and the NFS commons:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> sudo apt-get install nfs-kernel-server nfs-common
|
||||
|
||||
* On CentOS 7
|
||||
On CentOS 7
|
||||
^^^^^^^^^^^
|
||||
|
||||
Install the NFS utils and start the required services:
|
||||
Same steps as with the client: install the NFS utils and start the
|
||||
required services:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> yum install nfs-utils
|
||||
$> sudo systemctl enable rpcbind
|
||||
|
@ -112,88 +132,111 @@ NFS server and clients. In the present example, the server’s IP address is
|
|||
$> sudo systemctl start nfs-lock
|
||||
$> sudo systemctl start nfs-idmap
|
||||
|
||||
For both distributions:
|
||||
On Ubuntu 14.04 and CentOS 7
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
#. Choose where shared data and metadata from the local
|
||||
`CloudServer <http://www.zenko.io/cloudserver/>`__ shall be stored (The
|
||||
present example uses /var/nfs/data and /var/nfs/metadata). Set permissions
|
||||
for these folders for
|
||||
sharing over NFS:
|
||||
Choose where your shared data and metadata from your local `Zenko CloudServer
|
||||
<http://www.zenko.io/cloudserver/>`__ will be stored.
|
||||
We chose to go with /var/nfs/data and /var/nfs/metadata. You also need
|
||||
to set proper sharing permissions for these folders as they'll be shared
|
||||
over NFS:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> mkdir -p /var/nfs/data /var/nfs/metadata
|
||||
$> chmod -R 777 /var/nfs/
|
||||
|
||||
#. The /etc/exports file configures network permissions and r-w-x permissions
|
||||
for NFS access. Edit /etc/exports, adding the following lines:
|
||||
Now you need to update your **/etc/exports** file. This is the file that
|
||||
configures network permissions and rwx permissions for NFS access. By
|
||||
default, Ubuntu applies the no\_subtree\_check option, so we declared
|
||||
both folders with the same permissions, even though they're in the same
|
||||
tree:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> sudo vim /etc/exports
|
||||
|
||||
In this file, add the following lines:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
/var/nfs/data 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
|
||||
/var/nfs/metadata 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
|
||||
|
||||
Ubuntu applies the no\_subtree\_check option by default, so both
|
||||
folders are declared with the same permissions, even though they’re in
|
||||
the same tree.
|
||||
Export this new NFS table:
|
||||
|
||||
#. Export this new NFS table:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> sudo exportfs -a
|
||||
|
||||
#. Edit the ``MountFlags`` option in the Docker config in
|
||||
/lib/systemd/system/docker.service to enable NFS mount from Docker volumes
|
||||
on other machines:
|
||||
Eventually, you need to allow for NFS mount from Docker volumes on other
|
||||
machines. You need to change the Docker config in
|
||||
**/lib/systemd/system/docker.service**:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> sudo vim /lib/systemd/system/docker.service
|
||||
|
||||
In this file, change the **MountFlags** option:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
MountFlags=shared
|
||||
|
||||
#. Restart the NFS server and Docker daemons to apply these changes.
|
||||
Now you just need to restart the NFS server and docker daemons so your
|
||||
changes apply.
|
||||
|
||||
* On Ubuntu 14.04
|
||||
On Ubuntu 14.04
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
.. code:: sh
|
||||
Restart your NFS Server and docker services:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> sudo service nfs-kernel-server restart
|
||||
$> sudo service docker restart
|
||||
|
||||
* On CentOS 7
|
||||
On CentOS 7
|
||||
^^^^^^^^^^^
|
||||
|
||||
.. code:: sh
|
||||
Restart your NFS Server and docker daemons:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> sudo systemctl restart nfs-server
|
||||
$> sudo systemctl daemon-reload
|
||||
$> sudo systemctl restart docker
|
||||
|
||||
Set up your Docker Swarm service
|
||||
--------------------------------
|
||||
|
||||
4. Set Up a Docker Swarm
|
||||
On All Machines
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
* On all machines and distributions:
|
||||
On Ubuntu 14.04 and CentOS 7
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Set up the Docker volumes to be mounted to the NFS server for CloudServer’s
|
||||
data and metadata storage. The following commands must be replicated on all
|
||||
machines:
|
||||
We will now set up the Docker volumes that will be mounted to the NFS
|
||||
Server and serve as data and metadata storage for Zenko CloudServer. These two
|
||||
commands have to be replicated on all machines:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
|
||||
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
|
||||
|
||||
There is no need to ``docker exec`` these volumes to mount them: the
|
||||
Docker Swarm manager does this when the Docker service is started.
|
||||
There is no need to ""docker exec" these volumes to mount them: the
|
||||
Docker Swarm manager will do it when the Docker service will be started.
|
||||
|
||||
* On a server:
|
||||
On Server
|
||||
^^^^^^^^^
|
||||
|
||||
To start a Docker service on a Docker Swarm cluster, initialize the cluster
|
||||
(that is, define a manager), prompt workers/nodes to join in, and then start
|
||||
the service.
|
||||
To start a Docker service on a Docker Swarm cluster, you first have to
|
||||
initialize that cluster (i.e.: define a manager), then have the
|
||||
workers/nodes join in, and then start the service. Initialize the swarm
|
||||
cluster, and look at the response:
|
||||
|
||||
Initialize the swarm cluster, and review its response:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> docker swarm init --advertise-addr 10.200.15.113
|
||||
|
||||
|
@ -207,180 +250,204 @@ NFS server and clients. In the present example, the server’s IP address is
|
|||
|
||||
To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
|
||||
|
||||
* On clients:
|
||||
On Clients
|
||||
^^^^^^^^^^
|
||||
|
||||
Copy and paste the command provided by your Docker Swarm init. A successful
|
||||
request/response will resemble:
|
||||
Simply copy/paste the command provided by your docker swarm init. When
|
||||
all goes well, you'll get something like this:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
|
||||
|
||||
This node joined a swarm as a worker.
|
||||
|
||||
Set Up Docker Swarm on Clients on a Server
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
On Server
|
||||
^^^^^^^^^
|
||||
|
||||
Start the service on the Swarm cluster.
|
||||
Start the service on your swarm cluster!
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/cloudserver
|
||||
$> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/s3server
|
||||
|
||||
On a successful installation, ``docker service ls`` returns the following
|
||||
output:
|
||||
If you run a docker service ls, you should have the following output:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> docker service ls
|
||||
ID NAME MODE REPLICAS IMAGE
|
||||
ocmggza412ft s3 replicated 1/1 scality/cloudserver:latest
|
||||
ocmggza412ft s3 replicated 1/1 scality/s3server:latest
|
||||
|
||||
If the service does not start, consider disabling apparmor/SELinux.
|
||||
If your service won't start, consider disabling apparmor/SELinux.
|
||||
|
||||
Testing the High-Availability CloudServer
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Testing your High Availability S3Server
|
||||
---------------------------------------
|
||||
|
||||
On all machines (client/server) and distributions (Ubuntu and CentOS),
|
||||
determine where CloudServer is running using ``docker ps``. CloudServer can
|
||||
operate on any node of the Swarm cluster, manager or worker. When you find
|
||||
it, you can kill it with ``docker stop <container id>``. It will respawn
|
||||
on a different node. Now, if one server falls, or if Docker stops
|
||||
unexpectedly, the end user will still be able to access your the local CloudServer.
|
||||
|
||||
Troubleshooting
|
||||
On All Machines
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
To troubleshoot the service, run:
|
||||
On Ubuntu 14.04 and CentOS 7
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Try to find out where your Scality Zenko CloudServer is actually running using
|
||||
the **docker ps** command. It can be on any node of the swarm cluster,
|
||||
manager or worker. When you find it, you can kill it, with **docker stop
|
||||
<container id>** and you'll see it respawn on a different node of the
|
||||
swarm cluster. Now you see, if one of your servers falls, or if docker
|
||||
stops unexpectedly, your end user will still be able to access your
|
||||
local Zenko CloudServer.
|
||||
|
||||
Troubleshooting
|
||||
---------------
|
||||
|
||||
To troubleshoot the service you can run:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> docker service ps s3docker service ps s3
|
||||
ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR
|
||||
0ar81cw4lvv8chafm8pw48wbc s3.1 scality/cloudserver localhost.localdomain.localdomain Running Running 7 days ago
|
||||
cvmf3j3bz8w6r4h0lf3pxo6eu \_ s3.1 scality/cloudserver localhost.localdomain.localdomain Shutdown Failed 7 days ago "task: non-zero exit (137)"
|
||||
0ar81cw4lvv8chafm8pw48wbc s3.1 scality/s3server localhost.localdomain.localdomain Running Running 7 days ago
|
||||
cvmf3j3bz8w6r4h0lf3pxo6eu \_ s3.1 scality/s3server localhost.localdomain.localdomain Shutdown Failed 7 days ago "task: non-zero exit (137)"
|
||||
|
||||
If the error is truncated, view the error in detail by inspecting the
|
||||
Docker task ID:
|
||||
If the error is truncated it is possible to have a more detailed view of
|
||||
the error by inspecting the docker task ID:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu
|
||||
|
||||
Off you go!
|
||||
~~~~~~~~~~~
|
||||
-----------
|
||||
|
||||
Let us know what you use this functionality for, and if you'd like any
|
||||
specific developments around it. Or, even better: come and contribute to
|
||||
our `Github repository <https://github.com/scality/s3/>`__! We look
|
||||
forward to meeting you!
|
||||
|
||||
Let us know how you use this and if you'd like any specific developments
|
||||
around it. Even better: come and contribute to our `Github repository
|
||||
<https://github.com/scality/s3/>`__! We look forward to meeting you!
|
||||
|
||||
S3FS
|
||||
====
|
||||
|
||||
You can export buckets as a filesystem with s3fs on CloudServer.
|
||||
Export your buckets as a filesystem with s3fs on top of Zenko CloudServer
|
||||
|
||||
`s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source
|
||||
tool, available both on Debian and RedHat distributions, that enables
|
||||
you to mount an S3 bucket on a filesystem-like backend. This tutorial uses
|
||||
an Ubuntu 14.04 host to deploy and use s3fs over CloudServer.
|
||||
tool that allows you to mount an S3 bucket on a filesystem-like backend.
|
||||
It is available both on Debian and RedHat distributions. For this
|
||||
tutorial, we used an Ubuntu 14.04 host to deploy and use s3fs over
|
||||
Scality's Zenko CloudServer.
|
||||
|
||||
Deploying Zenko CloudServer with SSL
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
----------------------------
|
||||
|
||||
First, deploy CloudServer with a file backend using `our DockerHub page
|
||||
<https://hub.docker.com/r/zenko/cloudserver>`__.
|
||||
First, you need to deploy **Zenko CloudServer**. This can be done very easily
|
||||
via `our DockerHub
|
||||
page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
|
||||
with a file backend).
|
||||
|
||||
.. note::
|
||||
*Note:* *- If you don't have docker installed on your machine, here
|
||||
are the `instructions to install it for your
|
||||
distribution <https://docs.docker.com/engine/installation/>`__*
|
||||
|
||||
If Docker is not installed on your machine, follow
|
||||
`these instructions <https://docs.docker.com/engine/installation/>`__
|
||||
to install it for your distribution.
|
||||
You also necessarily have to set up SSL with Zenko CloudServer to use s3fs. We
|
||||
have a nice
|
||||
`tutorial <https://s3.scality.com/v1.0/page/scality-with-ssl>`__ to help
|
||||
you do it.
|
||||
|
||||
You must also set up SSL with CloudServer to use s3fs. See `Using SSL
|
||||
<./GETTING_STARTED#Using_SSL>`__ for instructions.
|
||||
|
||||
s3fs Setup
|
||||
~~~~~~~~~~
|
||||
s3fs setup
|
||||
----------
|
||||
|
||||
Installing s3fs
|
||||
---------------
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Follow the instructions in the s3fs `README
|
||||
<https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation-from-pre-built-packages>`__,
|
||||
s3fs has quite a few dependencies. As explained in their
|
||||
`README <https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation>`__,
|
||||
the following commands should install everything for Ubuntu 14.04:
|
||||
|
||||
Check that s3fs is properly installed. A version check should return
|
||||
a response resembling:
|
||||
.. code:: sh
|
||||
|
||||
$> sudo apt-get install automake autotools-dev g++ git libcurl4-gnutls-dev
|
||||
$> sudo apt-get install libfuse-dev libssl-dev libxml2-dev make pkg-config
|
||||
|
||||
Now you want to install s3fs per se:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> git clone https://github.com/s3fs-fuse/s3fs-fuse.git
|
||||
$> cd s3fs-fuse
|
||||
$> ./autogen.sh
|
||||
$> ./configure
|
||||
$> make
|
||||
$> sudo make install
|
||||
|
||||
Check that s3fs is properly installed by checking its version. it should
|
||||
answer as below:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> s3fs --version
|
||||
|
||||
Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL
|
||||
Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
|
||||
License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>
|
||||
This is free software: you are free to change and redistribute it.
|
||||
There is NO WARRANTY, to the extent permitted by law.
|
||||
|
||||
Configuring s3fs
|
||||
----------------
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
s3fs expects you to provide it with a password file. Our file is
|
||||
``/etc/passwd-s3fs``. The structure for this file is
|
||||
``ACCESSKEYID:SECRETKEYID``, so, for CloudServer, you can run:
|
||||
``ACCESSKEYID:SECRETKEYID``, so, for S3Server, you can run:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs
|
||||
$> chmod 600 /etc/passwd-s3fs
|
||||
|
||||
Using CloudServer with s3fs
|
||||
---------------------------
|
||||
Using Zenko CloudServer with s3fs
|
||||
------------------------
|
||||
|
||||
1. Use /mnt/tests3fs as a mount point.
|
||||
First, you're going to need a mountpoint; we chose ``/mnt/tests3fs``:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> mkdir /mnt/tests3fs
|
||||
|
||||
2. Create a bucket on your local CloudServer. In the present example it is
|
||||
named “tests3fs”.
|
||||
Then, you want to create a bucket on your local Zenko CloudServer; we named it
|
||||
``tests3fs``:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> s3cmd mb s3://tests3fs
|
||||
|
||||
3. Mount the bucket to your mount point with s3fs:
|
||||
*Note:* *- If you've never used s3cmd with our Zenko CloudServer, our README
|
||||
provides you with a `recommended
|
||||
config <https://github.com/scality/S3/blob/master/README.md#s3cmd>`__*
|
||||
|
||||
.. code:: sh
|
||||
Now you can mount your bucket to your mountpoint with s3fs:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style
|
||||
|
||||
The structure of this command is:
|
||||
``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``. Of these mandatory
|
||||
options:
|
||||
*If you're curious, the structure of this command is*
|
||||
``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``\ *, and the
|
||||
options are mandatory and serve the following purposes:
|
||||
* ``passwd_file``\ *: specifiy path to password file;
|
||||
* ``url``\ *: specify the hostname used by your SSL provider;
|
||||
* ``use_path_request_style``\ *: force path style (by default, s3fs
|
||||
uses subdomains (DNS style)).*
|
||||
|
||||
* ``passwd_file`` specifies the path to the password file.
|
||||
* ``url`` specifies the host name used by your SSL provider.
|
||||
* ``use_path_request_style`` forces the path style (by default,
|
||||
s3fs uses DNS-style subdomains).
|
||||
| From now on, you can either add files to your mountpoint, or add
|
||||
objects to your bucket, and they'll show in the other.
|
||||
| For example, let's' create two files, and then a directory with a file
|
||||
in our mountpoint:
|
||||
|
||||
Once the bucket is mounted, files added to the mount point or
|
||||
objects added to the bucket will appear in both locations.
|
||||
|
||||
Example
|
||||
-------
|
||||
|
||||
Create two files, and then a directory with a file in our mount point:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
|
||||
$> mkdir /mnt/tests3fs/dir1
|
||||
$> touch /mnt/tests3fs/dir1/file3
|
||||
|
||||
Now, use s3cmd to show what is in CloudServer:
|
||||
Now, I can use s3cmd to show me what is actually in S3Server:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: sh
|
||||
|
||||
$> s3cmd ls -r s3://tests3fs
|
||||
|
||||
|
@ -389,30 +456,27 @@ Example
|
|||
2017-02-28 17:28 0 s3://tests3fs/file1
|
||||
2017-02-28 17:28 0 s3://tests3fs/file2
|
||||
|
||||
Now you can enjoy a filesystem view on your local CloudServer.
|
||||
Now you can enjoy a filesystem view on your local Zenko CloudServer!
|
||||
|
||||
|
||||
Duplicity
|
||||
=========
|
||||
|
||||
How to back up your files with CloudServer.
|
||||
How to backup your files with Zenko CloudServer.
|
||||
|
||||
Installing Duplicity and its Dependencies
|
||||
Installing
|
||||
-----------
|
||||
|
||||
Installing Duplicity and its dependencies
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To install `Duplicity <http://duplicity.nongnu.org/>`__,
|
||||
go to `this site <https://code.launchpad.net/duplicity/0.7-series>`__.
|
||||
Download the latest tarball. Decompress it and follow the instructions
|
||||
in the README.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> tar zxvf duplicity-0.7.11.tar.gz
|
||||
$> cd duplicity-0.7.11
|
||||
$> python setup.py install
|
||||
|
||||
You may receive error messages indicating the need to install some or all
|
||||
of the following dependencies:
|
||||
Second, you want to install
|
||||
`Duplicity <http://duplicity.nongnu.org/index.html>`__. You have to
|
||||
download `this
|
||||
tarball <https://code.launchpad.net/duplicity/0.7-series/0.7.11/+download/duplicity-0.7.11.tar.gz>`__,
|
||||
decompress it, and then checkout the README inside, which will give you
|
||||
a list of dependencies to install. If you're using Ubuntu 14.04, this is
|
||||
your lucky day: here is a lazy step by step install.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
|
@ -420,20 +484,30 @@ of the following dependencies:
|
|||
$> apt-get install python-dev python-pip python-lockfile
|
||||
$> pip install -U boto
|
||||
|
||||
Testing the Installation
|
||||
------------------------
|
||||
Then you want to actually install Duplicity:
|
||||
|
||||
1. Check that CloudServer is running. Run ``$> docker ps``. You should
|
||||
see one container named ``scality/cloudserver``. If you do not, run
|
||||
``$> docker start cloudserver`` and check again.
|
||||
.. code:: sh
|
||||
|
||||
$> tar zxvf duplicity-0.7.11.tar.gz
|
||||
$> cd duplicity-0.7.11
|
||||
$> python setup.py install
|
||||
|
||||
2. Duplicity uses a module called “Boto” to send requests to S3. Boto
|
||||
requires a configuration file located in ``/etc/boto.cfg`` to store
|
||||
your credentials and preferences. A minimal configuration
|
||||
you can fine tune `following these instructions
|
||||
<http://boto.cloudhackers.com/en/latest/getting_started.html>`__ is
|
||||
shown here:
|
||||
Using
|
||||
------
|
||||
|
||||
Testing your installation
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
First, we're just going to quickly check that Zenko CloudServer is actually
|
||||
running. To do so, simply run ``$> docker ps`` . You should see one
|
||||
container named ``scality/s3server``. If that is not the case, try
|
||||
``$> docker start s3server``, and check again.
|
||||
|
||||
Secondly, as you probably know, Duplicity uses a module called **Boto**
|
||||
to send requests to S3. Boto requires a configuration file located in
|
||||
**``/etc/boto.cfg``** to have your credentials and preferences. Here is
|
||||
a minimalistic config `that you can finetune following these
|
||||
instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
|
||||
|
||||
::
|
||||
|
||||
|
@ -447,25 +521,26 @@ Testing the Installation
|
|||
# If using SSL, unmute and provide absolute path to local CA certificate
|
||||
# ca_certificates_file = /absolute/path/to/ca.crt
|
||||
|
||||
.. note:: To set up SSL with CloudServer, check out our `Using SSL
|
||||
<./GETTING_STARTED#Using_SSL>`__ in GETTING STARTED.
|
||||
*Note:* *If you want to set up SSL with Zenko CloudServer, check out our
|
||||
`tutorial <http://link/to/SSL/tutorial>`__*
|
||||
|
||||
3. At this point all requirements to run CloudServer as a backend to Duplicity
|
||||
have been met. A local folder/file should back up to the local S3.
|
||||
Try it with the decompressed Duplicity folder:
|
||||
At this point, we've met all the requirements to start running Zenko CloudServer
|
||||
as a backend to Duplicity. So we should be able to back up a local
|
||||
folder/file to local S3. Let's try with the duplicity decompressed
|
||||
folder:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
$> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/"
|
||||
|
||||
.. note:: Duplicity will prompt for a symmetric encryption passphrase.
|
||||
Save it carefully, as you will need it to recover your data.
|
||||
Alternatively, you can add the ``--no-encryption`` flag
|
||||
and the data will be stored plain.
|
||||
*Note:* *Duplicity will prompt you for a symmetric encryption
|
||||
passphrase. Save it somewhere as you will need it to recover your
|
||||
data. Alternatively, you can also add the ``--no-encryption`` flag
|
||||
and the data will be stored plain.*
|
||||
|
||||
If this command is successful, you will receive an output resembling:
|
||||
If this command is succesful, you will get an output looking like this:
|
||||
|
||||
.. code:: sh
|
||||
::
|
||||
|
||||
--------------[ Backup Statistics ]--------------
|
||||
StartTime 1486486547.13 (Tue Feb 7 16:55:47 2017)
|
||||
|
@ -485,13 +560,15 @@ Testing the Installation
|
|||
Errors 0
|
||||
-------------------------------------------------
|
||||
|
||||
Congratulations! You can now back up to your local S3 through Duplicity.
|
||||
Congratulations! You can now backup to your local S3 through duplicity
|
||||
:)
|
||||
|
||||
Automating Backups
|
||||
------------------
|
||||
Automating backups
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The easiest way to back up files periodically is to write a bash script
|
||||
and add it to your crontab. A suggested script follows.
|
||||
Now you probably want to back up your files periodically. The easiest
|
||||
way to do this is to write a bash script and add it to your crontab.
|
||||
Here is my suggestion for such a file:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
|
@ -500,33 +577,33 @@ and add it to your crontab. A suggested script follows.
|
|||
# Export your passphrase so you don't have to type anything
|
||||
export PASSPHRASE="mypassphrase"
|
||||
|
||||
# To use a GPG key, put it here and uncomment the line below
|
||||
# If you want to use a GPG Key, put it here and unmute the line below
|
||||
#GPG_KEY=
|
||||
|
||||
# Define your backup bucket, with localhost specified
|
||||
DEST="s3://127.0.0.1:8000/testbucketcloudserver/"
|
||||
DEST="s3://127.0.0.1:8000/testbuckets3server/"
|
||||
|
||||
# Define the absolute path to the folder to back up
|
||||
# Define the absolute path to the folder you want to backup
|
||||
SOURCE=/root/testfolder
|
||||
|
||||
# Set to "full" for full backups, and "incremental" for incremental backups
|
||||
# Warning: you must perform one full backup befor you can perform
|
||||
# Warning: you have to perform one full backup befor you can perform
|
||||
# incremental ones on top of it
|
||||
FULL=incremental
|
||||
|
||||
# How long to keep backups. If you don't want to delete old backups, keep
|
||||
# this value empty; otherwise, the syntax is "1Y" for one year, "1M" for
|
||||
# one month, "1D" for one day.
|
||||
# How long to keep backups for; if you don't want to delete old
|
||||
# backups, keep empty; otherwise, syntax is "1Y" for one year, "1M"
|
||||
# for one month, "1D" for one day
|
||||
OLDER_THAN="1Y"
|
||||
|
||||
# is_running checks whether Duplicity is currently completing a task
|
||||
# is_running checks whether duplicity is currently completing a task
|
||||
is_running=$(ps -ef | grep duplicity | grep python | wc -l)
|
||||
|
||||
# If Duplicity is already completing a task, this will not run
|
||||
# If duplicity is already completing a task, this will simply not run
|
||||
if [ $is_running -eq 0 ]; then
|
||||
echo "Backup for ${SOURCE} started"
|
||||
|
||||
# To delete backups older than a certain time, do it here
|
||||
# If you want to delete backups older than a certain time, we do it here
|
||||
if [ "$OLDER_THAN" != "" ]; then
|
||||
echo "Removing backups older than ${OLDER_THAN}"
|
||||
duplicity remove-older-than ${OLDER_THAN} ${DEST}
|
||||
|
@ -549,17 +626,17 @@ and add it to your crontab. A suggested script follows.
|
|||
# Forget the passphrase...
|
||||
unset PASSPHRASE
|
||||
|
||||
Put this file in ``/usr/local/sbin/backup.sh``. Run ``crontab -e`` and
|
||||
paste your configuration into the file that opens. If you're unfamiliar
|
||||
with Cron, here is a good `HowTo
|
||||
<https://help.ubuntu.com/community/CronHowto>`__. If the folder being
|
||||
backed up is a folder to be modified permanently during the work day,
|
||||
we can set incremental backups every 5 minutes from 8 AM to 9 PM Monday
|
||||
through Friday by pasting the following line into crontab:
|
||||
So let's say you put this file in ``/usr/local/sbin/backup.sh.`` Next
|
||||
you want to run ``crontab -e`` and paste your configuration in the file
|
||||
that opens. If you're unfamiliar with Cron, here is a good `How
|
||||
To <https://help.ubuntu.com/community/CronHowto>`__. The folder I'm
|
||||
backing up is a folder I modify permanently during my workday, so I want
|
||||
incremental backups every 5mn from 8AM to 9PM monday to friday. Here is
|
||||
the line I will paste in my crontab:
|
||||
|
||||
.. code:: sh
|
||||
.. code:: cron
|
||||
|
||||
*/5 8-20 * * 1-5 /usr/local/sbin/backup.sh
|
||||
|
||||
Adding or removing files from the folder being backed up will result in
|
||||
incremental backups in the bucket.
|
||||
Now I can try and add / remove files from the folder I'm backing up, and
|
||||
I will see incremental backups in my bucket.
|
||||
|
|
|
@ -1,263 +0,0 @@
|
|||
Metadata Search Documentation
|
||||
=============================
|
||||
|
||||
Description
|
||||
-----------
|
||||
|
||||
This feature enables metadata search to be performed on the metadata of objects
|
||||
stored in Zenko.
|
||||
|
||||
Requirements
|
||||
------------
|
||||
|
||||
* MongoDB
|
||||
|
||||
Design
|
||||
------
|
||||
|
||||
The Metadata Search feature expands on the existing :code:`GET Bucket` S3 API by
|
||||
enabling users to conduct metadata searches by adding the custom Zenko query
|
||||
string parameter, :code:`search`. The :code:`search` parameter is structured as a pseudo
|
||||
SQL WHERE clause, and supports basic SQL operators. For example:
|
||||
:code:`"A=1 AND B=2 OR C=3"` (complex queries can be built using nesting
|
||||
operators, :code:`(` and :code:`)`).
|
||||
|
||||
The search process is as follows:
|
||||
|
||||
* Zenko receives a :code:`GET` request.
|
||||
|
||||
.. code::
|
||||
|
||||
# regular getBucket request
|
||||
GET /bucketname HTTP/1.1
|
||||
Host: 127.0.0.1:8000
|
||||
Date: Wed, 18 Oct 2018 17:50:00 GMT
|
||||
Authorization: authorization string
|
||||
|
||||
# getBucket versions request
|
||||
GET /bucketname?versions HTTP/1.1
|
||||
Host: 127.0.0.1:8000
|
||||
Date: Wed, 18 Oct 2018 17:50:00 GMT
|
||||
Authorization: authorization string
|
||||
|
||||
# search getBucket request
|
||||
GET /bucketname?search=key%3Dsearch-item HTTP/1.1
|
||||
Host: 127.0.0.1:8000
|
||||
Date: Wed, 18 Oct 2018 17:50:00 GMT
|
||||
Authorization: authorization string
|
||||
|
||||
* If the request does *not* contain the :code:`search` query parameter, Zenko performs
|
||||
a normal bucket listing and returns an XML result containing the list of
|
||||
objects.
|
||||
* If the request *does* contain the :code:`search` query parameter, Zenko parses and
|
||||
validates the search string.
|
||||
|
||||
- If the search string is invalid, Zenko returns an :code:`InvalidArgument` error.
|
||||
|
||||
.. code::
|
||||
|
||||
<?xml version=\"1.0\" encoding=\"UTF-8\"?>
|
||||
<Error>
|
||||
<Code>InvalidArgument</Code>
|
||||
<Message>Invalid sql where clause sent as search query</Message>
|
||||
<Resource></Resource>
|
||||
<RequestId>d1d6afc64345a8e1198e</RequestId>
|
||||
</Error>
|
||||
|
||||
- If the search string is valid, Zenko parses it and generates an abstract
|
||||
syntax tree (AST). The AST is then passed to the MongoDB backend to be
|
||||
used as the query filter for retrieving objects from a bucket that
|
||||
satisfies the requested search conditions. Zenko parses the filtered
|
||||
results and returns them as the response.
|
||||
|
||||
Metadata search results have the same structure as a :code:`GET Bucket` response:
|
||||
|
||||
.. code:: xml
|
||||
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>bucketname</Name>
|
||||
<Prefix/>
|
||||
<Marker/>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>objectKey</Key>
|
||||
<LastModified>2018-04-19T18:31:49.426Z</LastModified>
|
||||
<ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
|
||||
<Size>0</Size>
|
||||
<Owner>
|
||||
<ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
|
||||
<DisplayName>Bart</DisplayName>
|
||||
</Owner>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
<Contents>
|
||||
...
|
||||
</Contents>
|
||||
</ListBucketResult>
|
||||
|
||||
Performing Metadata Searches with Zenko
|
||||
---------------------------------------
|
||||
|
||||
You can perform metadata searches by:
|
||||
|
||||
+ Using the :code:`search_bucket` tool in the
|
||||
`Scality/S3 <https://github.com/scality/S3>`_ GitHub repository.
|
||||
+ Creating a signed HTTP request to Zenko in your preferred programming
|
||||
language.
|
||||
|
||||
Using the S3 Tool
|
||||
+++++++++++++++++
|
||||
|
||||
After cloning the `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository
|
||||
and installing the necessary dependencies, run the following command in the S3
|
||||
project’s root directory to access the search tool:
|
||||
|
||||
.. code::
|
||||
|
||||
node bin/search_bucket
|
||||
|
||||
This generates the following output:
|
||||
|
||||
.. code::
|
||||
|
||||
Usage: search_bucket [options]
|
||||
|
||||
Options:
|
||||
|
||||
-V, --version output the version number
|
||||
-a, --access-key <accessKey> Access key id
|
||||
-k, --secret-key <secretKey> Secret access key
|
||||
-b, --bucket <bucket> Name of the bucket
|
||||
-q, --query <query> Search query
|
||||
-h, --host <host> Host of the server
|
||||
-p, --port <port> Port of the server
|
||||
-s --ssl
|
||||
-v, --verbose
|
||||
-h, --help output usage information
|
||||
|
||||
In the following examples, Zenko Server is accessible on endpoint
|
||||
:code:`http://127.0.0.1:8000` and contains the bucket :code:`zenkobucket`.
|
||||
|
||||
.. code::
|
||||
|
||||
# search for objects with metadata "blue"
|
||||
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
|
||||
-q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
|
||||
|
||||
# search for objects tagged with "type=color"
|
||||
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
|
||||
-q "tags.type=color" -h 127.0.0.1 -p 8000
|
||||
|
||||
Coding Examples
|
||||
+++++++++++++++
|
||||
|
||||
Search requests can be also performed by making HTTP requests authenticated
|
||||
with one of the AWS Signature schemes: version 2 or version 4. \
|
||||
For more about authentication scheme, see:
|
||||
|
||||
* https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
|
||||
* http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
|
||||
* http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
|
||||
|
||||
You can also view examples for making requests with Auth V4 in various
|
||||
languages `here <../../../examples>`__.
|
||||
|
||||
Specifying Metadata Fields
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To search system metadata headers:
|
||||
|
||||
.. code::
|
||||
|
||||
{system-metadata-key}{supported SQL op}{search value}
|
||||
# example
|
||||
key = blueObject
|
||||
size > 0
|
||||
key LIKE "blue.*"
|
||||
|
||||
To search custom user metadata:
|
||||
|
||||
.. code::
|
||||
|
||||
# metadata must be prefixed with "x-amz-meta-"
|
||||
x-amz-meta-{user-metadata-key}{supported SQL op}{search value}
|
||||
|
||||
# example
|
||||
x-amz-meta-color = blue
|
||||
x-amz-meta-color != red
|
||||
x-amz-meta-color LIKE "b.*"
|
||||
|
||||
To search tags:
|
||||
|
||||
.. code::
|
||||
|
||||
# tag searches must be prefixed with "tags."
|
||||
tags.{tag-key}{supported SQL op}{search value}
|
||||
# example
|
||||
tags.type = color
|
||||
|
||||
Examples queries:
|
||||
|
||||
.. code::
|
||||
|
||||
# searching for objects with custom metadata "color"=blue" and are tagged
|
||||
# "type"="color"
|
||||
|
||||
tags.type="color" AND x-amz-meta-color="blue"
|
||||
|
||||
# searching for objects with the object key containing the substring "blue"
|
||||
# or (custom metadata "color"=blue" and are tagged "type"="color")
|
||||
|
||||
key LIKE '.*blue.*' OR (x-amz-meta-color="blue" AND tags.type="color")
|
||||
|
||||
Differences from SQL
|
||||
++++++++++++++++++++
|
||||
|
||||
Zenko metadata search queries are similar to SQL-query :code:`WHERE` clauses, but
|
||||
differ in that:
|
||||
|
||||
* They follow the :code:`PCRE` format
|
||||
* They do not require values with hyphens to be enclosed in
|
||||
backticks, :code:``(`)``
|
||||
|
||||
.. code::
|
||||
|
||||
# SQL query
|
||||
`x-amz-meta-search-item` = `ice-cream-cone`
|
||||
|
||||
# MD Search query
|
||||
x-amz-meta-search-item = ice-cream-cone
|
||||
|
||||
* Search queries do not support all SQL operators.
|
||||
|
||||
.. code::
|
||||
|
||||
# Supported SQL operators:
|
||||
=, <, >, <=, >=, !=, AND, OR, LIKE, <>
|
||||
|
||||
# Unsupported SQL operators:
|
||||
NOT, BETWEEN, IN, IS, +, -, %, ^, /, *, !
|
||||
|
||||
Using Regular Expressions in Metadata Search
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Regular expressions in Zenko metadata search differ from SQL in the following
|
||||
ways:
|
||||
|
||||
+ Wildcards are represented with :code:`.*` instead of :code:`%`.
|
||||
+ Regex patterns must be wrapped in quotes. Failure to do this can lead to
|
||||
misinterpretation of patterns.
|
||||
+ As with :code:`PCRE`, regular expressions can be entered in either the
|
||||
:code:`/pattern/` syntax or as the pattern itself if regex options are
|
||||
not required.
|
||||
|
||||
Example regular expressions:
|
||||
|
||||
.. code::
|
||||
|
||||
# search for strings containing word substring "helloworld"
|
||||
".*helloworld.*"
|
||||
"/.*helloworld.*/"
|
||||
"/.*helloworld.*/i"
|
|
@ -1,21 +0,0 @@
|
|||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line.
|
||||
SPHINXOPTS =
|
||||
SPHINXBUILD = sphinx-build
|
||||
SPHINXPROJ = Zenko
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = _build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
# Cloudserver Release Plan
|
||||
|
||||
## Docker Image Generation
|
||||
|
||||
Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
|
||||
CloudServer has a few images there:
|
||||
|
||||
* Cloudserver container image: ghcr.io/scality/cloudserver
|
||||
* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
|
||||
* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
|
||||
|
||||
With every CI build, the CI will push images, tagging the
|
||||
content with the developer branch's short SHA-1 commit hash.
|
||||
This allows those images to be used by developers, CI builds,
|
||||
build chain and so on.
|
||||
|
||||
Tagged versions of cloudserver will be stored in the production namespace.
|
||||
|
||||
## How to Pull Docker Images
|
||||
|
||||
```sh
|
||||
docker pull ghcr.io/scality/cloudserver:<commit hash>
|
||||
docker pull ghcr.io/scality/cloudserver:<tag>
|
||||
```
|
||||
|
||||
## Release Process
|
||||
|
||||
To release a production image:
|
||||
|
||||
* Create a PR to bump the package version
|
||||
Update Cloudserver's `package.json` by bumping it to the relevant next
|
||||
version in a new PR. Per example if the last released version was
|
||||
`8.4.7`, the next version would be `8.4.8`.
|
||||
|
||||
```js
|
||||
{
|
||||
"name": "cloudserver",
|
||||
"version": "8.4.8", <--- Here
|
||||
[...]
|
||||
}
|
||||
```
|
||||
|
||||
* Review & merge the PR
|
||||
|
||||
* Create the release on GitHub
|
||||
|
||||
* Go the Release tab (https://github.com/scality/cloudserver/releases);
|
||||
* Click on the `Draft new release button`;
|
||||
* In the `tag` field, type the name of the release (`8.4.8`), and confirm
|
||||
to create the tag on publish;
|
||||
* Click on `Generate release notes` button to fill the fields;
|
||||
* Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
|
||||
* Click to `Publish the release` to create the GitHub release and git tag
|
||||
|
||||
Notes:
|
||||
* the Git tag will be created automatically.
|
||||
* this should be done as soon as the PR is merged, so that the tag
|
||||
is put on the "version bump" commit.
|
||||
|
||||
* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
|
||||
|
||||
* Branch Name: The one used for the tag earlier. In this example `development/8.4`
|
||||
* Override Stage: 'release'
|
||||
* Extra properties:
|
||||
* name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
|
||||
|
||||
* Release the release version on Jira
|
||||
|
||||
* Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
|
||||
* Create a next version
|
||||
* Name: `[next version]`, in this example `8.4.9`
|
||||
* Click `...` and select `Release` on the recently released version (`8.4.8`)
|
||||
* Fill in the field to move incomplete version to the next one
|
|
@ -1,12 +1,10 @@
|
|||
.. _use-public-cloud:
|
||||
|
||||
Using Public Clouds as data backends
|
||||
====================================
|
||||
|
||||
Introduction
|
||||
------------
|
||||
|
||||
As stated in our `GETTING STARTED guide <GETTING_STARTED.html#location-configuration>`__,
|
||||
As stated in our `GETTING STARTED guide <../GETTING_STARTED/#location-configuration>`__,
|
||||
new data backends can be added by creating a region (also called location
|
||||
constraint) with the right endpoint and credentials.
|
||||
This section of the documentation shows you how to set up our currently
|
||||
|
@ -162,8 +160,8 @@ CloudServer.
|
|||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
|
||||
-v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
|
||||
-v ~/.aws/credentials:/root/.aws/credentials \
|
||||
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 \
|
||||
-d scality/cloudserver
|
||||
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
|
||||
-d scality/s3server
|
||||
|
||||
Testing: put an object to AWS S3 using CloudServer
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
@ -245,7 +243,7 @@ There are a few configurable options here:
|
|||
this region should behave like any other AWS S3 region (in the case of MS Azure
|
||||
hosted data, this is mostly relevant for the format of errors);
|
||||
- :code:`azureStorageEndpoint` : set to your storage account's endpoint, usually
|
||||
:code:`https://{{storageAccountName}}.blob.core.windows.net`;
|
||||
:code:`https://{{storageAccountName}}.blob.core.windows.name`;
|
||||
- :code:`azureContainerName` : set to an *existing container* in your MS Azure
|
||||
storage account; this is the container in which your data will be stored for
|
||||
this location constraint;
|
||||
|
@ -324,7 +322,7 @@ CloudServer.
|
|||
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
|
||||
-v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
|
||||
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
|
||||
-d scality/cloudserver
|
||||
-d scality/s3server
|
||||
|
||||
Testing: put an object to MS Azure using CloudServer
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
|
@ -1,79 +0,0 @@
|
|||
============================================
|
||||
Add New Backend Storage To Zenko CloudServer
|
||||
============================================
|
||||
|
||||
This set of documents aims at bootstrapping developers with Zenko's CloudServer
|
||||
module, so they can then go on and contribute features.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
non-s3-compatible-backend
|
||||
s3-compatible-backend
|
||||
|
||||
We always encourage our community to offer new extensions to Zenko,
|
||||
and new backend support is paramount to meeting more community needs.
|
||||
If that is something you want to contribute (or just do on your own
|
||||
version of the cloudserver image), this is the guid to read. Please
|
||||
make sure you follow our `Contributing Guidelines`_/.
|
||||
|
||||
If you need help with anything, please search our `forum`_ for more
|
||||
information.
|
||||
|
||||
Add support for a new backend
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Currently the main public cloud protocols are supported and more can
|
||||
be added. There are two main types of backend: those compatible with
|
||||
Amazon's S3 protocol and those not compatible.
|
||||
|
||||
================= ========== ============ ===========
|
||||
Backend type Supported Active WIP Not started
|
||||
================= ========== ============ ===========
|
||||
Private disk/fs x
|
||||
AWS S3 x
|
||||
Microsoft Azure x
|
||||
Backblaze B2 x
|
||||
Google Cloud x
|
||||
Openstack Swift x
|
||||
================= ========== ============ ===========
|
||||
|
||||
.. important:: Should you want to request for a new backend to be
|
||||
supported, please do so by opening a `Github issue`_,
|
||||
and filling out the "Feature Request" section of our
|
||||
template.
|
||||
|
||||
To add support for a new backend support to CloudServer official
|
||||
repository, please follow these steps:
|
||||
|
||||
- familiarize yourself with our `Contributing Guidelines`_
|
||||
- open a `Github issue`_ and fill out Feature Request form, and
|
||||
specify you would like to contribute it yourself;
|
||||
- wait for our core team to get back to you with an answer on whether
|
||||
we are interested in taking that contribution in (and hence
|
||||
committing to maintaining it over time);
|
||||
- once approved, fork the repository and start your development;
|
||||
- use the `forum`_ with any question you may have during the
|
||||
development process;
|
||||
- when you think it's ready, let us know so that we create a feature
|
||||
branch against which we'll compare and review your code;
|
||||
- open a pull request with your changes against that dedicated feature
|
||||
branch;
|
||||
- once that pull request gets merged, you're done.
|
||||
|
||||
.. tip::
|
||||
|
||||
While we do take care of the final rebase (when we merge your feature
|
||||
branch on the latest default branch), we do ask that you keep up to date with our latest default branch
|
||||
until then.
|
||||
|
||||
.. important::
|
||||
|
||||
If we do not approve your feature request, you may of course still
|
||||
work on supporting a new backend: all our "no" means is that we do not
|
||||
have the resources, as part of our core development team, to maintain
|
||||
this feature for the moment.
|
||||
|
||||
.. _GitHub issue: https://github.com/scality/S3/issues
|
||||
.. _Contributing Guidelines: https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md
|
||||
.. _forum: https://forum.zenko.io
|
|
@ -1,53 +0,0 @@
|
|||
=================
|
||||
Add A New Backend
|
||||
=================
|
||||
|
||||
Supporting all possible public cloud storage APIs is CloudServer's
|
||||
ultimate goal. As an open source project, contributions are welcome.
|
||||
|
||||
The first step is to get familiar with building a custom Docker image
|
||||
for CloudServer.
|
||||
|
||||
Build a Custom Docker Image
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Clone Zenko's CloudServer, install all dependencies and start the
|
||||
service:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ git clone https://github.com/scality/cloudserver
|
||||
$ cd cloudserver
|
||||
$ yarn install
|
||||
$ yarn start
|
||||
|
||||
.. tip::
|
||||
|
||||
Some optional dependencies may fail, resulting in you seeing `yarn
|
||||
WARN` messages; these can safely be ignored. Refer to the User
|
||||
documentation for all available options.
|
||||
|
||||
Build the Docker image:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
# docker build . -t
|
||||
# {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
|
||||
|
||||
Push the newly created Docker image to your own hub:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
# docker push
|
||||
# {{YOUR_DOCKERHUB_ACCOUNT}}/cloudserver:{{OPTIONAL_VERSION_TAG}}
|
||||
|
||||
.. note::
|
||||
|
||||
To perform this last operation, you need to be authenticated with DockerHub
|
||||
|
||||
There are two main types of backend you could want Zenko to support:
|
||||
|
||||
== link:S3_COMPATIBLE_BACKENDS.adoc[S3 compatible data backends]
|
||||
|
||||
== link:NON_S3_COMPATIBLE_BACKENDS.adoc[Data backends using another protocol than the S3 protocol]
|
||||
|
|
@ -1,530 +0,0 @@
|
|||
==========================================================
|
||||
Adding support for data backends not supporting the S3 API
|
||||
==========================================================
|
||||
|
||||
These backends abstract the complexity of multiple APIs to let users
|
||||
work on a single common namespace across multiple clouds.
|
||||
|
||||
This documents aims at introducing you to the right files in
|
||||
CloudServer (the Zenko stack's subcomponent in charge of API
|
||||
translation, among other things) to add support to your own backend of
|
||||
choice.
|
||||
|
||||
General configuration
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
There are a number of constants and environment variables to define to support a
|
||||
new data backend; here is a list and where to find them:
|
||||
|
||||
:file:`/constants.js`
|
||||
---------------------
|
||||
|
||||
* give your backend type a name, as part of the `externalBackends` object;
|
||||
* specify whether versioning is implemented, as part of the
|
||||
`versioningNotImplemented` object;
|
||||
|
||||
:file:`/lib/Config.js`
|
||||
----------------------
|
||||
|
||||
* this is where you should put common utility functions, like the ones to parse
|
||||
the location object from `locationConfig.json`;
|
||||
* make sure you define environment variables (like `GCP_SERVICE_EMAIL` as we'll
|
||||
use those internally for the CI to test against the real remote backend;
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}Client.js`
|
||||
---------------------------------------------------
|
||||
|
||||
* this file is where you'll instantiate your backend client; this should be a
|
||||
class with a constructor taking the config object built in `/lib/Config.js` as
|
||||
parameter;
|
||||
* over time, you may need some utility functions which we've defined in the
|
||||
folder `/api/apiUtils`, and in the file `/lib/data/external/utils`;
|
||||
|
||||
:file:`/lib/data/external/utils.js`
|
||||
-----------------------------------
|
||||
|
||||
* make sure to add options for `sourceLocationConstraintType` to be equal to
|
||||
the name you gave your backend in :file:`/constants.js`;
|
||||
|
||||
:file:`/lib/data/external/{{BackendName}}_lib/`
|
||||
-----------------------------------------------
|
||||
|
||||
* this folder is where you'll put the functions needed for supporting your
|
||||
backend; keep your files as atomic as possible;
|
||||
|
||||
:file:`/tests/locationConfig/locationConfigTests.json`
|
||||
------------------------------------------------------
|
||||
|
||||
* this file is where you'll create location profiles to be used by your
|
||||
functional tests;
|
||||
|
||||
:file:`/lib/data/locationConstraintParser.js`
|
||||
---------------------------------------------
|
||||
|
||||
* this is where you'll instantiate your client if the operation the end user
|
||||
sent effectively writes to your backend; everything happens inside the
|
||||
function `parseLC()`; you should add a condition that executes if
|
||||
`locationObj.type` is the name of your backend (that you defined in
|
||||
`constants.js`), and instantiates a client of yours. See pseudocode below,
|
||||
assuming location type name is `ztore`:
|
||||
|
||||
|
||||
.. code-block:: js
|
||||
:linenos:
|
||||
:emphasize-lines: 12
|
||||
|
||||
(...) //<1>
|
||||
const ZtoreClient = require('./external/ZtoreClient');
|
||||
const { config } = require('../Config'); //<1>
|
||||
|
||||
function parseLC(){ //<1>
|
||||
(...) //<1>
|
||||
Object.keys(config.locationConstraints).forEach(location => { //<1>
|
||||
const locationObj = config.locationConstraints[location]; //<1>
|
||||
(...) //<1>
|
||||
if (locationObj.type === 'ztore' {
|
||||
const ztoreEndpoint = config.getZtoreEndpoint(location);
|
||||
const ztoreCredentials = config.getZtoreCredentials(location); //<2>
|
||||
clients[location] = new ZtoreClient({
|
||||
ztoreEndpoint,
|
||||
ztoreCredentials,
|
||||
ztoreBucketname: locationObj.details.ztoreBucketName,
|
||||
bucketMatch: locationObj.details.BucketMatch,
|
||||
dataStoreName: location,
|
||||
}); //<3>
|
||||
clients[location].clientType = 'ztore';
|
||||
});
|
||||
(...) //<1>
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
1. Code that is already there
|
||||
2. You may need more utility functions depending on your backend specs
|
||||
3. You may have more fields required in your constructor object depending on
|
||||
your backend specs
|
||||
|
||||
Operation of type PUT
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
PUT routes are usually where people get started, as it's the easiest to check!
|
||||
Simply go on your remote backend console and you'll be able to see whether your
|
||||
object actually went up in the cloud...
|
||||
|
||||
These are the files you'll need to edit:
|
||||
|
||||
:file:`/lib/data/external/{{BackendName}}Client.js`
|
||||
---------------------------------------------------
|
||||
|
||||
- the function that is going to call your `put()` function is also called
|
||||
`put()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
|
||||
- define a function with signature like
|
||||
`put(stream, size, keyContext, reqUids, callback)`; this is worth exploring a
|
||||
bit more as these parameters are the same for all backends:
|
||||
//TODO: generate this from jsdoc
|
||||
|
||||
- `stream`: the stream of data you want to put in the cloud; if you're
|
||||
unfamiliar with node.js streams, we suggest you start training, as we use
|
||||
them a lot !
|
||||
|
||||
- `size`: the size of the object you're trying to put;
|
||||
|
||||
- `keyContext`: an object with metadata about the operation; common entries are
|
||||
`namespace`, `buckerName`, `owner`, `cipherBundle`, and `tagging`; if these
|
||||
are not sufficient for your integration, contact us to get architecture
|
||||
validation before adding new entries;
|
||||
|
||||
- `reqUids`: the request unique ID used for logging;
|
||||
|
||||
- `callback`: your function's callback (should handle errors);
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}_lib/`
|
||||
-----------------------------------------------
|
||||
|
||||
- this is where you should put all utility functions for your PUT operation, and
|
||||
then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
|
||||
your code clean;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/put/put{{BackendName}}js`
|
||||
-----------------------------------------------------------------------------------
|
||||
|
||||
- every contribution should come with thorough functional tests, showing
|
||||
nominal context gives expected behaviour, and error cases are handled in a way
|
||||
that is standard with the backend (including error messages and code);
|
||||
- the ideal setup is if you simulate your backend locally, so as not to be
|
||||
subjected to network flakiness in the CI; however, we know there might not be
|
||||
mockups available for every client; if that is the case of your backend, you
|
||||
may test against the "real" endpoint of your data backend;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
|
||||
-------------------------------------------------------------------
|
||||
|
||||
- where you'll define a constant for your backend location matching your
|
||||
:file:`/tests/locationConfig/locationConfigTests.json`
|
||||
- depending on your backend, the sample `keys[]` and associated made up objects
|
||||
may not work for you (if your backend's key format is different, for example);
|
||||
if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
|
||||
function returning ajusted `keys[]` to your tests.
|
||||
|
||||
Operation of type GET
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
GET routes are easy to test after PUT routes are implemented, hence why we're
|
||||
covering them second.
|
||||
|
||||
These are the files you'll need to edit:
|
||||
|
||||
:file:`/lib/data/external/{{BackendName}}Client.js`
|
||||
---------------------------------------------------
|
||||
|
||||
- the function that is going to call your `get()` function is also called
|
||||
`get()`, and it's defined in `/lib/data/multipleBackendGateway.js`;
|
||||
- define a function with signature like
|
||||
`get(objectGetInfo, range, reqUids, callback)`; this is worth exploring a
|
||||
bit more as these parameters are the same for all backends:
|
||||
|
||||
//TODO: generate this from jsdoc
|
||||
|
||||
- `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
|
||||
data store, and `client`, the data store name;
|
||||
|
||||
- `range`: the range of bytes you will get, for "get-by-range" operations (we
|
||||
recommend you do simple GETs first, and then look at this);
|
||||
|
||||
- `reqUids`: the request unique ID used for logging;
|
||||
|
||||
- `callback`: your function's callback (should handle errors);
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}_lib/`
|
||||
-----------------------------------------------
|
||||
|
||||
- this is where you should put all utility functions for your GET operation, and
|
||||
then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
|
||||
your code clean;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
|
||||
-----------------------------------------------------------------------------------
|
||||
|
||||
- every contribution should come with thorough functional tests, showing
|
||||
nominal context gives expected behaviour, and error cases are handled in a way
|
||||
that is standard with the backend (including error messages and code);
|
||||
- the ideal setup is if you simulate your backend locally, so as not to be
|
||||
subjected to network flakiness in the CI; however, we know there might not be
|
||||
mockups available for every client; if that is the case of your backend, you
|
||||
may test against the "real" endpoint of your data backend;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
|
||||
-------------------------------------------------------------------
|
||||
|
||||
.. note:: You should need this section if you have followed the tutorial in order
|
||||
(that is, if you have covered the PUT operation already)
|
||||
|
||||
- where you'll define a constant for your backend location matching your
|
||||
:file:`/tests/locationConfig/locationConfigTests.json`
|
||||
- depending on your backend, the sample `keys[]` and associated made up objects
|
||||
may not work for you (if your backend's key format is different, for example);
|
||||
if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
|
||||
|
||||
Operation of type DELETE
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
DELETE routes are easy to test after PUT routes are implemented, and they are
|
||||
similar to GET routes in our implementation, hence why we're covering them
|
||||
third.
|
||||
|
||||
These are the files you'll need to edit:
|
||||
|
||||
:file:`/lib/data/external/{{BackendName}}Client.js`
|
||||
---------------------------------------------------
|
||||
|
||||
- the function that is going to call your `delete()` function is also called
|
||||
`delete()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
|
||||
- define a function with signature like
|
||||
`delete(objectGetInfo, reqUids, callback)`; this is worth exploring a
|
||||
bit more as these parameters are the same for all backends:
|
||||
|
||||
//TODO: generate this from jsdoc
|
||||
* `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
|
||||
data store, and `client`, the data store name;
|
||||
* `reqUids`: the request unique ID used for logging;
|
||||
* `callback`: your function's callback (should handle errors);
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}_lib/`
|
||||
-----------------------------------------------
|
||||
|
||||
- this is where you should put all utility functions for your DELETE operation,
|
||||
and then import then in `/lib/data/external/{{BackendName}}Client.js`, to keep
|
||||
your code clean;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/delete/delete{{BackendName}}js`
|
||||
-----------------------------------------------------------------------------------------
|
||||
|
||||
- every contribution should come with thorough functional tests, showing
|
||||
nominal context gives expected behaviour, and error cases are handled in a way
|
||||
that is standard with the backend (including error messages and code);
|
||||
- the ideal setup is if you simulate your backend locally, so as not to be
|
||||
subjected to network flakiness in the CI; however, we know there might not be
|
||||
mockups available for every client; if that is the case of your backend, you
|
||||
may test against the "real" endpoint of your data backend;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
|
||||
-------------------------------------------------------------------
|
||||
|
||||
.. note:: You should need this section if you have followed the
|
||||
tutorial in order (that is, if you have covered the PUT operation
|
||||
already)
|
||||
|
||||
- where you'll define a constant for your backend location matching your
|
||||
:file:`/tests/locationConfig/locationConfigTests.json`
|
||||
- depending on your backend, the sample `keys[]` and associated made up objects
|
||||
may not work for you (if your backend's key format is different, for example);
|
||||
if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
|
||||
|
||||
Operation of type HEAD
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
HEAD routes are very similar to DELETE routes in our implementation, hence why
|
||||
we're covering them fourth.
|
||||
|
||||
These are the files you'll need to edit:
|
||||
|
||||
:file:`/lib/data/external/{{BackendName}}Client.js`
|
||||
---------------------------------------------------
|
||||
|
||||
- the function that is going to call your `head()` function is also called
|
||||
`head()`, and it's defined in :file:`/lib/data/multipleBackendGateway.js`;
|
||||
- define a function with signature like
|
||||
`head(objectGetInfo, reqUids, callback)`; this is worth exploring a
|
||||
bit more as these parameters are the same for all backends:
|
||||
|
||||
// TODO:: generate this from jsdoc
|
||||
|
||||
* `objectGetInfo`: a dictionary with two entries: `key`, the object key in the
|
||||
data store, and `client`, the data store name;
|
||||
* `reqUids`: the request unique ID used for logging;
|
||||
* `callback`: your function's callback (should handle errors);
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}_lib/`
|
||||
-----------------------------------------------
|
||||
|
||||
- this is where you should put all utility functions for your HEAD operation,
|
||||
and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
|
||||
your code clean;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/get/get{{BackendName}}js`
|
||||
-----------------------------------------------------------------------------------
|
||||
|
||||
- every contribution should come with thorough functional tests, showing
|
||||
nominal context gives expected behaviour, and error cases are handled in a way
|
||||
that is standard with the backend (including error messages and code);
|
||||
- the ideal setup is if you simulate your backend locally, so as not to be
|
||||
subjected to network flakiness in the CI; however, we know there might not be
|
||||
mockups available for every client; if that is the case of your backend, you
|
||||
may test against the "real" endpoint of your data backend;
|
||||
|
||||
:file:`tests/functional/aws-node-sdk/test/multipleBackend/utils.js`
|
||||
-------------------------------------------------------------------
|
||||
|
||||
.. note:: You should need this section if you have followed the tutorial in order
|
||||
(that is, if you have covered the PUT operation already)
|
||||
|
||||
- where you'll define a constant for your backend location matching your
|
||||
:file:`/tests/locationConfig/locationConfigTests.json`
|
||||
- depending on your backend, the sample `keys[]` and associated made up objects
|
||||
may not work for you (if your backend's key format is different, for example);
|
||||
if that is the case, you should add a custom `utils.get{{BackendName}}keys()`
|
||||
|
||||
Healthcheck
|
||||
~~~~~~~~~~~
|
||||
|
||||
Healtchecks are used to make sure failure to write to a remote cloud is due to
|
||||
a problem on that remote cloud, an not on Zenko's side.
|
||||
This is usually done by trying to create a bucket that already exists, and
|
||||
making sure you get the expected answer.
|
||||
|
||||
These are the files you'll need to edit:
|
||||
|
||||
:file:`/lib/data/external/{{BackendName}}Client.js`
|
||||
---------------------------------------------------
|
||||
|
||||
- the function that is going to call your `healthcheck()` function is called
|
||||
`checkExternalBackend()` and it's defined in
|
||||
:file:`/lib/data/multipleBackendGateway.js`; you will need to add your own;
|
||||
- your healtcheck function should get `location` as a parameter, which is an
|
||||
object comprising:`
|
||||
|
||||
* `reqUids`: the request unique ID used for logging;
|
||||
* `callback`: your function's callback (should handle errors);
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}_lib/{{backendName}}_create_bucket.js`
|
||||
-------------------------------------------------------------------------------
|
||||
|
||||
- this is where you should write the function performing the actual bucket
|
||||
creation;
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}_lib/utils.js`
|
||||
-------------------------------------------------------
|
||||
|
||||
- add an object named per your backend's name to the `backendHealth` dictionary,
|
||||
with proper `response` and `time` entries;
|
||||
|
||||
:file:`lib/data/multipleBackendGateway.js`
|
||||
------------------------------------------
|
||||
|
||||
- edit the `healthcheck` function to add your location's array, and call your
|
||||
healthcheck; see pseudocode below for a sample implementation, provided your
|
||||
backend name is `ztore`
|
||||
|
||||
|
||||
.. code-block:: js
|
||||
:linenos:
|
||||
|
||||
(...) //<1>
|
||||
|
||||
healthcheck: (flightCheckOnStartUp, log, callback) => { //<1>
|
||||
(...) //<1>
|
||||
const ztoreArray = []; //<2>
|
||||
async.each(Object.keys(clients), (location, cb) => { //<1>
|
||||
(...) //<1>
|
||||
} else if (client.clientType === 'ztore' {
|
||||
ztoreArray.push(location); //<3>
|
||||
return cb();
|
||||
}
|
||||
(...) //<1>
|
||||
multBackendResp[location] = { code: 200, message: 'OK' }; //<1>
|
||||
return cb();
|
||||
}, () => { //<1>
|
||||
async.parallel([
|
||||
(...) //<1>
|
||||
next => checkExternalBackend( //<4>
|
||||
clients, ztoreArray, 'ztore', flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, next),
|
||||
] (...) //<1>
|
||||
});
|
||||
(...) //<1>
|
||||
});
|
||||
}
|
||||
|
||||
1. Code that is already there
|
||||
2. The array that will store all locations of type 'ztore'
|
||||
3. Where you add locations of type 'ztore' to the array
|
||||
4. Where you actually call the healthcheck function on all 'ztore' locations
|
||||
|
||||
Multipart upload (MPU)
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
This is the final part to supporting a new backend! MPU is far from
|
||||
the easiest subject, but you've come so far it shouldn't be a problem.
|
||||
|
||||
These are the files you'll need to edit:
|
||||
|
||||
:file:`/lib/data/external/{{BackendName}}Client.js`
|
||||
---------------------------------------------------
|
||||
|
||||
You'll be creating four functions with template signatures:
|
||||
|
||||
- `createMPU(Key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
|
||||
cacheControl, contentDisposition, contentEncoding, log, callback)` will
|
||||
initiate the multi part upload process; now, here, all parameters are
|
||||
metadata headers except for:
|
||||
|
||||
* `Key`, the key id for the final object (collection of all parts);
|
||||
* `bucketName`, the name of the bucket to which we will do an MPU;
|
||||
* `log`, the logger;
|
||||
|
||||
- `uploadPart(request, streamingV4Params, stream, size, key, uploadId, partNumber, bucketName, log, callback)`
|
||||
will be called for each part; the parameters can be explicited as follow:
|
||||
|
||||
* `request`, the request object for putting the part;
|
||||
* `streamingV4Params`, parameters for auth V4 parameters against S3;
|
||||
* `stream`, the node.js readable stream used to put the part;
|
||||
* `size`, the size of the part;
|
||||
* `key`, the key of the object;
|
||||
* `uploadId`, multipart upload id string;
|
||||
* `partNumber`, the number of the part in this MPU (ordered);
|
||||
* `bucketName`, the name of the bucket to which we will do an MPU;
|
||||
* `log`, the logger;
|
||||
|
||||
- `completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback)` will
|
||||
end the MPU process once all parts are uploaded; parameters can be explicited
|
||||
as follows:
|
||||
|
||||
* `jsonList`, user-sent list of parts to include in final mpu object;
|
||||
* `mdInfo`, object containing 3 keys: storedParts, mpuOverviewKey, and
|
||||
splitter;
|
||||
* `key`, the key of the object;
|
||||
* `uploadId`, multipart upload id string;
|
||||
* `bucketName`, name of bucket;
|
||||
* `log`, logger instance:
|
||||
|
||||
- `abortMPU(key, uploadId, bucketName, log, callback)` will handle errors, and
|
||||
make sure that all parts that may have been uploaded will be deleted if the
|
||||
MPU ultimately fails; the parameters are:
|
||||
|
||||
* `key`, the key of the object;
|
||||
* `uploadId`, multipart upload id string;
|
||||
* `bucketName`, name of bucket;
|
||||
* `log`, logger instance.
|
||||
|
||||
:file:`/lib/api/objectPutPart.js`
|
||||
---------------------------------
|
||||
|
||||
- you'll need to add your backend type in appropriate sections (simply look for
|
||||
other backends already implemented).
|
||||
|
||||
:file:`/lib/data/external/{{backendName}}_lib/`
|
||||
-----------------------------------------------
|
||||
|
||||
- this is where you should put all utility functions for your MPU operations,
|
||||
and then import then in :file:`/lib/data/external/{{BackendName}}Client.js`, to keep
|
||||
your code clean;
|
||||
|
||||
:file:`lib/data/multipleBackendGateway.js`
|
||||
------------------------------------------
|
||||
|
||||
- edit the `createMPU` function to add your location type, and call your
|
||||
`createMPU()`; see pseudocode below for a sample implementation, provided your
|
||||
backend name is `ztore`
|
||||
|
||||
.. code-block:: javascript
|
||||
:linenos:
|
||||
|
||||
(...) //<1>
|
||||
createMPU:(key, metaHeaders, bucketName, websiteRedirectHeader, //<1>
|
||||
location, contentType, cacheControl, contentDisposition,
|
||||
contentEncoding, log, cb) => {
|
||||
const client = clients[location]; //<1>
|
||||
if (client.clientType === 'aws_s3') { //<1>
|
||||
return client.createMPU(key, metaHeaders, bucketName,
|
||||
websiteRedirectHeader, contentType, cacheControl,
|
||||
contentDisposition, contentEncoding, log, cb);
|
||||
} else if (client.clientType === 'ztore') { //<2>
|
||||
return client.createMPU(key, metaHeaders, bucketName,
|
||||
websiteRedirectHeader, contentType, cacheControl,
|
||||
contentDisposition, contentEncoding, log, cb);
|
||||
}
|
||||
return cb();
|
||||
};
|
||||
(...) //<1>
|
||||
|
||||
1. Code that is already there
|
||||
2. Where the `createMPU()` of your client is actually called
|
||||
|
||||
Add functional tests
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* :file:`tests/functional/aws-node-sdk/test/multipleBackend/initMPU/{{BackendName}}InitMPU.js`
|
||||
* :file:`tests/functional/aws-node-sdk/test/multipleBackend/listParts/{{BackendName}}ListPart.js`
|
||||
* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/{{BackendName}}AbortMPU.js`
|
||||
* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/{{BackendName}}CompleteMPU.js`
|
||||
* :file:`tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/{{BackendName}}UploadPart.js`
|
||||
|
||||
Adding support in Orbit, Zenko's UI for simplified Multi Cloud Management
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
This can only be done by our core developers' team. Once your backend
|
||||
integration is merged, you may open a feature request on the
|
||||
`Zenko repository`_, and we will
|
||||
get back to you after we evaluate feasability and maintainability.
|
||||
|
||||
.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new
|
|
@ -1,43 +0,0 @@
|
|||
======================
|
||||
S3-Compatible Backends
|
||||
======================
|
||||
|
||||
|
||||
Adding Support in CloudServer
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
This is the easiest case for backend support integration: there is nothing to do
|
||||
but configuration! Follow the steps described in our
|
||||
:ref:`use-public-cloud` and make sure you:
|
||||
|
||||
- set ``details.awsEndpoint`` to your storage provider endpoint;
|
||||
|
||||
- use ``details.credentials`` and *not* ``details.credentialsProfile`` to set your
|
||||
credentials for that S3-compatible backend.
|
||||
|
||||
For example, if you’re using a Wasabi bucket as a backend, then your region
|
||||
definition for that backend will look something like:
|
||||
::
|
||||
|
||||
"wasabi-bucket-zenkobucket": {
|
||||
"type": "aws_s3",
|
||||
"legacyAwsBehavior": true,
|
||||
"details": {
|
||||
"awsEndpoint": "s3.wasabisys.com",
|
||||
"bucketName": "zenkobucket",
|
||||
"bucketMatch": true,
|
||||
"credentials": {
|
||||
"accessKey": "\\{YOUR_WASABI_ACCESS_KEY}",
|
||||
"secretKey": "\\{YOUR_WASABI_SECRET_KEY}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
Adding Support in Zenko Orbit
|
||||
#############################
|
||||
|
||||
This can only be done by our core developpers' team. If that’s what you’re
|
||||
after, open a feature request on the `Zenko repository`_, and we will
|
||||
get back to you after we evaluate feasability and maintainability.
|
||||
|
||||
.. _Zenko repository: https://www.github.com/scality/Zenko/issues/new
|
|
@ -1,12 +1,11 @@
|
|||
Scality Zenko CloudServer
|
||||
=========================
|
||||
==================
|
||||
|
||||
.. _user-docs:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Documentation
|
||||
:glob:
|
||||
|
||||
CONTRIBUTING
|
||||
GETTING_STARTED
|
||||
|
@ -15,4 +14,3 @@ Scality Zenko CloudServer
|
|||
DOCKER
|
||||
INTEGRATIONS
|
||||
ARCHITECTURE
|
||||
developers/*
|
||||
|
|
|
@ -1,2 +0,0 @@
|
|||
Sphinx >= 1.7.5
|
||||
recommonmark >= 0.4.0
|
|
@ -1,119 +0,0 @@
|
|||
#
|
||||
# This file is autogenerated by pip-compile
|
||||
# To update, run:
|
||||
#
|
||||
# tox -e pip-compile
|
||||
#
|
||||
alabaster==0.7.12 \
|
||||
--hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
|
||||
--hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 \
|
||||
# via sphinx
|
||||
babel==2.6.0 \
|
||||
--hash=sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669 \
|
||||
--hash=sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23 \
|
||||
# via sphinx
|
||||
certifi==2018.10.15 \
|
||||
--hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
|
||||
--hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a \
|
||||
# via requests
|
||||
chardet==3.0.4 \
|
||||
--hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
|
||||
--hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
|
||||
# via requests
|
||||
commonmark==0.5.4 \
|
||||
--hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5 \
|
||||
# via recommonmark
|
||||
docutils==0.14 \
|
||||
--hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
|
||||
--hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
|
||||
--hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
|
||||
# via recommonmark, sphinx
|
||||
idna==2.7 \
|
||||
--hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
|
||||
--hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
|
||||
# via requests
|
||||
imagesize==1.1.0 \
|
||||
--hash=sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8 \
|
||||
--hash=sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5 \
|
||||
# via sphinx
|
||||
jinja2==2.10 \
|
||||
--hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
|
||||
--hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
|
||||
# via sphinx
|
||||
markupsafe==1.1.0 \
|
||||
--hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \
|
||||
--hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \
|
||||
--hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \
|
||||
--hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \
|
||||
--hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \
|
||||
--hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \
|
||||
--hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \
|
||||
--hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \
|
||||
--hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \
|
||||
--hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \
|
||||
--hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \
|
||||
--hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \
|
||||
--hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \
|
||||
--hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \
|
||||
--hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \
|
||||
--hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \
|
||||
--hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \
|
||||
--hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \
|
||||
--hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \
|
||||
--hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \
|
||||
--hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \
|
||||
--hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \
|
||||
--hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \
|
||||
--hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \
|
||||
--hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \
|
||||
--hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \
|
||||
--hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \
|
||||
--hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 \
|
||||
# via jinja2
|
||||
packaging==18.0 \
|
||||
--hash=sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807 \
|
||||
--hash=sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9 \
|
||||
# via sphinx
|
||||
pygments==2.2.0 \
|
||||
--hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
|
||||
--hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc \
|
||||
# via sphinx
|
||||
pyparsing==2.3.0 \
|
||||
--hash=sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b \
|
||||
--hash=sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592 \
|
||||
# via packaging
|
||||
pytz==2018.7 \
|
||||
--hash=sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca \
|
||||
--hash=sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6 \
|
||||
# via babel
|
||||
recommonmark==0.4.0 \
|
||||
--hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d \
|
||||
--hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7
|
||||
requests==2.20.1 \
|
||||
--hash=sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54 \
|
||||
--hash=sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263 \
|
||||
# via sphinx
|
||||
six==1.11.0 \
|
||||
--hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
|
||||
--hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \
|
||||
# via packaging, sphinx
|
||||
snowballstemmer==1.2.1 \
|
||||
--hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 \
|
||||
--hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \
|
||||
# via sphinx
|
||||
sphinx==1.8.2 \
|
||||
--hash=sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea \
|
||||
--hash=sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64
|
||||
sphinxcontrib-websupport==1.1.0 \
|
||||
--hash=sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd \
|
||||
--hash=sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9 \
|
||||
# via sphinx
|
||||
typing==3.6.6 \
|
||||
--hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \
|
||||
--hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
|
||||
--hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
|
||||
# via sphinx
|
||||
urllib3==1.24.1 \
|
||||
--hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
|
||||
--hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
|
||||
# via requests
|
|
@ -1,46 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"io/ioutil"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Input AWS access key, secret key
|
||||
aws_access_key_id := "accessKey1"
|
||||
aws_secret_access_key := "verySecretKey1"
|
||||
endpoint := "http://localhost:8000"
|
||||
bucket_name := "bucketname"
|
||||
searchQuery := url.QueryEscape("x-amz-meta-color=blue")
|
||||
buf := bytes.NewBuffer([]byte{})
|
||||
|
||||
requestUrl := fmt.Sprintf("%s/%s?search=%s",
|
||||
endpoint, bucket_name, searchQuery)
|
||||
|
||||
request, err := http.NewRequest("GET", requestUrl, buf)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
reader := bytes.NewReader(buf.Bytes())
|
||||
credentials := credentials.NewStaticCredentials(aws_access_key_id,
|
||||
aws_secret_access_key, "")
|
||||
signer := v4.NewSigner(credentials)
|
||||
signer.Sign(request, reader, "s3", "us-east-1", time.Now())
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(request)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
}
|
|
@ -1,28 +0,0 @@
|
|||
const { S3 } = require('aws-sdk');
|
||||
const config = {
|
||||
sslEnabled: false,
|
||||
endpoint: 'http://127.0.0.1:8000',
|
||||
signatureCache: false,
|
||||
signatureVersion: 'v4',
|
||||
region: 'us-east-1',
|
||||
s3ForcePathStyle: true,
|
||||
accessKeyId: 'accessKey1',
|
||||
secretAccessKey: 'verySecretKey1',
|
||||
};
|
||||
const s3Client = new S3(config);
|
||||
|
||||
const encodedSearch =
|
||||
encodeURIComponent('x-amz-meta-color="blue"');
|
||||
const req = s3Client.listObjects({ Bucket: 'bucketname' });
|
||||
|
||||
// the build event
|
||||
req.on('build', () => {
|
||||
req.httpRequest.path = `${req.httpRequest.path}?search=${encodedSearch}`;
|
||||
});
|
||||
req.on('success', res => {
|
||||
process.stdout.write(`Result ${res.data}`);
|
||||
});
|
||||
req.on('error', err => {
|
||||
process.stdout.write(`Error ${err}`);
|
||||
});
|
||||
req.send();
|
|
@ -1,79 +0,0 @@
|
|||
import datetime
|
||||
import hashlib
|
||||
import hmac
|
||||
import urllib
|
||||
# pip install requests
|
||||
import requests
|
||||
|
||||
access_key = 'accessKey1'
|
||||
secret_key = 'verySecretKey1'
|
||||
|
||||
method = 'GET'
|
||||
service = 's3'
|
||||
host = 'localhost:8000'
|
||||
region = 'us-east-1'
|
||||
canonical_uri = '/bucketname'
|
||||
query = 'x-amz-meta-color=blue'
|
||||
canonical_querystring = 'search=%s' % (urllib.quote(query))
|
||||
algorithm = 'AWS4-HMAC-SHA256'
|
||||
|
||||
t = datetime.datetime.utcnow()
|
||||
amz_date = t.strftime('%Y%m%dT%H%M%SZ')
|
||||
date_stamp = t.strftime('%Y%m%d')
|
||||
|
||||
# Key derivation functions. See:
|
||||
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
|
||||
|
||||
|
||||
def sign(key, msg):
|
||||
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
|
||||
|
||||
|
||||
def getSignatureKey(key, date_stamp, regionName, serviceName):
|
||||
kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
|
||||
kRegion = sign(kDate, regionName)
|
||||
kService = sign(kRegion, serviceName)
|
||||
kSigning = sign(kService, 'aws4_request')
|
||||
return kSigning
|
||||
|
||||
|
||||
payload_hash = hashlib.sha256('').hexdigest()
|
||||
|
||||
canonical_headers = \
|
||||
'host:{0}\nx-amz-content-sha256:{1}\nx-amz-date:{2}\n' \
|
||||
.format(host, payload_hash, amz_date)
|
||||
|
||||
signed_headers = 'host;x-amz-content-sha256;x-amz-date'
|
||||
|
||||
canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
|
||||
.format(method, canonical_uri, canonical_querystring, canonical_headers,
|
||||
signed_headers, payload_hash)
|
||||
print(canonical_request)
|
||||
|
||||
credential_scope = '{0}/{1}/{2}/aws4_request' \
|
||||
.format(date_stamp, region, service)
|
||||
|
||||
string_to_sign = '{0}\n{1}\n{2}\n{3}' \
|
||||
.format(algorithm, amz_date, credential_scope,
|
||||
hashlib.sha256(canonical_request).hexdigest())
|
||||
|
||||
signing_key = getSignatureKey(secret_key, date_stamp, region, service)
|
||||
|
||||
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
|
||||
hashlib.sha256).hexdigest()
|
||||
|
||||
authorization_header = \
|
||||
'{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}' \
|
||||
.format(algorithm, access_key, credential_scope, signed_headers, signature)
|
||||
|
||||
# The 'host' header is added automatically by the Python 'requests' library.
|
||||
headers = {
|
||||
'X-Amz-Content-Sha256': payload_hash,
|
||||
'X-Amz-Date': amz_date,
|
||||
'Authorization': authorization_header
|
||||
}
|
||||
|
||||
endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
|
||||
|
||||
r = requests.get(endpoint, headers=headers)
|
||||
print(r.text)
|
|
@ -14,10 +14,8 @@ RUN rm -f ~/.gitconfig && \
|
|||
git config --global --add safe.directory . && \
|
||||
git lfs install && \
|
||||
GIT_LFS_SKIP_SMUDGE=1 && \
|
||||
yarn global add typescript && \
|
||||
yarn install --frozen-lockfile --production --network-concurrency 1 && \
|
||||
yarn cache clean --all && \
|
||||
yarn global remove typescript
|
||||
yarn cache clean --all
|
||||
|
||||
# run symlinking separately to avoid yarn installation errors
|
||||
# we might have to check if the symlinking is really needed!
|
||||
|
|
12
index.js
12
index.js
|
@ -1,10 +1,10 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
require('werelogs').stderrUtils.catchAndTimestampStderr(
|
||||
undefined,
|
||||
// Do not exit as workers have their own listener that will exit
|
||||
// But primary don't have another listener
|
||||
require('cluster').isPrimary ? 1 : null,
|
||||
);
|
||||
/**
|
||||
* Catch uncaught exceptions and add timestamp to aid debugging
|
||||
*/
|
||||
process.on('uncaughtException', err => {
|
||||
process.stderr.write(`${new Date().toISOString()}: Uncaught exception: \n${err.stack}`);
|
||||
});
|
||||
|
||||
require('./lib/server.js')();
|
||||
|
|
1046
lib/Config.js
1046
lib/Config.js
File diff suppressed because it is too large
Load Diff
141
lib/api/api.js
141
lib/api/api.js
|
@ -7,8 +7,7 @@ const bucketDeleteEncryption = require('./bucketDeleteEncryption');
|
|||
const bucketDeleteWebsite = require('./bucketDeleteWebsite');
|
||||
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
|
||||
const bucketDeletePolicy = require('./bucketDeletePolicy');
|
||||
const bucketDeleteQuota = require('./bucketDeleteQuota');
|
||||
const { bucketGet } = require('./bucketGet');
|
||||
const bucketGet = require('./bucketGet');
|
||||
const bucketGetACL = require('./bucketGetACL');
|
||||
const bucketGetCors = require('./bucketGetCors');
|
||||
const bucketGetVersioning = require('./bucketGetVersioning');
|
||||
|
@ -18,7 +17,6 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
|
|||
const bucketGetNotification = require('./bucketGetNotification');
|
||||
const bucketGetObjectLock = require('./bucketGetObjectLock');
|
||||
const bucketGetPolicy = require('./bucketGetPolicy');
|
||||
const bucketGetQuota = require('./bucketGetQuota');
|
||||
const bucketGetEncryption = require('./bucketGetEncryption');
|
||||
const bucketHead = require('./bucketHead');
|
||||
const { bucketPut } = require('./bucketPut');
|
||||
|
@ -35,7 +33,6 @@ const bucketPutNotification = require('./bucketPutNotification');
|
|||
const bucketPutEncryption = require('./bucketPutEncryption');
|
||||
const bucketPutPolicy = require('./bucketPutPolicy');
|
||||
const bucketPutObjectLock = require('./bucketPutObjectLock');
|
||||
const bucketUpdateQuota = require('./bucketUpdateQuota');
|
||||
const bucketGetReplication = require('./bucketGetReplication');
|
||||
const bucketDeleteReplication = require('./bucketDeleteReplication');
|
||||
const corsPreflight = require('./corsPreflight');
|
||||
|
@ -43,11 +40,10 @@ const completeMultipartUpload = require('./completeMultipartUpload');
|
|||
const initiateMultipartUpload = require('./initiateMultipartUpload');
|
||||
const listMultipartUploads = require('./listMultipartUploads');
|
||||
const listParts = require('./listParts');
|
||||
const metadataSearch = require('./metadataSearch');
|
||||
const { multiObjectDelete } = require('./multiObjectDelete');
|
||||
const multipartDelete = require('./multipartDelete');
|
||||
const objectCopy = require('./objectCopy');
|
||||
const { objectDelete } = require('./objectDelete');
|
||||
const objectDelete = require('./objectDelete');
|
||||
const objectDeleteTagging = require('./objectDeleteTagging');
|
||||
const objectGet = require('./objectGet');
|
||||
const objectGetACL = require('./objectGetACL');
|
||||
|
@ -56,13 +52,13 @@ const objectGetRetention = require('./objectGetRetention');
|
|||
const objectGetTagging = require('./objectGetTagging');
|
||||
const objectHead = require('./objectHead');
|
||||
const objectPut = require('./objectPut');
|
||||
const objectPost = require('./objectPost');
|
||||
const objectPutACL = require('./objectPutACL');
|
||||
const objectPutLegalHold = require('./objectPutLegalHold');
|
||||
const objectPutTagging = require('./objectPutTagging');
|
||||
const objectPutPart = require('./objectPutPart');
|
||||
const objectPutCopyPart = require('./objectPutCopyPart');
|
||||
const objectPutRetention = require('./objectPutRetention');
|
||||
const objectRestore = require('./objectRestore');
|
||||
const prepareRequestContexts
|
||||
= require('./apiUtils/authorization/prepareRequestContexts');
|
||||
const serviceGet = require('./serviceGet');
|
||||
|
@ -72,8 +68,9 @@ const writeContinue = require('../utilities/writeContinue');
|
|||
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
|
||||
const parseCopySource = require('./apiUtils/object/parseCopySource');
|
||||
const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
|
||||
const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
|
||||
const { checkAuthResults } = require('./apiUtils/authorization/permissionChecks');
|
||||
const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
|
||||
const { processPostForm } = require('./apiUtils/apiCallers/callPostObject');
|
||||
|
||||
const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
|
||||
|
||||
|
@ -85,10 +82,6 @@ const api = {
|
|||
// Attach the apiMethod method to the request, so it can used by monitoring in the server
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
request.apiMethod = apiMethod;
|
||||
// Array of end of API callbacks, used to perform some logic
|
||||
// at the end of an API.
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
request.finalizerHooks = [];
|
||||
|
||||
const actionLog = monitoringMap[apiMethod];
|
||||
if (!actionLog &&
|
||||
|
@ -152,72 +145,23 @@ const api = {
|
|||
// eslint-disable-next-line no-param-reassign
|
||||
request.apiMethods = apiMethods;
|
||||
|
||||
function checkAuthResults(authResults) {
|
||||
let returnTagCount = true;
|
||||
const isImplicitDeny = {};
|
||||
let isOnlyImplicitDeny = true;
|
||||
if (apiMethod === 'objectGet') {
|
||||
// first item checks s3:GetObject(Version) action
|
||||
if (!authResults[0].isAllowed && !authResults[0].isImplicit) {
|
||||
log.trace('get object authorization denial from Vault');
|
||||
return errors.AccessDenied;
|
||||
}
|
||||
// TODO add support for returnTagCount in the bucket policy
|
||||
// checks
|
||||
isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
|
||||
// second item checks s3:GetObject(Version)Tagging action
|
||||
if (!authResults[1].isAllowed) {
|
||||
log.trace('get tagging authorization denial ' +
|
||||
'from Vault');
|
||||
returnTagCount = false;
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < authResults.length; i++) {
|
||||
isImplicitDeny[authResults[i].action] = true;
|
||||
if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
|
||||
// Any explicit deny rejects the current API call
|
||||
log.trace('authorization denial from Vault');
|
||||
return errors.AccessDenied;
|
||||
}
|
||||
if (authResults[i].isAllowed) {
|
||||
// If the action is allowed, the result is not implicit
|
||||
// Deny.
|
||||
isImplicitDeny[authResults[i].action] = false;
|
||||
isOnlyImplicitDeny = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// These two APIs cannot use ACLs or Bucket Policies, hence, any
|
||||
// implicit deny from vault must be treated as an explicit deny.
|
||||
if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
|
||||
return errors.AccessDenied;
|
||||
}
|
||||
return { returnTagCount, isImplicitDeny };
|
||||
}
|
||||
|
||||
return async.waterfall([
|
||||
next => auth.server.doAuth(
|
||||
request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
|
||||
request, log, (err, userInfo, authorizationResults, streamingV4Params) => {
|
||||
if (err) {
|
||||
// VaultClient returns standard errors, but the route requires
|
||||
// Arsenal errors
|
||||
const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
|
||||
log.trace('authentication error', { error: err });
|
||||
return next(arsenalError);
|
||||
return next(err);
|
||||
}
|
||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
||||
return next(null, userInfo, authorizationResults, streamingV4Params);
|
||||
}, 's3', requestContexts),
|
||||
(userInfo, authorizationResults, streamingV4Params, infos, next) => {
|
||||
(userInfo, authorizationResults, streamingV4Params, next) => {
|
||||
const authNames = { accountName: userInfo.getAccountDisplayName() };
|
||||
if (userInfo.isRequesterAnIAMUser()) {
|
||||
authNames.userName = userInfo.getIAMdisplayName();
|
||||
}
|
||||
if (isRequesterASessionUser(userInfo)) {
|
||||
authNames.sessionName = userInfo.getShortid().split(':')[1];
|
||||
}
|
||||
log.addDefaultFields(authNames);
|
||||
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
||||
return next(null, userInfo, authorizationResults, streamingV4Params);
|
||||
}
|
||||
// issue 100 Continue to the client
|
||||
writeContinue(request, response);
|
||||
|
@ -248,12 +192,12 @@ const api = {
|
|||
}
|
||||
// Convert array of post buffers into one string
|
||||
request.post = Buffer.concat(post, postLength).toString();
|
||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
||||
return next(null, userInfo, authorizationResults, streamingV4Params);
|
||||
});
|
||||
return undefined;
|
||||
},
|
||||
// Tag condition keys require information from CloudServer for evaluation
|
||||
(userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
|
||||
(userInfo, authorizationResults, streamingV4Params, next) => tagConditionKeyAuth(
|
||||
authorizationResults,
|
||||
request,
|
||||
requestContexts,
|
||||
|
@ -264,16 +208,15 @@ const api = {
|
|||
log.trace('tag authentication error', { error: err });
|
||||
return next(err);
|
||||
}
|
||||
return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
|
||||
return next(null, userInfo, authResultsWithTags, streamingV4Params);
|
||||
},
|
||||
),
|
||||
], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
|
||||
], (err, userInfo, authorizationResults, streamingV4Params) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
request.accountQuotas = infos?.accountQuota;
|
||||
if (authorizationResults) {
|
||||
const checkedResults = checkAuthResults(authorizationResults);
|
||||
const checkedResults = checkAuthResults(apiMethod, authorizationResults, log);
|
||||
if (checkedResults instanceof Error) {
|
||||
return callback(checkedResults);
|
||||
}
|
||||
|
@ -288,23 +231,55 @@ const api = {
|
|||
return acc;
|
||||
}, {});
|
||||
}
|
||||
const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
|
||||
(hook, done) => hook(err, done),
|
||||
() => callback(err, ...results));
|
||||
|
||||
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
||||
request._response = response;
|
||||
return this[apiMethod](userInfo, request, streamingV4Params,
|
||||
log, methodCallback, authorizationResults);
|
||||
log, callback, authorizationResults);
|
||||
}
|
||||
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
|
||||
return this[apiMethod](userInfo, request, sourceBucket,
|
||||
sourceObject, sourceVersionId, log, methodCallback);
|
||||
sourceObject, sourceVersionId, log, callback);
|
||||
}
|
||||
if (apiMethod === 'objectGet') {
|
||||
return this[apiMethod](userInfo, request, returnTagCount, log, callback);
|
||||
}
|
||||
return this[apiMethod](userInfo, request, log, methodCallback);
|
||||
return this[apiMethod](userInfo, request, log, callback);
|
||||
});
|
||||
},
|
||||
callPostObject(request, response, log, callback) {
|
||||
request.apiMethod = 'objectPost';
|
||||
|
||||
const requestContexts = prepareRequestContexts('objectPost', request,
|
||||
undefined, undefined, undefined);
|
||||
// Extract all the _apiMethods and store them in an array
|
||||
const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : [];
|
||||
// Attach the names to the current request
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
request.apiMethods = apiMethods;
|
||||
|
||||
return processPostForm(request, response, requestContexts, log,
|
||||
(err, userInfo, authorizationResults, streamingV4Params) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
if (authorizationResults) {
|
||||
const checkedResults = checkAuthResults(authorizationResults);
|
||||
if (checkedResults instanceof Error) {
|
||||
return callback(checkedResults);
|
||||
}
|
||||
request.actionImplicitDenies = checkedResults.isImplicitDeny;
|
||||
} else {
|
||||
// create an object of keys apiMethods with all values to false:
|
||||
// for backward compatibility, all apiMethods are allowed by default
|
||||
// thus it is explicitly allowed, so implicit deny is false
|
||||
request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
|
||||
acc[curr] = false;
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
request._response = response;
|
||||
return objectPost(userInfo, request, streamingV4Params,
|
||||
log, callback, authorizationResults);
|
||||
});
|
||||
},
|
||||
bucketDelete,
|
||||
|
@ -325,21 +300,18 @@ const api = {
|
|||
bucketPutCors,
|
||||
bucketPutVersioning,
|
||||
bucketPutTagging,
|
||||
bucketDeleteTagging,
|
||||
bucketGetTagging,
|
||||
bucketPutWebsite,
|
||||
bucketPutReplication,
|
||||
bucketGetReplication,
|
||||
bucketDeleteReplication,
|
||||
bucketDeleteQuota,
|
||||
bucketPutLifecycle,
|
||||
bucketUpdateQuota,
|
||||
bucketGetLifecycle,
|
||||
bucketDeleteLifecycle,
|
||||
bucketPutPolicy,
|
||||
bucketGetPolicy,
|
||||
bucketGetQuota,
|
||||
bucketDeletePolicy,
|
||||
bucketDeleteTagging,
|
||||
bucketPutObjectLock,
|
||||
bucketPutNotification,
|
||||
bucketGetNotification,
|
||||
|
@ -349,7 +321,6 @@ const api = {
|
|||
initiateMultipartUpload,
|
||||
listMultipartUploads,
|
||||
listParts,
|
||||
metadataSearch,
|
||||
multiObjectDelete,
|
||||
multipartDelete,
|
||||
objectDelete,
|
||||
|
@ -362,13 +333,13 @@ const api = {
|
|||
objectCopy,
|
||||
objectHead,
|
||||
objectPut,
|
||||
objectPost,
|
||||
objectPutACL,
|
||||
objectPutLegalHold,
|
||||
objectPutTagging,
|
||||
objectPutPart,
|
||||
objectPutCopyPart,
|
||||
objectPutRetention,
|
||||
objectRestore,
|
||||
serviceGet,
|
||||
websiteGet: website,
|
||||
websiteHead: website,
|
||||
|
|
|
@ -0,0 +1,180 @@
|
|||
const { auth, errors } = require('arsenal');
|
||||
const busboy = require('@fastify/busboy');
|
||||
const writeContinue = require('../../../utilities/writeContinue');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
|
||||
// per doc: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTForms.html#HTTPPOSTFormDeclaration
|
||||
const MAX_FIELD_SIZE = 20 * 1024; // 20KB
|
||||
// per doc: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
|
||||
const MAX_KEY_SIZE = 1024;
|
||||
|
||||
async function authenticateRequest(request, requestContexts, log) {
|
||||
return new Promise(resolve => {
|
||||
// TODO RING-45960 remove ignore for POST object here
|
||||
auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) =>
|
||||
resolve({ userInfo, authorizationResults, streamingV4Params }), 's3', requestContexts);
|
||||
});
|
||||
}
|
||||
|
||||
async function parseFormData(request, response, requestContexts, log) {
|
||||
/* eslint-disable no-param-reassign */
|
||||
const formDataParser = busboy({ headers: request.headers });
|
||||
writeContinue(request, response);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
request.formData = {};
|
||||
let totalFieldSize = 0;
|
||||
let fileEventData = null;
|
||||
let tempFileStream;
|
||||
let tempFilePath;
|
||||
let authResponse;
|
||||
let fileWrittenPromiseResolve;
|
||||
let formParserFinishedPromiseResolve;
|
||||
|
||||
const fileWrittenPromise = new Promise((res) => { fileWrittenPromiseResolve = res; });
|
||||
const formParserFinishedPromise = new Promise((res) => { formParserFinishedPromiseResolve = res; });
|
||||
|
||||
formDataParser.on('field', (fieldname, val) => {
|
||||
totalFieldSize += Buffer.byteLength(val, 'utf8');
|
||||
if (totalFieldSize > MAX_FIELD_SIZE) {
|
||||
return reject(errors.MaxPostPreDataLengthExceeded);
|
||||
}
|
||||
const lowerFieldname = fieldname.toLowerCase();
|
||||
if (lowerFieldname === 'key') {
|
||||
if (val.length > MAX_KEY_SIZE) {
|
||||
return reject(errors.KeyTooLongError);
|
||||
} else if (val.length === 0) {
|
||||
return reject(errors.InvalidArgument
|
||||
.customizeDescription('User key must have a length greater than 0.'));
|
||||
}
|
||||
}
|
||||
request.formData[lowerFieldname] = val;
|
||||
return undefined;
|
||||
});
|
||||
|
||||
formDataParser.on('file', async (fieldname, file, filename, encoding, mimetype) => {
|
||||
if (fileEventData) {
|
||||
const err = errors.InvalidArgument
|
||||
.customizeDescription('POST requires exactly one file upload per request.');
|
||||
file.resume(); // Resume the stream to drain and discard the file
|
||||
if (tempFilePath) {
|
||||
fs.unlink(tempFilePath, unlinkErr => {
|
||||
if (unlinkErr) {
|
||||
log.error('Failed to delete temp file', { error: unlinkErr });
|
||||
}
|
||||
});
|
||||
}
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
fileEventData = { fieldname, file, filename, encoding, mimetype };
|
||||
if (!('key' in request.formData)) {
|
||||
const err = errors.InvalidArgument
|
||||
.customizeDescription('Bucket POST must contain a field named '
|
||||
+ "'key'. If it is specified, please check the order of the fields.");
|
||||
return reject(err);
|
||||
}
|
||||
// Replace `${filename}` with the actual filename
|
||||
request.formData.key = request.formData.key.replace('${filename}', filename);
|
||||
try {
|
||||
// Authenticate request before streaming file
|
||||
// TODO RING-45960 auth to be properly implemented
|
||||
authResponse = await authenticateRequest(request, requestContexts, log);
|
||||
|
||||
// Create a temporary file to stream the file data
|
||||
// This is to finalize validation on form data before storing the file
|
||||
tempFilePath = path.join(os.tmpdir(), filename);
|
||||
tempFileStream = fs.createWriteStream(tempFilePath);
|
||||
|
||||
file.pipe(tempFileStream);
|
||||
|
||||
tempFileStream.on('finish', () => {
|
||||
request.fileEventData = { ...fileEventData, file: tempFilePath };
|
||||
fileWrittenPromiseResolve();
|
||||
});
|
||||
|
||||
tempFileStream.on('error', (err) => {
|
||||
log.trace('Error streaming file to temporary location', { error: err });
|
||||
reject(errors.InternalError);
|
||||
});
|
||||
|
||||
// Wait for both file writing and form parsing to finish
|
||||
return Promise.all([fileWrittenPromise, formParserFinishedPromise])
|
||||
.then(() => resolve(authResponse))
|
||||
.catch(reject);
|
||||
} catch (err) {
|
||||
return reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
formDataParser.on('finish', () => {
|
||||
if (!fileEventData) {
|
||||
const err = errors.InvalidArgument
|
||||
.customizeDescription('POST requires exactly one file upload per request.');
|
||||
return reject(err);
|
||||
}
|
||||
return formParserFinishedPromiseResolve();
|
||||
});
|
||||
|
||||
formDataParser.on('error', (err) => {
|
||||
log.trace('Error processing form data:', { error: err.description });
|
||||
request.unpipe(formDataParser);
|
||||
// Following observed AWS behaviour
|
||||
reject(errors.MalformedPOSTRequest);
|
||||
});
|
||||
|
||||
request.pipe(formDataParser);
|
||||
return undefined;
|
||||
});
|
||||
}
|
||||
|
||||
function getFileStat(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.stat(filePath, (err, stats) => {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
}
|
||||
return resolve(stats);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function processPostForm(request, response, requestContexts, log, callback) {
|
||||
if (!request.headers || !request.headers['content-type'].includes('multipart/form-data')) {
|
||||
const contentTypeError = errors.PreconditionFailed
|
||||
.customizeDescription('Bucket POST must be of the enclosure-type multipart/form-data');
|
||||
return process.nextTick(callback, contentTypeError);
|
||||
}
|
||||
try {
|
||||
const { userInfo, authorizationResults, streamingV4Params } =
|
||||
await parseFormData(request, response, requestContexts, log);
|
||||
|
||||
const fileStat = await getFileStat(request.fileEventData.file);
|
||||
request.parsedContentLength = fileStat.size;
|
||||
request.fileEventData.file = fs.createReadStream(request.fileEventData.file);
|
||||
if (request.formData['content-type']) {
|
||||
request.headers['content-type'] = request.formData['content-type'];
|
||||
} else {
|
||||
request.headers['content-type'] = 'binary/octet-stream';
|
||||
}
|
||||
|
||||
const authNames = { accountName: userInfo.getAccountDisplayName() };
|
||||
if (userInfo.isRequesterAnIAMUser()) {
|
||||
authNames.userName = userInfo.getIAMdisplayName();
|
||||
}
|
||||
log.addDefaultFields(authNames);
|
||||
|
||||
return callback(null, userInfo, authorizationResults, streamingV4Params);
|
||||
} catch (err) {
|
||||
return callback(err);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
authenticateRequest,
|
||||
parseFormData,
|
||||
processPostForm,
|
||||
getFileStat,
|
||||
};
|
|
@ -5,13 +5,8 @@ const constants = require('../../../../constants');
|
|||
const { config } = require('../../../Config');
|
||||
|
||||
const {
|
||||
allAuthedUsersId,
|
||||
bucketOwnerActions,
|
||||
logId,
|
||||
publicId,
|
||||
arrayOfAllowed,
|
||||
assumedRoleArnResourceType,
|
||||
backbeatLifecycleSessionName,
|
||||
allAuthedUsersId, bucketOwnerActions, logId, publicId,
|
||||
assumedRoleArnResourceType, backbeatLifecycleSessionName, arrayOfAllowed,
|
||||
actionsToConsiderAsObjectPut,
|
||||
} = constants;
|
||||
|
||||
|
@ -19,25 +14,6 @@ const {
|
|||
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
|
||||
? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
|
||||
|
||||
function getServiceAccountProperties(canonicalID) {
|
||||
const canonicalIDArray = canonicalID.split('/');
|
||||
const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
|
||||
return constants.serviceAccountProperties[serviceName];
|
||||
}
|
||||
|
||||
function isServiceAccount(canonicalID) {
|
||||
return getServiceAccountProperties(canonicalID) !== undefined;
|
||||
}
|
||||
|
||||
function isRequesterASessionUser(authInfo) {
|
||||
const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
|
||||
return regexpAssumedRoleArn.test(authInfo.getArn());
|
||||
}
|
||||
|
||||
function isRequesterNonAccountUser(authInfo) {
|
||||
return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the access control for a given bucket based on the request type and user's canonical ID.
|
||||
*
|
||||
|
@ -388,11 +364,11 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo
|
|||
let requesterIsNotUser = true;
|
||||
let arn = null;
|
||||
if (authInfo) {
|
||||
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
|
||||
requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
|
||||
arn = authInfo.getArn();
|
||||
}
|
||||
// if the bucket owner is an account, users should not have default access
|
||||
if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
|
||||
if ((bucket.getOwner() === canonicalID) && requesterIsNotUser) {
|
||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
||||
return results[_requestType];
|
||||
}
|
||||
|
@ -463,11 +439,11 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI
|
|||
let arn = null;
|
||||
let isUserUnauthenticated = false;
|
||||
if (authInfo) {
|
||||
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
|
||||
requesterIsNotUser = !authInfo.isRequesterAnIAMUser();
|
||||
arn = authInfo.getArn();
|
||||
isUserUnauthenticated = arn === undefined;
|
||||
}
|
||||
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
|
||||
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser) {
|
||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
||||
return results[_requestType];
|
||||
}
|
||||
|
@ -538,7 +514,7 @@ function checkIp(value) {
|
|||
}
|
||||
|
||||
// Apply the existing IP validation logic to each element
|
||||
const validateIpRegex = ip => {
|
||||
const validateIpRegex = (ip) => {
|
||||
if (constants.ipv4Regex.test(ip)) {
|
||||
return ip.split('.').every(part => parseInt(part, 10) <= 255);
|
||||
}
|
||||
|
@ -577,7 +553,6 @@ function validatePolicyConditions(policy) {
|
|||
if (s.Condition) {
|
||||
const conditionOperators = Object.keys(s.Condition);
|
||||
// there can be multiple condition operations in the Condition enclosure
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
for (const conditionOperator of conditionOperators) {
|
||||
const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
|
||||
const conditionValue = s.Condition[conditionOperator][conditionKey];
|
||||
|
@ -601,6 +576,48 @@ function validatePolicyConditions(policy) {
|
|||
return null;
|
||||
}
|
||||
|
||||
function checkAuthResults(apiMethod, authResults, log) {
|
||||
let returnTagCount = true;
|
||||
const isImplicitDeny = {};
|
||||
let isOnlyImplicitDeny = true;
|
||||
if (apiMethod === 'objectGet') {
|
||||
// first item checks s3:GetObject(Version) action
|
||||
if (!authResults[0].isAllowed && !authResults[0].isImplicit) {
|
||||
log.trace('get object authorization denial from Vault');
|
||||
return errors.AccessDenied;
|
||||
}
|
||||
// TODO add support for returnTagCount in the bucket policy
|
||||
// checks
|
||||
isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
|
||||
// second item checks s3:GetObject(Version)Tagging action
|
||||
if (!authResults[1].isAllowed) {
|
||||
log.trace('get tagging authorization denial ' +
|
||||
'from Vault');
|
||||
returnTagCount = false;
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < authResults.length; i++) {
|
||||
isImplicitDeny[authResults[i].action] = true;
|
||||
if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
|
||||
// Any explicit deny rejects the current API call
|
||||
log.trace('authorization denial from Vault');
|
||||
return errors.AccessDenied;
|
||||
}
|
||||
if (authResults[i].isAllowed) {
|
||||
// If the action is allowed, the result is not implicit
|
||||
// Deny.
|
||||
isImplicitDeny[authResults[i].action] = false;
|
||||
isOnlyImplicitDeny = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// These two APIs cannot use ACLs or Bucket Policies, hence, any
|
||||
// implicit deny from vault must be treated as an explicit deny.
|
||||
if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
|
||||
return errors.AccessDenied;
|
||||
}
|
||||
return { returnTagCount, isImplicitDeny };
|
||||
}
|
||||
|
||||
/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
|
||||
* @param {string} arn - Amazon resource name - example:
|
||||
|
@ -628,14 +645,11 @@ function isLifecycleSession(arn) {
|
|||
module.exports = {
|
||||
isBucketAuthorized,
|
||||
isObjAuthorized,
|
||||
getServiceAccountProperties,
|
||||
isServiceAccount,
|
||||
isRequesterASessionUser,
|
||||
isRequesterNonAccountUser,
|
||||
checkBucketAcls,
|
||||
checkObjectAcls,
|
||||
validatePolicyResource,
|
||||
validatePolicyConditions,
|
||||
checkAuthResults,
|
||||
isLifecycleSession,
|
||||
evaluateBucketPolicyWithIAM,
|
||||
};
|
||||
|
|
|
@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
|||
apiMethod, 's3');
|
||||
}
|
||||
|
||||
if (apiMethod === 'bucketPut') {
|
||||
if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -65,17 +65,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
|||
|
||||
const requestContexts = [];
|
||||
|
||||
if (apiMethod === 'multiObjectDelete') {
|
||||
// MultiObjectDelete does not require any authorization when evaluating
|
||||
// the API. Instead, we authorize each object passed.
|
||||
// But in order to get any relevant information from the authorization service
|
||||
// for example, the account quota, we must send a request context object
|
||||
// with no `specificResource`. We expect the result to be an implicit deny.
|
||||
// In the API, we then ignore these authorization results, and we can use
|
||||
// any information returned, e.g., the quota.
|
||||
const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
|
||||
requestContexts.push(requestContextMultiObjectDelete);
|
||||
} else if (apiMethodAfterVersionCheck === 'objectCopy'
|
||||
if (apiMethodAfterVersionCheck === 'objectCopy'
|
||||
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') {
|
||||
const objectGetAction = sourceVersionId ? 'objectGetVersion' :
|
||||
'objectGet';
|
||||
|
@ -157,13 +147,6 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
|||
generateRequestContext('objectGet');
|
||||
requestContexts.push(getObjectRequestContext);
|
||||
} else if (apiMethodAfterVersionCheck === 'objectPut') {
|
||||
// if put object with version
|
||||
if (request.headers['x-scal-s3-version-id'] ||
|
||||
request.headers['x-scal-s3-version-id'] === '') {
|
||||
const putVersionRequestContext =
|
||||
generateRequestContext('objectPutVersion');
|
||||
requestContexts.push(putVersionRequestContext);
|
||||
} else {
|
||||
const putRequestContext =
|
||||
generateRequestContext(apiMethodAfterVersionCheck);
|
||||
requestContexts.push(putRequestContext);
|
||||
|
@ -178,38 +161,6 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
|||
generateRequestContext('objectPutLegalHold');
|
||||
requestContexts.push(putLegalHoldStatusAction);
|
||||
}
|
||||
// if put object (versioning) with ACL
|
||||
if (isHeaderAcl(request.headers)) {
|
||||
const putAclRequestContext =
|
||||
generateRequestContext('objectPutACL');
|
||||
requestContexts.push(putAclRequestContext);
|
||||
}
|
||||
if (request.headers['x-amz-object-lock-mode']) {
|
||||
const putObjectLockRequestContext =
|
||||
generateRequestContext('objectPutRetention');
|
||||
requestContexts.push(putObjectLockRequestContext);
|
||||
}
|
||||
if (request.headers['x-amz-version-id']) {
|
||||
const putObjectVersionRequestContext =
|
||||
generateRequestContext('objectPutTaggingVersion');
|
||||
requestContexts.push(putObjectVersionRequestContext);
|
||||
}
|
||||
}
|
||||
} else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
|
||||
apiMethodAfterVersionCheck === 'objectPutPart' ||
|
||||
apiMethodAfterVersionCheck === 'completeMultipartUpload'
|
||||
) {
|
||||
if (request.headers['x-scal-s3-version-id'] ||
|
||||
request.headers['x-scal-s3-version-id'] === '') {
|
||||
const putVersionRequestContext =
|
||||
generateRequestContext('objectPutVersion');
|
||||
requestContexts.push(putVersionRequestContext);
|
||||
} else {
|
||||
const putRequestContext =
|
||||
generateRequestContext(apiMethodAfterVersionCheck);
|
||||
requestContexts.push(putRequestContext);
|
||||
}
|
||||
|
||||
// if put object (versioning) with ACL
|
||||
if (isHeaderAcl(request.headers)) {
|
||||
const putAclRequestContext =
|
||||
|
|
|
@ -60,8 +60,6 @@ function updateRequestContextsWithTags(request, requestContexts, apiMethod, log,
|
|||
log.trace('error processing tag condition key evaluation');
|
||||
return cb(err);
|
||||
}
|
||||
// FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
for (const rc of requestContexts) {
|
||||
rc.setNeedTagEval(true);
|
||||
if (requestTagsQuery) {
|
||||
|
|
|
@ -10,15 +10,13 @@ const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
|
|||
const metadata = require('../../../metadata/wrapper');
|
||||
const kms = require('../../../kms/wrapper');
|
||||
const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
|
||||
const { isServiceAccount } = require('../authorization/permissionChecks');
|
||||
|
||||
const usersBucket = constants.usersBucket;
|
||||
const oldUsersBucket = constants.oldUsersBucket;
|
||||
const zenkoSeparator = constants.zenkoSeparator;
|
||||
const userBucketOwner = 'admin';
|
||||
|
||||
|
||||
function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
|
||||
function addToUsersBucket(canonicalID, bucketName, log, cb) {
|
||||
// BACKWARD: Simplify once do not have to deal with old
|
||||
// usersbucket name and old splitter
|
||||
|
||||
|
@ -30,10 +28,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
|
|||
const splitter = usersBucketAttrs ?
|
||||
constants.splitter : constants.oldSplitter;
|
||||
let key = createKeyForUserBucket(canonicalID, splitter, bucketName);
|
||||
const omVal = {
|
||||
creationDate: new Date().toJSON(),
|
||||
ingestion: bucketMD.getIngestion(),
|
||||
};
|
||||
const omVal = { creationDate: new Date().toJSON() };
|
||||
// If the new format usersbucket does not exist, try to put the
|
||||
// key in the old usersBucket using the old splitter.
|
||||
// Otherwise put the key in the new format usersBucket
|
||||
|
@ -41,7 +36,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
|
|||
usersBucket : oldUsersBucket;
|
||||
return metadata.putObjectMD(usersBucketBeingCalled, key,
|
||||
omVal, {}, log, err => {
|
||||
if (err?.is?.NoSuchBucket) {
|
||||
if (err && err.is.NoSuchBucket) {
|
||||
// There must be no usersBucket so createBucket
|
||||
// one using the new format
|
||||
log.trace('users bucket does not exist, ' +
|
||||
|
@ -96,7 +91,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
|
|||
return callback(err);
|
||||
}
|
||||
log.trace('created bucket in metadata');
|
||||
return addToUsersBucket(canonicalID, bucketName, bucket, log, err => {
|
||||
return addToUsersBucket(canonicalID, bucketName, log, err => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
@ -119,7 +114,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
|
|||
*/
|
||||
function cleanUpBucket(bucketMD, canonicalID, log, callback) {
|
||||
const bucketName = bucketMD.getName();
|
||||
return addToUsersBucket(canonicalID, bucketName, bucketMD, log, err => {
|
||||
return addToUsersBucket(canonicalID, bucketName, log, err => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
@ -170,28 +165,15 @@ function createBucket(authInfo, bucketName, headers,
|
|||
const ownerDisplayName =
|
||||
authInfo.getAccountDisplayName();
|
||||
const creationDate = new Date().toJSON();
|
||||
const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
|
||||
const headerObjectLock = headers['x-amz-bucket-object-lock-enabled'];
|
||||
const objectLockEnabled
|
||||
= headerObjectLock && headerObjectLock.toLowerCase() === 'true';
|
||||
const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName,
|
||||
creationDate, BucketInfo.currentModelVersion(), null, null, null, null,
|
||||
null, null, null, null, null, null, null, null, null, isNFSEnabled,
|
||||
null, null, objectLockEnabled);
|
||||
let locationConstraintVal = null;
|
||||
null, null, null, null, null, null, null, null, objectLockEnabled);
|
||||
|
||||
if (locationConstraint) {
|
||||
const [locationConstraintStr, ingestion] =
|
||||
locationConstraint.split(zenkoSeparator);
|
||||
if (locationConstraintStr) {
|
||||
locationConstraintVal = locationConstraintStr;
|
||||
bucket.setLocationConstraint(locationConstraintStr);
|
||||
}
|
||||
if (ingestion === 'ingest') {
|
||||
bucket.enableIngestion();
|
||||
//automatically enable versioning for ingestion buckets
|
||||
bucket.setVersioningConfiguration({ Status: 'Enabled' });
|
||||
}
|
||||
if (locationConstraint !== undefined) {
|
||||
bucket.setLocationConstraint(locationConstraint);
|
||||
}
|
||||
if (objectLockEnabled) {
|
||||
// default versioning configuration AWS sets
|
||||
|
@ -241,8 +223,7 @@ function createBucket(authInfo, bucketName, headers,
|
|||
}
|
||||
const existingBucketMD = results.getAnyExistingBucketInfo;
|
||||
if (existingBucketMD instanceof BucketInfo &&
|
||||
existingBucketMD.getOwner() !== canonicalID &&
|
||||
!isServiceAccount(canonicalID)) {
|
||||
existingBucketMD.getOwner() !== canonicalID) {
|
||||
// return existingBucketMD to collect cors headers
|
||||
return cb(errors.BucketAlreadyExists, existingBucketMD);
|
||||
}
|
||||
|
@ -273,7 +254,7 @@ function createBucket(authInfo, bucketName, headers,
|
|||
// error unless old AWS behavior (us-east-1)
|
||||
// Existing locationConstraint must have legacyAwsBehavior === true
|
||||
// New locationConstraint should have legacyAwsBehavior === true
|
||||
if (isLegacyAWSBehavior(locationConstraintVal) &&
|
||||
if (isLegacyAWSBehavior(locationConstraint) &&
|
||||
isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) {
|
||||
log.trace('returning 200 instead of 409 to mirror us-east-1');
|
||||
return cb(null, existingBucketMD);
|
||||
|
|
|
@ -93,7 +93,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log,
|
|||
log, (err, objectsListRes) => {
|
||||
// If no shadow bucket ever created, no ongoing MPU's, so
|
||||
// continue with deletion
|
||||
if (err?.is.NoSuchBucket) {
|
||||
if (err && err.is.NoSuchBucket) {
|
||||
return next();
|
||||
}
|
||||
if (err) {
|
||||
|
|
|
@ -1,26 +0,0 @@
|
|||
const { errors } = require('arsenal');
|
||||
|
||||
function checkPreferredLocations(location, locationConstraints, log) {
|
||||
const retError = loc => {
|
||||
const errMsg = 'value of the location you are attempting to set - ' +
|
||||
`${loc} - is not listed in the locationConstraint config`;
|
||||
log.trace(`locationConstraint is invalid - ${errMsg}`,
|
||||
{ locationConstraint: loc });
|
||||
return errors.InvalidLocationConstraint.customizeDescription(errMsg);
|
||||
};
|
||||
if (typeof location === 'string' && !locationConstraints[location]) {
|
||||
return retError(location);
|
||||
}
|
||||
if (typeof location === 'object') {
|
||||
const { read, write } = location;
|
||||
if (!locationConstraints[read]) {
|
||||
return retError(read);
|
||||
}
|
||||
if (!locationConstraints[write]) {
|
||||
return retError(write);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = checkPreferredLocations;
|
|
@ -11,11 +11,11 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
|
|||
metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
|
||||
// If the object representing the bucket is not in the
|
||||
// users bucket just continue
|
||||
if (error?.is.NoSuchKey) {
|
||||
if (error && error.is.NoSuchKey) {
|
||||
return cb(null);
|
||||
// BACKWARDS COMPATIBILITY: Remove this once no longer
|
||||
// have old user bucket format
|
||||
} else if (error?.is.NoSuchBucket) {
|
||||
} else if (error && error.NoSuchBucket) {
|
||||
const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
|
||||
oldSplitter, bucketName);
|
||||
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
|
||||
|
|
|
@ -15,21 +15,11 @@ function getNotificationConfiguration(parsedXml) {
|
|||
}
|
||||
const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource));
|
||||
const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]);
|
||||
// getting invalid targets
|
||||
const invalidTargets = [];
|
||||
notifConfigTargets.forEach((t, i) => {
|
||||
if (!targets.has(t)) {
|
||||
invalidTargets.push({
|
||||
ArgumentName: notifConfig.queueConfig[i].queueArn,
|
||||
ArgumentValue: 'The destination queue does not exist',
|
||||
});
|
||||
}
|
||||
});
|
||||
if (invalidTargets.length > 0) {
|
||||
const errDesc = 'Unable to validate the following destination configurations';
|
||||
let error = errors.InvalidArgument.customizeDescription(errDesc);
|
||||
error = error.addMetadataEntry('invalidArguments', invalidTargets);
|
||||
return { error };
|
||||
if (!notifConfigTargets.every(t => targets.has(t))) {
|
||||
// TODO: match the error message to AWS's response along with
|
||||
// the request destination name in the response
|
||||
const errDesc = 'Unable to validate the destination configuration';
|
||||
return { error: errors.InvalidArgument.customizeDescription(errDesc) };
|
||||
}
|
||||
return notifConfig;
|
||||
}
|
||||
|
|
|
@ -1,19 +0,0 @@
|
|||
/**
|
||||
* parse LIKE expressions
|
||||
* @param {string} regex - regex pattern
|
||||
* @return {object} MongoDB search object
|
||||
*/
|
||||
function parseLikeExpression(regex) {
|
||||
if (typeof regex !== 'string') {
|
||||
return null;
|
||||
}
|
||||
const split = regex.split('/');
|
||||
if (split.length < 3 || split[0] !== '') {
|
||||
return { $regex: regex };
|
||||
}
|
||||
const pattern = split.slice(1, split.length - 1).join('/');
|
||||
const regexOpt = split[split.length - 1];
|
||||
return { $regex: new RegExp(pattern), $options: regexOpt };
|
||||
}
|
||||
|
||||
module.exports = parseLikeExpression;
|
|
@ -1,85 +0,0 @@
|
|||
const parseLikeExpression = require('./parseLikeExpression');
|
||||
|
||||
/*
|
||||
This code is based on code from https://github.com/olehch/sqltomongo
|
||||
with the following license:
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2016 Oleh
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A helper object to map SQL-like naming to MongoDB query syntax
|
||||
*/
|
||||
const exprMapper = {
|
||||
'=': '$eq',
|
||||
'!=': '$ne',
|
||||
'<>': '$ne',
|
||||
'>': '$gt',
|
||||
'<': '$lt',
|
||||
'>=': '$gte',
|
||||
'<=': '$lte',
|
||||
'LIKE': '$regex',
|
||||
};
|
||||
|
||||
/*
|
||||
* Parses object with WHERE clause recursively
|
||||
* and generates MongoDB `find` query object
|
||||
*/
|
||||
function parseWhere(root) {
|
||||
const operator = Object.keys(root)[0];
|
||||
|
||||
// extract leaf binary expressions
|
||||
if (operator === 'AND') {
|
||||
const e1 = parseWhere(root[operator][0]);
|
||||
const e2 = parseWhere(root[operator][1]);
|
||||
|
||||
// eslint-disable-next-line
|
||||
return { '$and' : [
|
||||
e1,
|
||||
e2,
|
||||
] };
|
||||
} else if (operator === 'OR') {
|
||||
const e1 = parseWhere(root[operator][0]);
|
||||
const e2 = parseWhere(root[operator][1]);
|
||||
|
||||
// eslint-disable-next-line
|
||||
return { '$or' : [
|
||||
e1,
|
||||
e2,
|
||||
] };
|
||||
}
|
||||
const field = root[operator][0];
|
||||
const value = root[operator][1];
|
||||
const expr = exprMapper[operator];
|
||||
const obj = {};
|
||||
|
||||
if (operator === 'LIKE') {
|
||||
obj[`value.${field}`] = parseLikeExpression(value);
|
||||
} else {
|
||||
obj[`value.${field}`] = { [expr]: value };
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
module.exports = parseWhere;
|
|
@ -1,32 +0,0 @@
|
|||
const config = require('../../../Config').config;
|
||||
|
||||
/**
|
||||
* Validates that the replication configuration contains a preferred
|
||||
* read location if the bucket location is a transient source
|
||||
*
|
||||
* @param {object} repConfig - replication configuration
|
||||
* @param {object} bucket - bucket metadata
|
||||
*
|
||||
* @return {boolean} validity of replication configuration with
|
||||
* transient source
|
||||
*/
|
||||
function validateReplicationConfig(repConfig, bucket) {
|
||||
const bucketLocationName = bucket.getLocationConstraint();
|
||||
if (!repConfig || !repConfig.rules) {
|
||||
return false;
|
||||
}
|
||||
const bucketLocation = config.locationConstraints[bucketLocationName];
|
||||
if (!bucketLocation.isTransient) {
|
||||
return true;
|
||||
}
|
||||
return repConfig.rules.every(rule => {
|
||||
if (!rule.storageClass) {
|
||||
return true;
|
||||
}
|
||||
const storageClasses = rule.storageClass.split(',');
|
||||
return storageClasses.some(
|
||||
site => site.endsWith(':preferred_read'));
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = validateReplicationConfig;
|
|
@ -1,96 +0,0 @@
|
|||
const Parser = require('sql-where-parser');
|
||||
const { errors } = require('arsenal');
|
||||
const objModel = require('arsenal').models.ObjectMD;
|
||||
|
||||
const BINARY_OP = 2;
|
||||
const sqlConfig = {
|
||||
operators: [
|
||||
{
|
||||
'=': BINARY_OP,
|
||||
'<': BINARY_OP,
|
||||
'>': BINARY_OP,
|
||||
'<>': BINARY_OP,
|
||||
'<=': BINARY_OP,
|
||||
'>=': BINARY_OP,
|
||||
'!=': BINARY_OP,
|
||||
},
|
||||
{ LIKE: BINARY_OP },
|
||||
{ AND: BINARY_OP },
|
||||
{ OR: BINARY_OP },
|
||||
],
|
||||
tokenizer: {
|
||||
shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'],
|
||||
shouldMatch: ['"', '\'', '`'],
|
||||
shouldDelimitBy: [' ', '\n', '\r', '\t'],
|
||||
},
|
||||
};
|
||||
const parser = new Parser(sqlConfig);
|
||||
|
||||
function _validateTree(whereClause, possibleAttributes) {
|
||||
let invalidAttribute;
|
||||
|
||||
function _searchTree(node) {
|
||||
if (typeof node !== 'object') {
|
||||
invalidAttribute = node;
|
||||
} else {
|
||||
const operator = Object.keys(node)[0];
|
||||
if (operator === 'AND' || operator === 'OR') {
|
||||
_searchTree(node[operator][0]);
|
||||
_searchTree(node[operator][1]);
|
||||
} else {
|
||||
const field = node[operator][0];
|
||||
if (!field.startsWith('tags.') &&
|
||||
!possibleAttributes[field] &&
|
||||
!field.startsWith('replicationInfo.') &&
|
||||
!field.startsWith('x-amz-meta-')) {
|
||||
invalidAttribute = field;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_searchTree(whereClause);
|
||||
return invalidAttribute;
|
||||
}
|
||||
|
||||
/**
|
||||
* validateSearchParams - validate value of ?search= in request
|
||||
* @param {string} searchParams - value of search params in request
|
||||
* which should be jsu sql where clause
|
||||
* For metadata: x-amz-meta-color=\"blue\"
|
||||
* For tags: tags.x-amz-meta-color=\"blue\"
|
||||
* For replication status : replication-status=\"PENDING\"
|
||||
* For any other attribute: `content-length`=5
|
||||
* @return {undefined | error} undefined if validates or arsenal error if not
|
||||
*/
|
||||
function validateSearchParams(searchParams) {
|
||||
let ast;
|
||||
try {
|
||||
// allow using 'replicationStatus' as search param to increase
|
||||
// ease of use, pending metadata search rework
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
searchParams = searchParams.replace(
|
||||
'replication-status', 'replicationInfo.status');
|
||||
ast = parser.parse(searchParams);
|
||||
} catch (e) {
|
||||
if (e) {
|
||||
return {
|
||||
error: errors.InvalidArgument
|
||||
.customizeDescription('Invalid sql where clause ' +
|
||||
'sent as search query'),
|
||||
};
|
||||
}
|
||||
}
|
||||
const possibleAttributes = objModel.getAttributes();
|
||||
const invalidAttribute = _validateTree(ast, possibleAttributes);
|
||||
if (invalidAttribute) {
|
||||
return {
|
||||
error: errors.InvalidArgument
|
||||
.customizeDescription('Search param ' +
|
||||
`contains unknown attribute: ${invalidAttribute}`) };
|
||||
}
|
||||
return {
|
||||
ast,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = validateSearchParams;
|
|
@ -0,0 +1,233 @@
|
|||
const { config } = require('../../../Config');
|
||||
const { legacyLocations } = require('../../../../constants.js');
|
||||
const { locationConstraints } = config;
|
||||
const escapeForXml = require('arsenal').s3middleware.escapeForXml;
|
||||
|
||||
class BackendInfo {
|
||||
/**
|
||||
* Represents the info necessary to evaluate which data backend to use
|
||||
* on a data put call.
|
||||
* @constructor
|
||||
* @param {string | undefined} objectLocationConstraint - location constraint
|
||||
* for object based on user meta header
|
||||
* @param {string | undefined } bucketLocationConstraint - location
|
||||
* constraint for bucket based on bucket metadata
|
||||
* @param {string} requestEndpoint - endpoint to which request was made
|
||||
* @param {string | undefined } legacyLocationConstraint - legacy location
|
||||
* constraint
|
||||
*/
|
||||
constructor(objectLocationConstraint, bucketLocationConstraint,
|
||||
requestEndpoint, legacyLocationConstraint) {
|
||||
this._objectLocationConstraint = objectLocationConstraint;
|
||||
this._bucketLocationConstraint = bucketLocationConstraint;
|
||||
this._requestEndpoint = requestEndpoint;
|
||||
this._legacyLocationConstraint = legacyLocationConstraint;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* validate proposed location constraint against config
|
||||
* @param {string | undefined} locationConstraint - value of user
|
||||
* metadata location constraint header or bucket location constraint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if valid, false if not
|
||||
*/
|
||||
static isValidLocationConstraint(locationConstraint, log) {
|
||||
if (Object.keys(config.locationConstraints).
|
||||
indexOf(locationConstraint) < 0) {
|
||||
log.trace('proposed locationConstraint is invalid',
|
||||
{ locationConstraint });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* validate that request endpoint is listed in the restEndpoint config
|
||||
* @param {string} requestEndpoint - request endpoint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if present, false if not
|
||||
*/
|
||||
static isRequestEndpointPresent(requestEndpoint, log) {
|
||||
if (Object.keys(config.restEndpoints).indexOf(requestEndpoint) < 0) {
|
||||
log.trace('requestEndpoint does not match config restEndpoints',
|
||||
{ requestEndpoint });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* validate that locationConstraint for request Endpoint matches
|
||||
* one config locationConstraint
|
||||
* @param {string} requestEndpoint - request endpoint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if matches, false if not
|
||||
*/
|
||||
static isRequestEndpointValueValid(requestEndpoint, log) {
|
||||
if (Object.keys(config.locationConstraints).indexOf(config
|
||||
.restEndpoints[requestEndpoint]) < 0) {
|
||||
log.trace('the default locationConstraint for request' +
|
||||
'Endpoint does not match any config locationConstraint',
|
||||
{ requestEndpoint });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* validate that s3 server is running with a file or memory backend
|
||||
* @param {string} requestEndpoint - request endpoint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if running with file/mem backend, false if not
|
||||
*/
|
||||
static isMemOrFileBackend(requestEndpoint, log) {
|
||||
if (config.backends.data === 'mem' ||
|
||||
config.backends.data === 'file') {
|
||||
log.trace('use data backend for the location', {
|
||||
dataBackend: config.backends.data,
|
||||
method: 'isMemOrFileBackend',
|
||||
});
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* validate requestEndpoint against config or mem/file data backend
|
||||
* - if there is no match for the request endpoint in the config
|
||||
* restEndpoints and data backend is set to mem or file we will use this
|
||||
* data backend for the location.
|
||||
* - if locationConstraint for request Endpoint does not match
|
||||
* any config locationConstraint, we will return an error
|
||||
* @param {string} requestEndpoint - request endpoint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if valid, false if not
|
||||
*/
|
||||
static isValidRequestEndpointOrBackend(requestEndpoint, log) {
|
||||
if (!BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
|
||||
return BackendInfo.isMemOrFileBackend(requestEndpoint, log);
|
||||
}
|
||||
return BackendInfo.isRequestEndpointValueValid(requestEndpoint, log);
|
||||
}
|
||||
|
||||
/**
|
||||
* validate controlling BackendInfo Parameter
|
||||
* @param {string | undefined} objectLocationConstraint - value of user
|
||||
* metadata location constraint header
|
||||
* @param {string | null} bucketLocationConstraint - location
|
||||
* constraint from bucket metadata
|
||||
* @param {string} requestEndpoint - endpoint of request
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {object} - location constraint validity
|
||||
*/
|
||||
static controllingBackendParam(objectLocationConstraint,
|
||||
bucketLocationConstraint, requestEndpoint, log) {
|
||||
if (objectLocationConstraint) {
|
||||
if (BackendInfo.isValidLocationConstraint(objectLocationConstraint,
|
||||
log)) {
|
||||
log.trace('objectLocationConstraint is valid');
|
||||
return { isValid: true };
|
||||
}
|
||||
log.trace('objectLocationConstraint is invalid');
|
||||
return { isValid: false, description: 'Object Location Error - ' +
|
||||
`Your object location "${escapeForXml(objectLocationConstraint)}"` +
|
||||
'is not in your location config - Please update.' };
|
||||
}
|
||||
if (bucketLocationConstraint) {
|
||||
if (BackendInfo.isValidLocationConstraint(bucketLocationConstraint,
|
||||
log)) {
|
||||
log.trace('bucketLocationConstraint is valid');
|
||||
return { isValid: true };
|
||||
}
|
||||
log.trace('bucketLocationConstraint is invalid');
|
||||
return { isValid: false, description: 'Bucket Location Error - ' +
|
||||
`Your bucket location "${escapeForXml(bucketLocationConstraint)}"` +
|
||||
' is not in your location config - Please update.' };
|
||||
}
|
||||
const legacyLocationConstraint =
|
||||
BackendInfo.getLegacyLocationConstraint();
|
||||
if (legacyLocationConstraint) {
|
||||
log.trace('legacy location is valid');
|
||||
return { isValid: true, legacyLocationConstraint };
|
||||
}
|
||||
if (!BackendInfo.isValidRequestEndpointOrBackend(requestEndpoint,
|
||||
log)) {
|
||||
return { isValid: false, description: 'Endpoint Location Error - ' +
|
||||
`Your endpoint "${requestEndpoint}" is not in restEndpoints ` +
|
||||
'in your config OR the default location constraint for request ' +
|
||||
`endpoint "${escapeForXml(requestEndpoint)}" does not ` +
|
||||
'match any config locationConstraint - Please update.' };
|
||||
}
|
||||
if (BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
|
||||
return { isValid: true };
|
||||
}
|
||||
return { isValid: true, defaultedToDataBackend: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Return legacyLocationConstraint
|
||||
* @return {string | undefined} legacyLocationConstraint;
|
||||
*/
|
||||
static getLegacyLocationConstraint() {
|
||||
return legacyLocations.find(ll => locationConstraints[ll]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return objectLocationConstraint
|
||||
* @return {string | undefined} objectLocationConstraint;
|
||||
*/
|
||||
getObjectLocationConstraint() {
|
||||
return this._objectLocationConstraint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return bucketLocationConstraint
|
||||
* @return {string | undefined} bucketLocationConstraint;
|
||||
*/
|
||||
getBucketLocationConstraint() {
|
||||
return this._bucketLocationConstraint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return requestEndpoint
|
||||
* @return {string} requestEndpoint;
|
||||
*/
|
||||
getRequestEndpoint() {
|
||||
return this._requestEndpoint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return locationConstraint that should be used with put request
|
||||
* Order of priority is:
|
||||
* (1) objectLocationConstraint,
|
||||
* (2) bucketLocationConstraint,
|
||||
* (3) legacyLocationConstraint,
|
||||
* (4) default locationConstraint for requestEndpoint if requestEndpoint
|
||||
* is listed in restEndpoints in config.json
|
||||
* (5) default data backend
|
||||
* @return {string} locationConstraint;
|
||||
*/
|
||||
getControllingLocationConstraint() {
|
||||
const objectLC = this.getObjectLocationConstraint();
|
||||
const bucketLC = this.getBucketLocationConstraint();
|
||||
const reqEndpoint = this.getRequestEndpoint();
|
||||
if (objectLC) {
|
||||
return objectLC;
|
||||
}
|
||||
if (bucketLC) {
|
||||
return bucketLC;
|
||||
}
|
||||
if (this._legacyLocationConstraint) {
|
||||
return this._legacyLocationConstraint;
|
||||
}
|
||||
if (config.restEndpoints[reqEndpoint]) {
|
||||
return config.restEndpoints[reqEndpoint];
|
||||
}
|
||||
return config.backends.data;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
BackendInfo,
|
||||
};
|
|
@ -73,6 +73,15 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
|
|||
return next(null, mpuBucket, destBucket, skipDataDelete);
|
||||
});
|
||||
},
|
||||
function sendAbortPut(mpuBucket, destBucket, skipDataDelete, next) {
|
||||
services.sendAbortMPUPut(bucketName, objectKey, uploadId, log,
|
||||
err => {
|
||||
if (err) {
|
||||
return next(err, destBucket);
|
||||
}
|
||||
return next(null, mpuBucket, destBucket, skipDataDelete);
|
||||
});
|
||||
},
|
||||
function getPartLocations(mpuBucket, destBucket, skipDataDelete,
|
||||
next) {
|
||||
services.getMPUparts(mpuBucket.getName(), uploadId, log,
|
||||
|
@ -87,6 +96,7 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
|
|||
},
|
||||
function deleteData(mpuBucket, storedParts, destBucket,
|
||||
skipDataDelete, next) {
|
||||
// for Azure we do not need to delete data
|
||||
if (skipDataDelete) {
|
||||
return next(null, mpuBucket, storedParts, destBucket);
|
||||
}
|
||||
|
|
|
@ -1,19 +0,0 @@
|
|||
const { zenkoIDHeader } = require('arsenal').constants;
|
||||
|
||||
const _config = require('../../../Config').config;
|
||||
|
||||
/**
|
||||
* applyZenkoUserMD - if request is within a Zenko deployment, apply user
|
||||
* metadata called "zenko-source" to the object
|
||||
* @param {Object} metaHeaders - user metadata object
|
||||
* @return {undefined}
|
||||
*/
|
||||
function applyZenkoUserMD(metaHeaders) {
|
||||
if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' &&
|
||||
!metaHeaders[zenkoIDHeader]) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
metaHeaders[zenkoIDHeader] = _config.getPublicInstanceId();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = applyZenkoUserMD;
|
|
@ -1,27 +0,0 @@
|
|||
/**
|
||||
* checkReadLocation - verify that a bucket's default read location exists
|
||||
* for a specified read data locator
|
||||
* @param {Config} config - Config object
|
||||
* @param {string} locationName - location constraint
|
||||
* @param {string} objectKey - object key
|
||||
* @param {string} bucketName - bucket name
|
||||
* @return {Object | null} return object containing location information
|
||||
* if location exists; otherwise, null
|
||||
*/
|
||||
function checkReadLocation(config, locationName, objectKey, bucketName) {
|
||||
const readLocation = config.getLocationConstraint(locationName);
|
||||
if (readLocation) {
|
||||
const bucketMatch = readLocation.details &&
|
||||
readLocation.details.bucketMatch;
|
||||
const backendKey = bucketMatch ? objectKey :
|
||||
`${bucketName}/${objectKey}`;
|
||||
return {
|
||||
location: locationName,
|
||||
key: backendKey,
|
||||
locationType: readLocation.type,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = checkReadLocation;
|
|
@ -1,247 +0,0 @@
|
|||
/*
|
||||
* Code based on Yutaka Oishi (Fujifilm) contributions
|
||||
* Date: 11 Sep 2020
|
||||
*/
|
||||
const { ObjectMDArchive } = require('arsenal').models;
|
||||
const errors = require('arsenal').errors;
|
||||
const { config } = require('../../../Config');
|
||||
const { locationConstraints } = config;
|
||||
|
||||
const { scaledMsPerDay } = config.getTimeOptions();
|
||||
|
||||
/**
|
||||
* Get response header "x-amz-restore"
|
||||
* Be called by objectHead.js
|
||||
* @param {object} objMD - object's metadata
|
||||
* @returns {string|undefined} x-amz-restore
|
||||
*/
|
||||
function getAmzRestoreResHeader(objMD) {
|
||||
if (objMD.archive &&
|
||||
objMD.archive.restoreRequestedAt &&
|
||||
!objMD.archive.restoreCompletedAt) {
|
||||
// Avoid race condition by relying on the `archive` MD of the object
|
||||
// and return the right header after a RESTORE request.
|
||||
// eslint-disable-next-line
|
||||
return `ongoing-request="true"`;
|
||||
}
|
||||
if (objMD['x-amz-restore']) {
|
||||
if (objMD['x-amz-restore']['expiry-date']) {
|
||||
const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString();
|
||||
// eslint-disable-next-line
|
||||
return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if restore can be done.
|
||||
*
|
||||
* @param {ObjectMD} objectMD - object metadata
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
|
||||
*/
|
||||
function _validateStartRestore(objectMD, log) {
|
||||
if (objectMD.archive?.restoreCompletedAt) {
|
||||
if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
|
||||
// return InvalidObjectState error if the restored object is expired
|
||||
// but restore info md of this object has not yet been cleared
|
||||
log.debug('The restored object already expired.',
|
||||
{
|
||||
archive: objectMD.archive,
|
||||
method: '_validateStartRestore',
|
||||
});
|
||||
return errors.InvalidObjectState;
|
||||
}
|
||||
|
||||
// If object is already restored, no further check is needed
|
||||
// Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have
|
||||
// been reset.
|
||||
return undefined;
|
||||
}
|
||||
const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
|
||||
if (!isLocationCold) {
|
||||
// return InvalidObjectState error if the object is not in cold storage,
|
||||
// not in cold storage means either location cold flag not exists or cold flag is explicit false
|
||||
log.debug('The bucket of the object is not in a cold storage location.',
|
||||
{
|
||||
isLocationCold,
|
||||
method: '_validateStartRestore',
|
||||
});
|
||||
return errors.InvalidObjectState;
|
||||
}
|
||||
if (objectMD.archive?.restoreRequestedAt) {
|
||||
// return RestoreAlreadyInProgress error if the object is currently being restored
|
||||
// check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
|
||||
log.debug('The object is currently being restored.',
|
||||
{
|
||||
archive: objectMD.archive,
|
||||
method: '_validateStartRestore',
|
||||
});
|
||||
return errors.RestoreAlreadyInProgress;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if "put version id" is allowed
|
||||
*
|
||||
* @param {ObjectMD} objMD - object metadata
|
||||
* @param {string} versionId - object's version id
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {ArsenalError|undefined} - undefined if "put version id" is allowed
|
||||
*/
|
||||
function validatePutVersionId(objMD, versionId, log) {
|
||||
if (!objMD) {
|
||||
const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
|
||||
log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
|
||||
return err;
|
||||
}
|
||||
|
||||
if (objMD.isDeleteMarker) {
|
||||
log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
|
||||
return errors.MethodNotAllowed;
|
||||
}
|
||||
|
||||
const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
|
||||
if (!isLocationCold) {
|
||||
log.error('The object data is not stored in a cold storage location.',
|
||||
{
|
||||
isLocationCold,
|
||||
dataStoreName: objMD.dataStoreName,
|
||||
method: 'validatePutVersionId',
|
||||
});
|
||||
return errors.InvalidObjectState;
|
||||
}
|
||||
|
||||
// make sure object archive restoration is in progress
|
||||
// NOTE: we do not use putObjectVersion to update the restoration period.
|
||||
if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
|
||||
|| objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
|
||||
log.error('object archive restoration is not in progress',
|
||||
{ method: 'validatePutVersionId', versionId });
|
||||
return errors.InvalidObjectState;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the object is already restored, and update the expiration date accordingly:
|
||||
* > After restoring an archived object, you can update the restoration period by reissuing the
|
||||
* > request with a new period. Amazon S3 updates the restoration period relative to the current
|
||||
* > time.
|
||||
*
|
||||
* @param {ObjectMD} objectMD - object metadata
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if the object is already restored
|
||||
*/
|
||||
function _updateObjectExpirationDate(objectMD, log) {
|
||||
// Check if restoreCompletedAt field exists
|
||||
// Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
|
||||
// checked earlier in the process, so checking again here would create weird states
|
||||
const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
|
||||
log.debug('The restore status of the object.', {
|
||||
isObjectAlreadyRestored,
|
||||
method: 'isObjectAlreadyRestored'
|
||||
});
|
||||
if (isObjectAlreadyRestored) {
|
||||
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
|
||||
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
|
||||
|
||||
/* eslint-disable no-param-reassign */
|
||||
objectMD.archive.restoreWillExpireAt = expiryDate;
|
||||
objectMD['x-amz-restore'] = {
|
||||
'ongoing-request': false,
|
||||
'expiry-date': expiryDate,
|
||||
};
|
||||
/* eslint-enable no-param-reassign */
|
||||
}
|
||||
return isObjectAlreadyRestored;
|
||||
}
|
||||
|
||||
/**
|
||||
* update restore expiration date.
|
||||
*
|
||||
* @param {ObjectMD} objectMD - objectMD instance
|
||||
* @param {object} restoreParam - restore param
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {ArsenalError|undefined} internal error if object MD is not valid
|
||||
*
|
||||
*/
|
||||
function _updateRestoreInfo(objectMD, restoreParam, log) {
|
||||
if (!objectMD.archive) {
|
||||
log.debug('objectMD.archive doesn\'t exits', {
|
||||
objectMD,
|
||||
method: '_updateRestoreInfo'
|
||||
});
|
||||
return errors.InternalError.customizeDescription('Archive metadata is missing.');
|
||||
}
|
||||
/* eslint-disable no-param-reassign */
|
||||
objectMD.archive.restoreRequestedAt = new Date();
|
||||
objectMD.archive.restoreRequestedDays = restoreParam.days;
|
||||
objectMD.originOp = 's3:ObjectRestore:Post';
|
||||
/* eslint-enable no-param-reassign */
|
||||
if (!ObjectMDArchive.isValid(objectMD.archive)) {
|
||||
log.debug('archive is not valid', {
|
||||
archive: objectMD.archive,
|
||||
method: '_updateRestoreInfo'
|
||||
});
|
||||
return errors.InternalError.customizeDescription('Invalid archive metadata.');
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* start to restore object.
|
||||
* If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
|
||||
* calculate restore expiry-date and add it to objectMD.
|
||||
* Be called by objectRestore.js
|
||||
*
|
||||
* @param {ObjectMD} objectMD - objectMd instance
|
||||
* @param {object} restoreParam - bucket name
|
||||
* @param {object} log - werelogs logger
|
||||
* @param {function} cb - bucket name
|
||||
* @return {undefined}
|
||||
*
|
||||
*/
|
||||
function startRestore(objectMD, restoreParam, log, cb) {
|
||||
log.info('Validating if restore can be done or not.');
|
||||
const checkResultError = _validateStartRestore(objectMD, log);
|
||||
if (checkResultError) {
|
||||
return cb(checkResultError);
|
||||
}
|
||||
log.info('Updating restore information.');
|
||||
const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
|
||||
if (updateResultError) {
|
||||
return cb(updateResultError);
|
||||
}
|
||||
const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
|
||||
return cb(null, isObjectAlreadyRestored);
|
||||
}
|
||||
|
||||
/**
|
||||
* checks if object data is available or if it's in cold storage
|
||||
* @param {ObjectMD} objMD Object metadata
|
||||
* @returns {ArsenalError|null} error if object data is not available
|
||||
*/
|
||||
function verifyColdObjectAvailable(objMD) {
|
||||
// return error when object is cold
|
||||
if (objMD.archive &&
|
||||
// Object is in cold backend
|
||||
(!objMD.archive.restoreRequestedAt ||
|
||||
// Object is being restored
|
||||
(objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
|
||||
const err = errors.InvalidObjectState
|
||||
.customizeDescription('The operation is not valid for the object\'s storage class');
|
||||
return err;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startRestore,
|
||||
getAmzRestoreResHeader,
|
||||
validatePutVersionId,
|
||||
verifyColdObjectAvailable,
|
||||
};
|
|
@ -7,17 +7,43 @@ const { data } = require('../../../data/wrapper');
|
|||
const services = require('../../../services');
|
||||
const { dataStore } = require('./storeObject');
|
||||
const locationConstraintCheck = require('./locationConstraintCheck');
|
||||
const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
|
||||
const { versioningPreprocessing } = require('./versioning');
|
||||
const removeAWSChunked = require('./removeAWSChunked');
|
||||
const getReplicationInfo = require('./getReplicationInfo');
|
||||
const { config } = require('../../../Config');
|
||||
const validateWebsiteHeader = require('./websiteServing')
|
||||
.validateWebsiteHeader;
|
||||
const applyZenkoUserMD = require('./applyZenkoUserMD');
|
||||
const { externalBackends, versioningNotImplBackends } = constants;
|
||||
const {
|
||||
externalBackends, versioningNotImplBackends, zenkoIDHeader,
|
||||
} = constants;
|
||||
|
||||
const externalVersioningErrorMessage = 'We do not currently support putting ' +
|
||||
'a versioned object to a location-constraint of type Azure or GCP.';
|
||||
'a versioned object to a location-constraint of type Azure.';
|
||||
|
||||
/**
|
||||
* Retro-propagation is where S3C ingestion will re-ingest an object whose
|
||||
* request originated from Zenko.
|
||||
* To avoid this, Zenko requests which create objects/versions will be tagged
|
||||
* with a user-metadata header defined in constants.zenkoIDHeader. When
|
||||
* ingesting objects into Zenko, we can determine if this object has already
|
||||
* been created in Zenko.
|
||||
* Delete marker requests cannot specify user-metadata fields, so we instead
|
||||
* rely on checking the "user-agent" to see the origin of a request.
|
||||
* If delete marker, and user-agent came from a Zenko client, we add the
|
||||
* user-metadata field to the object metadata.
|
||||
* @param {Object} metaHeaders - user metadata object
|
||||
* @param {http.ClientRequest} request - client request with user-agent header
|
||||
* @param {Boolean} isDeleteMarker - delete marker indicator
|
||||
* @return {undefined}
|
||||
*/
|
||||
function _checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker) {
|
||||
const userAgent = request.headers['user-agent'];
|
||||
|
||||
if (isDeleteMarker && userAgent && userAgent.includes('Zenko')) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
metaHeaders[zenkoIDHeader] = 'zenko';
|
||||
}
|
||||
}
|
||||
|
||||
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
||||
metadataStoreParams, dataToDelete, log, requestMethod, callback) {
|
||||
|
@ -52,7 +78,6 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
|||
* credentialScope (to be used for streaming v4 auth if applicable)
|
||||
* @param {(object|null)} overheadField - fields to be included in metadata overhead
|
||||
* @param {RequestLogger} log - logger instance
|
||||
* @param {string} originOp - Origin operation
|
||||
* @param {function} callback - callback function
|
||||
* @return {undefined} and call callback with (err, result) -
|
||||
* result.contentMD5 - content md5 of new object or version
|
||||
|
@ -60,10 +85,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
|||
*/
|
||||
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
|
||||
overheadField, log, originOp, callback) {
|
||||
const putVersionId = request.headers['x-scal-s3-version-id'];
|
||||
const isPutVersion = putVersionId || putVersionId === '';
|
||||
|
||||
overheadField, log, callback) {
|
||||
const size = isDeleteMarker ? 0 : request.parsedContentLength;
|
||||
// although the request method may actually be 'DELETE' if creating a
|
||||
// delete marker, for our purposes we consider this to be a 'PUT'
|
||||
|
@ -86,9 +108,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
|||
});
|
||||
return process.nextTick(() => callback(metaHeaders));
|
||||
}
|
||||
// if the request occurs within a Zenko deployment, we place a user-metadata
|
||||
// field on the object
|
||||
applyZenkoUserMD(metaHeaders);
|
||||
// if receiving a request from Zenko for a delete marker, we place a
|
||||
// user-metadata field on the object
|
||||
_checkAndApplyZenkoMD(metaHeaders, request, isDeleteMarker);
|
||||
|
||||
log.trace('meta headers', { metaHeaders, method: 'objectPut' });
|
||||
const objectKeyContext = {
|
||||
|
@ -114,26 +136,11 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
|||
size,
|
||||
headers,
|
||||
isDeleteMarker,
|
||||
replicationInfo: getReplicationInfo(
|
||||
objectKey, bucketMD, false, size, null, null, authInfo),
|
||||
replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
|
||||
overheadField,
|
||||
log,
|
||||
};
|
||||
|
||||
// For Azure BlobStorage API compatability
|
||||
// If an object already exists copy/repair creation-time
|
||||
// creation-time must remain static after an object is created
|
||||
// --> EVEN FOR VERSIONS <--
|
||||
if (objMD) {
|
||||
if (objMD['creation-time']) {
|
||||
metadataStoreParams.creationTime = objMD['creation-time'];
|
||||
} else {
|
||||
// If creation-time is not set (for old objects)
|
||||
// fall back to the last modified and store it back to the db
|
||||
metadataStoreParams.creationTime = objMD['last-modified'];
|
||||
}
|
||||
}
|
||||
|
||||
if (!isDeleteMarker) {
|
||||
metadataStoreParams.contentType = request.headers['content-type'];
|
||||
metadataStoreParams.cacheControl = request.headers['cache-control'];
|
||||
|
@ -143,7 +150,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
|||
removeAWSChunked(request.headers['content-encoding']);
|
||||
metadataStoreParams.expires = request.headers.expires;
|
||||
metadataStoreParams.tagging = request.headers['x-amz-tagging'];
|
||||
metadataStoreParams.originOp = originOp;
|
||||
metadataStoreParams.originOp = 's3:ObjectCreated:Put';
|
||||
const defaultObjectLockConfiguration
|
||||
= bucketMD.getObjectLockConfiguration();
|
||||
if (defaultObjectLockConfiguration) {
|
||||
|
@ -158,7 +165,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
|||
// eslint-disable-next-line no-param-reassign
|
||||
request.headers[constants.objectLocationConstraintHeader] =
|
||||
objMD[constants.objectLocationConstraintHeader];
|
||||
metadataStoreParams.originOp = originOp;
|
||||
metadataStoreParams.originOp = 's3:ObjectRemoved:DeleteMarkerCreated';
|
||||
}
|
||||
|
||||
const backendInfoObj =
|
||||
|
@ -197,44 +204,24 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
|||
const dontSkipBackend = externalBackends;
|
||||
/* eslint-enable camelcase */
|
||||
|
||||
const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
|
||||
const mdOnlySize = request.headers['x-amz-meta-size'];
|
||||
|
||||
return async.waterfall([
|
||||
function storeData(next) {
|
||||
if (size === 0) {
|
||||
if (!dontSkipBackend[locationType]) {
|
||||
if (size === 0 && !dontSkipBackend[locationType]) {
|
||||
metadataStoreParams.contentMD5 = constants.emptyFileMd5;
|
||||
return next(null, null, null);
|
||||
}
|
||||
// Handle mdOnlyHeader as a metadata only operation. If
|
||||
// the object in question is actually 0 byte or has a body size
|
||||
// then handle normally.
|
||||
if (mdOnlyHeader === 'true' && mdOnlySize > 0) {
|
||||
log.debug('metadata only operation x-amz-meta-mdonly');
|
||||
const md5 = request.headers['x-amz-meta-md5chksum']
|
||||
? new Buffer(request.headers['x-amz-meta-md5chksum'],
|
||||
'base64').toString('hex') : null;
|
||||
const numParts = request.headers['x-amz-meta-md5numparts'];
|
||||
let _md5;
|
||||
if (numParts === undefined) {
|
||||
_md5 = md5;
|
||||
// Object Post receives a file stream.
|
||||
// This is to be used to store data instead of the request stream itself.
|
||||
|
||||
let stream;
|
||||
|
||||
if (request.apiMethod === 'objectPost') {
|
||||
stream = request.fileEventData ? request.fileEventData.file : undefined;
|
||||
} else {
|
||||
_md5 = `${md5}-${numParts}`;
|
||||
stream = request;
|
||||
}
|
||||
const versionId = request.headers['x-amz-meta-version-id'];
|
||||
const dataGetInfo = {
|
||||
key: objectKey,
|
||||
dataStoreName: location,
|
||||
dataStoreType: locationType,
|
||||
dataStoreVersionId: versionId,
|
||||
dataStoreMD5: _md5,
|
||||
};
|
||||
return next(null, dataGetInfo, _md5);
|
||||
}
|
||||
}
|
||||
return dataStore(objectKeyContext, cipherBundle, request, size,
|
||||
streamingV4Params, backendInfo, log, next);
|
||||
|
||||
return dataStore(objectKeyContext, cipherBundle, stream, size, streamingV4Params, backendInfo, log, next);
|
||||
},
|
||||
function processDataResult(dataGetInfo, calculatedHash, next) {
|
||||
if (dataGetInfo === null || dataGetInfo === undefined) {
|
||||
|
@ -256,19 +243,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
|||
dataGetInfoArr[0].cipheredDataKey =
|
||||
cipherBundle.cipheredDataKey;
|
||||
}
|
||||
if (mdOnlyHeader === 'true') {
|
||||
metadataStoreParams.size = mdOnlySize;
|
||||
dataGetInfoArr[0].size = mdOnlySize;
|
||||
}
|
||||
metadataStoreParams.contentMD5 = calculatedHash;
|
||||
return next(null, dataGetInfoArr);
|
||||
},
|
||||
function getVersioningInfo(infoArr, next) {
|
||||
// if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
|
||||
if (isPutVersion) {
|
||||
const options = overwritingVersioning(objMD, metadataStoreParams);
|
||||
return process.nextTick(() => next(null, options, infoArr));
|
||||
}
|
||||
return versioningPreprocessing(bucketName, bucketMD,
|
||||
metadataStoreParams.objectKey, objMD, log, (err, options) => {
|
||||
if (err) {
|
||||
|
|
|
@ -4,25 +4,23 @@ const {
|
|||
LifecycleDateTime,
|
||||
LifecycleUtils,
|
||||
} = require('arsenal').s3middleware.lifecycleHelpers;
|
||||
const { config } = require('../../../Config');
|
||||
|
||||
const {
|
||||
expireOneDayEarlier,
|
||||
transitionOneDayEarlier,
|
||||
timeProgressionFactor,
|
||||
scaledMsPerDay,
|
||||
} = config.getTimeOptions();
|
||||
// moves lifecycle transition deadlines 1 day earlier, mostly for testing
|
||||
const transitionOneDayEarlier = process.env.TRANSITION_ONE_DAY_EARLIER === 'true';
|
||||
// moves lifecycle expiration deadlines 1 day earlier, mostly for testing
|
||||
const expireOneDayEarlier = process.env.EXPIRE_ONE_DAY_EARLIER === 'true';
|
||||
|
||||
const lifecycleDateTime = new LifecycleDateTime({
|
||||
transitionOneDayEarlier,
|
||||
expireOneDayEarlier,
|
||||
timeProgressionFactor,
|
||||
});
|
||||
|
||||
const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor);
|
||||
const lifecycleUtils = new LifecycleUtils(supportedLifecycleRules, lifecycleDateTime);
|
||||
|
||||
const oneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
|
||||
|
||||
function calculateDate(objDate, expDays, datetime) {
|
||||
return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay));
|
||||
return new Date(datetime.getTimestamp(objDate) + expDays * oneDay);
|
||||
}
|
||||
|
||||
function formatExpirationHeader(date, id) {
|
||||
|
|
|
@ -1,51 +0,0 @@
|
|||
const { errors } = require('arsenal');
|
||||
|
||||
/**
|
||||
* getReplicationBackendDataLocator - compares given location constraint to
|
||||
* replication backends
|
||||
* @param {object} locationObj - object containing location information
|
||||
* @param {string} locationObj.location - name of location constraint
|
||||
* @param {string} locationObj.key - keyname of object in location constraint
|
||||
* @param {string} locationObj.locationType - type of location constraint
|
||||
* @param {object} replicationInfo - information about object replication
|
||||
* @param {array} replicationInfo.backends - array containing information about
|
||||
* each replication location
|
||||
* @param {string} replicationInfo.backends[].site - name of replication
|
||||
* location
|
||||
* @param {string} replicationInfo.backends[].status - status of replication
|
||||
* @param {string} replicationInfo.backends[].dataStoreVersionId - version id
|
||||
* of object at replication location
|
||||
* @return {object} res - response object
|
||||
* {array} [res.dataLocator] - if COMPLETED status: array
|
||||
* containing the cloud location,
|
||||
* undefined otherwise
|
||||
* {string} [res.status] - replication status if no error
|
||||
* {string} [res.reason] - reason message if PENDING/FAILED
|
||||
* {Error} [res.error] - defined if object is not replicated to
|
||||
* location passed in locationObj
|
||||
*/
|
||||
function getReplicationBackendDataLocator(locationObj, replicationInfo) {
|
||||
const repBackendResult = {};
|
||||
const locMatch = replicationInfo.backends.find(
|
||||
backend => backend.site === locationObj.location);
|
||||
if (!locMatch) {
|
||||
repBackendResult.error = errors.InvalidLocationConstraint.
|
||||
customizeDescription('Object is not replicated to location ' +
|
||||
'passed in location header');
|
||||
return repBackendResult;
|
||||
}
|
||||
repBackendResult.status = locMatch.status;
|
||||
if (['PENDING', 'FAILED'].includes(locMatch.status)) {
|
||||
repBackendResult.reason =
|
||||
`Object replication to specified backend is ${locMatch.status}`;
|
||||
return repBackendResult;
|
||||
}
|
||||
repBackendResult.dataLocator = [{
|
||||
key: locationObj.key,
|
||||
dataStoreName: locationObj.location,
|
||||
dataStoreType: locationObj.locationType,
|
||||
dataStoreVersionId: locMatch.dataStoreVersionId }];
|
||||
return repBackendResult;
|
||||
}
|
||||
|
||||
module.exports = getReplicationBackendDataLocator;
|
|
@ -1,7 +1,5 @@
|
|||
const s3config = require('../../../Config').config;
|
||||
const { isServiceAccount, getServiceAccountProperties } =
|
||||
require('../authorization/permissionChecks');
|
||||
const { replicationBackends } = require('arsenal').constants;
|
||||
const { isLifecycleSession } = require('../authorization/permissionChecks.js');
|
||||
|
||||
function _getBackend(objectMD, site) {
|
||||
const backends = objectMD ? objectMD.replicationInfo.backends : [];
|
||||
|
@ -23,31 +21,25 @@ function _getStorageClasses(rule) {
|
|||
}
|
||||
const { replicationEndpoints } = s3config;
|
||||
// If no storage class, use the given default endpoint or the sole endpoint
|
||||
if (replicationEndpoints.length > 0) {
|
||||
if (replicationEndpoints.length > 1) {
|
||||
const endPoint =
|
||||
replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
|
||||
replicationEndpoints.find(endpoint => endpoint.default);
|
||||
return [endPoint.site];
|
||||
}
|
||||
return undefined;
|
||||
return [replicationEndpoints[0].site];
|
||||
}
|
||||
|
||||
function _getReplicationInfo(rule, replicationConfig, content, operationType,
|
||||
objectMD, bucketMD) {
|
||||
objectMD) {
|
||||
const storageTypes = [];
|
||||
const backends = [];
|
||||
const storageClasses = _getStorageClasses(rule);
|
||||
if (!storageClasses) {
|
||||
return undefined;
|
||||
}
|
||||
storageClasses.forEach(storageClass => {
|
||||
const storageClassName =
|
||||
storageClass.endsWith(':preferred_read') ?
|
||||
storageClass.split(':')[0] : storageClass;
|
||||
const location = s3config.locationConstraints[storageClassName];
|
||||
if (location && replicationBackends[location.type]) {
|
||||
const location = s3config.locationConstraints[storageClass];
|
||||
if (location && ['aws_s3', 'azure'].includes(location.type)) {
|
||||
storageTypes.push(location.type);
|
||||
}
|
||||
backends.push(_getBackend(objectMD, storageClassName));
|
||||
backends.push(_getBackend(objectMD, storageClass));
|
||||
});
|
||||
if (storageTypes.length > 0 && operationType) {
|
||||
content.push(operationType);
|
||||
|
@ -60,7 +52,6 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
|
|||
storageClass: storageClasses.join(','),
|
||||
role: replicationConfig.role,
|
||||
storageType: storageTypes.join(','),
|
||||
isNFS: bucketMD.isNFS(),
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -74,45 +65,26 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
|
|||
* @param {string} operationType - The type of operation to replicate
|
||||
* @param {object} objectMD - The object metadata
|
||||
* @param {AuthInfo} [authInfo] - authentication info of object owner
|
||||
* @param {boolean} [isDeleteMarker] - whether creating a delete marker
|
||||
* @return {undefined}
|
||||
*/
|
||||
function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
|
||||
objectMD, authInfo) {
|
||||
objectMD, authInfo, isDeleteMarker) {
|
||||
const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
|
||||
const config = bucketMD.getReplicationConfiguration();
|
||||
|
||||
// Do not replicate object in the following cases:
|
||||
//
|
||||
// - bucket does not have a replication configuration
|
||||
//
|
||||
// - replication configuration does not apply to the object
|
||||
// (i.e. no rule matches object prefix)
|
||||
//
|
||||
// - replication configuration applies to the object (i.e. a rule matches
|
||||
// object prefix) but the status is disabled
|
||||
//
|
||||
// - object owner is an internal service account like Lifecycle,
|
||||
// unless the account properties explicitly allow it to
|
||||
// replicate like MD ingestion (because we do not want to
|
||||
// replicate objects created from actions triggered by internal
|
||||
// services, by design)
|
||||
|
||||
// If bucket does not have a replication configuration, do not replicate.
|
||||
if (config) {
|
||||
let doReplicate = false;
|
||||
if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) {
|
||||
doReplicate = true;
|
||||
} else {
|
||||
const serviceAccountProps = getServiceAccountProperties(
|
||||
authInfo.getCanonicalID());
|
||||
doReplicate = serviceAccountProps.canReplicate;
|
||||
// If delete an object due to a lifecycle action,
|
||||
// the delete marker is not replicated to the destination buckets.
|
||||
if (isDeleteMarker && authInfo && isLifecycleSession(authInfo.getArn())) {
|
||||
return undefined;
|
||||
}
|
||||
if (doReplicate) {
|
||||
const rule = config.rules.find(
|
||||
rule => (objKey.startsWith(rule.prefix) && rule.enabled));
|
||||
|
||||
const rule = config.rules.find(rule =>
|
||||
(objKey.startsWith(rule.prefix) && rule.enabled));
|
||||
if (rule) {
|
||||
return _getReplicationInfo(
|
||||
rule, config, content, operationType, objectMD, bucketMD);
|
||||
}
|
||||
return _getReplicationInfo(rule, config, content, operationType,
|
||||
objectMD);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
const { errors, models } = require('arsenal');
|
||||
const { errors } = require('arsenal');
|
||||
|
||||
const { BackendInfo } = models;
|
||||
const { config } = require('../../../Config');
|
||||
const { BackendInfo } = require('./BackendInfo');
|
||||
const constants = require('../../../../constants');
|
||||
|
||||
/**
|
||||
|
@ -30,7 +29,7 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) {
|
|||
const bucketLocationConstraint = bucket.getLocationConstraint();
|
||||
const requestEndpoint = request.parsedHost;
|
||||
|
||||
const controllingBackend = BackendInfo.controllingBackendParam(config,
|
||||
const controllingBackend = BackendInfo.controllingBackendParam(
|
||||
objectLocationConstraint, bucketLocationConstraint,
|
||||
requestEndpoint, log);
|
||||
if (!controllingBackend.isValid) {
|
||||
|
@ -40,7 +39,7 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) {
|
|||
};
|
||||
return backendInfoObj;
|
||||
}
|
||||
const backendInfo = new BackendInfo(config, objectLocationConstraint,
|
||||
const backendInfo = new BackendInfo(objectLocationConstraint,
|
||||
bucketLocationConstraint, requestEndpoint,
|
||||
controllingBackend.legacyLocationConstraint);
|
||||
backendInfoObj = {
|
||||
|
|
|
@ -1,37 +0,0 @@
|
|||
const { errors } = require('arsenal');
|
||||
|
||||
const { config } = require('../../../Config');
|
||||
|
||||
/**
|
||||
* locationHeaderCheck - compares 'x-amz-location-constraint' header
|
||||
* to location constraints in config
|
||||
* @param {object} headers - request headers
|
||||
* @param {string} objectKey - key name of object
|
||||
* @param {string} bucketName - name of bucket
|
||||
* @return {undefined|Object} returns error, object, or undefined
|
||||
* @return {string} return.location - name of location constraint
|
||||
* @return {string} return.key - name of object at location constraint
|
||||
* @return {string} - return.locationType - type of location constraint
|
||||
*/
|
||||
function locationHeaderCheck(headers, objectKey, bucketName) {
|
||||
const location = headers['x-amz-location-constraint'];
|
||||
if (location) {
|
||||
const validLocation = config.locationConstraints[location];
|
||||
if (!validLocation) {
|
||||
return errors.InvalidLocationConstraint.customizeDescription(
|
||||
'Invalid location constraint specified in header');
|
||||
}
|
||||
const bucketMatch = validLocation.details.bucketMatch;
|
||||
const backendKey = bucketMatch ? objectKey :
|
||||
`${bucketName}/${objectKey}`;
|
||||
return {
|
||||
location,
|
||||
key: backendKey,
|
||||
locationType: validLocation.type,
|
||||
};
|
||||
}
|
||||
// no location header was passed
|
||||
return undefined;
|
||||
}
|
||||
|
||||
module.exports = locationHeaderCheck;
|
|
@ -22,13 +22,8 @@ function locationKeysHaveChanged(prev, curr) {
|
|||
return curr.every(v => v.key !== prev);
|
||||
}
|
||||
const keysMap = {};
|
||||
prev.forEach(v => {
|
||||
if (!keysMap[v.dataStoreType]) {
|
||||
keysMap[v.dataStoreType] = {};
|
||||
}
|
||||
keysMap[v.dataStoreType][v.key] = true;
|
||||
});
|
||||
return curr.every(v => !(keysMap[v.dataStoreType] && keysMap[v.dataStoreType][v.key]));
|
||||
prev.forEach(v => { keysMap[v.key] = true; });
|
||||
return curr.every(v => !keysMap[v.key]);
|
||||
}
|
||||
|
||||
module.exports = locationKeysHaveChanged;
|
||||
|
|
|
@ -5,7 +5,6 @@ const { config } = require('../../../Config');
|
|||
const vault = require('../../../auth/vault');
|
||||
const { evaluateBucketPolicyWithIAM } = require('../authorization/permissionChecks');
|
||||
|
||||
const { scaledMsPerDay } = config.getTimeOptions();
|
||||
/**
|
||||
* Calculates retain until date for the locked object version
|
||||
* @param {object} retention - includes days or years retention period
|
||||
|
@ -21,9 +20,8 @@ function calculateRetainUntilDate(retention) {
|
|||
const date = moment();
|
||||
// Calculate the number of days to retain the lock on the object
|
||||
const retainUntilDays = days || years * 365;
|
||||
const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay;
|
||||
const retainUntilDate
|
||||
= date.add(retainUntilDaysInMs, 'ms');
|
||||
= date.add(retainUntilDays, 'days');
|
||||
return retainUntilDate.toISOString();
|
||||
}
|
||||
/**
|
||||
|
|
|
@ -1,172 +0,0 @@
|
|||
const async = require('async');
|
||||
const { errors, s3middleware } = require('arsenal');
|
||||
|
||||
const { allowedRestoreObjectRequestTierValues } = require('../../../../constants');
|
||||
const coldStorage = require('./coldStorage');
|
||||
const monitoring = require('../../../utilities/monitoringHandler');
|
||||
const { pushMetric } = require('../../../utapi/utilities');
|
||||
const { decodeVersionId } = require('./versioning');
|
||||
const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
|
||||
const { parseRestoreRequestXml } = s3middleware.objectRestore;
|
||||
const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
|
||||
|
||||
/**
|
||||
* Check if tier is supported
|
||||
* @param {object} restoreInfo - restore information
|
||||
* @returns {ArsenalError|undefined} return NotImplemented error if tier not support
|
||||
*/
|
||||
function checkTierSupported(restoreInfo) {
|
||||
if (!allowedRestoreObjectRequestTierValues.includes(restoreInfo.tier)) {
|
||||
return errors.NotImplemented;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* POST Object restore process
|
||||
*
|
||||
* @param {MetadataWrapper} metadata - metadata wrapper
|
||||
* @param {object} mdUtils - utility object to treat metadata
|
||||
* @param {AuthInfo} userInfo - Instance of AuthInfo class with requester's info
|
||||
* @param {IncomingMessage} request - request info
|
||||
* @param {object} log - Werelogs logger
|
||||
* @param {function} callback callback function
|
||||
* @return {undefined}
|
||||
*/
|
||||
function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
|
||||
const METHOD = 'objectRestore';
|
||||
|
||||
const { bucketName, objectKey } = request;
|
||||
|
||||
log.debug('processing request', { method: METHOD });
|
||||
|
||||
const decodedVidResult = decodeVersionId(request.query);
|
||||
if (decodedVidResult instanceof Error) {
|
||||
log.trace('invalid versionId query',
|
||||
{
|
||||
method: METHOD,
|
||||
versionId: request.query.versionId,
|
||||
error: decodedVidResult,
|
||||
});
|
||||
return process.nextTick(() => callback(decodedVidResult));
|
||||
}
|
||||
|
||||
let isObjectRestored = false;
|
||||
|
||||
const mdValueParams = {
|
||||
authInfo: userInfo,
|
||||
bucketName,
|
||||
objectKey,
|
||||
versionId: decodedVidResult,
|
||||
requestType: request.apiMethods || 'restoreObject',
|
||||
/**
|
||||
* Restoring an object might not cause any impact on
|
||||
* the storage, if the object is already restored: in
|
||||
* this case, the duration is extended. We disable the
|
||||
* quota evaluation and trigger it manually.
|
||||
*/
|
||||
checkQuota: false,
|
||||
request,
|
||||
};
|
||||
|
||||
return async.waterfall([
|
||||
// get metadata of bucket and object
|
||||
function validateBucketAndObject(next) {
|
||||
return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies,
|
||||
log, (err, bucketMD, objectMD) => {
|
||||
if (err) {
|
||||
log.trace('request authorization failed', { method: METHOD, error: err });
|
||||
return next(err);
|
||||
}
|
||||
// Call back error if object metadata could not be obtained
|
||||
if (!objectMD) {
|
||||
const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey;
|
||||
log.trace('error no object metadata found', { method: METHOD, error: err });
|
||||
return next(err, bucketMD);
|
||||
}
|
||||
// If object metadata is delete marker,
|
||||
// call back NoSuchKey or MethodNotAllowed depending on specifying versionId
|
||||
if (objectMD.isDeleteMarker) {
|
||||
let err = errors.NoSuchKey;
|
||||
if (decodedVidResult) {
|
||||
err = errors.MethodNotAllowed;
|
||||
}
|
||||
log.trace('version is a delete marker', { method: METHOD, error: err });
|
||||
return next(err, bucketMD, objectMD);
|
||||
}
|
||||
log.info('it acquired the object metadata.', {
|
||||
'method': METHOD,
|
||||
});
|
||||
return next(null, bucketMD, objectMD);
|
||||
});
|
||||
},
|
||||
|
||||
// generate restore param obj from xml of request body and check tier validity
|
||||
function parseRequestXmlAndCheckTier(bucketMD, objectMD, next) {
|
||||
log.trace('parsing object restore information');
|
||||
return parseRestoreRequestXml(request.post, log, (err, restoreInfo) => {
|
||||
if (err) {
|
||||
return next(err, bucketMD, objectMD, restoreInfo);
|
||||
}
|
||||
log.info('it parsed xml of the request body.', { method: METHOD, value: restoreInfo });
|
||||
const checkTierResult = checkTierSupported(restoreInfo);
|
||||
if (checkTierResult instanceof Error) {
|
||||
return next(checkTierResult);
|
||||
}
|
||||
return next(null, bucketMD, objectMD, restoreInfo);
|
||||
});
|
||||
},
|
||||
// start restore process
|
||||
function startRestore(bucketMD, objectMD, restoreInfo, next) {
|
||||
return coldStorage.startRestore(objectMD, restoreInfo, log,
|
||||
(err, _isObjectRestored) => {
|
||||
isObjectRestored = _isObjectRestored;
|
||||
return next(err, bucketMD, objectMD);
|
||||
});
|
||||
},
|
||||
function evaluateQuotas(bucketMD, objectMD, next) {
|
||||
if (isObjectRestored) {
|
||||
return next(null, bucketMD, objectMD);
|
||||
}
|
||||
const actions = Array.isArray(mdValueParams.requestType) ?
|
||||
mdValueParams.requestType : [mdValueParams.requestType];
|
||||
const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
|
||||
return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
|
||||
false, log, err => next(err, bucketMD, objectMD));
|
||||
},
|
||||
function updateObjectMD(bucketMD, objectMD, next) {
|
||||
const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
|
||||
metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
|
||||
log, err => next(err, bucketMD, objectMD));
|
||||
},
|
||||
],
|
||||
(err, bucketMD) => {
|
||||
// generate CORS response header
|
||||
const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD);
|
||||
if (err) {
|
||||
log.trace('error processing request',
|
||||
{
|
||||
method: METHOD,
|
||||
error: err,
|
||||
});
|
||||
monitoring.promMetrics(
|
||||
'POST', bucketName, err.code, 'restoreObject');
|
||||
return callback(err, err.code, responseHeaders);
|
||||
}
|
||||
pushMetric('restoreObject', log, {
|
||||
userInfo,
|
||||
bucket: bucketName,
|
||||
});
|
||||
if (isObjectRestored) {
|
||||
monitoring.promMetrics(
|
||||
'POST', bucketName, '200', 'restoreObject');
|
||||
return callback(null, 200, responseHeaders);
|
||||
}
|
||||
monitoring.promMetrics(
|
||||
'POST', bucketName, '202', 'restoreObject');
|
||||
return callback(null, 202, responseHeaders);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
module.exports = objectRestore;
|
|
@ -13,7 +13,7 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform');
|
|||
* the type of request requires them
|
||||
*/
|
||||
function prepareStream(stream, streamingV4Params, log, errCb) {
|
||||
if (stream.headers['x-amz-content-sha256'] ===
|
||||
if (stream && stream.headers && stream.headers['x-amz-content-sha256'] ===
|
||||
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
|
||||
if (typeof streamingV4Params !== 'object') {
|
||||
// this might happen if the user provided a valid V2
|
||||
|
|
|
@ -0,0 +1,241 @@
|
|||
const { errors } = require('arsenal');
|
||||
const crypto = require('crypto');
|
||||
const constants = require('../../../../constants');
|
||||
|
||||
/**
|
||||
* createAggregateETag - creates ETag from concatenated MPU part ETags to
|
||||
* mimic AWS
|
||||
* @param {string} concatETags - string of concatenated MPU part ETags
|
||||
* @param {array} partList - list of parts to complete MPU with
|
||||
* @return {string} aggregateETag - final complete MPU obj ETag
|
||||
*/
|
||||
function createAggregateETag(concatETags, partList) {
|
||||
// AWS documentation is unclear on what the MD5 is that it returns
|
||||
// in the response for a complete multipart upload request.
|
||||
// The docs state that they might or might not
|
||||
// return the MD5 of the complete object. It appears
|
||||
// they are returning the MD5 of the parts' MD5s so that is
|
||||
// what we have done here. We:
|
||||
// 1) concatenate the hex version of the
|
||||
// individual ETags
|
||||
// 2) convert the concatenated hex to binary
|
||||
// 3) take the md5 of the binary
|
||||
// 4) create the hex digest of the md5
|
||||
// 5) add '-' plus the number of parts at the end
|
||||
|
||||
// Convert the concatenated hex ETags to binary
|
||||
const bufferedHex = Buffer.from(concatETags, 'hex');
|
||||
// Convert the buffer to a binary string
|
||||
const binaryString = bufferedHex.toString('binary');
|
||||
// Get the md5 of the binary string
|
||||
const md5Hash = crypto.createHash('md5');
|
||||
md5Hash.update(binaryString, 'binary');
|
||||
// Get the hex digest of the md5
|
||||
let aggregateETag = md5Hash.digest('hex');
|
||||
// Add the number of parts at the end
|
||||
aggregateETag = `${aggregateETag}-${partList.length}`;
|
||||
|
||||
return aggregateETag;
|
||||
}
|
||||
|
||||
/**
|
||||
* generateMpuPartStorageInfo - generates info needed for storage of
|
||||
* completed MPU object
|
||||
* @param {array} filteredPartList - list of parts filtered from metadata
|
||||
* @return {object} partsInfo - contains three keys: aggregateETag,
|
||||
* dataLocations, and calculatedSize
|
||||
*/
|
||||
function generateMpuPartStorageInfo(filteredPartList) {
|
||||
// Assemble array of part locations, aggregate size
|
||||
// and build string to create aggregate ETag
|
||||
let calculatedSize = 0;
|
||||
const dataLocations = [];
|
||||
let concatETags = '';
|
||||
const partsInfo = {};
|
||||
|
||||
filteredPartList.forEach((storedPart, index) => {
|
||||
const partETagWithoutQuotes =
|
||||
storedPart.ETag.slice(1, -1);
|
||||
const dataStoreETag = `${index + 1}:${partETagWithoutQuotes}`;
|
||||
concatETags += partETagWithoutQuotes;
|
||||
|
||||
// If part was put by a regular put part rather than a
|
||||
// copy it is always one location. With a put part
|
||||
// copy, could be multiple locations so loop over array
|
||||
// of locations.
|
||||
for (let j = 0; j < storedPart.locations.length; j++) {
|
||||
// If the piece has parts (was a put part object
|
||||
// copy) each piece will have a size attribute.
|
||||
// Otherwise, the piece was put by a regular put
|
||||
// part and the size the of the piece is the full
|
||||
// part size.
|
||||
const location = storedPart.locations[j];
|
||||
// If there is no location, move on
|
||||
if (!location || typeof location !== 'object') {
|
||||
continue;
|
||||
}
|
||||
let pieceSize = Number.parseInt(storedPart.size, 10);
|
||||
if (location.size) {
|
||||
pieceSize = Number.parseInt(location.size, 10);
|
||||
}
|
||||
const pieceRetrievalInfo = {
|
||||
key: location.key,
|
||||
size: pieceSize,
|
||||
start: calculatedSize,
|
||||
dataStoreName: location.dataStoreName,
|
||||
dataStoreETag,
|
||||
cryptoScheme: location.sseCryptoScheme,
|
||||
cipheredDataKey: location.sseCipheredDataKey,
|
||||
};
|
||||
dataLocations.push(pieceRetrievalInfo);
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
calculatedSize += pieceSize;
|
||||
}
|
||||
});
|
||||
|
||||
partsInfo.aggregateETag =
|
||||
createAggregateETag(concatETags, filteredPartList);
|
||||
partsInfo.dataLocations = dataLocations;
|
||||
partsInfo.calculatedSize = calculatedSize;
|
||||
return partsInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* validateAndFilterMpuParts - validates part list sent by user and filters
|
||||
* parts stored in metadata against user part list
|
||||
* @param {array} storedParts - array of parts stored in metadata
|
||||
* @param {array} jsonList - array of parts sent by user for completion
|
||||
* @param {string} mpuOverviewKey - metadata mpu key
|
||||
* @param {string} splitter - mpu key divider
|
||||
* @param {object} log - Werelogs instance
|
||||
* @return {object} filtersPartsObj - contains 3 keys: partList, keysToDelete,
|
||||
* and extraPartLocations
|
||||
*/
|
||||
function validateAndFilterMpuParts(storedParts, jsonList, mpuOverviewKey,
|
||||
splitter, log) {
|
||||
let storedPartsCopy = [];
|
||||
const filteredPartsObj = {};
|
||||
filteredPartsObj.partList = [];
|
||||
|
||||
const keysToDelete = [];
|
||||
storedParts.forEach(item => {
|
||||
keysToDelete.push(item.key);
|
||||
storedPartsCopy.push({
|
||||
// In order to delete the part listing in the shadow
|
||||
// bucket, need the full key
|
||||
key: item.key,
|
||||
ETag: `"${item.value.ETag}"`,
|
||||
size: item.value.Size,
|
||||
locations: Array.isArray(item.value.partLocations) ?
|
||||
item.value.partLocations : [item.value.partLocations],
|
||||
});
|
||||
});
|
||||
keysToDelete.push(mpuOverviewKey);
|
||||
|
||||
// Check list sent to make sure valid
|
||||
const partLength = jsonList.Part.length;
|
||||
// A user can put more parts than they end up including
|
||||
// in the completed MPU but there cannot be more
|
||||
// parts in the complete message than were already put
|
||||
if (partLength > storedPartsCopy.length) {
|
||||
filteredPartsObj.error = errors.InvalidPart;
|
||||
return filteredPartsObj;
|
||||
}
|
||||
|
||||
let extraParts = [];
|
||||
const extraPartLocations = [];
|
||||
|
||||
for (let i = 0; i < partLength; i++) {
|
||||
const part = jsonList.Part[i];
|
||||
const partNumber = Number.parseInt(part.PartNumber[0], 10);
|
||||
// If the complete list of parts sent with
|
||||
// the complete multipart upload request is not
|
||||
// in ascending order return an error
|
||||
if (i > 0) {
|
||||
const previousPartNumber =
|
||||
Number.parseInt(jsonList.Part[i - 1].PartNumber[0], 10);
|
||||
if (partNumber <= previousPartNumber) {
|
||||
filteredPartsObj.error = errors.InvalidPartOrder;
|
||||
return filteredPartsObj;
|
||||
}
|
||||
}
|
||||
|
||||
let isPartUploaded = false;
|
||||
while (storedPartsCopy.length > 0 && !isPartUploaded) {
|
||||
const storedPart = storedPartsCopy[0];
|
||||
const storedPartNumber =
|
||||
Number.parseInt(storedPart.key.split(splitter)[1], 10);
|
||||
|
||||
if (storedPartNumber === partNumber) {
|
||||
isPartUploaded = true;
|
||||
filteredPartsObj.partList.push(storedPart);
|
||||
|
||||
let partETag = part.ETag[0].replace(/['"]/g, '');
|
||||
// some clients send base64, convert to hex
|
||||
// 32 chars = 16 bytes(2 chars-per-byte) = 128 bits of
|
||||
// MD5 hex
|
||||
if (partETag.length !== 32) {
|
||||
const buffered = Buffer.from(part.ETag[0], 'base64')
|
||||
.toString('hex');
|
||||
partETag = `${buffered}`;
|
||||
}
|
||||
partETag = `"${partETag}"`;
|
||||
// If list of parts sent with complete mpu request contains
|
||||
// a part ETag that does not match the ETag for the part
|
||||
// stored in metadata, return an error
|
||||
if (partETag !== storedPart.ETag) {
|
||||
filteredPartsObj.error = errors.InvalidPart;
|
||||
return filteredPartsObj;
|
||||
}
|
||||
|
||||
// If any part other than the last part is less than
|
||||
// 5MB, return an error
|
||||
const storedPartSize =
|
||||
Number.parseInt(storedPart.size, 10);
|
||||
// allow smaller parts for testing
|
||||
if (process.env.MPU_TESTING) {
|
||||
log.info('MPU_TESTING env variable setting',
|
||||
{ setting: process.env.MPU_TESTING });
|
||||
}
|
||||
if (process.env.MPU_TESTING !== 'yes' &&
|
||||
i < jsonList.Part.length - 1 &&
|
||||
storedPartSize < constants.minimumAllowedPartSize) {
|
||||
log.debug('part too small on complete mpu');
|
||||
filteredPartsObj.error = errors.EntityTooSmall;
|
||||
return filteredPartsObj;
|
||||
}
|
||||
|
||||
storedPartsCopy = storedPartsCopy.splice(1);
|
||||
} else {
|
||||
extraParts.push(storedPart);
|
||||
storedPartsCopy = storedPartsCopy.splice(1);
|
||||
}
|
||||
}
|
||||
if (!isPartUploaded) {
|
||||
filteredPartsObj.error = errors.InvalidPart;
|
||||
return filteredPartsObj;
|
||||
}
|
||||
}
|
||||
extraParts = extraParts.concat(storedPartsCopy);
|
||||
// if extra parts, need to delete the data when done with completing
|
||||
// mpu so extract the info to delete here
|
||||
if (extraParts.length > 0) {
|
||||
extraParts.forEach(part => {
|
||||
const locations = part.locations;
|
||||
locations.forEach(location => {
|
||||
if (!location || typeof location !== 'object') {
|
||||
return;
|
||||
}
|
||||
extraPartLocations.push(location);
|
||||
});
|
||||
});
|
||||
}
|
||||
filteredPartsObj.keysToDelete = keysToDelete;
|
||||
filteredPartsObj.extraPartLocations = extraPartLocations;
|
||||
return filteredPartsObj;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
generateMpuPartStorageInfo,
|
||||
validateAndFilterMpuParts,
|
||||
};
|
|
@ -4,40 +4,13 @@ const async = require('async');
|
|||
const metadata = require('../../../metadata/wrapper');
|
||||
const { config } = require('../../../Config');
|
||||
|
||||
const { scaledMsPerDay } = config.getTimeOptions();
|
||||
|
||||
const versionIdUtils = versioning.VersionID;
|
||||
// Use Arsenal function to generate a version ID used internally by metadata
|
||||
// for null versions that are created before bucket versioning is configured
|
||||
const nonVersionedObjId =
|
||||
versionIdUtils.getInfVid(config.replicationGroupId);
|
||||
|
||||
/** decodeVID - decode the version id
|
||||
* @param {string} versionId - version ID
|
||||
* @return {(Error|string|undefined)} - return Invalid Argument if decryption
|
||||
* fails due to improper format, otherwise undefined or the decoded version id
|
||||
*/
|
||||
function decodeVID(versionId) {
|
||||
if (versionId === 'null') {
|
||||
return versionId;
|
||||
}
|
||||
|
||||
let decoded;
|
||||
const invalidErr = errors.InvalidArgument.customizeDescription('Invalid version id specified');
|
||||
try {
|
||||
decoded = versionIdUtils.decode(versionId);
|
||||
} catch (err) {
|
||||
return invalidErr;
|
||||
}
|
||||
|
||||
if (decoded instanceof Error) {
|
||||
return invalidErr;
|
||||
}
|
||||
|
||||
return decoded;
|
||||
}
|
||||
|
||||
/** decodeVersionId - decode the version id from a query object
|
||||
/** decodedVidResult - decode the version id from a query object
|
||||
* @param {object} [reqQuery] - request query object
|
||||
* @param {string} [reqQuery.versionId] - version ID sent in request query
|
||||
* @return {(Error|string|undefined)} - return Invalid Argument if decryption
|
||||
|
@ -47,7 +20,16 @@ function decodeVersionId(reqQuery) {
|
|||
if (!reqQuery || !reqQuery.versionId) {
|
||||
return undefined;
|
||||
}
|
||||
return decodeVID(reqQuery.versionId);
|
||||
let versionId = reqQuery.versionId;
|
||||
if (versionId === 'null') {
|
||||
return versionId;
|
||||
}
|
||||
versionId = versionIdUtils.decode(versionId);
|
||||
if (versionId instanceof Error) {
|
||||
return errors.InvalidArgument
|
||||
.customizeDescription('Invalid version id specified');
|
||||
}
|
||||
return versionId;
|
||||
}
|
||||
|
||||
/** getVersionIdResHeader - return encrypted version ID if appropriate
|
||||
|
@ -61,7 +43,8 @@ function getVersionIdResHeader(verCfg, objectMD) {
|
|||
if (objectMD.isNull || !objectMD.versionId) {
|
||||
return 'null';
|
||||
}
|
||||
return versionIdUtils.encode(objectMD.versionId);
|
||||
return versionIdUtils.encode(objectMD.versionId,
|
||||
config.versionIdEncodingType);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
@ -460,93 +443,6 @@ function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersio
|
|||
return options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep metadatas when the object is restored from cold storage
|
||||
* but remove the specific ones we don't want to keep
|
||||
* @param {object} objMD - obj metadata
|
||||
* @param {object} metadataStoreParams - custom built object containing resource details.
|
||||
* @return {undefined}
|
||||
*/
|
||||
function restoreMetadata(objMD, metadataStoreParams) {
|
||||
/* eslint-disable no-param-reassign */
|
||||
const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt'];
|
||||
// We need to keep user metadata and tags
|
||||
Object.keys(objMD).forEach(key => {
|
||||
if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) {
|
||||
metadataStoreParams.metaHeaders[key] = objMD[key];
|
||||
}
|
||||
});
|
||||
|
||||
if (objMD['x-amz-website-redirect-location']) {
|
||||
if (!metadataStoreParams.headers) {
|
||||
metadataStoreParams.headers = {};
|
||||
}
|
||||
metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location'];
|
||||
}
|
||||
|
||||
if (objMD.replicationInfo) {
|
||||
metadataStoreParams.replicationInfo = objMD.replicationInfo;
|
||||
}
|
||||
|
||||
if (objMD.legalHold) {
|
||||
metadataStoreParams.legalHold = objMD.legalHold;
|
||||
}
|
||||
|
||||
if (objMD.acl) {
|
||||
metadataStoreParams.acl = objMD.acl;
|
||||
}
|
||||
|
||||
metadataStoreParams.creationTime = objMD['creation-time'];
|
||||
metadataStoreParams.lastModifiedDate = objMD['last-modified'];
|
||||
metadataStoreParams.taggingCopy = objMD.tags;
|
||||
}
|
||||
|
||||
/** overwritingVersioning - return versioning information for S3 to handle
|
||||
* storing version metadata with a specific version id.
|
||||
* @param {object} objMD - obj metadata
|
||||
* @param {object} metadataStoreParams - custom built object containing resource details.
|
||||
* @return {object} options
|
||||
* options.versionId - specific versionId to overwrite in metadata
|
||||
* options.isNull - (true/undefined) whether new version is null or not
|
||||
* options.nullVersionId - if storing a null version in version history, the
|
||||
* version id of the null version
|
||||
*/
|
||||
function overwritingVersioning(objMD, metadataStoreParams) {
|
||||
metadataStoreParams.updateMicroVersionId = true;
|
||||
metadataStoreParams.amzStorageClass = objMD['x-amz-storage-class'];
|
||||
|
||||
// set correct originOp
|
||||
metadataStoreParams.originOp = 's3:ObjectRestore:Completed';
|
||||
|
||||
// update restore
|
||||
const days = objMD.archive?.restoreRequestedDays;
|
||||
const now = Date.now();
|
||||
metadataStoreParams.archive = {
|
||||
archiveInfo: objMD.archive?.archiveInfo,
|
||||
restoreRequestedAt: objMD.archive?.restoreRequestedAt,
|
||||
restoreRequestedDays: objMD.archive?.restoreRequestedDays,
|
||||
restoreCompletedAt: new Date(now),
|
||||
restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
|
||||
};
|
||||
|
||||
/* eslint-enable no-param-reassign */
|
||||
|
||||
const versionId = objMD.versionId || undefined;
|
||||
const options = {
|
||||
versionId,
|
||||
isNull: objMD.isNull,
|
||||
};
|
||||
if (objMD.nullVersionId) {
|
||||
options.extraMD = {
|
||||
nullVersionId: objMD.nullVersionId,
|
||||
};
|
||||
}
|
||||
|
||||
restoreMetadata(objMD, metadataStoreParams);
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
decodeVersionId,
|
||||
getVersionIdResHeader,
|
||||
|
@ -556,6 +452,4 @@ module.exports = {
|
|||
versioningPreprocessing,
|
||||
getVersionSpecificMetadataOptions,
|
||||
preprocessingVersioningDelete,
|
||||
overwritingVersioning,
|
||||
decodeVID,
|
||||
};
|
||||
|
|
|
@ -1,314 +0,0 @@
|
|||
const async = require('async');
|
||||
const { errors } = require('arsenal');
|
||||
const monitoring = require('../../../utilities/monitoringHandler');
|
||||
const {
|
||||
actionNeedQuotaCheckCopy,
|
||||
actionNeedQuotaCheck,
|
||||
actionWithDataDeletion,
|
||||
} = require('arsenal').policies;
|
||||
const { config } = require('../../../Config');
|
||||
const QuotaService = require('../../../quotas/quotas');
|
||||
|
||||
/**
|
||||
* Process the bytes to write based on the request and object metadata
|
||||
* @param {string} apiMethod - api method
|
||||
* @param {BucketInfo} bucket - bucket info
|
||||
* @param {string} versionId - version id of the object
|
||||
* @param {number} contentLength - content length of the object
|
||||
* @param {object} objMD - object metadata
|
||||
* @param {object} destObjMD - destination object metadata
|
||||
* @return {number} processed content length
|
||||
*/
|
||||
function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
|
||||
let bytes = contentLength;
|
||||
if (apiMethod === 'objectRestore') {
|
||||
// object is being restored
|
||||
bytes = Number.parseInt(objMD['content-length'], 10);
|
||||
} else if (!bytes && objMD?.['content-length']) {
|
||||
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
|
||||
if (!destObjMD || bucket.isVersioningEnabled()) {
|
||||
// object is being copied
|
||||
bytes = Number.parseInt(objMD['content-length'], 10);
|
||||
} else if (!bucket.isVersioningEnabled()) {
|
||||
// object is being copied and replaces the target
|
||||
bytes = Number.parseInt(objMD['content-length'], 10) -
|
||||
Number.parseInt(destObjMD['content-length'], 10);
|
||||
}
|
||||
} else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
|
||||
// object is being deleted
|
||||
bytes = -Number.parseInt(objMD['content-length'], 10);
|
||||
}
|
||||
} else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
|
||||
// object is being replaced: store the diff, if the bucket is not versioned
|
||||
bytes = bytes - Number.parseInt(objMD['content-length'], 10);
|
||||
}
|
||||
return bytes || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a metric is stale based on the provided parameters.
|
||||
*
|
||||
* @param {Object} metric - The metric object to check.
|
||||
* @param {string} resourceType - The type of the resource.
|
||||
* @param {string} resourceName - The name of the resource.
|
||||
* @param {string} action - The action being performed.
|
||||
* @param {number} inflight - The number of inflight requests.
|
||||
* @param {Object} log - The logger object.
|
||||
* @returns {boolean} Returns true if the metric is stale, false otherwise.
|
||||
*/
|
||||
function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
|
||||
if (metric.date && Date.now() - new Date(metric.date).getTime() >
|
||||
QuotaService.maxStaleness) {
|
||||
log.warn('Stale metrics from the quota service, allowing the request', {
|
||||
resourceType,
|
||||
resourceName,
|
||||
action,
|
||||
inflight,
|
||||
});
|
||||
monitoring.requestWithQuotaMetricsUnavailable.inc();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates quotas for a bucket and an account and update inflight count.
|
||||
*
|
||||
* @param {number} bucketQuota - The quota limit for the bucket.
|
||||
* @param {number} accountQuota - The quota limit for the account.
|
||||
* @param {object} bucket - The bucket object.
|
||||
* @param {object} account - The account object.
|
||||
* @param {number} inflight - The number of inflight requests.
|
||||
* @param {number} inflightForCheck - The number of inflight requests for checking quotas.
|
||||
* @param {string} action - The action being performed.
|
||||
* @param {object} log - The logger object.
|
||||
* @param {function} callback - The callback function to be called when evaluation is complete.
|
||||
* @returns {object} - The result of the evaluation.
|
||||
*/
|
||||
function _evaluateQuotas(
|
||||
bucketQuota,
|
||||
accountQuota,
|
||||
bucket,
|
||||
account,
|
||||
inflight,
|
||||
inflightForCheck,
|
||||
action,
|
||||
log,
|
||||
callback,
|
||||
) {
|
||||
let bucketQuotaExceeded = false;
|
||||
let accountQuotaExceeded = false;
|
||||
const creationDate = new Date(bucket.getCreationDate()).getTime();
|
||||
return async.parallel({
|
||||
bucketQuota: parallelDone => {
|
||||
if (bucketQuota > 0) {
|
||||
return QuotaService.getUtilizationMetrics('bucket',
|
||||
`${bucket.getName()}_${creationDate}`, null, {
|
||||
action,
|
||||
inflight,
|
||||
}, (err, bucketMetrics) => {
|
||||
if (err || inflight < 0) {
|
||||
return parallelDone(err);
|
||||
}
|
||||
if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
|
||||
bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
|
||||
log.debug('Bucket quota exceeded', {
|
||||
bucket: bucket.getName(),
|
||||
action,
|
||||
inflight,
|
||||
quota: bucketQuota,
|
||||
bytesTotal: bucketMetrics.bytesTotal,
|
||||
});
|
||||
bucketQuotaExceeded = true;
|
||||
}
|
||||
return parallelDone();
|
||||
});
|
||||
}
|
||||
return parallelDone();
|
||||
},
|
||||
accountQuota: parallelDone => {
|
||||
if (accountQuota > 0 && account?.account) {
|
||||
return QuotaService.getUtilizationMetrics('account',
|
||||
account.account, null, {
|
||||
action,
|
||||
inflight,
|
||||
}, (err, accountMetrics) => {
|
||||
if (err || inflight < 0) {
|
||||
return parallelDone(err);
|
||||
}
|
||||
if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
|
||||
accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
|
||||
log.debug('Account quota exceeded', {
|
||||
accountId: account.account,
|
||||
action,
|
||||
inflight,
|
||||
quota: accountQuota,
|
||||
bytesTotal: accountMetrics.bytesTotal,
|
||||
});
|
||||
accountQuotaExceeded = true;
|
||||
}
|
||||
return parallelDone();
|
||||
});
|
||||
}
|
||||
return parallelDone();
|
||||
},
|
||||
}, err => {
|
||||
if (err) {
|
||||
log.warn('Error evaluating quotas', {
|
||||
error: err.name,
|
||||
description: err.message,
|
||||
isInflightDeletion: inflight < 0,
|
||||
});
|
||||
}
|
||||
return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Monitors the duration of quota evaluation for a specific API method.
|
||||
*
|
||||
* @param {string} apiMethod - The name of the API method being monitored.
|
||||
* @param {string} type - The type of quota being evaluated.
|
||||
* @param {string} code - The code associated with the quota being evaluated.
|
||||
* @param {number} duration - The duration of the quota evaluation in nanoseconds.
|
||||
* @returns {undefined} - Returns nothing.
|
||||
*/
|
||||
function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
|
||||
monitoring.quotaEvaluationDuration.labels({
|
||||
action: apiMethod,
|
||||
type,
|
||||
code,
|
||||
}).observe(duration / 1e9);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Request} request - request object
|
||||
* @param {BucketInfo} bucket - bucket object
|
||||
* @param {Account} account - account object
|
||||
* @param {array} apiNames - action names: operations to authorize
|
||||
* @param {string} apiMethod - the main API call
|
||||
* @param {number} inflight - inflight bytes
|
||||
* @param {boolean} isStorageReserved - Flag to check if the current quota, minus
|
||||
* the incoming bytes, are under the limit.
|
||||
* @param {Logger} log - logger
|
||||
* @param {function} callback - callback function
|
||||
* @returns {boolean} - true if the quota is valid, false otherwise
|
||||
*/
|
||||
function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
|
||||
if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
|
||||
return callback(null);
|
||||
}
|
||||
let type;
|
||||
let bucketQuotaExceeded = false;
|
||||
let accountQuotaExceeded = false;
|
||||
let quotaEvaluationDuration;
|
||||
const requestStartTime = process.hrtime.bigint();
|
||||
const bucketQuota = bucket.getQuota();
|
||||
const accountQuota = account?.quota || 0;
|
||||
const shouldSendInflights = config.isQuotaInflightEnabled();
|
||||
|
||||
if (bucketQuota && accountQuota) {
|
||||
type = 'bucket+account';
|
||||
} else if (bucketQuota) {
|
||||
type = 'bucket';
|
||||
} else {
|
||||
type = 'account';
|
||||
}
|
||||
|
||||
if (actionWithDataDeletion[apiMethod]) {
|
||||
type = 'delete';
|
||||
}
|
||||
|
||||
if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
|
||||
if (bucketQuota > 0 || accountQuota > 0) {
|
||||
log.warn('quota is set for a bucket, but the quota service is disabled', {
|
||||
bucketName: bucket.getName(),
|
||||
});
|
||||
monitoring.requestWithQuotaMetricsUnavailable.inc();
|
||||
}
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
if (isStorageReserved) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
inflight = 0;
|
||||
}
|
||||
|
||||
return async.forEach(apiNames, (apiName, done) => {
|
||||
// Object copy operations first check the target object,
|
||||
// meaning the source object, containing the current bytes,
|
||||
// is checked second. This logic handles these APIs calls by
|
||||
// ensuring the bytes are positives (i.e., not an object
|
||||
// replacement).
|
||||
if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
inflight = Math.abs(inflight);
|
||||
} else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
|
||||
return done();
|
||||
}
|
||||
// When inflights are disabled, the sum of the current utilization metrics
|
||||
// and the current bytes are compared with the quota. The current bytes
|
||||
// are not sent to the utilization service. When inflights are enabled,
|
||||
// the sum of the current utilization metrics only are compared with the
|
||||
// quota. They include the current inflight bytes sent in the request.
|
||||
let _inflights = shouldSendInflights ? inflight : undefined;
|
||||
const inflightForCheck = shouldSendInflights ? 0 : inflight;
|
||||
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
|
||||
inflightForCheck, apiName, log,
|
||||
(err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
bucketQuotaExceeded = _bucketQuotaExceeded;
|
||||
accountQuotaExceeded = _accountQuotaExceeded;
|
||||
|
||||
// Inflights are inverted: in case of cleanup, we just re-issue
|
||||
// the same API call.
|
||||
if (_inflights) {
|
||||
_inflights = -_inflights;
|
||||
}
|
||||
|
||||
request.finalizerHooks.push((errorFromAPI, _done) => {
|
||||
const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
|
||||
const quotaCleanUpStartTime = process.hrtime.bigint();
|
||||
// Quotas are cleaned only in case of error in the API
|
||||
async.waterfall([
|
||||
cb => {
|
||||
if (errorFromAPI) {
|
||||
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
|
||||
null, apiName, log, cb);
|
||||
}
|
||||
return cb();
|
||||
},
|
||||
], () => {
|
||||
monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
|
||||
Number(process.hrtime.bigint() - quotaCleanUpStartTime));
|
||||
return _done();
|
||||
});
|
||||
});
|
||||
|
||||
return done();
|
||||
});
|
||||
}, err => {
|
||||
quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
|
||||
if (err) {
|
||||
log.warn('Error getting metrics from the quota service, allowing the request', {
|
||||
error: err.name,
|
||||
description: err.message,
|
||||
});
|
||||
}
|
||||
if (!actionWithDataDeletion[apiMethod] &&
|
||||
(bucketQuotaExceeded || accountQuotaExceeded)) {
|
||||
return callback(errors.QuotaExceeded);
|
||||
}
|
||||
return callback();
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
processBytesToWrite,
|
||||
isMetricStale,
|
||||
validateQuotas,
|
||||
};
|
|
@ -3,7 +3,7 @@ const constants = require('../../../constants');
|
|||
const services = require('../../services');
|
||||
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
|
||||
const { pushMetric } = require('../../utapi/utilities');
|
||||
const monitoring = require('../../utilities/monitoringHandler');
|
||||
const monitoring = require('../../utilities/metrics');
|
||||
const { getLocationConstraintErrorMessage, processCurrents,
|
||||
validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
|
||||
const { config } = require('../../Config');
|
||||
|
|
|
@ -4,7 +4,7 @@ const services = require('../../services');
|
|||
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
|
||||
const { pushMetric } = require('../../utapi/utilities');
|
||||
const versionIdUtils = versioning.VersionID;
|
||||
const monitoring = require('../../utilities/monitoringHandler');
|
||||
const monitoring = require('../../utilities/metrics');
|
||||
const { getLocationConstraintErrorMessage, processNonCurrents,
|
||||
validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
|
||||
const { config } = require('../../Config');
|
||||
|
|
|
@ -3,7 +3,7 @@ const constants = require('../../../constants');
|
|||
const services = require('../../services');
|
||||
const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils');
|
||||
const { pushMetric } = require('../../utapi/utilities');
|
||||
const monitoring = require('../../utilities/monitoringHandler');
|
||||
const monitoring = require('../../utilities/metrics');
|
||||
const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle');
|
||||
const { config } = require('../../Config');
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|||
const deleteBucket = require('./apiUtils/bucket/bucketDeletion');
|
||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
||||
const { pushMetric } = require('../utapi/utilities');
|
||||
const monitoring = require('../utilities/monitoringHandler');
|
||||
const monitoring = require('../utilities/metrics');
|
||||
|
||||
/**
|
||||
* bucketDelete - DELETE bucket (currently supports only non-versioned buckets)
|
||||
|
|
|
@ -6,7 +6,7 @@ const { isBucketAuthorized } =
|
|||
require('./apiUtils/authorization/permissionChecks');
|
||||
const metadata = require('../metadata/wrapper');
|
||||
const { pushMetric } = require('../utapi/utilities');
|
||||
const monitoring = require('../utilities/monitoringHandler');
|
||||
const monitoring = require('../utilities/metrics');
|
||||
|
||||
const requestType = 'bucketDeleteCors';
|
||||
|
||||
|
|
|
@ -2,7 +2,7 @@ const metadata = require('../metadata/wrapper');
|
|||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
||||
const { pushMetric } = require('../utapi/utilities');
|
||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||
const monitoring = require('../utilities/monitoringHandler');
|
||||
const monitoring = require('../utilities/metrics');
|
||||
|
||||
/**
|
||||
* bucketDeleteLifecycle - Delete the bucket Lifecycle configuration
|
||||
|
|
|
@ -1,58 +0,0 @@
|
|||
const { waterfall } = require('async');
|
||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
||||
const metadata = require('../metadata/wrapper');
|
||||
const { pushMetric } = require('../utapi/utilities');
|
||||
const monitoring = require('../utilities/monitoringHandler');
|
||||
|
||||
const requestType = 'bucketDeleteQuota';
|
||||
|
||||
/**
|
||||
* Bucket Update Quota - Update bucket quota
|
||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
|
||||
* @param {object} request - http request object
|
||||
* @param {object} log - Werelogs logger
|
||||
* @param {function} callback - callback to server
|
||||
* @return {undefined}
|
||||
*/
|
||||
function bucketDeleteQuota(authInfo, request, log, callback) {
|
||||
log.debug('processing request', { method: 'bucketDeleteQuota' });
|
||||
|
||||
const { bucketName } = request;
|
||||
const metadataValParams = {
|
||||
authInfo,
|
||||
bucketName,
|
||||
requestType: request.apiMethods || requestType,
|
||||
request,
|
||||
};
|
||||
return waterfall([
|
||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
||||
(err, bucket) => next(err, bucket)),
|
||||
(bucket, next) => {
|
||||
bucket.setQuota(0);
|
||||
metadata.updateBucket(bucket.getName(), bucket, log, err =>
|
||||
next(err, bucket));
|
||||
},
|
||||
], (err, bucket) => {
|
||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||
request.method, bucket);
|
||||
if (err) {
|
||||
log.debug('error processing request', {
|
||||
error: err,
|
||||
method: 'bucketDeleteQuota'
|
||||
});
|
||||
monitoring.promMetrics('DELETE', bucketName, err.code,
|
||||
'bucketDeleteQuota');
|
||||
return callback(err, err.code, corsHeaders);
|
||||
}
|
||||
monitoring.promMetrics(
|
||||
'DELETE', bucketName, '204', 'bucketDeleteQuota');
|
||||
pushMetric('bucketDeleteQuota', log, {
|
||||
authInfo,
|
||||
bucket: bucketName,
|
||||
});
|
||||
return callback(null, 204, corsHeaders);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = bucketDeleteQuota;
|
|
@ -2,7 +2,7 @@ const metadata = require('../metadata/wrapper');
|
|||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
||||
const { pushMetric } = require('../utapi/utilities');
|
||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||
const monitoring = require('../utilities/monitoringHandler');
|
||||
const monitoring = require('../utilities/metrics');
|
||||
|
||||
/**
|
||||
* bucketDeleteReplication - Delete the bucket replication configuration
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
const { waterfall } = require('async');
|
||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
||||
const { pushMetric } = require('../utapi/utilities');
|
||||
const monitoring = require('../utilities/monitoringHandler');
|
||||
const metadata = require('../metadata/wrapper');
|
||||
const util = require('node:util');
|
||||
const monitoring = require('../utilities/metrics');
|
||||
|
||||
/**
|
||||
* Bucket Delete Tagging - Delete a bucket's Tagging
|
||||
|
@ -13,10 +13,16 @@ const metadata = require('../metadata/wrapper');
|
|||
* @param {function} callback - callback to server
|
||||
* @return {undefined}
|
||||
*/
|
||||
function bucketDeleteTagging(authInfo, request, log, callback) {
|
||||
async function bucketDeleteTagging(authInfo, request, log, callback) {
|
||||
const bucketName = request.bucketName;
|
||||
let error = null;
|
||||
log.debug('processing request', { method: 'bucketDeleteTagging', bucketName });
|
||||
|
||||
let bucket;
|
||||
const metadataValidateBucketPromise = util.promisify(standardMetadataValidateBucket);
|
||||
let updateBucketPromise = util.promisify(metadata.updateBucket);
|
||||
// necessary to bind metadata as updateBucket calls 'this', causing undefined otherwise
|
||||
updateBucketPromise = updateBucketPromise.bind(metadata);
|
||||
const metadataValParams = {
|
||||
authInfo,
|
||||
bucketName,
|
||||
|
@ -24,39 +30,27 @@ function bucketDeleteTagging(authInfo, request, log, callback) {
|
|||
request,
|
||||
};
|
||||
|
||||
let bucket = null;
|
||||
return waterfall([
|
||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
||||
(err, b) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
bucket = b;
|
||||
try {
|
||||
bucket = await metadataValidateBucketPromise(metadataValParams, request.actionImplicitDenies, log);
|
||||
bucket.setTags([]);
|
||||
return next();
|
||||
}),
|
||||
next => metadata.updateBucket(bucket.getName(), bucket, log, next),
|
||||
], err => {
|
||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||
request.method, bucket);
|
||||
if (err) {
|
||||
log.error('error processing request', {
|
||||
error: err,
|
||||
method: 'deleteBucketTagging',
|
||||
bucketName
|
||||
});
|
||||
monitoring.promMetrics('DELETE', bucketName, err.code,
|
||||
'deleteBucketTagging');
|
||||
return callback(err, corsHeaders);
|
||||
}
|
||||
// eslint-disable-next-line no-unused-expressions
|
||||
await updateBucketPromise(bucket.getName(), bucket, log);
|
||||
pushMetric('deleteBucketTagging', log, {
|
||||
authInfo,
|
||||
bucket: bucketName,
|
||||
});
|
||||
monitoring.promMetrics(
|
||||
'DELETE', bucketName, '200', 'deleteBucketTagging');
|
||||
return callback(err, corsHeaders);
|
||||
});
|
||||
} catch (err) {
|
||||
error = err;
|
||||
log.error('error processing request', { error: err,
|
||||
method: 'deleteBucketTagging', bucketName });
|
||||
monitoring.promMetrics('DELETE', bucketName, err.code,
|
||||
'deleteBucketTagging');
|
||||
}
|
||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||
request.method, bucket);
|
||||
return callback(error, corsHeaders);
|
||||
}
|
||||
|
||||
module.exports = bucketDeleteTagging;
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue