Compare commits

..

2 Commits

Author SHA1 Message Date
williamlardier 345fa52c27 wip 2024-02-13 00:38:41 +01:00
williamlardier 9f1dd45671 wip 2024-02-13 00:13:26 +01:00
117 changed files with 8930 additions and 6880 deletions

View File

@ -46,6 +46,7 @@
"new-parens": "off", "new-parens": "off",
"no-multi-spaces": "off", "no-multi-spaces": "off",
"quote-props": "off", "quote-props": "off",
"no-console": "off",
"mocha/no-exclusive-tests": "error", "mocha/no-exclusive-tests": "error",
}, },
"parserOptions": { "parserOptions": {

View File

@ -16,7 +16,7 @@ runs:
run: |- run: |-
set -exu; set -exu;
mkdir -p /tmp/artifacts/${JOB_NAME}/; mkdir -p /tmp/artifacts/${JOB_NAME}/;
- uses: actions/setup-node@v4 - uses: actions/setup-node@v2
with: with:
node-version: '16' node-version: '16'
cache: 'yarn' cache: 'yarn'

View File

@ -40,11 +40,6 @@ services:
- DEFAULT_BUCKET_KEY_FORMAT - DEFAULT_BUCKET_KEY_FORMAT
- METADATA_MAX_CACHED_BUCKETS - METADATA_MAX_CACHED_BUCKETS
- ENABLE_NULL_VERSION_COMPAT_MODE - ENABLE_NULL_VERSION_COMPAT_MODE
- SCUBA_HOST
- SCUBA_PORT
- SCUBA_HEALTHCHECK_FREQUENCY
- S3QUOTA
- QUOTA_ENABLE_INFLIGHTS
env_file: env_file:
- creds.env - creds.env
depends_on: depends_on:
@ -72,7 +67,7 @@ services:
pykmip: pykmip:
network_mode: "host" network_mode: "host"
profiles: ['pykmip'] profiles: ['pykmip']
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip} image: registry.scality.com/cloudserver-dev/pykmip
volumes: volumes:
- /tmp/artifacts/${JOB_NAME}:/artifacts - /tmp/artifacts/${JOB_NAME}:/artifacts
mongo: mongo:

View File

@ -1,3 +1,3 @@
FROM ghcr.io/scality/federation/sproxyd:7.10.6.8 FROM registry.scality.com/federation/sproxyd:7.10.6.8
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/ ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
RUN chown root:root /conf/sproxyd0.conf RUN chown root:root /conf/sproxyd0.conf

View File

@ -1,10 +1,7 @@
name: Test alerts name: Test alerts
on: on:
push: push
branches-ignore:
- 'development/**'
- 'q/*/**'
jobs: jobs:
run-alert-tests: run-alert-tests:
@ -20,16 +17,13 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Render and test ${{ matrix.tests.name }} - name: Render and test ${{ matrix.tests.name }}
uses: scality/action-prom-render-test@1.0.3 uses: scality/action-prom-render-test@1.0.1
with: with:
alert_file_path: monitoring/alerts.yaml alert_file_path: monitoring/alerts.yaml
test_file_path: ${{ matrix.tests.file }} test_file_path: ${{ matrix.tests.file }}
alert_inputs: | alert_inputs: >-
namespace=zenko namespace=zenko,service=artesca-data-connector-s3api-metrics,replicas=3
service=artesca-data-connector-s3api-metrics
reportJob=artesca-data-ops-report-handler
replicas=3
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}

View File

@ -3,7 +3,7 @@ name: codeQL
on: on:
push: push:
branches: [w/**, q/*] branches: [development/*, stabilization/*, hotfix/*]
pull_request: pull_request:
branches: [development/*, stabilization/*, hotfix/*] branches: [development/*, stabilization/*, hotfix/*]
workflow_dispatch: workflow_dispatch:
@ -14,12 +14,12 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@v3 uses: github/codeql-action/init@v2
with: with:
languages: javascript, python, ruby languages: javascript, python, ruby
- name: Build and analyze - name: Build and analyze
uses: github/codeql-action/analyze@v3 uses: github/codeql-action/analyze@v2

View File

@ -10,7 +10,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: 'Checkout Repository' - name: 'Checkout Repository'
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: 'Dependency Review' - name: 'Dependency Review'
uses: actions/dependency-review-action@v4 uses: actions/dependency-review-action@v3

View File

@ -10,69 +10,58 @@ on:
required: true required: true
env: env:
REGISTRY_NAME: registry.scality.com
PROJECT_NAME: ${{ github.event.repository.name }} PROJECT_NAME: ${{ github.event.repository.name }}
jobs: jobs:
build-federation-image: build-federation-image:
runs-on: ubuntu-20.04 uses: scality/workflows/.github/workflows/docker-build.yaml@v1
steps: secrets: inherit
- name: Checkout with:
uses: actions/checkout@v4 push: true
- name: Set up Docker Buildx registry: registry.scality.com
uses: docker/setup-buildx-action@v3 namespace: ${{ github.event.repository.name }}
- name: Login to GitHub Registry name: ${{ github.event.repository.name }}
uses: docker/login-action@v3 context: .
with: file: images/svc-base/Dockerfile
registry: ghcr.io tag: ${{ github.event.inputs.tag }}-svc-base
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Build and push image for federation
uses: docker/build-push-action@v5
with:
push: true
context: .
file: images/svc-base/Dockerfile
tags: |
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
cache-from: type=gha,scope=federation
cache-to: type=gha,mode=max,scope=federation
release: release:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Set up Docker Buildk - name: Set up Docker Buildk
uses: docker/setup-buildx-action@v3 uses: docker/setup-buildx-action@v2
- name: Login to Registry - name: Login to Registry
uses: docker/login-action@v3 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ${{ env.REGISTRY_NAME }}
username: ${{ github.repository_owner }} username: ${{ secrets.REGISTRY_LOGIN }}
password: ${{ github.token }} password: ${{ secrets.REGISTRY_PASSWORD }}
- name: Push dashboards into the production namespace - name: Push dashboards into the production namespace
run: | run: |
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \ oras push ${{ env.REGISTRY_NAME }}/${{ env.PROJECT_NAME }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
dashboard.json:application/grafana-dashboard+json \ dashboard.json:application/grafana-dashboard+json \
alerts.yaml:application/prometheus-alerts+yaml alerts.yaml:application/prometheus-alerts+yaml
working-directory: monitoring working-directory: monitoring
- name: Build and push - name: Build and push
uses: docker/build-push-action@v5 uses: docker/build-push-action@v4
with: with:
context: . context: .
push: true push: true
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }} tags: ${{ env.REGISTRY_NAME }}/${{ env.PROJECT_NAME }}/${{ env.PROJECT_NAME }}:${{ github.event.inputs.tag }}
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
- name: Create Release - name: Create Release
uses: softprops/action-gh-release@v2 uses: softprops/action-gh-release@v1
env: env:
GITHUB_TOKEN: ${{ github.token }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with: with:
name: Release ${{ github.event.inputs.tag }} name: Release ${{ github.event.inputs.tag }}
tag_name: ${{ github.event.inputs.tag }} tag_name: ${{ github.event.inputs.tag }}

View File

@ -67,24 +67,23 @@ env:
ENABLE_LOCAL_CACHE: "true" ENABLE_LOCAL_CACHE: "true"
REPORT_TOKEN: "report-token-1" REPORT_TOKEN: "report-token-1"
REMOTE_MANAGEMENT_DISABLE: "1" REMOTE_MANAGEMENT_DISABLE: "1"
# https://github.com/git-lfs/git-lfs/issues/5749
GIT_CLONE_PROTECTION_ACTIVE: 'false'
jobs: jobs:
linting-coverage: linting-coverage:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- uses: actions/setup-node@v4 - uses: actions/setup-node@v2
with: with:
node-version: '16' node-version: '16'
cache: yarn cache: yarn
- name: install dependencies - name: install dependencies
run: yarn install --frozen-lockfile --network-concurrency 1 run: yarn install --frozen-lockfile --network-concurrency 1
- uses: actions/setup-python@v5 - uses: actions/setup-python@v4
with: with:
python-version: '3.9' python-version: '3.9'
- uses: actions/cache@v4 - uses: actions/cache@v3
with: with:
path: ~/.cache/pip path: ~/.cache/pip
key: ${{ runner.os }}-pip key: ${{ runner.os }}-pip
@ -117,7 +116,7 @@ jobs:
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";" find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
if: always() if: always()
- name: Upload files to artifacts - name: Upload files to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v2
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net
@ -133,54 +132,46 @@ jobs:
packages: write packages: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3 uses: docker/setup-buildx-action@v2
- name: Login to GitHub Registry - name: Login to GitHub Registry
uses: docker/login-action@v3 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.repository_owner }} username: ${{ github.repository_owner }}
password: ${{ github.token }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Registry
uses: docker/login-action@v2
with:
registry: registry.scality.com
username: ${{ secrets.REGISTRY_LOGIN }}
password: ${{ secrets.REGISTRY_PASSWORD }}
- name: Build and push cloudserver image - name: Build and push cloudserver image
uses: docker/build-push-action@v5 uses: docker/build-push-action@v4
with: with:
push: true push: true
context: . context: .
provenance: false provenance: false
tags: | tags: |
ghcr.io/${{ github.repository }}:${{ github.sha }} ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
labels: | registry.scality.com/cloudserver-dev/cloudserver:${{ github.sha }}
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=cloudserver cache-from: type=gha,scope=cloudserver
cache-to: type=gha,mode=max,scope=cloudserver cache-to: type=gha,mode=max,scope=cloudserver
- name: Build and push pykmip image
uses: docker/build-push-action@v5
with:
push: true
context: .github/pykmip
tags: |
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
labels: |
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=pykmip
cache-to: type=gha,mode=max,scope=pykmip
- name: Build and push MongoDB - name: Build and push MongoDB
uses: docker/build-push-action@v5 uses: docker/build-push-action@v4
with: with:
push: true push: true
context: .github/docker/mongodb context: .github/docker/mongodb
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
cache-from: type=gha,scope=mongodb cache-from: type=gha
cache-to: type=gha,mode=max,scope=mongodb cache-to: type=gha,mode=max
multiple-backend: multiple-backend:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: build needs: build
env: env:
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
S3BACKEND: mem S3BACKEND: mem
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
@ -188,13 +179,13 @@ jobs:
JOB_NAME: ${{ github.job }} JOB_NAME: ${{ github.job }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Login to Registry - name: Login to Registry
uses: docker/login-action@v3 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: registry.scality.com
username: ${{ github.repository_owner }} username: ${{ secrets.REGISTRY_LOGIN }}
password: ${{ github.token }} password: ${{ secrets.REGISTRY_PASSWORD }}
- name: Setup CI environment - name: Setup CI environment
uses: ./.github/actions/setup-ci uses: ./.github/actions/setup-ci
- name: Setup CI services - name: Setup CI services
@ -209,7 +200,7 @@ jobs:
env: env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts - name: Upload logs to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v3
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net
@ -229,11 +220,11 @@ jobs:
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
DEFAULT_BUCKET_KEY_FORMAT: v0 DEFAULT_BUCKET_KEY_FORMAT: v0
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
JOB_NAME: ${{ github.job }} JOB_NAME: ${{ github.job }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Setup CI environment - name: Setup CI environment
uses: ./.github/actions/setup-ci uses: ./.github/actions/setup-ci
- name: Setup CI services - name: Setup CI services
@ -247,7 +238,7 @@ jobs:
env: env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts - name: Upload logs to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v3
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net
@ -268,11 +259,11 @@ jobs:
DEFAULT_BUCKET_KEY_FORMAT: v1 DEFAULT_BUCKET_KEY_FORMAT: v1
METADATA_MAX_CACHED_BUCKETS: 1 METADATA_MAX_CACHED_BUCKETS: 1
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
JOB_NAME: ${{ github.job }} JOB_NAME: ${{ github.job }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Setup CI environment - name: Setup CI environment
uses: ./.github/actions/setup-ci uses: ./.github/actions/setup-ci
- name: Setup CI services - name: Setup CI services
@ -287,7 +278,7 @@ jobs:
env: env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts - name: Upload logs to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v3
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net
@ -307,13 +298,13 @@ jobs:
env: env:
S3BACKEND: file S3BACKEND: file
S3VAULT: mem S3VAULT: mem
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
MPU_TESTING: "yes" MPU_TESTING: "yes"
JOB_NAME: ${{ matrix.job-name }} JOB_NAME: ${{ matrix.job-name }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Setup CI environment - name: Setup CI environment
uses: ./.github/actions/setup-ci uses: ./.github/actions/setup-ci
- name: Setup matrix job artifacts directory - name: Setup matrix job artifacts directory
@ -330,7 +321,7 @@ jobs:
bash wait_for_local_port.bash 8000 40 bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
- name: Upload logs to artifacts - name: Upload logs to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v3
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net
@ -344,14 +335,14 @@ jobs:
needs: build needs: build
env: env:
ENABLE_UTAPI_V2: t ENABLE_UTAPI_V2: t
S3BACKEND: mem S3BACKEND: mem
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }} JOB_NAME: ${{ github.job }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Setup CI environment - name: Setup CI environment
uses: ./.github/actions/setup-ci uses: ./.github/actions/setup-ci
- name: Setup CI services - name: Setup CI services
@ -363,51 +354,7 @@ jobs:
bash wait_for_local_port.bash 8000 40 bash wait_for_local_port.bash 8000 40
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts - name: Upload logs to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v3
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
quota-tests:
runs-on: ubuntu-latest
needs: build
strategy:
matrix:
inflights:
- name: "With Inflights"
value: "true"
- name: "Without Inflights"
value: "false"
env:
S3METADATA: mongodb
S3BACKEND: mem
S3QUOTA: scuba
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
SCUBA_HOST: localhost
SCUBA_PORT: 8100
SCUBA_HEALTHCHECK_FREQUENCY: 100
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run quota tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net
@ -423,13 +370,12 @@ jobs:
S3BACKEND: file S3BACKEND: file
S3VAULT: mem S3VAULT: mem
MPU_TESTING: "yes" MPU_TESTING: "yes"
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }} JOB_NAME: ${{ github.job }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Setup CI environment - name: Setup CI environment
uses: ./.github/actions/setup-ci uses: ./.github/actions/setup-ci
- name: Copy KMIP certs - name: Copy KMIP certs
@ -445,7 +391,7 @@ jobs:
bash wait_for_local_port.bash 5696 40 bash wait_for_local_port.bash 5696 40
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts - name: Upload logs to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v3
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net
@ -453,7 +399,7 @@ jobs:
password: ${{ secrets.ARTIFACTS_PASSWORD }} password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts source: /tmp/artifacts
if: always() if: always()
ceph-backend-test: ceph-backend-test:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: build needs: build
@ -465,17 +411,17 @@ jobs:
MPU_TESTING: "yes" MPU_TESTING: "yes"
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
JOB_NAME: ${{ github.job }} JOB_NAME: ${{ github.job }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Login to GitHub Registry - name: Login to GitHub Registry
uses: docker/login-action@v3 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.repository_owner }} username: ${{ github.repository_owner }}
password: ${{ github.token }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Setup CI environment - name: Setup CI environment
uses: ./.github/actions/setup-ci uses: ./.github/actions/setup-ci
- uses: ruby/setup-ruby@v1 - uses: ruby/setup-ruby@v1
@ -483,7 +429,7 @@ jobs:
ruby-version: '2.5.9' ruby-version: '2.5.9'
- name: Install Ruby dependencies - name: Install Ruby dependencies
run: | run: |
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5 gem install nokogiri:1.12.5 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
- name: Install Java dependencies - name: Install Java dependencies
run: | run: |
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
@ -523,7 +469,7 @@ jobs:
S3VAULT: mem S3VAULT: mem
S3METADATA: mongodb S3METADATA: mongodb
- name: Upload logs to artifacts - name: Upload logs to artifacts
uses: scality/action-artifacts@v4 uses: scality/action-artifacts@v3
with: with:
method: upload method: upload
url: https://artifacts.scality.net url: https://artifacts.scality.net

View File

@ -23,7 +23,6 @@ RUN apt-get update \
ENV PYTHON=python3 ENV PYTHON=python3
COPY package.json yarn.lock /usr/src/app/ COPY package.json yarn.lock /usr/src/app/
RUN npm install typescript -g
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1 RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
################################################################################ ################################################################################

175
README.md
View File

@ -1,7 +1,10 @@
# Zenko CloudServer with Vitastor Backend # Zenko CloudServer
![Zenko CloudServer logo](res/scality-cloudserver-logo.png) ![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/zenko/cloudserver)
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
## Overview ## Overview
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
@ -11,71 +14,137 @@ Scalitys Open Source Multi-Cloud Data Controller.
CloudServer provides a single AWS S3 API interface to access multiple CloudServer provides a single AWS S3 API interface to access multiple
backend data storage both on-premise or public in the cloud. backend data storage both on-premise or public in the cloud.
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor) CloudServer is useful for Developers, either to run as part of a
backend support. continous integration test environment to emulate the AWS S3 service locally
or as an abstraction layer to develop object storage enabled
application on the go.
## Quick Start with Vitastor ## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
Vitastor Backend is in experimental status, however you can already try to ## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
it works too 😊.
Installation instructions: ## Docker
### Install Vitastor [Run your Zenko CloudServer with Docker](https://hub.docker.com/r/zenko/cloudserver/)
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md). ## Contributing
### Install Zenko with Vitastor Backend In order to contribute, please follow the
[Contributing Guidelines](
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor` ## Installation
- Install dependencies: `npm install --omit dev` or just `npm install`
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
### Install and Configure MongoDB ### Dependencies
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/). Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
. Up-to-date versions can be found at
[Nodesource](https://github.com/nodesource/distributions).
### Setup Zenko ### Clone source code
- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data` ```shell
- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data` git clone https://github.com/scality/S3.git
- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
access keys, but it's not published, so let's use a file for now.
- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
in this file.
Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
### Start Zenko
Start the S3 server with: `node index.js`
If you use default settings, Zenko CloudServer starts on port 8000.
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
Now you can access your S3 with `s3cmd` or `geesefs`:
```
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
``` ```
``` ### Install js dependencies
AWS_ACCESS_KEY_ID=accessKey1 \
AWS_SECRET_ACCESS_KEY=verySecretKey1 \ Go to the ./S3 folder,
geesefs --endpoint http://localhost:8000 testbucket mountdir
```shell
yarn install --frozen-lockfile
``` ```
# Author & License If you get an error regarding installation of the diskUsage module,
please install g++.
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0) If you get an error regarding level-down bindings, try clearing your yarn cache:
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way) ```shell
yarn cache clean
```
## Run it with a file backend
```shell
yarn start
```
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
9991 are also open locally for internal transfer of metadata and data,
respectively.
The default access key is accessKey1 with
a secret key of verySecretKey1.
By default the metadata files will be saved in the
localMetadata directory and the data files will be saved
in the localData directory within the ./S3 directory on your
machine. These directories have been pre-created within the
repository. If you would like to save the data or metadata in
different locations of your choice, you must specify them with absolute paths.
So, when starting the server:
```shell
mkdir -m 700 $(pwd)/myFavoriteDataPath
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
yarn start
```
## Run it with multiple data backends
```shell
export S3DATA='multiple'
yarn start
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
With multiple backends, you have the ability to
choose where each object will be saved by setting
the following header with a locationConstraint on
a PUT request:
```shell
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
```
If no header is sent with a PUT object request, the
location constraint of the bucket will determine
where the data is saved. If the bucket has no location
constraint, the endpoint of the PUT request will be
used to determine location.
See the Configuration section in our documentation
[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
to learn how to set location constraints.
## Run it with an in-memory backend
```shell
yarn run mem_backend
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
## Run it with Vault user management
Note: Vault is proprietary and must be accessed separately.
```shell
export S3VAULT=vault
yarn start
```
This starts a Zenko CloudServer using Vault for user management.
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae

46
bin/metrics_server.js Executable file
View File

@ -0,0 +1,46 @@
#!/usr/bin/env node
'use strict'; // eslint-disable-line strict
const {
startWSManagementClient,
startPushConnectionHealthCheckServer,
} = require('../lib/management/push');
const logger = require('../lib/utilities/logger');
const {
PUSH_ENDPOINT: pushEndpoint,
INSTANCE_ID: instanceId,
MANAGEMENT_TOKEN: managementToken,
} = process.env;
if (!pushEndpoint) {
logger.error('missing push endpoint env var');
process.exit(1);
}
if (!instanceId) {
logger.error('missing instance id env var');
process.exit(1);
}
if (!managementToken) {
logger.error('missing management token env var');
process.exit(1);
}
startPushConnectionHealthCheckServer(err => {
if (err) {
logger.error('could not start healthcheck server', { error: err });
process.exit(1);
}
const url = `${pushEndpoint}/${instanceId}/ws?metrics=1`;
startWSManagementClient(url, managementToken, err => {
if (err) {
logger.error('connection failed, exiting', { error: err });
process.exit(1);
}
logger.info('no more connection, exiting');
process.exit(0);
});
});

46
bin/secure_channel_proxy.js Executable file
View File

@ -0,0 +1,46 @@
#!/usr/bin/env node
'use strict'; // eslint-disable-line strict
const {
startWSManagementClient,
startPushConnectionHealthCheckServer,
} = require('../lib/management/push');
const logger = require('../lib/utilities/logger');
const {
PUSH_ENDPOINT: pushEndpoint,
INSTANCE_ID: instanceId,
MANAGEMENT_TOKEN: managementToken,
} = process.env;
if (!pushEndpoint) {
logger.error('missing push endpoint env var');
process.exit(1);
}
if (!instanceId) {
logger.error('missing instance id env var');
process.exit(1);
}
if (!managementToken) {
logger.error('missing management token env var');
process.exit(1);
}
startPushConnectionHealthCheckServer(err => {
if (err) {
logger.error('could not start healthcheck server', { error: err });
process.exit(1);
}
const url = `${pushEndpoint}/${instanceId}/ws?proxy=1`;
startWSManagementClient(url, managementToken, err => {
if (err) {
logger.error('connection failed, exiting', { error: err });
process.exit(1);
}
logger.info('no more connection, exiting');
process.exit(0);
});
});

View File

@ -4,7 +4,6 @@
"metricsPort": 8002, "metricsPort": 8002,
"metricsListenOn": [], "metricsListenOn": [],
"replicationGroupId": "RG001", "replicationGroupId": "RG001",
"workers": 4,
"restEndpoints": { "restEndpoints": {
"localhost": "us-east-1", "localhost": "us-east-1",
"127.0.0.1": "us-east-1", "127.0.0.1": "us-east-1",
@ -102,14 +101,6 @@
"readPreference": "primary", "readPreference": "primary",
"database": "metadata" "database": "metadata"
}, },
"authdata": "authdata.json",
"backends": {
"auth": "file",
"data": "file",
"metadata": "mongodb",
"kms": "file",
"quota": "none"
},
"externalBackends": { "externalBackends": {
"aws_s3": { "aws_s3": {
"httpAgent": { "httpAgent": {

View File

@ -1,71 +0,0 @@
{
"port": 8000,
"listenOn": [],
"metricsPort": 8002,
"metricsListenOn": [],
"replicationGroupId": "RG001",
"restEndpoints": {
"localhost": "STANDARD",
"127.0.0.1": "STANDARD",
"yourhostname.ru": "STANDARD"
},
"websiteEndpoints": [
"static.yourhostname.ru"
],
"replicationEndpoints": [ {
"site": "zenko",
"servers": ["127.0.0.1:8000"],
"default": true
} ],
"log": {
"logLevel": "info",
"dumpLevel": "error"
},
"healthChecks": {
"allowFrom": ["127.0.0.1/8", "::1"]
},
"backends": {
"metadata": "mongodb"
},
"mongodb": {
"replicaSetHosts": "127.0.0.1:27017",
"writeConcern": "majority",
"replicaSet": "rs0",
"readPreference": "primary",
"database": "s3",
"authCredentials": {
"username": "s3",
"password": ""
}
},
"externalBackends": {
"aws_s3": {
"httpAgent": {
"keepAlive": false,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
},
"gcp": {
"httpAgent": {
"keepAlive": true,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
}
},
"requests": {
"viaProxy": false,
"trustedProxyCIDRs": [],
"extractClientIPFromHeader": ""
},
"bucketNotificationDestinations": [
{
"resource": "target1",
"type": "dummy",
"host": "localhost:6000"
}
]
}

View File

@ -116,7 +116,7 @@ const constants = {
], ],
// user metadata header to set object locationConstraint // user metadata header to set object locationConstraint
objectLocationConstraintHeader: 'x-amz-storage-class', objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified', lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
legacyLocations: ['sproxyd', 'legacy'], legacyLocations: ['sproxyd', 'legacy'],
// declare here all existing service accounts and their properties // declare here all existing service accounts and their properties
@ -205,6 +205,9 @@ const constants = {
], ],
allowedUtapiEventFilterStates: ['allow', 'deny'], allowedUtapiEventFilterStates: ['allow', 'deny'],
allowedRestoreObjectRequestTierValues: ['Standard'], allowedRestoreObjectRequestTierValues: ['Standard'],
validStorageClasses: [
'STANDARD',
],
lifecycleListing: { lifecycleListing: {
CURRENT_TYPE: 'current', CURRENT_TYPE: 'current',
NON_CURRENT_TYPE: 'noncurrent', NON_CURRENT_TYPE: 'noncurrent',
@ -235,14 +238,6 @@ const constants = {
assumedRoleArnResourceType: 'assumed-role', assumedRoleArnResourceType: 'assumed-role',
// Session name of the backbeat lifecycle assumed role session. // Session name of the backbeat lifecycle assumed role session.
backbeatLifecycleSessionName: 'backbeat-lifecycle', backbeatLifecycleSessionName: 'backbeat-lifecycle',
actionsToConsiderAsObjectPut: [
'initiateMultipartUpload',
'objectPutPart',
'completeMultipartUpload',
],
// if requester is not bucket owner, bucket policy actions should be denied with
// MethodNotAllowed error
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
}; };
module.exports = constants; module.exports = constants;

View File

@ -2,12 +2,11 @@
## Docker Image Generation ## Docker Image Generation
Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages). Docker images are hosted on [registry.scality.com](registry.scality.com).
CloudServer has a few images there: CloudServer has two namespaces there:
* Cloudserver container image: ghcr.io/scality/cloudserver * Production Namespace: registry.scality.com/cloudserver
* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard * Dev Namespace: registry.scality.com/cloudserver-dev
* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
With every CI build, the CI will push images, tagging the With every CI build, the CI will push images, tagging the
content with the developer branch's short SHA-1 commit hash. content with the developer branch's short SHA-1 commit hash.
@ -19,8 +18,8 @@ Tagged versions of cloudserver will be stored in the production namespace.
## How to Pull Docker Images ## How to Pull Docker Images
```sh ```sh
docker pull ghcr.io/scality/cloudserver:<commit hash> docker pull registry.scality.com/cloudserver-dev/cloudserver:<commit hash>
docker pull ghcr.io/scality/cloudserver:<tag> docker pull registry.scality.com/cloudserver/cloudserver:<tag>
``` ```
## Release Process ## Release Process

View File

@ -1,4 +1,4 @@
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0 FROM registry.scality.com/federation/nodesvc-base:7.10.6.0
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
@ -14,10 +14,8 @@ RUN rm -f ~/.gitconfig && \
git config --global --add safe.directory . && \ git config --global --add safe.directory . && \
git lfs install && \ git lfs install && \
GIT_LFS_SKIP_SMUDGE=1 && \ GIT_LFS_SKIP_SMUDGE=1 && \
yarn global add typescript && \
yarn install --frozen-lockfile --production --network-concurrency 1 && \ yarn install --frozen-lockfile --production --network-concurrency 1 && \
yarn cache clean --all && \ yarn cache clean --all
yarn global remove typescript
# run symlinking separately to avoid yarn installation errors # run symlinking separately to avoid yarn installation errors
# we might have to check if the symlinking is really needed! # we might have to check if the symlinking is really needed!

View File

@ -1,10 +1,10 @@
'use strict'; // eslint-disable-line strict 'use strict'; // eslint-disable-line strict
require('werelogs').stderrUtils.catchAndTimestampStderr( /**
undefined, * Catch uncaught exceptions and add timestamp to aid debugging
// Do not exit as workers have their own listener that will exit */
// But primary don't have another listener process.on('uncaughtException', err => {
require('cluster').isPrimary ? 1 : null, process.stderr.write(`${new Date().toISOString()}: Uncaught exception: \n${err.stack}`);
); });
require('./lib/server.js')(); require('./lib/server.js')();

View File

@ -107,47 +107,6 @@ function parseSproxydConfig(configSproxyd) {
return joi.attempt(configSproxyd, joiSchema, 'bad config'); return joi.attempt(configSproxyd, joiSchema, 'bad config');
} }
function parseRedisConfig(redisConfig) {
const joiSchema = joi.object({
password: joi.string().allow(''),
host: joi.string(),
port: joi.number(),
retry: joi.object({
connectBackoff: joi.object({
min: joi.number().required(),
max: joi.number().required(),
jitter: joi.number().required(),
factor: joi.number().required(),
deadline: joi.number().required(),
}),
}),
// sentinel config
sentinels: joi.alternatives().try(
joi.string()
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
.custom(hosts => hosts.split(',').map(item => {
const [host, port] = item.split(':');
return { host, port: Number.parseInt(port, 10) };
})),
joi.array().items(
joi.object({
host: joi.string().required(),
port: joi.number().required(),
})
).min(1),
),
name: joi.string(),
sentinelPassword: joi.string().allow(''),
})
.and('host', 'port')
.and('sentinels', 'name')
.xor('host', 'sentinels')
.without('sentinels', ['host', 'port'])
.without('host', ['sentinels', 'sentinelPassword']);
return joi.attempt(redisConfig, joiSchema, 'bad config');
}
function restEndpointsAssert(restEndpoints, locationConstraints) { function restEndpointsAssert(restEndpoints, locationConstraints) {
assert(typeof restEndpoints === 'object', assert(typeof restEndpoints === 'object',
'bad config: restEndpoints must be an object of endpoints'); 'bad config: restEndpoints must be an object of endpoints');
@ -377,7 +336,7 @@ function dmfLocationConstraintAssert(locationObj) {
function locationConstraintAssert(locationConstraints) { function locationConstraintAssert(locationConstraints) {
const supportedBackends = const supportedBackends =
['mem', 'file', 'scality', ['mem', 'file', 'scality',
'mongodb', 'dmf', 'azure_archive', 'vitastor'].concat(Object.keys(validExternalBackends)); 'mongodb', 'dmf', 'azure_archive'].concat(Object.keys(validExternalBackends));
assert(typeof locationConstraints === 'object', assert(typeof locationConstraints === 'object',
'bad config: locationConstraints must be an object'); 'bad config: locationConstraints must be an object');
Object.keys(locationConstraints).forEach(l => { Object.keys(locationConstraints).forEach(l => {
@ -502,23 +461,27 @@ function locationConstraintAssert(locationConstraints) {
locationConstraints[l].details.connector.hdclient); locationConstraints[l].details.connector.hdclient);
} }
}); });
assert(Object.keys(locationConstraints)
.includes('us-east-1'), 'bad locationConfig: must ' +
'include us-east-1 as a locationConstraint');
} }
function parseUtapiReindex(config) { function parseUtapiReindex(config) {
const { const {
enabled, enabled,
schedule, schedule,
redis, sentinel,
bucketd, bucketd,
onlyCountLatestWhenObjectLocked, onlyCountLatestWhenObjectLocked,
} = config; } = config;
assert(typeof enabled === 'boolean', assert(typeof enabled === 'boolean',
'bad config: utapi.reindex.enabled must be a boolean'); 'bad config: utapi.reindex.enabled must be a boolean');
assert(typeof sentinel === 'object',
const parsedRedis = parseRedisConfig(redis); 'bad config: utapi.reindex.sentinel must be an object');
assert(Array.isArray(parsedRedis.sentinels), assert(typeof sentinel.port === 'number',
'bad config: utapi reindex redis config requires a list of sentinels'); 'bad config: utapi.reindex.sentinel.port must be a number');
assert(typeof sentinel.name === 'string',
'bad config: utapi.reindex.sentinel.name must be a string');
assert(typeof bucketd === 'object', assert(typeof bucketd === 'object',
'bad config: utapi.reindex.bucketd must be an object'); 'bad config: utapi.reindex.bucketd must be an object');
assert(typeof bucketd.port === 'number', assert(typeof bucketd.port === 'number',
@ -536,13 +499,6 @@ function parseUtapiReindex(config) {
'bad config: utapi.reindex.schedule must be a valid ' + 'bad config: utapi.reindex.schedule must be a valid ' +
`cron schedule. ${e.message}.`); `cron schedule. ${e.message}.`);
} }
return {
enabled,
schedule,
redis: parsedRedis,
bucketd,
onlyCountLatestWhenObjectLocked,
};
} }
function requestsConfigAssert(requestsConfig) { function requestsConfigAssert(requestsConfig) {
@ -630,6 +586,7 @@ class Config extends EventEmitter {
// Read config automatically // Read config automatically
this._getLocationConfig(); this._getLocationConfig();
this._getConfig(); this._getConfig();
this._configureBackends();
} }
_getLocationConfig() { _getLocationConfig() {
@ -841,11 +798,11 @@ class Config extends EventEmitter {
this.websiteEndpoints = config.websiteEndpoints; this.websiteEndpoints = config.websiteEndpoints;
} }
this.workers = false; this.clusters = false;
if (config.workers !== undefined) { if (config.clusters !== undefined) {
assert(Number.isInteger(config.workers) && config.workers > 0, assert(Number.isInteger(config.clusters) && config.clusters > 0,
'bad config: workers must be a positive integer'); 'bad config: clusters must be a positive integer');
this.workers = config.workers; this.clusters = config.clusters;
} }
if (config.usEastBehavior !== undefined) { if (config.usEastBehavior !== undefined) {
@ -1083,7 +1040,8 @@ class Config extends EventEmitter {
assert(typeof config.localCache.port === 'number', assert(typeof config.localCache.port === 'number',
'config: bad port for localCache. port must be a number'); 'config: bad port for localCache. port must be a number');
if (config.localCache.password !== undefined) { if (config.localCache.password !== undefined) {
assert(typeof config.localCache.password === 'string', assert(
this._verifyRedisPassword(config.localCache.password),
'config: vad password for localCache. password must' + 'config: vad password for localCache. password must' +
' be a string'); ' be a string');
} }
@ -1109,46 +1067,56 @@ class Config extends EventEmitter {
} }
if (config.redis) { if (config.redis) {
this.redis = parseRedisConfig(config.redis); if (config.redis.sentinels) {
} this.redis = { sentinels: [], name: null };
if (config.scuba) {
this.scuba = {}; assert(typeof config.redis.name === 'string',
if (config.scuba.host) { 'bad config: redis sentinel name must be a string');
assert(typeof config.scuba.host === 'string', this.redis.name = config.redis.name;
'bad config: scuba host must be a string'); assert(Array.isArray(config.redis.sentinels) ||
this.scuba.host = config.scuba.host; typeof config.redis.sentinels === 'string',
'bad config: redis sentinels must be an array or string');
if (typeof config.redis.sentinels === 'string') {
config.redis.sentinels.split(',').forEach(item => {
const [host, port] = item.split(':');
this.redis.sentinels.push({ host,
port: Number.parseInt(port, 10) });
});
} else if (Array.isArray(config.redis.sentinels)) {
config.redis.sentinels.forEach(item => {
const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.redis.sentinels.push({ host, port });
});
}
if (config.redis.sentinelPassword !== undefined) {
assert(
this._verifyRedisPassword(config.redis.sentinelPassword));
this.redis.sentinelPassword = config.redis.sentinelPassword;
}
} else {
// check for standalone configuration
this.redis = {};
assert(typeof config.redis.host === 'string',
'bad config: redis.host must be a string');
assert(typeof config.redis.port === 'number',
'bad config: redis.port must be a number');
this.redis.host = config.redis.host;
this.redis.port = config.redis.port;
} }
if (config.scuba.port) { if (config.redis.password !== undefined) {
assert(Number.isInteger(config.scuba.port) assert(
&& config.scuba.port > 0, this._verifyRedisPassword(config.redis.password),
'bad config: scuba port must be a positive integer'); 'bad config: invalid password for redis. password must ' +
this.scuba.port = config.scuba.port; 'be a string');
this.redis.password = config.redis.password;
} }
} }
if (process.env.SCUBA_HOST && process.env.SCUBA_PORT) {
assert(typeof process.env.SCUBA_HOST === 'string',
'bad config: scuba host must be a string');
assert(Number.isInteger(Number(process.env.SCUBA_PORT))
&& Number(process.env.SCUBA_PORT) > 0,
'bad config: scuba port must be a positive integer');
this.scuba = {
host: process.env.SCUBA_HOST,
port: Number(process.env.SCUBA_PORT),
};
}
if (this.scuba) {
this.quotaEnabled = true;
}
const maxStaleness = Number(process.env.QUOTA_MAX_STALENESS_MS) ||
config.quota?.maxStatenessMS ||
24 * 60 * 60 * 1000;
assert(Number.isInteger(maxStaleness), 'bad config: maxStalenessMS must be an integer');
const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' ||
config.quota?.enableInflights || false;
this.quota = {
maxStaleness,
enableInflights,
};
if (config.utapi) { if (config.utapi) {
this.utapi = { component: 's3' }; this.utapi = { component: 's3' };
if (config.utapi.host) { if (config.utapi.host) {
@ -1177,8 +1145,50 @@ class Config extends EventEmitter {
assert(config.redis, 'missing required property of utapi ' + assert(config.redis, 'missing required property of utapi ' +
'configuration: redis'); 'configuration: redis');
if (config.utapi.redis) { if (config.utapi.redis) {
this.utapi.redis = parseRedisConfig(config.utapi.redis); if (config.utapi.redis.sentinels) {
if (this.utapi.redis.retry === undefined) { this.utapi.redis = { sentinels: [], name: null };
assert(typeof config.utapi.redis.name === 'string',
'bad config: redis sentinel name must be a string');
this.utapi.redis.name = config.utapi.redis.name;
assert(Array.isArray(config.utapi.redis.sentinels),
'bad config: redis sentinels must be an array');
config.utapi.redis.sentinels.forEach(item => {
const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.utapi.redis.sentinels.push({ host, port });
});
} else {
// check for standalone configuration
this.utapi.redis = {};
assert(typeof config.utapi.redis.host === 'string',
'bad config: redis.host must be a string');
assert(typeof config.utapi.redis.port === 'number',
'bad config: redis.port must be a number');
this.utapi.redis.host = config.utapi.redis.host;
this.utapi.redis.port = config.utapi.redis.port;
}
if (config.utapi.redis.retry !== undefined) {
if (config.utapi.redis.retry.connectBackoff !== undefined) {
const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
assert.strictEqual(typeof min, 'number',
'utapi.redis.retry.connectBackoff: min must be a number');
assert.strictEqual(typeof max, 'number',
'utapi.redis.retry.connectBackoff: max must be a number');
assert.strictEqual(typeof jitter, 'number',
'utapi.redis.retry.connectBackoff: jitter must be a number');
assert.strictEqual(typeof factor, 'number',
'utapi.redis.retry.connectBackoff: factor must be a number');
assert.strictEqual(typeof deadline, 'number',
'utapi.redis.retry.connectBackoff: deadline must be a number');
}
this.utapi.redis.retry = config.utapi.redis.retry;
} else {
this.utapi.redis.retry = { this.utapi.redis.retry = {
connectBackoff: { connectBackoff: {
min: 10, min: 10,
@ -1189,6 +1199,22 @@ class Config extends EventEmitter {
}, },
}; };
} }
if (config.utapi.redis.password !== undefined) {
assert(
this._verifyRedisPassword(config.utapi.redis.password),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.password = config.utapi.redis.password;
}
if (config.utapi.redis.sentinelPassword !== undefined) {
assert(
this._verifyRedisPassword(
config.utapi.redis.sentinelPassword),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.sentinelPassword =
config.utapi.redis.sentinelPassword;
}
} }
if (config.utapi.metrics) { if (config.utapi.metrics) {
this.utapi.metrics = config.utapi.metrics; this.utapi.metrics = config.utapi.metrics;
@ -1258,7 +1284,8 @@ class Config extends EventEmitter {
} }
if (config.utapi && config.utapi.reindex) { if (config.utapi && config.utapi.reindex) {
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex); parseUtapiReindex(config.utapi.reindex);
this.utapi.reindex = config.utapi.reindex;
} }
} }
@ -1303,8 +1330,6 @@ class Config extends EventEmitter {
} }
} }
this.authdata = config.authdata || 'authdata.json';
this.kms = {}; this.kms = {};
if (config.kms) { if (config.kms) {
assert(typeof config.kms.userName === 'string'); assert(typeof config.kms.userName === 'string');
@ -1524,6 +1549,25 @@ class Config extends EventEmitter {
this.outboundProxy.certs = certObj.certs; this.outboundProxy.certs = certObj.certs;
} }
this.managementAgent = {};
this.managementAgent.port = 8010;
this.managementAgent.host = 'localhost';
if (config.managementAgent !== undefined) {
if (config.managementAgent.port !== undefined) {
assert(Number.isInteger(config.managementAgent.port)
&& config.managementAgent.port > 0,
'bad config: managementAgent port must be a positive ' +
'integer');
this.managementAgent.port = config.managementAgent.port;
}
if (config.managementAgent.host !== undefined) {
assert.strictEqual(typeof config.managementAgent.host, 'string',
'bad config: management agent host must ' +
'be a string');
this.managementAgent.host = config.managementAgent.host;
}
}
// Ephemeral token to protect the reporting endpoint: // Ephemeral token to protect the reporting endpoint:
// try inherited from parent first, then hardcoded in conf file, // try inherited from parent first, then hardcoded in conf file,
// then create a fresh one as last resort. // then create a fresh one as last resort.
@ -1613,8 +1657,6 @@ class Config extends EventEmitter {
'bad config: maxScannedLifecycleListingEntries must be greater than 2'); 'bad config: maxScannedLifecycleListingEntries must be greater than 2');
this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries; this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
} }
this._configureBackends(config);
} }
_setTimeOptions() { _setTimeOptions() {
@ -1653,43 +1695,40 @@ class Config extends EventEmitter {
} }
_getAuthData() { _getAuthData() {
return JSON.parse(fs.readFileSync(findConfigFile(process.env.S3AUTH_CONFIG || this.authdata), { encoding: 'utf-8' })); return require(findConfigFile(process.env.S3AUTH_CONFIG || 'authdata.json'));
} }
_configureBackends(config) { _configureBackends() {
const backends = config.backends || {};
/** /**
* Configure the backends for Authentication, Data and Metadata. * Configure the backends for Authentication, Data and Metadata.
*/ */
let auth = backends.auth || 'mem'; let auth = 'mem';
let data = backends.data || 'multiple'; let data = 'multiple';
let metadata = backends.metadata || 'file'; let metadata = 'file';
let kms = backends.kms || 'file'; let kms = 'file';
let quota = backends.quota || 'none';
if (process.env.S3BACKEND) { if (process.env.S3BACKEND) {
const validBackends = ['mem', 'file', 'scality', 'cdmi']; const validBackends = ['mem', 'file', 'scality', 'cdmi'];
assert(validBackends.indexOf(process.env.S3BACKEND) > -1, assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
'bad environment variable: S3BACKEND environment variable ' + 'bad environment variable: S3BACKEND environment variable ' +
'should be one of mem/file/scality/cdmi' 'should be one of mem/file/scality/cdmi'
); );
auth = process.env.S3BACKEND == 'scality' ? 'scality' : 'mem'; auth = process.env.S3BACKEND;
data = process.env.S3BACKEND; data = process.env.S3BACKEND;
metadata = process.env.S3BACKEND; metadata = process.env.S3BACKEND;
kms = process.env.S3BACKEND; kms = process.env.S3BACKEND;
} }
if (process.env.S3VAULT) { if (process.env.S3VAULT) {
auth = process.env.S3VAULT; auth = process.env.S3VAULT;
auth = (auth === 'file' || auth === 'mem' || auth === 'cdmi' ? 'mem' : auth);
} }
if (auth === 'file' || auth === 'mem' || auth === 'cdmi') { if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
// Auth only checks for 'mem' since mem === file // Auth only checks for 'mem' since mem === file
auth = 'mem';
let authData; let authData;
if (process.env.SCALITY_ACCESS_KEY_ID && if (process.env.SCALITY_ACCESS_KEY_ID &&
process.env.SCALITY_SECRET_ACCESS_KEY) { process.env.SCALITY_SECRET_ACCESS_KEY) {
authData = buildAuthDataAccount( authData = buildAuthDataAccount(
process.env.SCALITY_ACCESS_KEY_ID, process.env.SCALITY_ACCESS_KEY_ID,
process.env.SCALITY_SECRET_ACCESS_KEY); process.env.SCALITY_SECRET_ACCESS_KEY);
} else { } else {
authData = this._getAuthData(); authData = this._getAuthData();
} }
@ -1697,7 +1736,7 @@ class Config extends EventEmitter {
throw new Error('bad config: invalid auth config file.'); throw new Error('bad config: invalid auth config file.');
} }
this.authData = authData; this.authData = authData;
} else if (auth === 'multiple') { } else if (auth === 'multiple') {
const authData = this._getAuthData(); const authData = this._getAuthData();
if (validateAuthConfig(authData)) { if (validateAuthConfig(authData)) {
throw new Error('bad config: invalid auth config file.'); throw new Error('bad config: invalid auth config file.');
@ -1712,9 +1751,9 @@ class Config extends EventEmitter {
'should be one of mem/file/scality/multiple' 'should be one of mem/file/scality/multiple'
); );
data = process.env.S3DATA; data = process.env.S3DATA;
if (data === 'scality' || data === 'multiple') { }
data = 'multiple'; if (data === 'scality' || data === 'multiple') {
} data = 'multiple';
} }
assert(this.locationConstraints !== undefined && assert(this.locationConstraints !== undefined &&
this.restEndpoints !== undefined, this.restEndpoints !== undefined,
@ -1727,18 +1766,18 @@ class Config extends EventEmitter {
if (process.env.S3KMS) { if (process.env.S3KMS) {
kms = process.env.S3KMS; kms = process.env.S3KMS;
} }
if (process.env.S3QUOTA) {
quota = process.env.S3QUOTA;
}
this.backends = { this.backends = {
auth, auth,
data, data,
metadata, metadata,
kms, kms,
quota,
}; };
} }
_verifyRedisPassword(password) {
return typeof password === 'string';
}
setAuthDataAccounts(accounts) { setAuthDataAccounts(accounts) {
this.authData.accounts = accounts; this.authData.accounts = accounts;
this.emit('authdata-update'); this.emit('authdata-update');
@ -1861,19 +1900,10 @@ class Config extends EventEmitter {
.update(instanceId) .update(instanceId)
.digest('hex'); .digest('hex');
} }
isQuotaEnabled() {
return !!this.quotaEnabled;
}
isQuotaInflightEnabled() {
return this.quota.enableInflights;
}
} }
module.exports = { module.exports = {
parseSproxydConfig, parseSproxydConfig,
parseRedisConfig,
locationConstraintAssert, locationConstraintAssert,
ConfigObject: Config, ConfigObject: Config,
config: new Config(), config: new Config(),

View File

@ -7,7 +7,6 @@ const bucketDeleteEncryption = require('./bucketDeleteEncryption');
const bucketDeleteWebsite = require('./bucketDeleteWebsite'); const bucketDeleteWebsite = require('./bucketDeleteWebsite');
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle'); const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
const bucketDeletePolicy = require('./bucketDeletePolicy'); const bucketDeletePolicy = require('./bucketDeletePolicy');
const bucketDeleteQuota = require('./bucketDeleteQuota');
const { bucketGet } = require('./bucketGet'); const { bucketGet } = require('./bucketGet');
const bucketGetACL = require('./bucketGetACL'); const bucketGetACL = require('./bucketGetACL');
const bucketGetCors = require('./bucketGetCors'); const bucketGetCors = require('./bucketGetCors');
@ -18,7 +17,6 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
const bucketGetNotification = require('./bucketGetNotification'); const bucketGetNotification = require('./bucketGetNotification');
const bucketGetObjectLock = require('./bucketGetObjectLock'); const bucketGetObjectLock = require('./bucketGetObjectLock');
const bucketGetPolicy = require('./bucketGetPolicy'); const bucketGetPolicy = require('./bucketGetPolicy');
const bucketGetQuota = require('./bucketGetQuota');
const bucketGetEncryption = require('./bucketGetEncryption'); const bucketGetEncryption = require('./bucketGetEncryption');
const bucketHead = require('./bucketHead'); const bucketHead = require('./bucketHead');
const { bucketPut } = require('./bucketPut'); const { bucketPut } = require('./bucketPut');
@ -35,7 +33,6 @@ const bucketPutNotification = require('./bucketPutNotification');
const bucketPutEncryption = require('./bucketPutEncryption'); const bucketPutEncryption = require('./bucketPutEncryption');
const bucketPutPolicy = require('./bucketPutPolicy'); const bucketPutPolicy = require('./bucketPutPolicy');
const bucketPutObjectLock = require('./bucketPutObjectLock'); const bucketPutObjectLock = require('./bucketPutObjectLock');
const bucketUpdateQuota = require('./bucketUpdateQuota');
const bucketGetReplication = require('./bucketGetReplication'); const bucketGetReplication = require('./bucketGetReplication');
const bucketDeleteReplication = require('./bucketDeleteReplication'); const bucketDeleteReplication = require('./bucketDeleteReplication');
const corsPreflight = require('./corsPreflight'); const corsPreflight = require('./corsPreflight');
@ -85,10 +82,6 @@ const api = {
// Attach the apiMethod method to the request, so it can used by monitoring in the server // Attach the apiMethod method to the request, so it can used by monitoring in the server
// eslint-disable-next-line no-param-reassign // eslint-disable-next-line no-param-reassign
request.apiMethod = apiMethod; request.apiMethod = apiMethod;
// Array of end of API callbacks, used to perform some logic
// at the end of an API.
// eslint-disable-next-line no-param-reassign
request.finalizerHooks = [];
const actionLog = monitoringMap[apiMethod]; const actionLog = monitoringMap[apiMethod];
if (!actionLog && if (!actionLog &&
@ -197,17 +190,14 @@ const api = {
return async.waterfall([ return async.waterfall([
next => auth.server.doAuth( next => auth.server.doAuth(
request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => { request, log, (err, userInfo, authorizationResults, streamingV4Params) => {
if (err) { if (err) {
// VaultClient returns standard errors, but the route requires
// Arsenal errors
const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
log.trace('authentication error', { error: err }); log.trace('authentication error', { error: err });
return next(arsenalError); return next(err);
} }
return next(null, userInfo, authorizationResults, streamingV4Params, infos); return next(null, userInfo, authorizationResults, streamingV4Params);
}, 's3', requestContexts), }, 's3', requestContexts),
(userInfo, authorizationResults, streamingV4Params, infos, next) => { (userInfo, authorizationResults, streamingV4Params, next) => {
const authNames = { accountName: userInfo.getAccountDisplayName() }; const authNames = { accountName: userInfo.getAccountDisplayName() };
if (userInfo.isRequesterAnIAMUser()) { if (userInfo.isRequesterAnIAMUser()) {
authNames.userName = userInfo.getIAMdisplayName(); authNames.userName = userInfo.getIAMdisplayName();
@ -217,7 +207,7 @@ const api = {
} }
log.addDefaultFields(authNames); log.addDefaultFields(authNames);
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') { if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
return next(null, userInfo, authorizationResults, streamingV4Params, infos); return next(null, userInfo, authorizationResults, streamingV4Params);
} }
// issue 100 Continue to the client // issue 100 Continue to the client
writeContinue(request, response); writeContinue(request, response);
@ -248,12 +238,12 @@ const api = {
} }
// Convert array of post buffers into one string // Convert array of post buffers into one string
request.post = Buffer.concat(post, postLength).toString(); request.post = Buffer.concat(post, postLength).toString();
return next(null, userInfo, authorizationResults, streamingV4Params, infos); return next(null, userInfo, authorizationResults, streamingV4Params);
}); });
return undefined; return undefined;
}, },
// Tag condition keys require information from CloudServer for evaluation // Tag condition keys require information from CloudServer for evaluation
(userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth( (userInfo, authorizationResults, streamingV4Params, next) => tagConditionKeyAuth(
authorizationResults, authorizationResults,
request, request,
requestContexts, requestContexts,
@ -264,14 +254,13 @@ const api = {
log.trace('tag authentication error', { error: err }); log.trace('tag authentication error', { error: err });
return next(err); return next(err);
} }
return next(null, userInfo, authResultsWithTags, streamingV4Params, infos); return next(null, userInfo, authResultsWithTags, streamingV4Params);
}, },
), ),
], (err, userInfo, authorizationResults, streamingV4Params, infos) => { ], (err, userInfo, authorizationResults, streamingV4Params) => {
if (err) { if (err) {
return callback(err); return callback(err);
} }
request.accountQuotas = infos?.accountQuota;
if (authorizationResults) { if (authorizationResults) {
const checkedResults = checkAuthResults(authorizationResults); const checkedResults = checkAuthResults(authorizationResults);
if (checkedResults instanceof Error) { if (checkedResults instanceof Error) {
@ -288,23 +277,19 @@ const api = {
return acc; return acc;
}, {}); }, {});
} }
const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
(hook, done) => hook(err, done),
() => callback(err, ...results));
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') { if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
request._response = response; request._response = response;
return this[apiMethod](userInfo, request, streamingV4Params, return this[apiMethod](userInfo, request, streamingV4Params,
log, methodCallback, authorizationResults); log, callback, authorizationResults);
} }
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') { if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
return this[apiMethod](userInfo, request, sourceBucket, return this[apiMethod](userInfo, request, sourceBucket,
sourceObject, sourceVersionId, log, methodCallback); sourceObject, sourceVersionId, log, callback);
} }
if (apiMethod === 'objectGet') { if (apiMethod === 'objectGet') {
return this[apiMethod](userInfo, request, returnTagCount, log, callback); return this[apiMethod](userInfo, request, returnTagCount, log, callback);
} }
return this[apiMethod](userInfo, request, log, methodCallback); return this[apiMethod](userInfo, request, log, callback);
}); });
}, },
bucketDelete, bucketDelete,
@ -331,14 +316,11 @@ const api = {
bucketPutReplication, bucketPutReplication,
bucketGetReplication, bucketGetReplication,
bucketDeleteReplication, bucketDeleteReplication,
bucketDeleteQuota,
bucketPutLifecycle, bucketPutLifecycle,
bucketUpdateQuota,
bucketGetLifecycle, bucketGetLifecycle,
bucketDeleteLifecycle, bucketDeleteLifecycle,
bucketPutPolicy, bucketPutPolicy,
bucketGetPolicy, bucketGetPolicy,
bucketGetQuota,
bucketDeletePolicy, bucketDeletePolicy,
bucketPutObjectLock, bucketPutObjectLock,
bucketPutNotification, bucketPutNotification,

View File

@ -12,7 +12,6 @@ const {
arrayOfAllowed, arrayOfAllowed,
assumedRoleArnResourceType, assumedRoleArnResourceType,
backbeatLifecycleSessionName, backbeatLifecycleSessionName,
actionsToConsiderAsObjectPut,
} = constants; } = constants;
// whitelist buckets to allow public read on objects // whitelist buckets to allow public read on objects
@ -49,24 +48,32 @@ function isRequesterNonAccountUser(authInfo) {
* @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false. * @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
*/ */
const considerAsObjectPut = ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'];
function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) { function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
// Same logic applies on the Versioned APIs, so let's simplify it. // Same logic applies on the Versioned APIs, so let's simplify it.
let requestTypeParsed = requestType.endsWith('Version') ? let requestTypeParsed = requestType.endsWith('Version') ?
requestType.slice(0, 'Version'.length * -1) : requestType; requestType.slice(0, 'Version'.length * -1) : requestType;
requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ? requestTypeParsed = considerAsObjectPut.includes(requestTypeParsed) ? 'objectPut' : requestTypeParsed;
'objectPut' : requestTypeParsed; const parsedMainApiCall = considerAsObjectPut.includes(mainApiCall) ? 'objectPut' : mainApiCall;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? console.log({
'objectPut' : mainApiCall; requestType,
requestTypeParsed,
parsedMainApiCall,
});
if (bucket.getOwner() === canonicalID) { if (bucket.getOwner() === canonicalID) {
console.log('acl true because bucket owner');
return true; return true;
} }
if (parsedMainApiCall === 'objectGet') { if (parsedMainApiCall === 'objectGet') {
if (requestTypeParsed === 'objectGetTagging') { if (requestTypeParsed === 'objectGetTagging') {
console.log('acl true because objectGetTagging');
return true; return true;
} }
} }
if (parsedMainApiCall === 'objectPut') { if (parsedMainApiCall === 'objectPut') {
if (arrayOfAllowed.includes(requestTypeParsed)) { if (arrayOfAllowed.includes(requestTypeParsed)) {
console.log('acl true because objectPut');
return true; return true;
} }
} }
@ -140,19 +147,16 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
// objectPutACL, objectGetACL, objectHead or objectGet, the bucket // objectPutACL, objectGetACL, objectHead or objectGet, the bucket
// authorization check should just return true so can move on to check // authorization check should just return true so can move on to check
// rights at the object level. // rights at the object level.
console.log('acls checks end');
return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL' return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
|| requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); || requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
} }
function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser, function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
isUserUnauthenticated, mainApiCall) { isUserUnauthenticated, mainApiCall) {
const bucketOwner = bucket.getOwner(); const bucketOwner = bucket.getOwner();
const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
'objectPut' : requestType;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
'objectPut' : mainApiCall;
// acls don't distinguish between users and accounts, so both should be allowed // acls don't distinguish between users and accounts, so both should be allowed
if (bucketOwnerActions.includes(requestTypeParsed) if (bucketOwnerActions.includes(requestType)
&& (bucketOwner === canonicalID)) { && (bucketOwner === canonicalID)) {
return true; return true;
} }
@ -161,9 +165,9 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
} }
// Backward compatibility // Backward compatibility
if (parsedMainApiCall === 'objectGet') { if (mainApiCall === 'objectGet') {
if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id'])) if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
&& requestTypeParsed === 'objectGetTagging') { && requestType === 'objectGetTagging') {
return true; return true;
} }
} }
@ -172,7 +176,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
return false; return false;
} }
if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') { if (requestType === 'objectGet' || requestType === 'objectHead') {
if (objectMD.acl.Canned === 'public-read' if (objectMD.acl.Canned === 'public-read'
|| objectMD.acl.Canned === 'public-read-write' || objectMD.acl.Canned === 'public-read-write'
|| (objectMD.acl.Canned === 'authenticated-read' || (objectMD.acl.Canned === 'authenticated-read'
@ -198,11 +202,11 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
// User is already authorized on the bucket for FULL_CONTROL or WRITE or // User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write // bucket has canned ACL public-read-write
if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') { if (requestType === 'objectPut' || requestType === 'objectDelete') {
return true; return true;
} }
if (requestTypeParsed === 'objectPutACL') { if (requestType === 'objectPutACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control' if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID) && bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
@ -218,7 +222,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
} }
} }
if (requestTypeParsed === 'objectGetACL') { if (requestType === 'objectGetACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control' if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID) && bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
@ -239,7 +243,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
const bucketAcl = bucket.getAcl(); const bucketAcl = bucket.getAcl();
const allowPublicReads = publicReadBuckets.includes(bucket.getName()) const allowPublicReads = publicReadBuckets.includes(bucket.getName())
&& bucketAcl.Canned === 'public-read' && bucketAcl.Canned === 'public-read'
&& (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); && (requestType === 'objectGet' || requestType === 'objectHead');
if (allowPublicReads) { if (allowPublicReads) {
return true; return true;
} }
@ -338,6 +342,16 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l
const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log); const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log); const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log); const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
console.log('checkBucketPolicy', {
principalMatch,
actionMatch,
resourceMatch,
conditionsMatch,
canonicalID,
requestType,
arn,
s,
});
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') { if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') {
// explicit deny trumps any allows, so return immediately // explicit deny trumps any allows, so return immediately
@ -356,11 +370,15 @@ function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner,
const bucketPolicy = bucket.getBucketPolicy(); const bucketPolicy = bucket.getBucketPolicy();
let processedResult = results[requestType]; let processedResult = results[requestType];
if (!bucketPolicy) { if (!bucketPolicy) {
console.log('no bucket policy', { actionImplicitDenies }, actionImplicitDenies[requestType], { aclPermission });
// must also check that this is indeed an allow, not an explicit deny
processedResult = actionImplicitDenies[requestType] === false && aclPermission; processedResult = actionImplicitDenies[requestType] === false && aclPermission;
} else { } else {
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn, const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
bucketOwner, log, request, actionImplicitDenies); bucketOwner, log, request, actionImplicitDenies);
console.log('bk results!', bucketPolicyPermission, actionImplicitDenies[requestType]);
if (bucketPolicyPermission === 'explicitDeny') { if (bucketPolicyPermission === 'explicitDeny') {
processedResult = false; processedResult = false;
} else if (bucketPolicyPermission === 'allow') { } else if (bucketPolicyPermission === 'allow') {
@ -369,6 +387,15 @@ function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner,
processedResult = actionImplicitDenies[requestType] === false && aclPermission; processedResult = actionImplicitDenies[requestType] === false && aclPermission;
} }
} }
console.log('processBP result', {
requestType,
bucketPolicy,
canonicalID,
arn,
bucketOwner,
aclPermission,
processedResult,
});
return processedResult; return processedResult;
} }
@ -394,14 +421,22 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo
// if the bucket owner is an account, users should not have default access // if the bucket owner is an account, users should not have default access
if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) { if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
results[_requestType] = actionImplicitDenies[_requestType] === false; results[_requestType] = actionImplicitDenies[_requestType] === false;
console.log('result bucket authz case account or service account', results[_requestType]);
return results[_requestType]; return results[_requestType];
} }
const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall); const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
console.log('acl checks on bucket', {
aclPermission,
bucketOwner: bucket.getOwner(),
canonicalID,
requesterIsNotUser,
isServiceAccount: isServiceAccount(canonicalID),
});
// In case of error bucket access is checked with bucketGet // In case of error bucket access is checked with bucketGet
// For website, bucket policy only uses objectGet and ignores bucketGet // For website, bucket policy only uses objectGet and ignores bucketGet
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html // https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
// bucketGet should be used to check acl but switched to objectGet for bucket policy // bucketGet should be used to check acl but switched to objectGet for bucket policy
if (isWebsite && _requestType === 'bucketGet') { if (isWebsite && (_requestType === 'bucketGet' || _requestType === 'listObjectVersions')) {
// eslint-disable-next-line no-param-reassign // eslint-disable-next-line no-param-reassign
_requestType = 'objectGet'; _requestType = 'objectGet';
actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false; actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
@ -425,7 +460,7 @@ function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, aut
arn = authInfo.getArn(); arn = authInfo.getArn();
} }
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log, return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
request, true, results, actionImplicitDenies); request, true, results, actionImplicitDenies);
}); });
} }
@ -441,34 +476,50 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false; actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
const parsedMethodName = _requestType.endsWith('Version') const parsedMethodName = _requestType.endsWith('Version')
? _requestType.slice(0, -7) : _requestType; ? _requestType.slice(0, -7) : _requestType;
console.log('parsedMethodName', parsedMethodName);
const bucketOwner = bucket.getOwner(); const bucketOwner = bucket.getOwner();
if (!objectMD) { if (!objectMD) {
// check bucket has read access // check bucket has read access
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions // 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
let permission = 'bucketGet'; let permision = 'bucketGet';
if (actionsToConsiderAsObjectPut.includes(_requestType)) { if (considerAsObjectPut.includes(_requestType)) {
permission = 'objectPut'; permision = 'objectPut';
} }
results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request, results[_requestType] = isBucketAuthorized(bucket, permision, canonicalID, authInfo, log, request,
actionImplicitDenies, isWebsite); actionImplicitDenies, isWebsite);
// User is already authorized on the bucket for FULL_CONTROL or WRITE or // User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write // bucket has canned ACL public-read-write
console.log('checking parsedMethodName', parsedMethodName, results[_requestType]);
if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete') if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
&& results[_requestType] === false) { && results[_requestType] === false) {
results[_requestType] = actionImplicitDenies[_requestType] === false; results[_requestType] = actionImplicitDenies[_requestType] === false;
console.log('object authorized? case objectPut or objectDelete', {
requestType: _requestType,
results: results[_requestType],
});
} }
// If the API is MPU related, we need to check if the user has
// permissions to the bucket
console.log('object authorized? case no objectmd', {
requestType: _requestType,
results: results[_requestType],
});
return results[_requestType]; return results[_requestType];
} }
let requesterIsNotUser = true; let requesterIsNotUser = true;
let arn = null; let arn = null;
let isUserUnauthenticated = false; let isUserUnauthenticated = false;
if (authInfo) { if (authInfo) {
requesterIsNotUser = !isRequesterNonAccountUser(authInfo); requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
arn = authInfo.getArn(); arn = authInfo.getArn();
isUserUnauthenticated = arn === undefined; isUserUnauthenticated = arn === undefined;
} }
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) { if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
results[_requestType] = actionImplicitDenies[_requestType] === false; results[_requestType] = actionImplicitDenies[_requestType] === false;
console.log('object authorized? case account or service account', {
requestType: _requestType,
results: results[_requestType],
});
return results[_requestType]; return results[_requestType];
} }
// account is authorized if: // account is authorized if:
@ -476,9 +527,13 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI
// - account is the bucket owner // - account is the bucket owner
// - requester is account, not user // - requester is account, not user
if (bucketOwnerActions.includes(parsedMethodName) if (bucketOwnerActions.includes(parsedMethodName)
&& (bucketOwner === canonicalID) && (bucketOwner === canonicalID)
&& requesterIsNotUser) { && requesterIsNotUser) {
results[_requestType] = actionImplicitDenies[_requestType] === false; results[_requestType] = actionImplicitDenies[_requestType] === false;
console.log('object authorized? case bucketOwnerActions', {
requestType: _requestType,
results: results[_requestType],
});
return results[_requestType]; return results[_requestType];
} }
const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName, const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName,
@ -517,44 +572,40 @@ function validatePolicyResource(bucketName, policy) {
function checkIp(value) { function checkIp(value) {
const errString = 'Invalid IP address in Conditions'; const errString = 'Invalid IP address in Conditions';
const values = Array.isArray(value) ? value : [value]; // these preliminary checks are validating the provided
// ip address against ipaddr.js, the library we use when
for (let i = 0; i < values.length; i++) { // evaluating IP condition keys. It ensures compatibility,
// these preliminary checks are validating the provided // but additional checks are required to enforce the right
// ip address against ipaddr.js, the library we use when // notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
// evaluating IP condition keys. It ensures compatibility, // we would accept different ip formats, which is not
// but additional checks are required to enforce the right // standard in an AWS use case.
// notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise, try {
// we would accept different ip formats, which is not
// standard in an AWS use case.
try { try {
try { parseCIDR(value);
parseCIDR(values[i]);
} catch (err) {
isValid(values[i]);
}
} catch (err) { } catch (err) {
return errString; isValid(value);
}
// Apply the existing IP validation logic to each element
const validateIpRegex = ip => {
if (constants.ipv4Regex.test(ip)) {
return ip.split('.').every(part => parseInt(part, 10) <= 255);
}
if (constants.ipv6Regex.test(ip)) {
return ip.split(':').every(part => part.length <= 4);
}
return false;
};
if (validateIpRegex(values[i]) !== true) {
return errString;
} }
} catch (err) {
return errString;
} }
// If the function hasn't returned by now, all elements are valid // credit to Theodore John.S
return null; // Medium article: Validating IPv4 and IPv6 Addresses with Ease
// — Unveiling the Power of Validation in JavaScript
const validateIpRegex = ip => {
if (constants.ipv4Regex.test(ip)) {
return ip.split('.').every(part => parseInt(part, 10) <= 255);
}
if (constants.ipv6Regex.test(ip)) {
return ip.split(':').every(part => part.length <= 4);
}
return false;
};
if (validateIpRegex(value) === true) {
return null;
}
return errString;
} }
// This function checks all bucket policy conditions if the values provided // This function checks all bucket policy conditions if the values provided

View File

@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
apiMethod, 's3'); apiMethod, 's3');
} }
if (apiMethod === 'bucketPut') { if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
return null; return null;
} }
@ -65,17 +65,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
const requestContexts = []; const requestContexts = [];
if (apiMethod === 'multiObjectDelete') { if (apiMethodAfterVersionCheck === 'objectCopy'
// MultiObjectDelete does not require any authorization when evaluating
// the API. Instead, we authorize each object passed.
// But in order to get any relevant information from the authorization service
// for example, the account quota, we must send a request context object
// with no `specificResource`. We expect the result to be an implicit deny.
// In the API, we then ignore these authorization results, and we can use
// any information returned, e.g., the quota.
const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
requestContexts.push(requestContextMultiObjectDelete);
} else if (apiMethodAfterVersionCheck === 'objectCopy'
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') { || apiMethodAfterVersionCheck === 'objectPutCopyPart') {
const objectGetAction = sourceVersionId ? 'objectGetVersion' : const objectGetAction = sourceVersionId ? 'objectGetVersion' :
'objectGet'; 'objectGet';

View File

@ -7,8 +7,6 @@ const errors = require('arsenal').errors;
const { config } = require('../../../Config'); const { config } = require('../../../Config');
const { locationConstraints } = config; const { locationConstraints } = config;
const { scaledMsPerDay } = config.getTimeOptions();
/** /**
* Get response header "x-amz-restore" * Get response header "x-amz-restore"
* Be called by objectHead.js * Be called by objectHead.js
@ -147,7 +145,7 @@ function _updateObjectExpirationDate(objectMD, log) {
}); });
if (isObjectAlreadyRestored) { if (isObjectAlreadyRestored) {
const expiryDate = new Date(objectMD.archive.restoreRequestedAt); const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay)); expiryDate.setDate(expiryDate.getDate() + objectMD.archive.restoreRequestedDays);
/* eslint-disable no-param-reassign */ /* eslint-disable no-param-reassign */
objectMD.archive.restoreWillExpireAt = expiryDate; objectMD.archive.restoreWillExpireAt = expiryDate;

View File

@ -5,6 +5,7 @@ const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
const constants = require('../../../../constants'); const constants = require('../../../../constants');
const { data } = require('../../../data/wrapper'); const { data } = require('../../../data/wrapper');
const services = require('../../../services'); const services = require('../../../services');
const logger = require('../../../utilities/logger');
const { dataStore } = require('./storeObject'); const { dataStore } = require('./storeObject');
const locationConstraintCheck = require('./locationConstraintCheck'); const locationConstraintCheck = require('./locationConstraintCheck');
const { versioningPreprocessing, overwritingVersioning } = require('./versioning'); const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
@ -20,7 +21,7 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' +
'a versioned object to a location-constraint of type Azure or GCP.'; 'a versioned object to a location-constraint of type Azure or GCP.';
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
metadataStoreParams, dataToDelete, log, requestMethod, callback) { metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
services.metadataStoreObject(bucketName, dataGetInfo, services.metadataStoreObject(bucketName, dataGetInfo,
cipherBundle, metadataStoreParams, (err, result) => { cipherBundle, metadataStoreParams, (err, result) => {
if (err) { if (err) {
@ -30,7 +31,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
const newDataStoreName = Array.isArray(dataGetInfo) ? const newDataStoreName = Array.isArray(dataGetInfo) ?
dataGetInfo[0].dataStoreName : null; dataGetInfo[0].dataStoreName : null;
return data.batchDelete(dataToDelete, requestMethod, return data.batchDelete(dataToDelete, requestMethod,
newDataStoreName, log, err => callback(err, result)); newDataStoreName, deleteLog, err => callback(err, result));
} }
return callback(null, result); return callback(null, result);
}); });
@ -197,9 +198,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
const dontSkipBackend = externalBackends; const dontSkipBackend = externalBackends;
/* eslint-enable camelcase */ /* eslint-enable camelcase */
const requestLogger =
logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
const mdOnlyHeader = request.headers['x-amz-meta-mdonly']; const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
const mdOnlySize = request.headers['x-amz-meta-size']; const mdOnlySize = request.headers['x-amz-meta-size'];
return async.waterfall([ return async.waterfall([
function storeData(next) { function storeData(next) {
if (size === 0) { if (size === 0) {
@ -294,7 +296,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
} }
return _storeInMDandDeleteData(bucketName, infoArr, return _storeInMDandDeleteData(bucketName, infoArr,
cipherBundle, metadataStoreParams, cipherBundle, metadataStoreParams,
options.dataToDelete, log, requestMethod, next); options.dataToDelete, requestLogger, requestMethod, next);
}, },
], callback); ], callback);
} }

View File

@ -300,13 +300,16 @@ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log,
signatureAge: authParams.params.data.signatureAge, signatureAge: authParams.params.data.signatureAge,
}, },
}; };
console.log('checking for s3:BypassGovernanceRetention permission');
return vault.checkPolicies(requestContextParams, return vault.checkPolicies(requestContextParams,
authInfo.getArn(), log, (err, authorizationResults) => { authInfo.getArn(), log, (err, authorizationResults) => {
console.log('==================================', err, authorizationResults, vault.checkPolicies);
if (err) { if (err) {
return cb(err); return cb(err);
} }
const explicitDenyExists = authorizationResults.some( const explicitDenyExists = authorizationResults.some(
authzResult => authzResult.isAllowed === false && !authzResult.isImplicit); authzResult => authzResult.isAllowed === false && !authzResult.isImplicit);
console.log('++++++++++++=', explicitDenyExists);
if (explicitDenyExists) { if (explicitDenyExists) {
log.trace('authorization check failed for user', log.trace('authorization check failed for user',
{ {
@ -333,6 +336,13 @@ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log,
log, log,
request); request);
console.log('actionImplicitDenies', actionImplicitDenies, areAllActionsAllowed, {
bucketMD,
actionImplicitDeniesKeys: Object.keys(actionImplicitDenies),
authInfo,
actionImplicitDenies,
});
return cb(areAllActionsAllowed === true ? null : errors.AccessDenied); return cb(areAllActionsAllowed === true ? null : errors.AccessDenied);
}); });
} }

View File

@ -8,7 +8,7 @@ const { pushMetric } = require('../../../utapi/utilities');
const { decodeVersionId } = require('./versioning'); const { decodeVersionId } = require('./versioning');
const collectCorsHeaders = require('../../../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
const { parseRestoreRequestXml } = s3middleware.objectRestore; const { parseRestoreRequestXml } = s3middleware.objectRestore;
const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
/** /**
* Check if tier is supported * Check if tier is supported
@ -59,14 +59,6 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
objectKey, objectKey,
versionId: decodedVidResult, versionId: decodedVidResult,
requestType: request.apiMethods || 'restoreObject', requestType: request.apiMethods || 'restoreObject',
/**
* Restoring an object might not cause any impact on
* the storage, if the object is already restored: in
* this case, the duration is extended. We disable the
* quota evaluation and trigger it manually.
*/
checkQuota: false,
request,
}; };
return async.waterfall([ return async.waterfall([
@ -124,16 +116,6 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
return next(err, bucketMD, objectMD); return next(err, bucketMD, objectMD);
}); });
}, },
function evaluateQuotas(bucketMD, objectMD, next) {
if (isObjectRestored) {
return next(null, bucketMD, objectMD);
}
const actions = Array.isArray(mdValueParams.requestType) ?
mdValueParams.requestType : [mdValueParams.requestType];
const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
false, log, err => next(err, bucketMD, objectMD));
},
function updateObjectMD(bucketMD, objectMD, next) { function updateObjectMD(bucketMD, objectMD, next) {
const params = objectMD.versionId ? { versionId: objectMD.versionId } : {}; const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params, metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,

View File

@ -210,7 +210,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
// null keys are used, which is used as an optimization to // null keys are used, which is used as an optimization to
// avoid having to check the versioned key since there can // avoid having to check the versioned key since there can
// be no more versioned key to clean up // be no more versioned key to clean up
if (mst.isNull && mst.versionId && !mst.isNull2) { if (mst.isNull && !mst.isNull2) {
const delOptions = { versionId: mst.versionId }; const delOptions = { versionId: mst.versionId };
return { options, delOptions }; return { options, delOptions };
} }
@ -241,7 +241,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
if (masterIsNull) { if (masterIsNull) {
// if master is a null version or a non-versioned key, // if master is a null version or a non-versioned key,
// copy it to a new null key // copy it to a new null key
const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId; const nullVersionId = mst.isNull ? mst.versionId : nonVersionedObjId;
if (nullVersionCompatMode) { if (nullVersionCompatMode) {
options.extraMD = { options.extraMD = {
nullVersionId, nullVersionId,

View File

@ -1,314 +0,0 @@
const async = require('async');
const { errors } = require('arsenal');
const monitoring = require('../../../utilities/monitoringHandler');
const {
actionNeedQuotaCheckCopy,
actionNeedQuotaCheck,
actionWithDataDeletion,
} = require('arsenal').policies;
const { config } = require('../../../Config');
const QuotaService = require('../../../quotas/quotas');
/**
* Process the bytes to write based on the request and object metadata
* @param {string} apiMethod - api method
* @param {BucketInfo} bucket - bucket info
* @param {string} versionId - version id of the object
* @param {number} contentLength - content length of the object
* @param {object} objMD - object metadata
* @param {object} destObjMD - destination object metadata
* @return {number} processed content length
*/
function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
let bytes = contentLength;
if (apiMethod === 'objectRestore') {
// object is being restored
bytes = Number.parseInt(objMD['content-length'], 10);
} else if (!bytes && objMD?.['content-length']) {
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
if (!destObjMD || bucket.isVersioningEnabled()) {
// object is being copied
bytes = Number.parseInt(objMD['content-length'], 10);
} else if (!bucket.isVersioningEnabled()) {
// object is being copied and replaces the target
bytes = Number.parseInt(objMD['content-length'], 10) -
Number.parseInt(destObjMD['content-length'], 10);
}
} else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
// object is being deleted
bytes = -Number.parseInt(objMD['content-length'], 10);
}
} else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
// object is being replaced: store the diff, if the bucket is not versioned
bytes = bytes - Number.parseInt(objMD['content-length'], 10);
}
return bytes || 0;
}
/**
* Checks if a metric is stale based on the provided parameters.
*
* @param {Object} metric - The metric object to check.
* @param {string} resourceType - The type of the resource.
* @param {string} resourceName - The name of the resource.
* @param {string} action - The action being performed.
* @param {number} inflight - The number of inflight requests.
* @param {Object} log - The logger object.
* @returns {boolean} Returns true if the metric is stale, false otherwise.
*/
function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
if (metric.date && Date.now() - new Date(metric.date).getTime() >
QuotaService.maxStaleness) {
log.warn('Stale metrics from the quota service, allowing the request', {
resourceType,
resourceName,
action,
inflight,
});
monitoring.requestWithQuotaMetricsUnavailable.inc();
return true;
}
return false;
}
/**
* Evaluates quotas for a bucket and an account and update inflight count.
*
* @param {number} bucketQuota - The quota limit for the bucket.
* @param {number} accountQuota - The quota limit for the account.
* @param {object} bucket - The bucket object.
* @param {object} account - The account object.
* @param {number} inflight - The number of inflight requests.
* @param {number} inflightForCheck - The number of inflight requests for checking quotas.
* @param {string} action - The action being performed.
* @param {object} log - The logger object.
* @param {function} callback - The callback function to be called when evaluation is complete.
* @returns {object} - The result of the evaluation.
*/
function _evaluateQuotas(
bucketQuota,
accountQuota,
bucket,
account,
inflight,
inflightForCheck,
action,
log,
callback,
) {
let bucketQuotaExceeded = false;
let accountQuotaExceeded = false;
const creationDate = new Date(bucket.getCreationDate()).getTime();
return async.parallel({
bucketQuota: parallelDone => {
if (bucketQuota > 0) {
return QuotaService.getUtilizationMetrics('bucket',
`${bucket.getName()}_${creationDate}`, null, {
action,
inflight,
}, (err, bucketMetrics) => {
if (err || inflight < 0) {
return parallelDone(err);
}
if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
log.debug('Bucket quota exceeded', {
bucket: bucket.getName(),
action,
inflight,
quota: bucketQuota,
bytesTotal: bucketMetrics.bytesTotal,
});
bucketQuotaExceeded = true;
}
return parallelDone();
});
}
return parallelDone();
},
accountQuota: parallelDone => {
if (accountQuota > 0 && account?.account) {
return QuotaService.getUtilizationMetrics('account',
account.account, null, {
action,
inflight,
}, (err, accountMetrics) => {
if (err || inflight < 0) {
return parallelDone(err);
}
if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
log.debug('Account quota exceeded', {
accountId: account.account,
action,
inflight,
quota: accountQuota,
bytesTotal: accountMetrics.bytesTotal,
});
accountQuotaExceeded = true;
}
return parallelDone();
});
}
return parallelDone();
},
}, err => {
if (err) {
log.warn('Error evaluating quotas', {
error: err.name,
description: err.message,
isInflightDeletion: inflight < 0,
});
}
return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
});
}
/**
* Monitors the duration of quota evaluation for a specific API method.
*
* @param {string} apiMethod - The name of the API method being monitored.
* @param {string} type - The type of quota being evaluated.
* @param {string} code - The code associated with the quota being evaluated.
* @param {number} duration - The duration of the quota evaluation in nanoseconds.
* @returns {undefined} - Returns nothing.
*/
function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
monitoring.quotaEvaluationDuration.labels({
action: apiMethod,
type,
code,
}).observe(duration / 1e9);
}
/**
*
* @param {Request} request - request object
* @param {BucketInfo} bucket - bucket object
* @param {Account} account - account object
* @param {array} apiNames - action names: operations to authorize
* @param {string} apiMethod - the main API call
* @param {number} inflight - inflight bytes
* @param {boolean} isStorageReserved - Flag to check if the current quota, minus
* the incoming bytes, are under the limit.
* @param {Logger} log - logger
* @param {function} callback - callback function
* @returns {boolean} - true if the quota is valid, false otherwise
*/
function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
return callback(null);
}
let type;
let bucketQuotaExceeded = false;
let accountQuotaExceeded = false;
let quotaEvaluationDuration;
const requestStartTime = process.hrtime.bigint();
const bucketQuota = bucket.getQuota();
const accountQuota = account?.quota || 0;
const shouldSendInflights = config.isQuotaInflightEnabled();
if (bucketQuota && accountQuota) {
type = 'bucket+account';
} else if (bucketQuota) {
type = 'bucket';
} else {
type = 'account';
}
if (actionWithDataDeletion[apiMethod]) {
type = 'delete';
}
if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
if (bucketQuota > 0 || accountQuota > 0) {
log.warn('quota is set for a bucket, but the quota service is disabled', {
bucketName: bucket.getName(),
});
monitoring.requestWithQuotaMetricsUnavailable.inc();
}
return callback(null);
}
if (isStorageReserved) {
// eslint-disable-next-line no-param-reassign
inflight = 0;
}
return async.forEach(apiNames, (apiName, done) => {
// Object copy operations first check the target object,
// meaning the source object, containing the current bytes,
// is checked second. This logic handles these APIs calls by
// ensuring the bytes are positives (i.e., not an object
// replacement).
if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
// eslint-disable-next-line no-param-reassign
inflight = Math.abs(inflight);
} else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
return done();
}
// When inflights are disabled, the sum of the current utilization metrics
// and the current bytes are compared with the quota. The current bytes
// are not sent to the utilization service. When inflights are enabled,
// the sum of the current utilization metrics only are compared with the
// quota. They include the current inflight bytes sent in the request.
let _inflights = shouldSendInflights ? inflight : undefined;
const inflightForCheck = shouldSendInflights ? 0 : inflight;
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
inflightForCheck, apiName, log,
(err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
if (err) {
return done(err);
}
bucketQuotaExceeded = _bucketQuotaExceeded;
accountQuotaExceeded = _accountQuotaExceeded;
// Inflights are inverted: in case of cleanup, we just re-issue
// the same API call.
if (_inflights) {
_inflights = -_inflights;
}
request.finalizerHooks.push((errorFromAPI, _done) => {
const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
const quotaCleanUpStartTime = process.hrtime.bigint();
// Quotas are cleaned only in case of error in the API
async.waterfall([
cb => {
if (errorFromAPI) {
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
null, apiName, log, cb);
}
return cb();
},
], () => {
monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
Number(process.hrtime.bigint() - quotaCleanUpStartTime));
return _done();
});
});
return done();
});
}, err => {
quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
if (err) {
log.warn('Error getting metrics from the quota service, allowing the request', {
error: err.name,
description: err.message,
});
}
if (!actionWithDataDeletion[apiMethod] &&
(bucketQuotaExceeded || accountQuotaExceeded)) {
return callback(errors.QuotaExceeded);
}
return callback();
});
}
module.exports = {
processBytesToWrite,
isMetricStale,
validateQuotas,
};

View File

@ -31,15 +31,15 @@ function bucketDeleteCors(authInfo, request, log, callback) {
'deleteBucketCors'); 'deleteBucketCors');
return callback(err); return callback(err);
} }
if (bucketShield(bucket, requestType)) { if (bucketShield(bucket, request.apiMethods || requestType)) {
monitoring.promMetrics('DELETE', bucketName, 400, monitoring.promMetrics('DELETE', bucketName, 400,
'deleteBucketCors'); 'deleteBucketCors');
return callback(errors.NoSuchBucket); return callback(errors.NoSuchBucket);
} }
log.trace('found bucket in metadata'); log.trace('found bucket in metadata');
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo, log, request,
authInfo, log, request, request.actionImplicitDenies)) { request.actionImplicitDenies)) {
log.debug('access denied for user on bucket', { log.debug('access denied for user on bucket', {
requestType, requestType,
method: 'bucketDeleteCors', method: 'bucketDeleteCors',

View File

@ -1,58 +0,0 @@
const { waterfall } = require('async');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
const metadata = require('../metadata/wrapper');
const { pushMetric } = require('../utapi/utilities');
const monitoring = require('../utilities/monitoringHandler');
const requestType = 'bucketDeleteQuota';
/**
* Bucket Update Quota - Update bucket quota
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
* @param {object} request - http request object
* @param {object} log - Werelogs logger
* @param {function} callback - callback to server
* @return {undefined}
*/
function bucketDeleteQuota(authInfo, request, log, callback) {
log.debug('processing request', { method: 'bucketDeleteQuota' });
const { bucketName } = request;
const metadataValParams = {
authInfo,
bucketName,
requestType: request.apiMethods || requestType,
request,
};
return waterfall([
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
(err, bucket) => next(err, bucket)),
(bucket, next) => {
bucket.setQuota(0);
metadata.updateBucket(bucket.getName(), bucket, log, err =>
next(err, bucket));
},
], (err, bucket) => {
const corsHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket);
if (err) {
log.debug('error processing request', {
error: err,
method: 'bucketDeleteQuota'
});
monitoring.promMetrics('DELETE', bucketName, err.code,
'bucketDeleteQuota');
return callback(err, err.code, corsHeaders);
}
monitoring.promMetrics(
'DELETE', bucketName, '204', 'bucketDeleteQuota');
pushMetric('bucketDeleteQuota', log, {
authInfo,
bucket: bucketName,
});
return callback(null, 204, corsHeaders);
});
}
module.exports = bucketDeleteQuota;

View File

@ -21,19 +21,15 @@ function bucketDeleteTagging(authInfo, request, log, callback) {
authInfo, authInfo,
bucketName, bucketName,
requestType: request.apiMethods || 'bucketDeleteTagging', requestType: request.apiMethods || 'bucketDeleteTagging',
request,
}; };
let bucket = null; let bucket = null;
return waterfall([ return waterfall([
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
(err, b) => { (err, b) => {
if (err) {
return next(err);
}
bucket = b; bucket = b;
bucket.setTags([]); bucket.setTags([]);
return next(); return next(err);
}), }),
next => metadata.updateBucket(bucket.getName(), bucket, log, next), next => metadata.updateBucket(bucket.getName(), bucket, log, next),
], err => { ], err => {

View File

@ -30,8 +30,8 @@ function bucketDeleteWebsite(authInfo, request, log, callback) {
} }
log.trace('found bucket in metadata'); log.trace('found bucket in metadata');
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo, log, request,
authInfo, log, request, request.actionImplicitDenies)) { request.actionImplicitDenies)) {
log.debug('access denied for user on bucket', { log.debug('access denied for user on bucket', {
requestType, requestType,
method: 'bucketDeleteWebsite', method: 'bucketDeleteWebsite',

View File

@ -39,8 +39,8 @@ function bucketGetCors(authInfo, request, log, callback) {
const corsHeaders = collectCorsHeaders(request.headers.origin, const corsHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket); request.method, bucket);
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo, log,
authInfo, log, request, request.actionImplicitDenies)) { request, request.actionImplicitDenies)) {
log.debug('access denied for user on bucket', { log.debug('access denied for user on bucket', {
requestType, requestType,
method: 'bucketGetCors', method: 'bucketGetCors',

View File

@ -41,8 +41,8 @@ function bucketGetLocation(authInfo, request, log, callback) {
const corsHeaders = collectCorsHeaders(request.headers.origin, const corsHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket); request.method, bucket);
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo, log, request,
authInfo, log, request, request.actionImplicitDenies)) { request.actionImplicitDenies)) {
log.debug('access denied for account on bucket', { log.debug('access denied for account on bucket', {
requestType, requestType,
method: 'bucketGetLocation', method: 'bucketGetLocation',

View File

@ -1,58 +0,0 @@
const { errors } = require('arsenal');
const { pushMetric } = require('../utapi/utilities');
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
/**
* bucketGetQuota - Get the bucket quota
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
* @param {object} request - http request object
* @param {object} log - Werelogs logger
* @param {function} callback - callback to server
* @return {undefined}
*/
function bucketGetQuota(authInfo, request, log, callback) {
log.debug('processing request', { method: 'bucketGetQuota' });
const { bucketName, headers, method } = request;
const metadataValParams = {
authInfo,
bucketName,
requestType: request.apiMethods || 'bucketGetQuota',
request,
};
const xml = [];
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
if (err) {
log.debug('error processing request', {
error: err,
method: 'bucketGetQuota',
});
return callback(err, null, corsHeaders);
}
xml.push(
'<?xml version="1.0" encoding="UTF-8"?>',
'<GetBucketQuota>',
'<Name>', bucket.getName(), '</Name>',
);
const bucketQuota = bucket.getQuota();
if (!bucketQuota) {
log.debug('bucket has no quota', {
method: 'bucketGetQuota',
});
return callback(errors.NoSuchQuota, null,
corsHeaders);
}
xml.push('<Quota>', bucketQuota, '</Quota>',
'</GetBucketQuota>');
pushMetric('getBucketQuota', log, {
authInfo,
bucket: bucketName,
});
return callback(null, xml.join(''), corsHeaders);
});
}
module.exports = bucketGetQuota;

View File

@ -37,7 +37,7 @@ const escapeForXml = s3middleware.escapeForXml;
function tagsToXml(tags) { function tagsToXml(tags) {
const xml = []; const xml = [];
xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Tagging> <TagSet>'); xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Tagging><TagSet>');
tags.forEach(tag => { tags.forEach(tag => {
xml.push('<Tag>'); xml.push('<Tag>');
@ -46,7 +46,7 @@ function tagsToXml(tags) {
xml.push('</Tag>'); xml.push('</Tag>');
}); });
xml.push('</TagSet> </Tagging>'); xml.push('</TagSet></Tagging>');
return xml.join(''); return xml.join('');
} }

View File

@ -39,8 +39,8 @@ function bucketGetWebsite(authInfo, request, log, callback) {
const corsHeaders = collectCorsHeaders(request.headers.origin, const corsHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket); request.method, bucket);
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo, log,
authInfo, log, request, request.actionImplicitDenies)) { request, request.actionImplicitDenies)) {
log.debug('access denied for user on bucket', { log.debug('access denied for user on bucket', {
requestType, requestType,
method: 'bucketGetWebsite', method: 'bucketGetWebsite',

View File

@ -45,8 +45,9 @@ function checkLocationConstraint(request, locationConstraint, log) {
} else if (parsedHost && restEndpoints[parsedHost]) { } else if (parsedHost && restEndpoints[parsedHost]) {
locationConstraintChecked = restEndpoints[parsedHost]; locationConstraintChecked = restEndpoints[parsedHost];
} else { } else {
locationConstraintChecked = Object.keys(locationConstrains)[0]; log.trace('no location constraint provided on bucket put;' +
log.trace('no location constraint provided on bucket put; setting '+locationConstraintChecked); 'setting us-east-1');
locationConstraintChecked = 'us-east-1';
} }
if (!locationConstraints[locationConstraintChecked]) { if (!locationConstraints[locationConstraintChecked]) {

View File

@ -70,8 +70,8 @@ function bucketPutCors(authInfo, request, log, callback) {
}); });
}, },
function validateBucketAuthorization(bucket, rules, corsHeaders, next) { function validateBucketAuthorization(bucket, rules, corsHeaders, next) {
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo,
authInfo, log, request, request.actionImplicitDenies)) { log, request, request.actionImplicitDenies)) {
log.debug('access denied for account on bucket', { log.debug('access denied for account on bucket', {
requestType, requestType,
}); });

View File

@ -39,7 +39,6 @@ function bucketPutTagging(authInfo, request, log, callback) {
authInfo, authInfo,
bucketName, bucketName,
requestType: request.apiMethods || 'bucketPutTagging', requestType: request.apiMethods || 'bucketPutTagging',
request,
}; };
let bucket = null; let bucket = null;
return waterfall([ return waterfall([

View File

@ -49,8 +49,8 @@ function bucketPutWebsite(authInfo, request, log, callback) {
}); });
}, },
function validateBucketAuthorization(bucket, config, next) { function validateBucketAuthorization(bucket, config, next) {
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo,
authInfo, log, request, request.actionImplicitDenies)) { log, request, request.actionImplicitDenies)) {
log.debug('access denied for user on bucket', { log.debug('access denied for user on bucket', {
requestType, requestType,
method: 'bucketPutWebsite', method: 'bucketPutWebsite',

View File

@ -1,85 +0,0 @@
const { waterfall } = require('async');
const { errors } = require('arsenal');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
const metadata = require('../metadata/wrapper');
const { pushMetric } = require('../utapi/utilities');
const monitoring = require('../utilities/monitoringHandler');
const { parseString } = require('xml2js');
function validateBucketQuotaProperty(requestBody, next) {
const quota = requestBody.quota;
const quotaValue = parseInt(quota, 10);
if (Number.isNaN(quotaValue)) {
return next(errors.InvalidArgument.customizeDescription('Quota Value should be a number'));
}
if (quotaValue <= 0) {
return next(errors.InvalidArgument.customizeDescription('Quota value must be a positive number'));
}
return next(null, quotaValue);
}
function parseRequestBody(requestBody, next) {
try {
const jsonData = JSON.parse(requestBody);
if (typeof jsonData !== 'object') {
throw new Error('Invalid JSON');
}
return next(null, jsonData);
} catch (jsonError) {
return parseString(requestBody, (xmlError, xmlData) => {
if (xmlError) {
return next(errors.InvalidArgument.customizeDescription('Request body must be a JSON object'));
}
return next(null, xmlData);
});
}
}
function bucketUpdateQuota(authInfo, request, log, callback) {
log.debug('processing request', { method: 'bucketUpdateQuota' });
const { bucketName } = request;
const metadataValParams = {
authInfo,
bucketName,
requestType: request.apiMethods || 'bucketUpdateQuota',
request,
};
let bucket = null;
return waterfall([
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
(err, b) => {
bucket = b;
return next(err, bucket);
}),
(bucket, next) => parseRequestBody(request.post, (err, requestBody) => next(err, bucket, requestBody)),
(bucket, requestBody, next) => validateBucketQuotaProperty(requestBody, (err, quotaValue) =>
next(err, bucket, quotaValue)),
(bucket, quotaValue, next) => {
bucket.setQuota(quotaValue);
return metadata.updateBucket(bucket.getName(), bucket, log, next);
},
], (err, bucket) => {
const corsHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket);
if (err) {
log.debug('error processing request', {
error: err,
method: 'bucketUpdateQuota'
});
monitoring.promMetrics('PUT', bucketName, err.code,
'updateBucketQuota');
return callback(err, err.code, corsHeaders);
}
monitoring.promMetrics(
'PUT', bucketName, '200', 'updateBucketQuota');
pushMetric('updateBucketQuota', log, {
authInfo,
bucket: bucketName,
});
return callback(null, corsHeaders);
});
}
module.exports = bucketUpdateQuota;

View File

@ -21,6 +21,8 @@ const { validateAndFilterMpuParts, generateMpuPartStorageInfo } =
const locationKeysHaveChanged const locationKeysHaveChanged
= require('./apiUtils/object/locationKeysHaveChanged'); = require('./apiUtils/object/locationKeysHaveChanged');
const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
const logger = require('../utilities/logger');
const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage');
const versionIdUtils = versioning.VersionID; const versionIdUtils = versioning.VersionID;
@ -80,7 +82,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
uploadId, uploadId,
// Note: permissions for completing a multipart upload are the // Note: permissions for completing a multipart upload are the
// same as putting a part. // same as putting a part.
requestType: request.apiMethods || 'putPart or complete', requestType: 'putPart or complete',
log, log,
request, request,
}; };
@ -474,9 +476,12 @@ function completeMultipartUpload(authInfo, request, log, callback) {
const newDataStoreName = const newDataStoreName =
Array.isArray(dataLocations) && dataLocations[0] ? Array.isArray(dataLocations) && dataLocations[0] ?
dataLocations[0].dataStoreName : null; dataLocations[0].dataStoreName : null;
const delLog =
logger.newRequestLoggerFromSerializedUids(log
.getSerializedUids());
return data.batchDelete(dataToDelete, return data.batchDelete(dataToDelete,
request.method, request.method,
newDataStoreName, log, err => { newDataStoreName, delLog, err => {
if (err) { if (err) {
return next(err); return next(err);
} }
@ -499,8 +504,10 @@ function completeMultipartUpload(authInfo, request, log, callback) {
function batchDeleteExtraParts(extraPartLocations, destinationBucket, function batchDeleteExtraParts(extraPartLocations, destinationBucket,
aggregateETag, generatedVersionId, next) { aggregateETag, generatedVersionId, next) {
if (extraPartLocations && extraPartLocations.length > 0) { if (extraPartLocations && extraPartLocations.length > 0) {
const delLog = logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids());
return data.batchDelete(extraPartLocations, request.method, return data.batchDelete(extraPartLocations, request.method,
null, log, err => { null, delLog, err => {
if (err) { if (err) {
return next(err); return next(err);
} }

View File

@ -6,7 +6,6 @@ const convertToXml = s3middleware.convertToXml;
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const { hasNonPrintables } = require('../utilities/stringChecks'); const { hasNonPrintables } = require('../utilities/stringChecks');
const { config } = require('../Config');
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation'); const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
const constants = require('../../constants'); const constants = require('../../constants');
const services = require('../services'); const services = require('../services');
@ -66,7 +65,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
const websiteRedirectHeader = const websiteRedirectHeader =
request.headers['x-amz-website-redirect-location']; request.headers['x-amz-website-redirect-location'];
if (request.headers['x-amz-storage-class'] && if (request.headers['x-amz-storage-class'] &&
!config.locationConstraints[request.headers['x-amz-storage-class']]) { !constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) {
log.trace('invalid storage-class header'); log.trace('invalid storage-class header');
monitoring.promMetrics('PUT', bucketName, monitoring.promMetrics('PUT', bucketName,
errors.InvalidStorageClass.code, 'initiateMultipartUpload'); errors.InvalidStorageClass.code, 'initiateMultipartUpload');

View File

@ -31,7 +31,6 @@ const { overheadField } = require('../../constants');
const versionIdUtils = versioning.VersionID; const versionIdUtils = versioning.VersionID;
const { data } = require('../data/wrapper'); const { data } = require('../data/wrapper');
const logger = require('../utilities/logger'); const logger = require('../utilities/logger');
const { validateQuotas } = require('./apiUtils/quotas/quotaUtils');
/* /*
Format of xml request: Format of xml request:
@ -333,9 +332,6 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
return callback(null, objMD, versionId); return callback(null, objMD, versionId);
}, },
(objMD, versionId, callback) => validateQuotas(
request, bucket, request.accountQuotas, ['objectDelete'], 'objectDelete',
-objMD?.['content-length'] || 0, false, log, err => callback(err, objMD, versionId)),
(objMD, versionId, callback) => { (objMD, versionId, callback) => {
const options = preprocessingVersioningDelete( const options = preprocessingVersioningDelete(
bucketName, bucket, objMD, versionId, config.nullVersionCompatMode); bucketName, bucket, objMD, versionId, config.nullVersionCompatMode);
@ -508,9 +504,8 @@ function multiObjectDelete(authInfo, request, log, callback) {
if (bucketShield(bucketMD, 'objectDelete')) { if (bucketShield(bucketMD, 'objectDelete')) {
return next(errors.NoSuchBucket); return next(errors.NoSuchBucket);
} }
// The implicit deny flag is ignored in the DeleteObjects API, as authorization only if (!isBucketAuthorized(bucketMD, 'objectDelete', canonicalID, authInfo, log, request,
// affects the objects. request.actionImplicitDenies)) {
if (!isBucketAuthorized(bucketMD, 'objectDelete', canonicalID, authInfo, log, request)) {
log.trace("access denied due to bucket acl's"); log.trace("access denied due to bucket acl's");
// if access denied at the bucket level, no access for // if access denied at the bucket level, no access for
// any of the objects so all results will be error results // any of the objects so all results will be error results

View File

@ -12,6 +12,7 @@ const { checkQueryVersionId, versioningPreprocessing }
= require('./apiUtils/object/versioning'); = require('./apiUtils/object/versioning');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
const { data } = require('../data/wrapper'); const { data } = require('../data/wrapper');
const logger = require('../utilities/logger');
const services = require('../services'); const services = require('../services');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const removeAWSChunked = require('./apiUtils/object/removeAWSChunked'); const removeAWSChunked = require('./apiUtils/object/removeAWSChunked');
@ -220,14 +221,6 @@ function objectCopy(authInfo, request, sourceBucket,
versionId: sourceVersionId, versionId: sourceVersionId,
getDeleteMarker: true, getDeleteMarker: true,
requestType: 'objectGet', requestType: 'objectGet',
/**
* Authorization will first check the target object, with an objectPut
* action. But in this context, the source object metadata is still
* unknown. In the context of quotas, to know the number of bytes that
* are being written, we explicitly enable the quota evaluation logic
* during the objectGet action instead.
*/
checkQuota: true,
request, request,
}; };
const valPutParams = { const valPutParams = {
@ -235,7 +228,6 @@ function objectCopy(authInfo, request, sourceBucket,
bucketName: destBucketName, bucketName: destBucketName,
objectKey: destObjectKey, objectKey: destObjectKey,
requestType: 'objectPut', requestType: 'objectPut',
checkQuota: false,
request, request,
}; };
const dataStoreContext = { const dataStoreContext = {
@ -249,7 +241,7 @@ function objectCopy(authInfo, request, sourceBucket,
const responseHeaders = {}; const responseHeaders = {};
if (request.headers['x-amz-storage-class'] && if (request.headers['x-amz-storage-class'] &&
!config.locationConstraints[request.headers['x-amz-storage-class']]) { !constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) {
log.trace('invalid storage-class header'); log.trace('invalid storage-class header');
monitoring.promMetrics('PUT', destBucketName, monitoring.promMetrics('PUT', destBucketName,
errors.InvalidStorageClass.code, 'copyObject'); errors.InvalidStorageClass.code, 'copyObject');
@ -287,10 +279,7 @@ function objectCopy(authInfo, request, sourceBucket,
}); });
}, },
function checkSourceAuthorization(destBucketMD, destObjMD, next) { function checkSourceAuthorization(destBucketMD, destObjMD, next) {
return standardMetadataValidateBucketAndObj({ return standardMetadataValidateBucketAndObj(valGetParams, request.actionImplicitDenies, log,
...valGetParams,
destObjMD,
}, request.actionImplicitDenies, log,
(err, sourceBucketMD, sourceObjMD) => { (err, sourceBucketMD, sourceObjMD) => {
if (err) { if (err) {
log.debug('error validating get part of request', log.debug('error validating get part of request',
@ -544,8 +533,10 @@ function objectCopy(authInfo, request, sourceBucket,
// the same as the destination // the same as the destination
if (!sourceIsDestination && dataToDelete) { if (!sourceIsDestination && dataToDelete) {
const newDataStoreName = storeMetadataParams.dataStoreName; const newDataStoreName = storeMetadataParams.dataStoreName;
const delLog = logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids());
return data.batchDelete(dataToDelete, request.method, return data.batchDelete(dataToDelete, request.method,
newDataStoreName, log, err => { newDataStoreName, delLog, err => {
if (err) { if (err) {
// if error, log the error and move on as it is not // if error, log the error and move on as it is not
// relevant to the client as the client's // relevant to the client as the client's

View File

@ -3,7 +3,6 @@ const { errors, versioning } = require('arsenal');
const constants = require('../../constants'); const constants = require('../../constants');
const aclUtils = require('../utilities/aclUtils'); const aclUtils = require('../utilities/aclUtils');
const { config } = require('../Config');
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation'); const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption'); const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const collectCorsHeaders = require('../utilities/collectCorsHeaders');
@ -72,7 +71,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
query, query,
} = request; } = request;
if (headers['x-amz-storage-class'] && if (headers['x-amz-storage-class'] &&
!config.locationConstraints[headers['x-amz-storage-class']]) { !constants.validStorageClasses.includes(headers['x-amz-storage-class'])) {
log.trace('invalid storage-class header'); log.trace('invalid storage-class header');
monitoring.promMetrics('PUT', request.bucketName, monitoring.promMetrics('PUT', request.bucketName,
errors.InvalidStorageClass.code, 'putObject'); errors.InvalidStorageClass.code, 'putObject');
@ -99,7 +98,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
'The encryption method specified is not supported'); 'The encryption method specified is not supported');
const requestType = request.apiMethods || 'objectPut'; const requestType = request.apiMethods || 'objectPut';
const valParams = { authInfo, bucketName, objectKey, versionId, const valParams = { authInfo, bucketName, objectKey, versionId,
requestType, request, withVersionId: isPutVersion }; requestType, request };
const canonicalID = authInfo.getCanonicalID(); const canonicalID = authInfo.getCanonicalID();
if (hasNonPrintables(objectKey)) { if (hasNonPrintables(objectKey)) {

View File

@ -9,12 +9,12 @@ const locationConstraintCheck =
require('./apiUtils/object/locationConstraintCheck'); require('./apiUtils/object/locationConstraintCheck');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const logger = require('../utilities/logger');
const services = require('../services'); const services = require('../services');
const setUpCopyLocator = require('./apiUtils/object/setUpCopyLocator'); const setUpCopyLocator = require('./apiUtils/object/setUpCopyLocator');
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const monitoring = require('../utilities/monitoringHandler'); const monitoring = require('../utilities/monitoringHandler');
const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage'); const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage');
const { validateQuotas } = require('./apiUtils/quotas/quotaUtils');
const versionIdUtils = versioning.VersionID; const versionIdUtils = versioning.VersionID;
@ -46,14 +46,6 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
versionId: reqVersionId, versionId: reqVersionId,
getDeleteMarker: true, getDeleteMarker: true,
requestType: 'objectGet', requestType: 'objectGet',
/**
* Authorization will first check the target object, with an objectPut
* action. But in this context, the source object metadata is still
* unknown. In the context of quotas, to know the number of bytes that
* are being written, we explicitly enable the quota evaluation logic
* during the objectGet action instead.
*/
checkQuota: true,
request, request,
}; };
@ -77,7 +69,6 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
bucketName: destBucketName, bucketName: destBucketName,
objectKey: destObjectKey, objectKey: destObjectKey,
requestType: 'objectPutPart', requestType: 'objectPutPart',
checkQuota: false,
request, request,
}; };
@ -98,7 +89,6 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
objectKey: destObjectKey, objectKey: destObjectKey,
partNumber: paddedPartNumber, partNumber: paddedPartNumber,
uploadId, uploadId,
enableQuota: true,
}; };
return async.waterfall([ return async.waterfall([
@ -192,16 +182,9 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
} }
return next(null, copyLocator.dataLocator, destBucketMD, return next(null, copyLocator.dataLocator, destBucketMD,
copyLocator.copyObjectSize, sourceVerId, copyLocator.copyObjectSize, sourceVerId,
sourceLocationConstraintName, sourceObjMD); sourceLocationConstraintName);
}); });
}, },
function _validateQuotas(dataLocator, destBucketMD,
copyObjectSize, sourceVerId,
sourceLocationConstraintName, sourceObjMD, next) {
return validateQuotas(request, destBucketMD, request.accountQuotas, valPutParams.requestType,
request.apiMethod, sourceObjMD?.['content-length'] || 0, false, log, err =>
next(err, dataLocator, destBucketMD, copyObjectSize, sourceVerId, sourceLocationConstraintName));
},
// get MPU shadow bucket to get splitter based on MD version // get MPU shadow bucket to get splitter based on MD version
function getMpuShadowBucket(dataLocator, destBucketMD, function getMpuShadowBucket(dataLocator, destBucketMD,
copyObjectSize, sourceVerId, copyObjectSize, sourceVerId,
@ -399,8 +382,10 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
// Clean up the old data now that new metadata (with new // Clean up the old data now that new metadata (with new
// data locations) has been stored // data locations) has been stored
if (oldLocationsToDelete) { if (oldLocationsToDelete) {
const delLog = logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids());
return data.batchDelete(oldLocationsToDelete, request.method, null, return data.batchDelete(oldLocationsToDelete, request.method, null,
log, err => { delLog, err => {
if (err) { if (err) {
// if error, log the error and move on as it is not // if error, log the error and move on as it is not
// relevant to the client as the client's // relevant to the client as the client's

View File

@ -11,6 +11,7 @@ const { isBucketAuthorized } =
const kms = require('../kms/wrapper'); const kms = require('../kms/wrapper');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const { pushMetric } = require('../utapi/utilities'); const { pushMetric } = require('../utapi/utilities');
const logger = require('../utilities/logger');
const services = require('../services'); const services = require('../services');
const locationConstraintCheck const locationConstraintCheck
= require('./apiUtils/object/locationConstraintCheck'); = require('./apiUtils/object/locationConstraintCheck');
@ -21,7 +22,6 @@ const { BackendInfo } = models;
const writeContinue = require('../utilities/writeContinue'); const writeContinue = require('../utilities/writeContinue');
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption'); const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders'); const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders');
const { validateQuotas } = require('./apiUtils/quotas/quotaUtils');
const skipError = new Error('skip'); const skipError = new Error('skip');
@ -61,9 +61,6 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
log.debug('processing request', { method: 'objectPutPart' }); log.debug('processing request', { method: 'objectPutPart' });
const size = request.parsedContentLength; const size = request.parsedContentLength;
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) { if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) {
log.debug('put part size too large', { size }); log.debug('put part size too large', { size });
monitoring.promMetrics('PUT', request.bucketName, 400, monitoring.promMetrics('PUT', request.bucketName, 400,
@ -107,9 +104,6 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`; const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
const { objectKey } = request; const { objectKey } = request;
const originalIdentityAuthzResults = request.actionImplicitDenies; const originalIdentityAuthzResults = request.actionImplicitDenies;
// For validating the request at the destinationBucket level the
// `requestType` is the general 'objectPut'.
const requestType = request.apiMethods || 'objectPutPart';
return async.waterfall([ return async.waterfall([
// Get the destination bucket. // Get the destination bucket.
@ -129,6 +123,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
}), }),
// Check the bucket authorization. // Check the bucket authorization.
(destinationBucket, next) => { (destinationBucket, next) => {
// For validating the request at the destinationBucket level the
// `requestType` is the general 'objectPut'.
const requestType = request.apiMethods || 'objectPutPart';
if (!isBucketAuthorized(destinationBucket, requestType, canonicalID, authInfo, if (!isBucketAuthorized(destinationBucket, requestType, canonicalID, authInfo,
log, request, request.actionImplicitDenies)) { log, request, request.actionImplicitDenies)) {
log.debug('access denied for user on bucket', { requestType }); log.debug('access denied for user on bucket', { requestType });
@ -136,8 +133,6 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
} }
return next(null, destinationBucket); return next(null, destinationBucket);
}, },
(destinationBucket, next) => validateQuotas(request, destinationBucket, request.accountQuotas,
requestType, request.apiMethod, size, isPutVersion, log, err => next(err, destinationBucket)),
// Get bucket server-side encryption, if it exists. // Get bucket server-side encryption, if it exists.
(destinationBucket, next) => getObjectSSEConfiguration( (destinationBucket, next) => getObjectSSEConfiguration(
request.headers, destinationBucket, log, request.headers, destinationBucket, log,
@ -385,8 +380,10 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
prevObjectSize, next) => { prevObjectSize, next) => {
if (oldLocationsToDelete) { if (oldLocationsToDelete) {
log.trace('overwriting mpu part, deleting data'); log.trace('overwriting mpu part, deleting data');
const delLog = logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids());
return data.batchDelete(oldLocationsToDelete, request.method, return data.batchDelete(oldLocationsToDelete, request.method,
objectLocationConstraint, log, err => { objectLocationConstraint, delLog, err => {
if (err) { if (err) {
// if error, log the error and move on as it is not // if error, log the error and move on as it is not
// relevant to the client as the client's // relevant to the client as the client's

View File

@ -1,3 +1,4 @@
const vaultclient = require('vaultclient');
const { auth } = require('arsenal'); const { auth } = require('arsenal');
const { config } = require('../Config'); const { config } = require('../Config');
@ -20,7 +21,6 @@ function getVaultClient(config) {
port, port,
https: true, https: true,
}); });
const vaultclient = require('vaultclient');
vaultClient = new vaultclient.Client(host, port, true, key, cert, ca); vaultClient = new vaultclient.Client(host, port, true, key, cert, ca);
} else { } else {
logger.info('vaultclient configuration', { logger.info('vaultclient configuration', {
@ -28,7 +28,6 @@ function getVaultClient(config) {
port, port,
https: false, https: false,
}); });
const vaultclient = require('vaultclient');
vaultClient = new vaultclient.Client(host, port); vaultClient = new vaultclient.Client(host, port);
} }
@ -50,6 +49,10 @@ function getMemBackend(config) {
} }
switch (config.backends.auth) { switch (config.backends.auth) {
case 'mem':
implName = 'vaultMem';
client = getMemBackend(config);
break;
case 'multiple': case 'multiple':
implName = 'vaultChain'; implName = 'vaultChain';
client = new ChainBackend('s3', [ client = new ChainBackend('s3', [
@ -57,14 +60,9 @@ case 'multiple':
getVaultClient(config), getVaultClient(config),
]); ]);
break; break;
case 'vault': default: // vault
implName = 'vault'; implName = 'vault';
client = getVaultClient(config); client = getVaultClient(config);
break;
default: // mem
implName = 'vaultMem';
client = getMemBackend(config);
break;
} }
module.exports = new Vault(client, implName); module.exports = new Vault(client, implName);

View File

@ -8,6 +8,20 @@ const inMemory = require('./in_memory/backend').backend;
const file = require('./file/backend'); const file = require('./file/backend');
const KMIPClient = require('arsenal').network.kmipClient; const KMIPClient = require('arsenal').network.kmipClient;
const Common = require('./common'); const Common = require('./common');
let scalityKMS;
let scalityKMSImpl;
try {
// eslint-disable-next-line import/no-unresolved
const ScalityKMS = require('scality-kms');
scalityKMS = new ScalityKMS(config.kms);
scalityKMSImpl = 'scalityKms';
} catch (error) {
logger.warn('scality kms unavailable. ' +
'Using file kms backend unless mem specified.',
{ error });
scalityKMS = file;
scalityKMSImpl = 'fileKms';
}
let client; let client;
let implName; let implName;
@ -19,9 +33,8 @@ if (config.backends.kms === 'mem') {
client = file; client = file;
implName = 'fileKms'; implName = 'fileKms';
} else if (config.backends.kms === 'scality') { } else if (config.backends.kms === 'scality') {
const ScalityKMS = require('scality-kms'); client = scalityKMS;
client = new ScalityKMS(config.kms); implName = scalityKMSImpl;
implName = 'scalityKms';
} else if (config.backends.kms === 'kmip') { } else if (config.backends.kms === 'kmip') {
const kmipConfig = { kmip: config.kmip }; const kmipConfig = { kmip: config.kmip };
if (!kmipConfig.kmip) { if (!kmipConfig.kmip) {

View File

@ -0,0 +1,131 @@
/**
* Target service that should handle a message
* @readonly
* @enum {number}
*/
const MessageType = {
/** Message that contains a configuration overlay */
CONFIG_OVERLAY_MESSAGE: 1,
/** Message that requests a metrics report */
METRICS_REQUEST_MESSAGE: 2,
/** Message that contains a metrics report */
METRICS_REPORT_MESSAGE: 3,
/** Close the virtual TCP socket associated to the channel */
CHANNEL_CLOSE_MESSAGE: 4,
/** Write data to the virtual TCP socket associated to the channel */
CHANNEL_PAYLOAD_MESSAGE: 5,
};
/**
* Target service that should handle a message
* @readonly
* @enum {number}
*/
const TargetType = {
/** Let the dispatcher choose the most appropriate message */
TARGET_ANY: 0,
};
const headerSize = 3;
class ChannelMessageV0 {
/**
* @param {Buffer} buffer Message bytes
*/
constructor(buffer) {
this.messageType = buffer.readUInt8(0);
this.channelNumber = buffer.readUInt8(1);
this.target = buffer.readUInt8(2);
this.payload = buffer.slice(headerSize);
}
/**
* @returns {number} Message type
*/
getType() {
return this.messageType;
}
/**
* @returns {number} Channel number if applicable
*/
getChannelNumber() {
return this.channelNumber;
}
/**
* @returns {number} Target service, or 0 to choose automatically
*/
getTarget() {
return this.target;
}
/**
* @returns {Buffer} Message payload if applicable
*/
getPayload() {
return this.payload;
}
/**
* Creates a wire representation of a channel close message
*
* @param {number} channelId Channel number
*
* @returns {Buffer} wire representation
*/
static encodeChannelCloseMessage(channelId) {
const buf = Buffer.alloc(headerSize);
buf.writeUInt8(MessageType.CHANNEL_CLOSE_MESSAGE, 0);
buf.writeUInt8(channelId, 1);
buf.writeUInt8(TargetType.TARGET_ANY, 2);
return buf;
}
/**
* Creates a wire representation of a channel data message
*
* @param {number} channelId Channel number
* @param {Buffer} data Payload
*
* @returns {Buffer} wire representation
*/
static encodeChannelDataMessage(channelId, data) {
const buf = Buffer.alloc(data.length + headerSize);
buf.writeUInt8(MessageType.CHANNEL_PAYLOAD_MESSAGE, 0);
buf.writeUInt8(channelId, 1);
buf.writeUInt8(TargetType.TARGET_ANY, 2);
data.copy(buf, headerSize);
return buf;
}
/**
* Creates a wire representation of a metrics message
*
* @param {object} body Metrics report
*
* @returns {Buffer} wire representation
*/
static encodeMetricsReportMessage(body) {
const report = JSON.stringify(body);
const buf = Buffer.alloc(report.length + headerSize);
buf.writeUInt8(MessageType.METRICS_REPORT_MESSAGE, 0);
buf.writeUInt8(0, 1);
buf.writeUInt8(TargetType.TARGET_ANY, 2);
buf.write(report, headerSize);
return buf;
}
/**
* Protocol name used for subprotocol negociation
*/
static get protocolName() {
return 'zenko-secure-channel-v0';
}
}
module.exports = {
ChannelMessageV0,
MessageType,
TargetType,
};

View File

@ -0,0 +1,94 @@
const WebSocket = require('ws');
const arsenal = require('arsenal');
const logger = require('../utilities/logger');
const _config = require('../Config').config;
const { patchConfiguration } = require('./configuration');
const { reshapeExceptionError } = arsenal.errorUtils;
const managementAgentMessageType = {
/** Message that contains the loaded overlay */
NEW_OVERLAY: 1,
};
const CONNECTION_RETRY_TIMEOUT_MS = 5000;
function initManagementClient() {
const { host, port } = _config.managementAgent;
const ws = new WebSocket(`ws://${host}:${port}/watch`);
ws.on('open', () => {
logger.info('connected with management agent');
});
ws.on('close', (code, reason) => {
logger.info('disconnected from management agent', { reason });
setTimeout(initManagementClient, CONNECTION_RETRY_TIMEOUT_MS);
});
ws.on('error', error => {
logger.error('error on connection with management agent', { error });
});
ws.on('message', data => {
const method = 'initManagementclient::onMessage';
const log = logger.newRequestLogger();
let msg;
if (!data) {
log.error('message without data', { method });
return;
}
try {
msg = JSON.parse(data);
} catch (err) {
log.error('data is an invalid json', { method, err, data });
return;
}
if (msg.payload === undefined) {
log.error('message without payload', { method });
return;
}
if (typeof msg.messageType !== 'number') {
log.error('messageType is not an integer', {
type: typeof msg.messageType,
method,
});
return;
}
switch (msg.messageType) {
case managementAgentMessageType.NEW_OVERLAY:
patchConfiguration(msg.payload, log, err => {
if (err) {
log.error('failed to patch overlay', {
error: reshapeExceptionError(err),
method,
});
}
});
return;
default:
log.error('new overlay message with unmanaged message type', {
method,
type: msg.messageType,
});
return;
}
});
}
function isManagementAgentUsed() {
return process.env.MANAGEMENT_USE_AGENT === '1';
}
module.exports = {
managementAgentMessageType,
initManagementClient,
isManagementAgentUsed,
};

View File

@ -0,0 +1,240 @@
const arsenal = require('arsenal');
const { buildAuthDataAccount } = require('../auth/in_memory/builder');
const _config = require('../Config').config;
const metadata = require('../metadata/wrapper');
const { getStoredCredentials } = require('./credentials');
const latestOverlayVersionKey = 'configuration/overlay-version';
const managementDatabaseName = 'PENSIEVE';
const replicatorEndpoint = 'zenko-cloudserver-replicator';
const { decryptSecret } = arsenal.pensieve.credentialUtils;
const { patchLocations } = arsenal.patches.locationConstraints;
const { reshapeExceptionError } = arsenal.errorUtils;
const { replicationBackends } = require('arsenal').constants;
function overlayHasVersion(overlay) {
return overlay && overlay.version !== undefined;
}
function remoteOverlayIsNewer(cachedOverlay, remoteOverlay) {
return (overlayHasVersion(remoteOverlay) &&
(!overlayHasVersion(cachedOverlay) ||
remoteOverlay.version > cachedOverlay.version));
}
/**
* Updates the live {Config} object with the new overlay configuration.
*
* No-op if this version was already applied to the live {Config}.
*
* @param {object} newConf Overlay configuration to apply
* @param {werelogs~Logger} log Request-scoped logger
* @param {function} cb Function to call with (error, newConf)
*
* @returns {undefined}
*/
function patchConfiguration(newConf, log, cb) {
if (newConf.version === undefined) {
log.debug('no remote configuration created yet');
return process.nextTick(cb, null, newConf);
}
if (_config.overlayVersion !== undefined &&
newConf.version <= _config.overlayVersion) {
log.debug('configuration version already applied',
{ configurationVersion: newConf.version });
return process.nextTick(cb, null, newConf);
}
return getStoredCredentials(log, (err, creds) => {
if (err) {
return cb(err);
}
const accounts = [];
if (newConf.users) {
newConf.users.forEach(u => {
if (u.secretKey && u.secretKey.length > 0) {
const secretKey = decryptSecret(creds, u.secretKey);
// accountType will be service-replication or service-clueso
let serviceName;
if (u.accountType && u.accountType.startsWith('service-')) {
serviceName = u.accountType.split('-')[1];
}
const newAccount = buildAuthDataAccount(
u.accessKey, secretKey, u.canonicalId, serviceName,
u.userName);
accounts.push(newAccount.accounts[0]);
}
});
}
const restEndpoints = Object.assign({}, _config.restEndpoints);
if (newConf.endpoints) {
newConf.endpoints.forEach(e => {
restEndpoints[e.hostname] = e.locationName;
});
}
if (!restEndpoints[replicatorEndpoint]) {
restEndpoints[replicatorEndpoint] = 'us-east-1';
}
const locations = patchLocations(newConf.locations, creds, log);
if (Object.keys(locations).length !== 0) {
try {
_config.setLocationConstraints(locations);
} catch (error) {
const exceptionError = reshapeExceptionError(error);
log.error('could not apply configuration version location ' +
'constraints', { error: exceptionError,
method: 'getStoredCredentials' });
return cb(exceptionError);
}
try {
const locationsWithReplicationBackend = Object.keys(locations)
// NOTE: In Orbit, we don't need to have Scality location in our
// replication endpoind config, since we do not replicate to
// any Scality Instance yet.
.filter(key => replicationBackends
[locations[key].type])
.reduce((obj, key) => {
/* eslint no-param-reassign:0 */
obj[key] = locations[key];
return obj;
}, {});
_config.setReplicationEndpoints(
locationsWithReplicationBackend);
} catch (error) {
const exceptionError = reshapeExceptionError(error);
log.error('could not apply replication endpoints',
{ error: exceptionError, method: 'getStoredCredentials' });
return cb(exceptionError);
}
}
_config.setAuthDataAccounts(accounts);
_config.setRestEndpoints(restEndpoints);
_config.setPublicInstanceId(newConf.instanceId);
if (newConf.browserAccess) {
if (Boolean(_config.browserAccessEnabled) !==
Boolean(newConf.browserAccess.enabled)) {
_config.browserAccessEnabled =
Boolean(newConf.browserAccess.enabled);
_config.emit('browser-access-enabled-change');
}
}
_config.overlayVersion = newConf.version;
log.info('applied configuration version',
{ configurationVersion: _config.overlayVersion });
return cb(null, newConf);
});
}
/**
* Writes configuration version to the management database
*
* @param {object} cachedOverlay Latest stored configuration version
* for freshness comparison purposes
* @param {object} remoteOverlay New configuration version
* @param {werelogs~Logger} log Request-scoped logger
* @param {function} cb Function to call with (error, remoteOverlay)
*
* @returns {undefined}
*/
function saveConfigurationVersion(cachedOverlay, remoteOverlay, log, cb) {
if (remoteOverlayIsNewer(cachedOverlay, remoteOverlay)) {
const objName = `configuration/overlay/${remoteOverlay.version}`;
metadata.putObjectMD(managementDatabaseName, objName, remoteOverlay,
{}, log, error => {
if (error) {
const exceptionError = reshapeExceptionError(error);
log.error('could not save configuration',
{ error: exceptionError,
method: 'saveConfigurationVersion',
configurationVersion: remoteOverlay.version });
cb(exceptionError);
return;
}
metadata.putObjectMD(managementDatabaseName,
latestOverlayVersionKey, remoteOverlay.version, {}, log,
error => {
if (error) {
log.error('could not save configuration version', {
configurationVersion: remoteOverlay.version,
});
}
cb(error, remoteOverlay);
});
});
} else {
log.debug('no remote configuration to cache yet');
process.nextTick(cb, null, remoteOverlay);
}
}
/**
* Loads the latest cached configuration overlay from the management
* database, without contacting the Orbit API.
*
* @param {werelogs~Logger} log Request-scoped logger
* @param {function} callback Function called with (error, cachedOverlay)
*
* @returns {undefined}
*/
function loadCachedOverlay(log, callback) {
return metadata.getObjectMD(managementDatabaseName,
latestOverlayVersionKey, {}, log, (err, version) => {
if (err) {
if (err.is.NoSuchKey) {
return process.nextTick(callback, null, {});
}
return callback(err);
}
return metadata.getObjectMD(managementDatabaseName,
`configuration/overlay/${version}`, {}, log, (err, conf) => {
if (err) {
if (err.is.NoSuchKey) {
return process.nextTick(callback, null, {});
}
return callback(err);
}
return callback(null, conf);
});
});
}
function applyAndSaveOverlay(overlay, log) {
patchConfiguration(overlay, log, err => {
if (err) {
log.error('could not apply pushed overlay', {
error: reshapeExceptionError(err),
method: 'applyAndSaveOverlay',
});
return;
}
saveConfigurationVersion(null, overlay, log, err => {
if (err) {
log.error('could not cache overlay version', {
error: reshapeExceptionError(err),
method: 'applyAndSaveOverlay',
});
return;
}
log.info('overlay push processed');
});
});
}
module.exports = {
loadCachedOverlay,
managementDatabaseName,
patchConfiguration,
saveConfigurationVersion,
remoteOverlayIsNewer,
applyAndSaveOverlay,
};

View File

@ -0,0 +1,145 @@
const arsenal = require('arsenal');
const forge = require('node-forge');
const request = require('../utilities/request');
const metadata = require('../metadata/wrapper');
const managementDatabaseName = 'PENSIEVE';
const tokenConfigurationKey = 'auth/zenko/remote-management-token';
const tokenRotationDelay = 3600 * 24 * 7 * 1000; // 7 days
const { reshapeExceptionError } = arsenal.errorUtils;
/**
* Retrieves Orbit API token from the management database.
*
* The token is used to authenticate stat posting and
*
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
* initialization process
* @param {function} callback Function called with (error, result)
*
* @returns {undefined}
*/
function getStoredCredentials(log, callback) {
metadata.getObjectMD(managementDatabaseName, tokenConfigurationKey, {},
log, callback);
}
function issueCredentials(managementEndpoint, instanceId, log, callback) {
log.info('registering with API to get token');
const keyPair = forge.pki.rsa.generateKeyPair({ bits: 2048, e: 0x10001 });
const privateKey = forge.pki.privateKeyToPem(keyPair.privateKey);
const publicKey = forge.pki.publicKeyToPem(keyPair.publicKey);
const postData = {
publicKey,
};
request.post(`${managementEndpoint}/${instanceId}/register`,
{ body: postData, json: true }, (error, response, body) => {
if (error) {
return callback(error);
}
if (response.statusCode !== 201) {
log.error('could not register instance', {
statusCode: response.statusCode,
});
return callback(arsenal.errors.InternalError);
}
/* eslint-disable no-param-reassign */
body.privateKey = privateKey;
/* eslint-enable no-param-reassign */
return callback(null, body);
});
}
function confirmInstanceCredentials(
managementEndpoint, instanceId, creds, log, callback) {
const postData = {
serial: creds.serial || 0,
publicKey: creds.publicKey,
};
const opts = {
headers: {
'x-instance-authentication-token': creds.token,
},
body: postData,
};
request.post(`${managementEndpoint}/${instanceId}/confirm`,
opts, (error, response) => {
if (error) {
return callback(error);
}
if (response.statusCode === 200) {
return callback(null, instanceId, creds.token);
}
return callback(arsenal.errors.InternalError);
});
}
/**
* Initializes credentials and PKI in the management database.
*
* In case the management database is new and empty, the instance
* is registered as new against the Orbit API with newly-generated
* RSA key pair.
*
* @param {string} managementEndpoint API endpoint
* @param {string} instanceId UUID of this deployment
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
* initialization process
* @param {function} callback Function called with (error, result)
*
* @returns {undefined}
*/
function initManagementCredentials(
managementEndpoint, instanceId, log, callback) {
getStoredCredentials(log, (error, value) => {
if (error) {
if (error.is.NoSuchKey) {
return issueCredentials(managementEndpoint, instanceId, log,
(error, value) => {
if (error) {
log.error('could not issue token',
{ error: reshapeExceptionError(error),
method: 'initManagementCredentials' });
return callback(error);
}
log.debug('saving token');
return metadata.putObjectMD(managementDatabaseName,
tokenConfigurationKey, value, {}, log, error => {
if (error) {
log.error('could not save token',
{ error: reshapeExceptionError(error),
method: 'initManagementCredentials',
});
return callback(error);
}
log.info('saved token locally, ' +
'confirming instance');
return confirmInstanceCredentials(
managementEndpoint, instanceId, value, log,
callback);
});
});
}
log.debug('could not get token', { error });
return callback(error);
}
log.info('returning existing token');
if (Date.now() - value.issueDate > tokenRotationDelay) {
log.warn('management API token is too old, should re-issue');
}
return callback(null, instanceId, value.token);
});
}
module.exports = {
getStoredCredentials,
initManagementCredentials,
};

138
lib/management/index.js Normal file
View File

@ -0,0 +1,138 @@
const arsenal = require('arsenal');
const async = require('async');
const metadata = require('../metadata/wrapper');
const logger = require('../utilities/logger');
const {
loadCachedOverlay,
managementDatabaseName,
patchConfiguration,
} = require('./configuration');
const { initManagementCredentials } = require('./credentials');
const { startWSManagementClient } = require('./push');
const { startPollingManagementClient } = require('./poll');
const { reshapeExceptionError } = arsenal.errorUtils;
const { isManagementAgentUsed } = require('./agentClient');
const initRemoteManagementRetryDelay = 10000;
const managementEndpointRoot =
process.env.MANAGEMENT_ENDPOINT ||
'https://api.zenko.io';
const managementEndpoint = `${managementEndpointRoot}/api/v1/instance`;
const pushEndpointRoot =
process.env.PUSH_ENDPOINT ||
'https://push.api.zenko.io';
const pushEndpoint = `${pushEndpointRoot}/api/v1/instance`;
function initManagementDatabase(log, callback) {
// XXX choose proper owner names
const md = new arsenal.models.BucketInfo(managementDatabaseName, 'owner',
'owner display name', new Date().toJSON());
metadata.createBucket(managementDatabaseName, md, log, error => {
if (error) {
if (error.is.BucketAlreadyExists) {
log.info('created management database');
return callback();
}
log.error('could not initialize management database',
{ error: reshapeExceptionError(error),
method: 'initManagementDatabase' });
return callback(error);
}
log.info('initialized management database');
return callback();
});
}
function startManagementListeners(instanceId, token) {
const mode = process.env.MANAGEMENT_MODE || 'push';
if (mode === 'push') {
const url = `${pushEndpoint}/${instanceId}/ws`;
startWSManagementClient(url, token);
} else {
startPollingManagementClient(managementEndpoint, instanceId, token);
}
}
/**
* Initializes Orbit-based management by:
* - creating the management database in metadata
* - generating a key pair for credentials encryption
* - generating an instance-unique ID
* - getting an authentication token for the API
* - loading and applying the latest cached overlay configuration
* - starting a configuration update and metrics push background task
*
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
* initialization process
* @param {function} callback Function to call once the overlay is loaded
* (overlay)
*
* @returns {undefined}
*/
function initManagement(log, callback) {
if ((process.env.REMOTE_MANAGEMENT_DISABLE &&
process.env.REMOTE_MANAGEMENT_DISABLE !== '0')
|| process.env.S3BACKEND === 'mem') {
log.info('remote management disabled');
return;
}
/* Temporary check before to fully move to the process management agent. */
if (isManagementAgentUsed() ^ typeof callback === 'function') {
let msg = 'misuse of initManagement function: ';
msg += `MANAGEMENT_USE_AGENT: ${process.env.MANAGEMENT_USE_AGENT}`;
msg += `, callback type: ${typeof callback}`;
throw new Error(msg);
}
async.waterfall([
// eslint-disable-next-line arrow-body-style
cb => { return isManagementAgentUsed() ? metadata.setup(cb) : cb(); },
cb => initManagementDatabase(log, cb),
cb => metadata.getUUID(log, cb),
(instanceId, cb) => initManagementCredentials(
managementEndpoint, instanceId, log, cb),
(instanceId, token, cb) => {
if (!isManagementAgentUsed()) {
cb(null, instanceId, token, {});
return;
}
loadCachedOverlay(log, (err, overlay) => cb(err, instanceId,
token, overlay));
},
(instanceId, token, overlay, cb) => {
if (!isManagementAgentUsed()) {
cb(null, instanceId, token, overlay);
return;
}
patchConfiguration(overlay, log,
err => cb(err, instanceId, token, overlay));
},
], (error, instanceId, token, overlay) => {
if (error) {
log.error('could not initialize remote management, retrying later',
{ error: reshapeExceptionError(error),
method: 'initManagement' });
setTimeout(initManagement,
initRemoteManagementRetryDelay,
logger.newRequestLogger());
} else {
log.info(`this deployment's Instance ID is ${instanceId}`);
log.end('management init done');
startManagementListeners(instanceId, token);
if (callback) {
callback(overlay);
}
}
});
}
module.exports = {
initManagement,
initManagementDatabase,
};

157
lib/management/poll.js Normal file
View File

@ -0,0 +1,157 @@
const arsenal = require('arsenal');
const async = require('async');
const request = require('../utilities/request');
const _config = require('../Config').config;
const logger = require('../utilities/logger');
const metadata = require('../metadata/wrapper');
const {
loadCachedOverlay,
patchConfiguration,
saveConfigurationVersion,
} = require('./configuration');
const { reshapeExceptionError } = arsenal.errorUtils;
const pushReportDelay = 30000;
const pullConfigurationOverlayDelay = 60000;
function loadRemoteOverlay(
managementEndpoint, instanceId, remoteToken, cachedOverlay, log, cb) {
log.debug('loading remote overlay');
const opts = {
headers: {
'x-instance-authentication-token': remoteToken,
'x-scal-request-id': log.getSerializedUids(),
},
json: true,
};
request.get(`${managementEndpoint}/${instanceId}/config/overlay`, opts,
(error, response, body) => {
if (error) {
return cb(error);
}
if (response.statusCode === 200) {
return cb(null, cachedOverlay, body);
}
if (response.statusCode === 404) {
return cb(null, cachedOverlay, {});
}
return cb(arsenal.errors.AccessForbidden, cachedOverlay, {});
});
}
// TODO save only after successful patch
function applyConfigurationOverlay(
managementEndpoint, instanceId, remoteToken, log) {
async.waterfall([
wcb => loadCachedOverlay(log, wcb),
(cachedOverlay, wcb) => patchConfiguration(cachedOverlay,
log, wcb),
(cachedOverlay, wcb) =>
loadRemoteOverlay(managementEndpoint, instanceId, remoteToken,
cachedOverlay, log, wcb),
(cachedOverlay, remoteOverlay, wcb) =>
saveConfigurationVersion(cachedOverlay, remoteOverlay, log, wcb),
(remoteOverlay, wcb) => patchConfiguration(remoteOverlay,
log, wcb),
], error => {
if (error) {
log.error('could not apply managed configuration',
{ error: reshapeExceptionError(error),
method: 'applyConfigurationOverlay' });
}
setTimeout(applyConfigurationOverlay, pullConfigurationOverlayDelay,
managementEndpoint, instanceId, remoteToken,
logger.newRequestLogger());
});
}
function postStats(managementEndpoint, instanceId, remoteToken, report, next) {
const toURL = `${managementEndpoint}/${instanceId}/stats`;
const toOptions = {
json: true,
headers: {
'content-type': 'application/json',
'x-instance-authentication-token': remoteToken,
},
body: report,
};
const toCallback = (err, response, body) => {
if (err) {
logger.info('could not post stats', { error: err });
}
if (response && response.statusCode !== 201) {
logger.info('could not post stats', {
body,
statusCode: response.statusCode,
});
}
if (next) {
next(null, instanceId, remoteToken);
}
};
return request.post(toURL, toOptions, toCallback);
}
function getStats(next) {
const fromURL = `http://localhost:${_config.port}/_/report`;
const fromOptions = {
headers: {
'x-scal-report-token': process.env.REPORT_TOKEN,
},
};
return request.get(fromURL, fromOptions, next);
}
function pushStats(managementEndpoint, instanceId, remoteToken, next) {
if (process.env.PUSH_STATS === 'false') {
return;
}
getStats((err, res, report) => {
if (err) {
logger.info('could not retrieve stats', { error: err });
return;
}
logger.debug('report', { report });
postStats(
managementEndpoint,
instanceId,
remoteToken,
report,
next
);
return;
});
setTimeout(pushStats, pushReportDelay,
managementEndpoint, instanceId, remoteToken);
}
/**
* Starts background task that updates configuration and pushes stats.
*
* Periodically polls for configuration updates, and pushes stats at
* a fixed interval.
*
* @param {string} managementEndpoint API endpoint
* @param {string} instanceId UUID of this deployment
* @param {string} remoteToken API authentication token
*
* @returns {undefined}
*/
function startPollingManagementClient(
managementEndpoint, instanceId, remoteToken) {
metadata.notifyBucketChange(() => {
pushStats(managementEndpoint, instanceId, remoteToken);
});
pushStats(managementEndpoint, instanceId, remoteToken);
applyConfigurationOverlay(managementEndpoint, instanceId, remoteToken,
logger.newRequestLogger());
}
module.exports = {
startPollingManagementClient,
};

301
lib/management/push.js Normal file
View File

@ -0,0 +1,301 @@
const arsenal = require('arsenal');
const HttpsProxyAgent = require('https-proxy-agent');
const net = require('net');
const request = require('../utilities/request');
const { URL } = require('url');
const WebSocket = require('ws');
const assert = require('assert');
const http = require('http');
const _config = require('../Config').config;
const logger = require('../utilities/logger');
const metadata = require('../metadata/wrapper');
const { reshapeExceptionError } = arsenal.errorUtils;
const { isManagementAgentUsed } = require('./agentClient');
const { applyAndSaveOverlay } = require('./configuration');
const {
ChannelMessageV0,
MessageType,
} = require('./ChannelMessageV0');
const {
CONFIG_OVERLAY_MESSAGE,
METRICS_REQUEST_MESSAGE,
CHANNEL_CLOSE_MESSAGE,
CHANNEL_PAYLOAD_MESSAGE,
} = MessageType;
const PING_INTERVAL_MS = 10000;
const subprotocols = [ChannelMessageV0.protocolName];
const cloudServerHost = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_HOST
|| 'localhost';
const cloudServerPort = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_PORT
|| _config.port;
let overlayMessageListener = null;
let connected = false;
// No wildcard nor cidr/mask match for now
function createWSAgent(pushEndpoint, env, log) {
const url = new URL(pushEndpoint);
const noProxy = (env.NO_PROXY || env.no_proxy
|| '').split(',');
if (noProxy.includes(url.hostname)) {
log.info('push server ws has proxy exclusion', { noProxy });
return null;
}
if (url.protocol === 'https:' || url.protocol === 'wss:') {
const httpsProxy = (env.HTTPS_PROXY || env.https_proxy);
if (httpsProxy) {
log.info('push server ws using https proxy', { httpsProxy });
return new HttpsProxyAgent(httpsProxy);
}
} else if (url.protocol === 'http:' || url.protocol === 'ws:') {
const httpProxy = (env.HTTP_PROXY || env.http_proxy);
if (httpProxy) {
log.info('push server ws using http proxy', { httpProxy });
return new HttpsProxyAgent(httpProxy);
}
}
const allProxy = (env.ALL_PROXY || env.all_proxy);
if (allProxy) {
log.info('push server ws using wildcard proxy', { allProxy });
return new HttpsProxyAgent(allProxy);
}
log.info('push server ws not using proxy');
return null;
}
/**
* Starts background task that updates configuration and pushes stats.
*
* Receives pushed Websocket messages on configuration updates, and
* sends stat messages in response to API sollicitations.
*
* @param {string} url API endpoint
* @param {string} token API authentication token
* @param {function} cb end-of-connection callback
*
* @returns {undefined}
*/
function startWSManagementClient(url, token, cb) {
logger.info('connecting to push server', { url });
function _logError(error, errorMessage, method) {
if (error) {
logger.error(`management client error: ${errorMessage}`,
{ error: reshapeExceptionError(error), method });
}
}
const socketsByChannelId = [];
const headers = {
'x-instance-authentication-token': token,
};
const agent = createWSAgent(url, process.env, logger);
const ws = new WebSocket(url, subprotocols, { headers, agent });
let pingTimeout = null;
function sendPing() {
if (ws.readyState === ws.OPEN) {
ws.ping(err => _logError(err, 'failed to send a ping', 'sendPing'));
}
pingTimeout = setTimeout(() => ws.terminate(), PING_INTERVAL_MS);
}
function initiatePing() {
clearTimeout(pingTimeout);
setTimeout(sendPing, PING_INTERVAL_MS);
}
function pushStats(options) {
if (process.env.PUSH_STATS === 'false') {
return;
}
const fromURL = `http://${cloudServerHost}:${cloudServerPort}/_/report`;
const fromOptions = {
json: true,
headers: {
'x-scal-report-token': process.env.REPORT_TOKEN,
'x-scal-report-skip-cache': Boolean(options && options.noCache),
},
};
request.get(fromURL, fromOptions, (err, response, body) => {
if (err) {
_logError(err, 'failed to get metrics report', 'pushStats');
return;
}
ws.send(ChannelMessageV0.encodeMetricsReportMessage(body),
err => _logError(err, 'failed to send metrics report message',
'pushStats'));
});
}
function closeChannel(channelId) {
const socket = socketsByChannelId[channelId];
if (socket) {
socket.destroy();
delete socketsByChannelId[channelId];
}
}
function receiveChannelData(channelId, payload) {
let socket = socketsByChannelId[channelId];
if (!socket) {
socket = net.createConnection(cloudServerPort, cloudServerHost);
socket.on('data', data => {
ws.send(ChannelMessageV0.
encodeChannelDataMessage(channelId, data), err =>
_logError(err, 'failed to send channel data message',
'receiveChannelData'));
});
socket.on('connect', () => {
});
socket.on('drain', () => {
});
socket.on('error', error => {
logger.error('failed to connect to S3', {
code: error.code,
host: error.address,
port: error.port,
});
});
socket.on('end', () => {
socket.destroy();
socketsByChannelId[channelId] = null;
ws.send(ChannelMessageV0.encodeChannelCloseMessage(channelId),
err => _logError(err,
'failed to send channel close message',
'receiveChannelData'));
});
socketsByChannelId[channelId] = socket;
}
socket.write(payload);
}
function browserAccessChangeHandler() {
if (!_config.browserAccessEnabled) {
socketsByChannelId.forEach(s => s.close());
}
}
ws.on('open', () => {
connected = true;
logger.info('connected to push server');
metadata.notifyBucketChange(() => {
pushStats({ noCache: true });
});
_config.on('browser-access-enabled-change', browserAccessChangeHandler);
initiatePing();
});
const cbOnce = cb ? arsenal.jsutil.once(cb) : null;
ws.on('close', () => {
logger.info('disconnected from push server, reconnecting in 10s');
metadata.notifyBucketChange(null);
_config.removeListener('browser-access-enabled-change',
browserAccessChangeHandler);
setTimeout(startWSManagementClient, 10000, url, token);
connected = false;
if (cbOnce) {
process.nextTick(cbOnce);
}
});
ws.on('error', err => {
connected = false;
logger.error('error from push server connection', {
error: err,
errorMessage: err.message,
});
if (cbOnce) {
process.nextTick(cbOnce, err);
}
});
ws.on('ping', () => {
ws.pong(err => _logError(err, 'failed to send a pong'));
});
ws.on('pong', () => {
initiatePing();
});
ws.on('message', data => {
const log = logger.newRequestLogger();
const message = new ChannelMessageV0(data);
switch (message.getType()) {
case CONFIG_OVERLAY_MESSAGE:
if (!isManagementAgentUsed()) {
applyAndSaveOverlay(JSON.parse(message.getPayload()), log);
} else {
if (overlayMessageListener) {
overlayMessageListener(message.getPayload().toString());
}
}
break;
case METRICS_REQUEST_MESSAGE:
pushStats();
break;
case CHANNEL_CLOSE_MESSAGE:
closeChannel(message.getChannelNumber());
break;
case CHANNEL_PAYLOAD_MESSAGE:
// browserAccessEnabled defaults to true unless explicitly false
if (_config.browserAccessEnabled !== false) {
receiveChannelData(
message.getChannelNumber(), message.getPayload());
}
break;
default:
logger.error('unknown message type from push server',
{ messageType: message.getType() });
}
});
}
function addOverlayMessageListener(callback) {
assert(typeof callback === 'function');
overlayMessageListener = callback;
}
function startPushConnectionHealthCheckServer(cb) {
const server = http.createServer((req, res) => {
if (req.url !== '/_/healthcheck') {
res.writeHead(404);
res.write('Not Found');
} else if (connected) {
res.writeHead(200);
res.write('Connected');
} else {
res.writeHead(503);
res.write('Not Connected');
}
res.end();
});
server.listen(_config.port, cb);
}
module.exports = {
createWSAgent,
startWSManagementClient,
startPushConnectionHealthCheckServer,
addOverlayMessageListener,
};

View File

@ -6,9 +6,6 @@ const BucketInfo = require('arsenal').models.BucketInfo;
const { isBucketAuthorized, isObjAuthorized } = const { isBucketAuthorized, isObjAuthorized } =
require('../api/apiUtils/authorization/permissionChecks'); require('../api/apiUtils/authorization/permissionChecks');
const bucketShield = require('../api/apiUtils/bucket/bucketShield'); const bucketShield = require('../api/apiUtils/bucket/bucketShield');
const { onlyOwnerAllowed } = require('../../constants');
const { actionNeedQuotaCheck, actionWithDataDeletion } = require('arsenal/build/lib/policyEvaluator/RequestContext');
const { processBytesToWrite, validateQuotas } = require('../api/apiUtils/quotas/quotaUtils');
/** getNullVersionFromMaster - retrieves the null version /** getNullVersionFromMaster - retrieves the null version
* metadata via retrieving the master key * metadata via retrieving the master key
@ -155,6 +152,9 @@ function validateBucket(bucket, params, log, actionImplicitDenies = {}) {
}); });
return errors.NoSuchBucket; return errors.NoSuchBucket;
} }
// if requester is not bucket owner, bucket policy actions should be denied with
// MethodNotAllowed error
const onlyOwnerAllowed = ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'];
const canonicalID = authInfo.getCanonicalID(); const canonicalID = authInfo.getCanonicalID();
if (!Array.isArray(requestType)) { if (!Array.isArray(requestType)) {
requestType = [requestType]; requestType = [requestType];
@ -184,11 +184,13 @@ function validateBucket(bucket, params, log, actionImplicitDenies = {}) {
* @return {undefined} - and call callback with params err, bucket md * @return {undefined} - and call callback with params err, bucket md
*/ */
function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log, callback) { function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log, callback) {
const { authInfo, bucketName, objectKey, versionId, getDeleteMarker, request, withVersionId } = params; const { authInfo, bucketName, objectKey, versionId, getDeleteMarker, request } = params;
let requestType = params.requestType; let requestType = params.requestType;
if (!Array.isArray(requestType)) { if (!Array.isArray(requestType)) {
requestType = [requestType]; requestType = [requestType];
} }
console.log('entering standardMetadataValidateBucketAndObj',
{ authInfo, bucketName, objectKey, versionId, getDeleteMarker, requestType });
async.waterfall([ async.waterfall([
next => { next => {
// versionId may be 'null', which asks metadata to fetch the null key specifically // versionId may be 'null', which asks metadata to fetch the null key specifically
@ -197,6 +199,7 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
getOptions.getDeleteMarker = true; getOptions.getDeleteMarker = true;
} }
return metadata.getBucketAndObjectMD(bucketName, objectKey, getOptions, log, (err, getResult) => { return metadata.getBucketAndObjectMD(bucketName, objectKey, getOptions, log, (err, getResult) => {
console.log('metadata.getBucketAndObjectMD', { err, getResult });
if (err) { if (err) {
// if some implicit iamAuthzResults, return AccessDenied // if some implicit iamAuthzResults, return AccessDenied
// before leaking any state information // before leaking any state information
@ -219,6 +222,7 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
return next(errors.NoSuchBucket); return next(errors.NoSuchBucket);
} }
const validationError = validateBucket(bucket, params, log, actionImplicitDenies); const validationError = validateBucket(bucket, params, log, actionImplicitDenies);
console.log('validateBucket', { validationError, bucket });
if (validationError) { if (validationError) {
return next(validationError, bucket); return next(validationError, bucket);
} }
@ -236,24 +240,11 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
log.debug('access denied for user on object', { requestType }); log.debug('access denied for user on object', { requestType });
return next(errors.AccessDenied, bucket); return next(errors.AccessDenied, bucket);
} }
console.log('object authorize', { bucket, objMD });
return next(null, bucket, objMD); return next(null, bucket, objMD);
}, },
(bucket, objMD, next) => {
const needQuotaCheck = requestType => requestType.some(type => actionNeedQuotaCheck[type] ||
actionWithDataDeletion[type]);
const checkQuota = params.checkQuota === undefined ? needQuotaCheck(requestType) : params.checkQuota;
// withVersionId cover cases when an object is being restored with a specific version ID.
// In this case, the storage space was already accounted for when the RestoreObject API call
// was made, so we don't need to add any inflight, but quota must be evaluated.
if (!checkQuota) {
return next(null, bucket, objMD);
}
const contentLength = processBytesToWrite(request.apiMethod, bucket, versionId,
request?.parsedContentLength || 0, objMD, params.destObjMD);
return validateQuotas(request, bucket, request.accountQuotas, requestType, request.apiMethod,
contentLength, withVersionId, log, err => next(err, bucket, objMD));
},
], (err, bucket, objMD) => { ], (err, bucket, objMD) => {
console.log('end of bucket checks', { err, bucket, objMD });
if (err) { if (err) {
// still return bucket for cors headers // still return bucket for cors headers
return callback(err, bucket); return callback(err, bucket);
@ -276,6 +267,7 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
function standardMetadataValidateBucket(params, actionImplicitDenies, log, callback) { function standardMetadataValidateBucket(params, actionImplicitDenies, log, callback) {
const { bucketName } = params; const { bucketName } = params;
return metadata.getBucket(bucketName, log, (err, bucket) => { return metadata.getBucket(bucketName, log, (err, bucket) => {
console.log('.getBucket', { err, bucket });
if (err) { if (err) {
// if some implicit actionImplicitDenies, return AccessDenied before // if some implicit actionImplicitDenies, return AccessDenied before
// leaking any state information // leaking any state information
@ -294,7 +286,6 @@ module.exports = {
validateBucket, validateBucket,
metadataGetObject, metadataGetObject,
metadataGetObjects, metadataGetObjects,
processBytesToWrite,
standardMetadataValidateBucketAndObj, standardMetadataValidateBucketAndObj,
standardMetadataValidateBucket, standardMetadataValidateBucket,
}; };

View File

@ -2,9 +2,9 @@ const MetadataWrapper = require('arsenal').storage.metadata.MetadataWrapper;
const { config } = require('../Config'); const { config } = require('../Config');
const logger = require('../utilities/logger'); const logger = require('../utilities/logger');
const constants = require('../../constants'); const constants = require('../../constants');
const bucketclient = require('bucketclient');
const clientName = config.backends.metadata; const clientName = config.backends.metadata;
let bucketclient;
let params; let params;
if (clientName === 'mem') { if (clientName === 'mem') {
params = {}; params = {};
@ -21,7 +21,6 @@ if (clientName === 'mem') {
noDbOpen: null, noDbOpen: null,
}; };
} else if (clientName === 'scality') { } else if (clientName === 'scality') {
bucketclient = require('bucketclient');
params = { params = {
bucketdBootstrap: config.bucketd.bootstrap, bucketdBootstrap: config.bucketd.bootstrap,
bucketdLog: config.bucketd.log, bucketdLog: config.bucketd.log,

View File

@ -1,17 +0,0 @@
const { config } = require('../Config');
const { ScubaClientImpl } = require('./scuba/wrapper');
let instance = null;
switch (config.backends.quota) {
case 'scuba':
instance = new ScubaClientImpl(config);
break;
default:
instance = {
enabled: false,
};
break;
}
module.exports = instance;

View File

@ -1,80 +0,0 @@
const util = require('util');
const { default: ScubaClient } = require('scubaclient');
const { externalBackendHealthCheckInterval } = require('../../../constants');
const monitoring = require('../../utilities/monitoringHandler');
class ScubaClientImpl extends ScubaClient {
constructor(config) {
super(config.scuba);
this.enabled = false;
this.maxStaleness = config.quota.maxStaleness;
this._healthCheckTimer = null;
this._log = null;
this._getLatestMetricsCallback = util.callbackify(this.getLatestMetrics);
if (config.scuba) {
this.enabled = true;
} else {
this.enabled = false;
}
}
setup(log) {
this._log = log;
if (this.enabled) {
this.periodicHealthCheck();
}
}
_healthCheck() {
return this.healthCheck().then(data => {
if (data?.date) {
const date = new Date(data.date);
if (Date.now() - date.getTime() > this.maxStaleness) {
throw new Error('Data is stale, disabling quotas');
}
}
if (!this.enabled) {
this._log.info('Scuba health check passed, enabling quotas');
}
monitoring.utilizationServiceAvailable.set(1);
this.enabled = true;
}).catch(err => {
if (this.enabled) {
this._log.warn('Scuba health check failed, disabling quotas', {
err: err.name,
description: err.message,
});
}
monitoring.utilizationServiceAvailable.set(0);
this.enabled = false;
});
}
periodicHealthCheck() {
if (this._healthCheckTimer) {
clearInterval(this._healthCheckTimer);
}
this._healthCheck();
this._healthCheckTimer = setInterval(async () => {
this._healthCheck();
}, Number(process.env.SCUBA_HEALTHCHECK_FREQUENCY)
|| externalBackendHealthCheckInterval);
}
getUtilizationMetrics(metricsClass, resourceName, options, body, callback) {
const requestStartTime = process.hrtime.bigint();
return this._getLatestMetricsCallback(metricsClass, resourceName, options, body, (err, data) => {
const responseTimeInNs = Number(process.hrtime.bigint() - requestStartTime);
monitoring.utilizationMetricsRetrievalDuration.labels({
code: err ? (err.statusCode || 500) : 200,
class: metricsClass,
}).observe(responseTimeInNs / 1e9);
return callback(err, data);
});
}
}
module.exports = {
ScubaClientImpl,
};

View File

@ -52,7 +52,6 @@ const NAMESPACE = 'default';
const CIPHER = null; // replication/lifecycle does not work on encrypted objects const CIPHER = null; // replication/lifecycle does not work on encrypted objects
let { locationConstraints } = config; let { locationConstraints } = config;
const { nullVersionCompatMode } = config;
const { implName } = dataWrapper; const { implName } = dataWrapper;
let dataClient = dataWrapper.client; let dataClient = dataWrapper.client;
config.on('location-constraints-update', () => { config.on('location-constraints-update', () => {
@ -505,7 +504,9 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
} }
if (versionId === 'null') { if (versionId === 'null') {
isNull = true; if (!config.nullVersionCompatMode) {
isNull = true;
}
// Retrieve the null version id from the object metadata. // Retrieve the null version id from the object metadata.
versionId = objMd && objMd.versionId; versionId = objMd && objMd.versionId;
if (!versionId) { if (!versionId) {
@ -514,16 +515,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
// the flag is needed to allow cloudserver to know that the version // the flag is needed to allow cloudserver to know that the version
// is a null version and allow access to it using the "null" versionId. // is a null version and allow access to it using the "null" versionId.
omVal.isNull = true; omVal.isNull = true;
// If the new null keys logic (S3C-7352) is supported (not compatibility mode),
// create a null key with the isNull2 flag.
if (!nullVersionCompatMode) {
omVal.isNull2 = true;
}
// Delete the version id from the version metadata payload to prevent issues
// with creating a non-version object (versioning set to false) that includes a version id.
// For example, this version ID might come from a null version of a suspended bucket being
// replicated to this bucket.
delete omVal.versionId;
if (versioning) { if (versioning) {
// If the null version does not have a version id, it is a current null version. // If the null version does not have a version id, it is a current null version.
// To update the metadata of a current version, versioning is set to false. // To update the metadata of a current version, versioning is set to false.
@ -558,6 +549,8 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
} }
const options = { const options = {
versionId,
isNull,
overheadField: constants.overheadField, overheadField: constants.overheadField,
}; };
@ -570,19 +563,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
options.versioning = true; options.versioning = true;
} }
// NOTE: When options fields are sent to Metadata through the query string,
// they are converted to strings. As a result, Metadata interprets the value undefined
// in the versionId field as an empty string ('').
// To prevent this, the versionId field is only included in options when it is defined.
if (versionId !== undefined) {
options.versionId = versionId;
}
// If the new null keys logic (S3C-7352) is not supported (compatibility mode), 'isNull' remains undefined.
if (!nullVersionCompatMode) {
options.isNull = isNull;
}
log.trace('putting object version', { log.trace('putting object version', {
objectKey: request.objectKey, omVal, options }); objectKey: request.objectKey, omVal, options });
return metadata.putObjectMD(bucketName, objectKey, omVal, options, log, return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,

View File

@ -18,9 +18,13 @@ const locationStorageCheck =
require('./api/apiUtils/object/locationStorageCheck'); require('./api/apiUtils/object/locationStorageCheck');
const vault = require('./auth/vault'); const vault = require('./auth/vault');
const metadata = require('./metadata/wrapper'); const metadata = require('./metadata/wrapper');
const { initManagement } = require('./management');
const {
initManagementClient,
isManagementAgentUsed,
} = require('./management/agentClient');
const HttpAgent = require('agentkeepalive'); const HttpAgent = require('agentkeepalive');
const QuotaService = require('./quotas/quotas');
const routes = arsenal.s3routes.routes; const routes = arsenal.s3routes.routes;
const { parseLC, MultipleBackendGateway } = arsenal.storage.data; const { parseLC, MultipleBackendGateway } = arsenal.storage.data;
const websiteEndpoints = _config.websiteEndpoints; const websiteEndpoints = _config.websiteEndpoints;
@ -51,6 +55,7 @@ const STATS_INTERVAL = 5; // 5 seconds
const STATS_EXPIRY = 30; // 30 seconds const STATS_EXPIRY = 30; // 30 seconds
const statsClient = new StatsClient(localCacheClient, STATS_INTERVAL, const statsClient = new StatsClient(localCacheClient, STATS_INTERVAL,
STATS_EXPIRY); STATS_EXPIRY);
const enableRemoteManagement = true;
class S3Server { class S3Server {
/** /**
@ -316,9 +321,16 @@ class S3Server {
this._startServer(this.routeAdminRequest, _config.metricsPort); this._startServer(this.routeAdminRequest, _config.metricsPort);
} }
// Start quota service health checks // TODO this should wait for metadata healthcheck to be ok
if (QuotaService.enabled) { // TODO only do this in cluster master
QuotaService?.setup(log); if (enableRemoteManagement) {
if (!isManagementAgentUsed()) {
setTimeout(() => {
initManagement(logger.newRequestLogger());
}, 5000);
} else {
initManagementClient();
}
} }
this.started = true; this.started = true;
@ -327,7 +339,8 @@ class S3Server {
} }
function main() { function main() {
let workers = _config.workers || 1; // TODO: change config to use workers prop. name for clarity
let workers = _config.clusters || 1;
if (process.env.S3BACKEND === 'mem') { if (process.env.S3BACKEND === 'mem') {
workers = 1; workers = 1;
} }

View File

@ -11,6 +11,7 @@ const constants = require('../constants');
const { config } = require('./Config'); const { config } = require('./Config');
const { data } = require('./data/wrapper'); const { data } = require('./data/wrapper');
const metadata = require('./metadata/wrapper'); const metadata = require('./metadata/wrapper');
const logger = require('./utilities/logger');
const { setObjectLockInformation } const { setObjectLockInformation }
= require('./api/apiUtils/object/objectLockHelpers'); = require('./api/apiUtils/object/objectLockHelpers');
const removeAWSChunked = require('./api/apiUtils/object/removeAWSChunked'); const removeAWSChunked = require('./api/apiUtils/object/removeAWSChunked');
@ -349,6 +350,9 @@ const services = {
return cb(err, res); return cb(err, res);
} }
log.trace('deleteObject: metadata delete OK'); log.trace('deleteObject: metadata delete OK');
const deleteLog =
logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids());
if (objectMD.location === null) { if (objectMD.location === null) {
return cb(null, res); return cb(null, res);
} }
@ -359,11 +363,11 @@ const services = {
} }
if (!Array.isArray(objectMD.location)) { if (!Array.isArray(objectMD.location)) {
data.delete(objectMD.location, log); data.delete(objectMD.location, deleteLog);
return cb(null, res); return cb(null, res);
} }
return data.batchDelete(objectMD.location, null, null, log, err => { return data.batchDelete(objectMD.location, null, null, deleteLog, err => {
if (err) { if (err) {
return cb(err); return cb(err);
} }

View File

@ -1,12 +1,16 @@
require('werelogs').stderrUtils.catchAndTimestampStderr();
const _config = require('../Config').config; const _config = require('../Config').config;
const { utapiVersion, UtapiServer: utapiServer } = require('utapi'); const { utapiVersion, UtapiServer: utapiServer } = require('utapi');
const vault = require('../auth/vault');
// start utapi server // start utapi server
if (utapiVersion === 1 && _config.utapi) { if (utapiVersion === 1 && _config.utapi) {
const fullConfig = Object.assign({}, _config.utapi, const fullConfig = Object.assign({}, _config.utapi,
{ redis: _config.redis, vaultclient: vault }); { redis: _config.redis });
if (_config.vaultd) {
Object.assign(fullConfig, { vaultd: _config.vaultd });
}
if (_config.https) {
Object.assign(fullConfig, { https: _config.https });
}
// copy healthcheck IPs // copy healthcheck IPs
if (_config.healthChecks) { if (_config.healthChecks) {
Object.assign(fullConfig, { healthChecks: _config.healthChecks }); Object.assign(fullConfig, { healthChecks: _config.healthChecks });

View File

@ -1,4 +1,3 @@
require('werelogs').stderrUtils.catchAndTimestampStderr();
const UtapiReindex = require('utapi').UtapiReindex; const UtapiReindex = require('utapi').UtapiReindex;
const { config } = require('../Config'); const { config } = require('../Config');

View File

@ -1,4 +1,3 @@
require('werelogs').stderrUtils.catchAndTimestampStderr();
const UtapiReplay = require('utapi').UtapiReplay; const UtapiReplay = require('utapi').UtapiReplay;
const _config = require('../Config').config; const _config = require('../Config').config;

View File

@ -1,11 +1,7 @@
const { configure, Werelogs } = require('werelogs'); const { Werelogs } = require('werelogs');
const _config = require('../Config.js').config; const _config = require('../Config.js').config;
configure({
level: _config.log.logLevel,
dump: _config.log.dumpLevel,
});
const werelogs = new Werelogs({ const werelogs = new Werelogs({
level: _config.log.logLevel, level: _config.log.logLevel,
dump: _config.log.dumpLevel, dump: _config.log.dumpLevel,

View File

@ -1,6 +1,5 @@
const { errors } = require('arsenal'); const { errors } = require('arsenal');
const client = require('prom-client'); const client = require('prom-client');
const { config } = require('../Config');
const collectDefaultMetrics = client.collectDefaultMetrics; const collectDefaultMetrics = client.collectDefaultMetrics;
const numberOfBuckets = new client.Gauge({ const numberOfBuckets = new client.Gauge({
@ -65,49 +64,6 @@ const httpResponseSizeBytes = new client.Summary({
help: 'Cloudserver HTTP response sizes in bytes', help: 'Cloudserver HTTP response sizes in bytes',
}); });
let quotaEvaluationDuration;
let utilizationMetricsRetrievalDuration;
let utilizationServiceAvailable;
let bucketsWithQuota;
let accountsWithQuota;
let requestWithQuotaMetricsUnavailable;
if (config.isQuotaEnabled) {
quotaEvaluationDuration = new client.Histogram({
name: 's3_cloudserver_quota_evaluation_duration_seconds',
help: 'Duration of the quota evaluation operation',
labelNames: ['action', 'code', 'type'],
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1],
});
utilizationMetricsRetrievalDuration = new client.Histogram({
name: 's3_cloudserver_quota_metrics_retrieval_duration_seconds',
help: 'Duration of the utilization metrics retrieval operation',
labelNames: ['code', 'class'],
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5],
});
utilizationServiceAvailable = new client.Gauge({
name: 's3_cloudserver_quota_utilization_service_available',
help: 'Availability of the utilization service',
});
bucketsWithQuota = new client.Gauge({
name: 's3_cloudserver_quota_buckets_count',
help: 'Total number of buckets quota',
});
accountsWithQuota = new client.Gauge({
name: 's3_cloudserver_quota_accounts_count',
help: 'Total number of account quota',
});
requestWithQuotaMetricsUnavailable = new client.Counter({
name: 's3_cloudserver_quota_unavailable_count',
help: 'Total number of requests with quota metrics unavailable',
});
}
// Lifecycle duration metric, to track the completion of restore. // Lifecycle duration metric, to track the completion of restore.
// This metric is used to track the time it takes to complete the lifecycle operation (restore). // This metric is used to track the time it takes to complete the lifecycle operation (restore).
// NOTE : this metric is the same as the one defined in Backbeat, and must keep the same name, // NOTE : this metric is the same as the one defined in Backbeat, and must keep the same name,
@ -187,10 +143,6 @@ function crrCacheToProm(crrResults) {
numberOfBuckets.set(crrResults.getObjectCount.buckets || 0); numberOfBuckets.set(crrResults.getObjectCount.buckets || 0);
numberOfObjects.set(crrResults.getObjectCount.objects || 0); numberOfObjects.set(crrResults.getObjectCount.objects || 0);
} }
if (config.isQuotaEnabled) {
bucketsWithQuota.set(crrResults?.getObjectCount?.bucketWithQuotaCount || 0);
accountsWithQuota.set(crrResults?.getVaultReport?.accountWithQuotaCount || 0);
}
if (crrResults.getDataDiskUsage) { if (crrResults.getDataDiskUsage) {
dataDiskAvailable.set(crrResults.getDataDiskUsage.available || 0); dataDiskAvailable.set(crrResults.getDataDiskUsage.available || 0);
dataDiskFree.set(crrResults.getDataDiskUsage.free || 0); dataDiskFree.set(crrResults.getDataDiskUsage.free || 0);
@ -268,9 +220,4 @@ module.exports = {
httpRequestsTotal, httpRequestsTotal,
httpActiveRequests, httpActiveRequests,
lifecycleDuration, lifecycleDuration,
quotaEvaluationDuration,
utilizationMetricsRetrievalDuration,
utilizationServiceAvailable,
bucketsWithQuota,
requestWithQuotaMetricsUnavailable,
}; };

View File

@ -10,7 +10,6 @@ const config = require('../Config').config;
const { data } = require('../data/wrapper'); const { data } = require('../data/wrapper');
const metadata = require('../metadata/wrapper'); const metadata = require('../metadata/wrapper');
const monitoring = require('../utilities/monitoringHandler'); const monitoring = require('../utilities/monitoringHandler');
const vault = require('../auth/vault');
const REPORT_MODEL_VERSION = 1; const REPORT_MODEL_VERSION = 1;
const ASYNCLIMIT = 5; const ASYNCLIMIT = 5;
@ -462,7 +461,6 @@ function reportHandler(clientIP, req, res, log) {
getCRRMetrics: cb => getCRRMetrics(log, cb), getCRRMetrics: cb => getCRRMetrics(log, cb),
getReplicationStates: cb => getReplicationStates(log, cb), getReplicationStates: cb => getReplicationStates(log, cb),
getIngestionInfo: cb => getIngestionInfo(log, cb), getIngestionInfo: cb => getIngestionInfo(log, cb),
getVaultReport: cb => vault.report(log, cb),
}, },
(err, results) => { (err, results) => {
if (err) { if (err) {
@ -490,7 +488,6 @@ function reportHandler(clientIP, req, res, log) {
capabilities: getCapabilities(), capabilities: getCapabilities(),
ingestStats: results.getIngestionInfo.metrics, ingestStats: results.getIngestionInfo.metrics,
ingestStatus: results.getIngestionInfo.status, ingestStatus: results.getIngestionInfo.status,
vaultReport: results.getVaultReport,
}; };
monitoring.crrCacheToProm(results); monitoring.crrCacheToProm(results);
res.writeHead(200, { 'Content-Type': 'application/json' }); res.writeHead(200, { 'Content-Type': 'application/json' });

View File

@ -1,12 +0,0 @@
{
"STANDARD": {
"type": "vitastor",
"objectId": "std",
"legacyAwsBehavior": true,
"details": {
"config_path": "/etc/vitastor/vitastor.conf",
"pool_id": 3,
"metadata_image": "s3-volume-meta"
}
}
}

179
managementAgent.js Normal file
View File

@ -0,0 +1,179 @@
const Uuid = require('uuid');
const WebSocket = require('ws');
const logger = require('./lib/utilities/logger');
const { initManagement } = require('./lib/management');
const _config = require('./lib/Config').config;
const { managementAgentMessageType } = require('./lib/management/agentClient');
const { addOverlayMessageListener } = require('./lib/management/push');
const { saveConfigurationVersion } = require('./lib/management/configuration');
// TODO: auth?
// TODO: werelogs with a specific name.
const CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS = 15000;
class ManagementAgentServer {
constructor() {
this.port = _config.managementAgent.port || 8010;
this.wss = null;
this.loadedOverlay = null;
this.stop = this.stop.bind(this);
process.on('SIGINT', this.stop);
process.on('SIGHUP', this.stop);
process.on('SIGQUIT', this.stop);
process.on('SIGTERM', this.stop);
process.on('SIGPIPE', () => {});
}
start(_cb) {
const cb = _cb || function noop() {};
/* Define REPORT_TOKEN env variable needed by the management
* module. */
process.env.REPORT_TOKEN = process.env.REPORT_TOKEN
|| _config.reportToken
|| Uuid.v4();
initManagement(logger.newRequestLogger(), overlay => {
let error = null;
if (overlay) {
this.loadedOverlay = overlay;
this.startServer();
} else {
error = new Error('failed to init management');
}
return cb(error);
});
}
stop() {
if (!this.wss) {
process.exit(0);
return;
}
this.wss.close(() => {
logger.info('server shutdown');
process.exit(0);
});
}
startServer() {
this.wss = new WebSocket.Server({
port: this.port,
clientTracking: true,
path: '/watch',
});
this.wss.on('connection', this.onConnection.bind(this));
this.wss.on('listening', this.onListening.bind(this));
this.wss.on('error', this.onError.bind(this));
setInterval(this.checkBrokenConnections.bind(this),
CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS);
addOverlayMessageListener(this.onNewOverlay.bind(this));
}
onConnection(socket, request) {
function hearthbeat() {
this.isAlive = true;
}
logger.info('client connected to watch route', {
ip: request.connection.remoteAddress,
});
/* eslint-disable no-param-reassign */
socket.isAlive = true;
socket.on('pong', hearthbeat.bind(socket));
if (socket.readyState !== socket.OPEN) {
logger.error('client socket not in ready state', {
state: socket.readyState,
client: socket._socket._peername,
});
return;
}
const msg = {
messageType: managementAgentMessageType.NEW_OVERLAY,
payload: this.loadedOverlay,
};
socket.send(JSON.stringify(msg), error => {
if (error) {
logger.error('failed to send remoteOverlay to client', {
error,
client: socket._socket._peername,
});
}
});
}
onListening() {
logger.info('websocket server listening',
{ port: this.port });
}
onError(error) {
logger.error('websocket server error', { error });
}
_sendNewOverlayToClient(client) {
if (client.readyState !== client.OPEN) {
logger.error('client socket not in ready state', {
state: client.readyState,
client: client._socket._peername,
});
return;
}
const msg = {
messageType: managementAgentMessageType.NEW_OVERLAY,
payload: this.loadedOverlay,
};
client.send(JSON.stringify(msg), error => {
if (error) {
logger.error(
'failed to send remoteOverlay to management agent client', {
error, client: client._socket._peername,
});
}
});
}
onNewOverlay(remoteOverlay) {
const remoteOverlayObj = JSON.parse(remoteOverlay);
saveConfigurationVersion(
this.loadedOverlay, remoteOverlayObj, logger, err => {
if (err) {
logger.error('failed to save remote overlay', { err });
return;
}
this.loadedOverlay = remoteOverlayObj;
this.wss.clients.forEach(
this._sendNewOverlayToClient.bind(this)
);
});
}
checkBrokenConnections() {
this.wss.clients.forEach(client => {
if (!client.isAlive) {
logger.info('close broken connection', {
client: client._socket._peername,
});
client.terminate();
return;
}
client.isAlive = false;
client.ping();
});
}
}
const server = new ManagementAgentServer();
server.start();

View File

@ -192,163 +192,3 @@ tests:
summary: Very high delete latency summary: Very high delete latency
exp_labels: exp_labels:
severity: critical severity: critical
# QuotaMetricsNotAvailable (case with bucket quota)
##################################################################################################
- name: Quota metrics not available (bucket quota)
interval: 1m
input_series:
- series: s3_cloudserver_quota_utilization_service_available{namespace="zenko",service="artesca-data-connector-s3api-metrics"}
values: 1+1x6 0+0x20 1+1x6
- series: s3_cloudserver_quota_buckets_count{namespace="zenko",job="artesca-data-ops-report-handler"}
values: 1+1x32
alert_rule_test:
- alertname: QuotaMetricsNotAvailable
eval_time: 6m
exp_alerts: []
- alertname: QuotaMetricsNotAvailable
eval_time: 15m
exp_alerts:
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: warning
- alertname: QuotaMetricsNotAvailable
eval_time: 20m
exp_alerts:
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: warning
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: critical
- alertname: QuotaMetricsNotAvailable
eval_time: 28m
exp_alerts: []
# QuotaMetricsNotAvailable (case with account quota)
##################################################################################################
- name: Quota metrics not available (account quota)
interval: 1m
input_series:
- series: s3_cloudserver_quota_utilization_service_available{namespace="zenko",service="artesca-data-connector-s3api-metrics"}
values: 1+1x6 0+0x20 1+1x6
- series: s3_cloudserver_quota_accounts_count{namespace="zenko",job="artesca-data-ops-report-handler"}
values: 1+1x32
alert_rule_test:
- alertname: QuotaMetricsNotAvailable
eval_time: 6m
exp_alerts: []
- alertname: QuotaMetricsNotAvailable
eval_time: 15m
exp_alerts:
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: warning
- alertname: QuotaMetricsNotAvailable
eval_time: 20m
exp_alerts:
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: warning
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: critical
- alertname: QuotaMetricsNotAvailable
eval_time: 28m
exp_alerts: []
# QuotaMetricsNotAvailable (case with both quota quota)
##################################################################################################
- name: Quota metrics not available (account quota)
interval: 1m
input_series:
- series: s3_cloudserver_quota_utilization_service_available{namespace="zenko",service="artesca-data-connector-s3api-metrics"}
values: 1+1x6 0+0x20 1+1x6
- series: s3_cloudserver_quota_accounts_count{namespace="zenko",job="artesca-data-ops-report-handler"}
values: 1+1x32
- series: s3_cloudserver_quota_buckets_count{namespace="zenko",job="artesca-data-ops-report-handler"}
values: 1+1x32
alert_rule_test:
- alertname: QuotaMetricsNotAvailable
eval_time: 6m
exp_alerts: []
- alertname: QuotaMetricsNotAvailable
eval_time: 15m
exp_alerts:
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: warning
- alertname: QuotaMetricsNotAvailable
eval_time: 20m
exp_alerts:
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: warning
- exp_annotations:
description: The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.
summary: Utilization metrics service not available
exp_labels:
severity: critical
- alertname: QuotaMetricsNotAvailable
eval_time: 28m
exp_alerts: []
# QuotaMetricsNotAvailable (case without quota)
##################################################################################################
- name: Utilization service Latency
interval: 1m
input_series:
- series: s3_cloudserver_quota_utilization_service_available{namespace="zenko",service="artesca-data-connector-s3api-metrics"}
values: 1+1x6 0+0x20 1+1x6
alert_rule_test:
- alertname: QuotaMetricsNotAvailable
eval_time: 6m
exp_alerts: []
- alertname: QuotaMetricsNotAvailable
eval_time: 15m
exp_alerts: []
- alertname: QuotaMetricsNotAvailable
eval_time: 20m
exp_alerts: []
- alertname: QuotaMetricsNotAvailable
eval_time: 28m
exp_alerts: []
# QuotaUnavailable
##################################################################################################
- name: Quota evaluation disabled
interval: 1m
input_series:
- series: s3_cloudserver_quota_unavailable_count{namespace="zenko",service="artesca-data-connector-s3api-metrics"}
values: 0+0x6 1+1x20 0+0x6
alert_rule_test:
- alertname: QuotaUnavailable
eval_time: 6m
exp_alerts: []
- alertname: QuotaUnavailable
eval_time: 20m
exp_alerts:
- exp_annotations:
description: Quotas were not honored due to metrics being unavailable. If the S3 Bucket or Account was created recently, the metrics may not be available yet.
summary: High number of quota requests with metrics unavailable
exp_labels:
severity: critical
- alertname: QuotaUnavailable
eval_time: 30m
exp_alerts: []

View File

@ -6,9 +6,6 @@ x-inputs:
- name: service - name: service
type: constant type: constant
value: artesca-data-connector-s3api-metrics value: artesca-data-connector-s3api-metrics
- name: reportJob
type: constant
value: artesca-data-ops-report-handler
- name: replicas - name: replicas
type: constant type: constant
- name: systemErrorsWarningThreshold - name: systemErrorsWarningThreshold
@ -29,9 +26,6 @@ x-inputs:
- name: deleteLatencyCriticalThreshold - name: deleteLatencyCriticalThreshold
type: config type: config
value: 1.000 value: 1.000
- name: quotaUnavailabilityThreshold
type: config
value: 0.500
groups: groups:
- name: CloudServer - name: CloudServer
@ -138,45 +132,3 @@ groups:
annotations: annotations:
description: "Latency of delete object operations is more than 1s" description: "Latency of delete object operations is more than 1s"
summary: "Very high delete latency" summary: "Very high delete latency"
# As a platform admin I want to be alerted (warning) when the utilization metrics service is enabled
# but not available for at least half of the S3 services during the last minute
- alert: QuotaMetricsNotAvailable
expr: |
avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"})
< ${quotaUnavailabilityThreshold} and
(max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or
max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0)
labels:
severity: warning
annotations:
description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled."
summary: "Utilization metrics service not available"
# As a platform admin I want to be alerted (critical) when the utilization metrics service is enabled
# but not available during the last 10 minutes
- alert: QuotaMetricsNotAvailable
expr: |
avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"})
< ${quotaUnavailabilityThreshold} and
(max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or
max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0)
for: 10m
labels:
severity: critical
annotations:
description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled."
summary: "Utilization metrics service not available"
# As a platform admin I want to be alerted (critical) when quotas were not honored due to metrics
# being unavailable
- alert: QuotaUnavailable
expr: |
sum(increase(s3_cloudserver_quota_unavailable_count{namespace="${namespace}",service="${service}"}[2m]))
> 0
for: 5m
labels:
severity: critical
annotations:
description: "Quotas were not honored due to metrics being unavailable. If the S3 Bucket or Account was created recently, the metrics may not be available yet."
summary: "High number of quota requests with metrics unavailable"

View File

@ -1931,7 +1931,7 @@
"targets": [ "targets": [
{ {
"datasource": null, "datasource": null,
"expr": "sum by(le) (increase(s3_cloudserver_http_request_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", "expr": "sum by(le) (increase(s3_cloudserver_http_request_duration_seconds_bucket{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))",
"format": "heatmap", "format": "heatmap",
"hide": false, "hide": false,
"instant": false, "instant": false,
@ -1960,7 +1960,7 @@
}, },
"yAxis": { "yAxis": {
"decimals": null, "decimals": null,
"format": "s", "format": "dtdurations",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@ -2182,7 +2182,7 @@
"targets": [ "targets": [
{ {
"datasource": null, "datasource": null,
"expr": "sum(rate(s3_cloudserver_http_response_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", "expr": "sum(rate(s3_cloudserver_http_response_size_bytes_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))",
"format": "time_series", "format": "time_series",
"hide": false, "hide": false,
"instant": false, "instant": false,
@ -2196,7 +2196,7 @@
}, },
{ {
"datasource": null, "datasource": null,
"expr": "sum(rate(s3_cloudserver_http_request_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", "expr": "sum(rate(s3_cloudserver_http_request_size_bytes_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))",
"format": "time_series", "format": "time_series",
"hide": false, "hide": false,
"instant": false, "instant": false,
@ -2665,865 +2665,6 @@
"transformations": [], "transformations": [],
"transparent": false, "transparent": false,
"type": "piechart" "type": "piechart"
},
{
"collapsed": false,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": []
}
}
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 65
},
"hideTimeOverride": false,
"id": 34,
"links": [],
"maxDataPoints": 100,
"panels": [],
"targets": [],
"title": "Quotas",
"transformations": [],
"transparent": false,
"type": "row"
},
{
"datasource": "${DS_PROMETHEUS}",
"description": "Number of S3 buckets with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {},
"decimals": null,
"mappings": [],
"noValue": "-",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "#808080",
"index": 0,
"line": true,
"op": "gt",
"value": "null",
"yaxis": "left"
},
{
"color": "blue",
"index": 1,
"line": true,
"op": "gt",
"value": 0.0,
"yaxis": "left"
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 66
},
"hideTimeOverride": false,
"id": 35,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": null,
"expr": "max(s3_cloudserver_quota_buckets_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Buckets with quota",
"transformations": [],
"transparent": false,
"type": "stat"
},
{
"datasource": "${DS_PROMETHEUS}",
"description": "Number of accounts with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {},
"decimals": null,
"mappings": [],
"noValue": "-",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "#808080",
"index": 0,
"line": true,
"op": "gt",
"value": "null",
"yaxis": "left"
},
{
"color": "blue",
"index": 1,
"line": true,
"op": "gt",
"value": 0.0,
"yaxis": "left"
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 70
},
"hideTimeOverride": false,
"id": 36,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": null,
"expr": "max(s3_cloudserver_quota_accounts_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Accounts with quota",
"transformations": [],
"transparent": false,
"type": "stat"
},
{
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 2,
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 66
},
"hideTimeOverride": false,
"id": 37,
"links": [],
"maxDataPoints": 100,
"options": {
"legend": {
"calcs": [],
"displayMode": "hidden",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_unavailable_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Operations with unavailable metrics",
"transformations": [],
"transparent": false,
"type": "timeseries"
},
{
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 2,
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 66
},
"hideTimeOverride": false,
"id": 38,
"links": [],
"maxDataPoints": 100,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max"
],
"displayMode": "table",
"placement": "right"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval])) by(action)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{action}}",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Quota evaluaton rate per S3 action",
"transformations": [],
"transparent": false,
"type": "timeseries"
},
{
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "stepAfter",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 2,
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"index": 0,
"line": true,
"op": "gt",
"value": "null",
"yaxis": "left"
},
{
"color": "orange",
"index": 1,
"line": true,
"op": "gt",
"value": 90.0,
"yaxis": "left"
},
{
"color": "red",
"index": 2,
"line": true,
"op": "gt",
"value": 0.0,
"yaxis": "left"
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 74
},
"hideTimeOverride": false,
"id": 39,
"links": [],
"maxDataPoints": 100,
"options": {
"legend": {
"calcs": [],
"displayMode": "hidden",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": null,
"expr": "avg(avg_over_time(s3_cloudserver_quota_utilization_service_available{namespace=\"${namespace}\",job=\"${job}\"}[$__rate_interval])) * 100",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Quota service uptime",
"transformations": [],
"transparent": false,
"type": "timeseries"
},
{
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 2,
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 74
},
"hideTimeOverride": false,
"id": 40,
"links": [],
"maxDataPoints": 100,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=~\"2..\", job=\"${job}\"}[$__rate_interval]))",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "Success",
"metric": "",
"refId": "",
"step": 10,
"target": ""
},
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=\"429\", job=\"${job}\"}[$__rate_interval]))",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "Quota Exceeded",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Quota evaluation status code over time",
"transformations": [],
"transparent": false,
"type": "timeseries"
},
{
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 2,
"type": "linear"
},
"showPoints": "auto",
"spanNulls": 180000,
"stacking": {},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 74
},
"hideTimeOverride": false,
"id": 41,
"links": [],
"maxDataPoints": 100,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max"
],
"displayMode": "table",
"placement": "right"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ type }} (success)",
"metric": "",
"refId": "",
"step": 10,
"target": ""
},
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ type }} (exceeded)",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Average quota evaluation latencies",
"transformations": [],
"transparent": false,
"type": "timeseries"
},
{
"cards": {
"cardPadding": null,
"cardRound": null
},
"color": {
"cardColor": "#b4ff00",
"colorScale": "sqrt",
"colorScheme": "interpolateOranges",
"exponent": 0.5,
"max": null,
"min": null,
"mode": "opacity"
},
"dataFormat": "tsbuckets",
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": []
}
}
},
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 82
},
"heatmap": {},
"hideTimeOverride": false,
"hideZeroBuckets": false,
"highlightCards": true,
"id": 42,
"legend": {
"show": false
},
"links": [],
"maxDataPoints": 25,
"reverseYBuckets": false,
"targets": [
{
"datasource": null,
"expr": "sum by(le) (increase(s3_cloudserver_quota_evaluation_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))",
"format": "heatmap",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ le }}",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Quota evaluation duration",
"tooltip": {
"show": true,
"showHistogram": true
},
"transformations": [],
"transparent": false,
"type": "heatmap",
"xAxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yAxis": {
"decimals": null,
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
},
{
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 2,
"type": "linear"
},
"showPoints": "auto",
"spanNulls": 180000,
"stacking": {},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 18,
"x": 6,
"y": 82
},
"hideTimeOverride": false,
"id": 43,
"links": [],
"maxDataPoints": 100,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ class }} (success)",
"metric": "",
"refId": "",
"step": 10,
"target": ""
},
{
"datasource": null,
"expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ class }} (error)",
"metric": "",
"refId": "",
"step": 10,
"target": ""
}
],
"title": "Average utilization metrics retrieval latencies",
"transformations": [],
"transparent": false,
"type": "timeseries"
} }
], ],
"refresh": "30s", "refresh": "30s",
@ -3625,5 +2766,5 @@
"timezone": "", "timezone": "",
"title": "S3 service", "title": "S3 service",
"uid": null, "uid": null,
"version": 110 "version": 31
} }

View File

@ -43,7 +43,7 @@ httpRequests = Stat(
noValue="0", noValue="0",
reduceCalc="mean", reduceCalc="mean",
targets=[Target( targets=[Target(
expr='sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501 expr='sum(rate(http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("green", 0, 0.0), Threshold("green", 0, 0.0),
@ -60,9 +60,9 @@ successRate = GaugePanel(
noValue="-", noValue="-",
targets=[Target( targets=[Target(
expr="\n".join([ expr="\n".join([
'sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job", code=~"2.."}[$__rate_interval])) * 100', # noqa: E501 'sum(rate(http_requests_total{namespace="${namespace}", job=~"$job", code=~"2.."}[$__rate_interval])) * 100', # noqa: E501
" /", " /",
'sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]) > 0)', # noqa: E501 'sum(rate(http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]) > 0)', # noqa: E501
]), ]),
legendFormat="Success rate", legendFormat="Success rate",
)], )],
@ -86,7 +86,7 @@ dataIngestionRate = Stat(
format="binBps", format="binBps",
reduceCalc="mean", reduceCalc="mean",
targets=[Target( targets=[Target(
expr='-sum(deriv(s3_cloudserver_disk_available_bytes{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501 expr='-sum(deriv(cloud_server_data_disk_available{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("dark-purple", 0, 0.0), Threshold("dark-purple", 0, 0.0),
@ -105,7 +105,7 @@ objectIngestionRate = Stat(
format="O/s", format="O/s",
reduceCalc="mean", reduceCalc="mean",
targets=[Target( targets=[Target(
expr='sum(deriv(s3_cloudserver_objects_count{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501 expr='sum(deriv(cloud_server_number_of_objects{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("dark-purple", 0, 0.0), Threshold("dark-purple", 0, 0.0),
@ -125,7 +125,7 @@ bucketsCounter = Stat(
noValue="-", noValue="-",
reduceCalc="lastNotNull", reduceCalc="lastNotNull",
targets=[Target( targets=[Target(
expr='sum(s3_cloudserver_buckets_count{namespace="${namespace}", job="${reportJob}"})', # noqa: E501 expr='sum(cloud_server_number_of_buckets{namespace="${namespace}", job="${reportJob}"})', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("#808080", 0, 0.0), Threshold("#808080", 0, 0.0),
@ -146,7 +146,7 @@ objectsCounter = Stat(
noValue="-", noValue="-",
reduceCalc="lastNotNull", reduceCalc="lastNotNull",
targets=[Target( targets=[Target(
expr='sum(s3_cloudserver_objects_count{namespace="${namespace}", job="${reportJob}"})', # noqa: E501 expr='sum(cloud_server_number_of_objects{namespace="${namespace}", job="${reportJob}"})', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("#808080", 0, 0.0), Threshold("#808080", 0, 0.0),
@ -181,8 +181,8 @@ lastReport = Stat(
targets=[Target( targets=[Target(
expr="\n".join([ expr="\n".join([
'time()', 'time()',
'- max(s3_cloudserver_last_report_timestamp{namespace="${namespace}", job="${reportJob}"})', # noqa: E501 '- max(cloud_server_last_report_timestamp{namespace="${namespace}", job="${reportJob}"})', # noqa: E501
'+ (max(s3_cloudserver_last_report_timestamp{namespace="${namespace}", job="${reportJob}"})', # noqa: E501 '+ (max(cloud_server_last_report_timestamp{namespace="${namespace}", job="${reportJob}"})', # noqa: E501
' - max(kube_cronjob_status_last_schedule_time{namespace="${namespace}", cronjob="${countItemsJob}"})', # noqa: E501 ' - max(kube_cronjob_status_last_schedule_time{namespace="${namespace}", cronjob="${countItemsJob}"})', # noqa: E501
' > 0 or vector(0))', ' > 0 or vector(0))',
]) ])
@ -207,7 +207,7 @@ def http_status_panel(title, code):
noValue="0", noValue="0",
reduceCalc="mean", reduceCalc="mean",
targets=[Target( targets=[Target(
expr='sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job",code=' + code + "}[$__rate_interval]))", # noqa: E501 expr='sum(rate(http_requests_total{namespace="${namespace}", job=~"$job",code=' + code + "}[$__rate_interval]))", # noqa: E501
)], )],
thresholds=[Threshold("semi-dark-blue", 0, 0.)], thresholds=[Threshold("semi-dark-blue", 0, 0.)],
) )
@ -222,7 +222,7 @@ activeRequests = Stat(
dataSource="${DS_PROMETHEUS}", dataSource="${DS_PROMETHEUS}",
reduceCalc="lastNotNull", reduceCalc="lastNotNull",
targets=[Target( targets=[Target(
expr='sum(s3_cloudserver_http_active_requests{namespace="${namespace}", job=~"$job"})', # noqa: E501 expr='sum(http_active_requests{namespace="${namespace}", job=~"$job"})', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("green", 0, 0.0), Threshold("green", 0, 0.0),
@ -242,7 +242,7 @@ oobDataIngestionRate = Stat(
format="binBps", format="binBps",
reduceCalc="mean", reduceCalc="mean",
targets=[Target( targets=[Target(
expr='sum(deriv(s3_cloudserver_ingested_bytes{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501 expr='sum(deriv(cloud_server_data_ingested{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("purple", 0, 0.0), Threshold("purple", 0, 0.0),
@ -261,7 +261,7 @@ oobObjectIngestionRate = Stat(
format="O/s", format="O/s",
reduceCalc="mean", reduceCalc="mean",
targets=[Target( targets=[Target(
expr='sum(deriv(s3_cloudserver_ingested_objects_count{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501 expr='sum(deriv(cloud_server_number_of_ingested_objects{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
)], )],
thresholds=[ thresholds=[
Threshold("purple", 0, 0.0), Threshold("purple", 0, 0.0),
@ -275,7 +275,7 @@ httpStatusCodes = TimeSeries(
lineInterpolation="smooth", lineInterpolation="smooth",
unit=UNITS.OPS_PER_SEC, unit=UNITS.OPS_PER_SEC,
targets=[Target( targets=[Target(
expr='sum by (code) (rate(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501 expr='sum by (code) (rate(http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
legendFormat="{{code}}", legendFormat="{{code}}",
)], )],
) )
@ -284,7 +284,7 @@ httpStatusCodes = TimeSeries(
def http_aggregated_request_target(title, code): def http_aggregated_request_target(title, code):
# type: (str, str) -> Target # type: (str, str) -> Target
return Target( return Target(
expr='sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job", code=' + code + "}[$__rate_interval]))", # noqa: E501 expr='sum(rate(http_requests_total{namespace="${namespace}", job=~"$job", code=' + code + "}[$__rate_interval]))", # noqa: E501
legendFormat=title, legendFormat=title,
) )
@ -331,7 +331,7 @@ requestsByAction = TimeSeries(
unit=UNITS.OPS_PER_SEC, unit=UNITS.OPS_PER_SEC,
targets=[ targets=[
Target( Target(
expr='sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval])) by(action)', # noqa: E501 expr='sum(rate(http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval])) by(action)', # noqa: E501
legendFormat="{{action}}", legendFormat="{{action}}",
) )
] ]
@ -345,7 +345,7 @@ requestsByMethod = PieChart(
unit=UNITS.SHORT, unit=UNITS.SHORT,
targets=[ targets=[
Target( Target(
expr='sum(round(increase(s3_cloudserver_http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]))) by(method)', # noqa: E501 expr='sum(round(increase(http_requests_total{namespace="${namespace}", job=~"$job"}[$__rate_interval]))) by(method)', # noqa: E501
legendFormat="{{method}}", legendFormat="{{method}}",
), ),
], ],
@ -358,36 +358,14 @@ def average_latency_target(title, action="", by=""):
by = " by (" + by + ")" if by else "" by = " by (" + by + ")" if by else ""
return Target( return Target(
expr="\n".join([ expr="\n".join([
'sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}", job=~"$job"' + extra + "}[$__rate_interval]))" + by, # noqa: E501 'sum(rate(http_request_duration_seconds_sum{namespace="${namespace}", job=~"$job"' + extra + "}[$__rate_interval]))" + by, # noqa: E501
" /", " /",
'sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}", job=~"$job"' + extra + "}[$__rate_interval]))" + by, # noqa: E501, 'sum(rate(http_request_duration_seconds_count{namespace="${namespace}", job=~"$job"' + extra + "}[$__rate_interval]))" + by, # noqa: E501,
]), ]),
legendFormat=title, legendFormat=title,
) )
def average_quota_latency_target(code="", by=""):
# type: (str, str) -> Target
extra = ', code=' + code if code else ""
by = " by (" + by + ")" if by else ""
return "\n".join([
'sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace="${namespace}", job="${job}"' + extra + "}[$__rate_interval]))" + by, # noqa: E501
" /",
'sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace="${namespace}", job="${job}"' + extra + "}[$__rate_interval]))" + by, # noqa: E501,
])
def average_quota_retrieval_latency(code="", by=""):
# type: (str, str) -> Target
extra = ', code=' + code if code else ""
by = " by (" + by + ")" if by else ""
return "\n".join([
'sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace="${namespace}", job="${job}"' + extra + "}[$__rate_interval]))" + by, # noqa: E501
" /",
'sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace="${namespace}", job="${job}"' + extra + "}[$__rate_interval]))" + by, # noqa: E501,
])
averageLatencies = TimeSeries( averageLatencies = TimeSeries(
title="Average latencies", title="Average latencies",
dataSource="${DS_PROMETHEUS}", dataSource="${DS_PROMETHEUS}",
@ -423,15 +401,15 @@ latenciesByAction = TimeSeries(
) )
requestTime = Heatmap( requestTime = Heatmap(
title="Request duration", title="Request time",
dataSource="${DS_PROMETHEUS}", dataSource="${DS_PROMETHEUS}",
dataFormat="tsbuckets", dataFormat="tsbuckets",
maxDataPoints=25, maxDataPoints=25,
tooltip=Tooltip(show=True, showHistogram=True), tooltip=Tooltip(show=True, showHistogram=True),
yAxis=YAxis(format=UNITS.SECONDS), yAxis=YAxis(format=UNITS.DURATION_SECONDS),
color=HeatmapColor(mode="opacity"), color=HeatmapColor(mode="opacity"),
targets=[Target( targets=[Target(
expr='sum by(le) (increase(s3_cloudserver_http_request_duration_seconds_bucket{namespace="${namespace}", job="${job}"}[$__rate_interval]))', # noqa: E501 expr='sum by(le) (increase(http_request_duration_seconds_bucket{namespace="${namespace}", job=~"$job"}[$__interval]))', # noqa: E501
format="heatmap", format="heatmap",
legendFormat="{{ le }}", legendFormat="{{ le }}",
)], )],
@ -455,11 +433,11 @@ bandWidth = TimeSeries(
unit="binBps", unit="binBps",
targets=[ targets=[
Target( Target(
expr='sum(rate(s3_cloudserver_http_response_size_bytes_sum{namespace="${namespace}", job="${job}"}[$__rate_interval]))', # noqa: E501 expr='sum(rate(http_response_size_bytes_sum{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
legendFormat="Out" legendFormat="Out"
), ),
Target( Target(
expr='sum(rate(s3_cloudserver_http_request_size_bytes_sum{namespace="${namespace}", job="${job}"}[$__rate_interval]))', # noqa: E501 expr='sum(rate(http_request_size_bytes_sum{namespace="${namespace}", job=~"$job"}[$__rate_interval]))', # noqa: E501
legendFormat="In" legendFormat="In"
) )
], ],
@ -483,7 +461,7 @@ uploadChunkSize = BarGauge(
noValue="-", noValue="-",
orientation="vertical", orientation="vertical",
targets=[Target( targets=[Target(
expr='avg(s3_cloudserver_http_request_size_bytes{namespace="${namespace}", job=~"$job"}) by (quantile)', # noqa: E501 expr='avg(http_request_size_bytes{namespace="${namespace}", job=~"$job"}) by (quantile)', # noqa: E501
legendFormat='{{ quantile }}', legendFormat='{{ quantile }}',
)], )],
thresholds=[ thresholds=[
@ -503,7 +481,7 @@ downloadChunkSize = BarGauge(
noValue="-", noValue="-",
orientation="vertical", orientation="vertical",
targets=[Target( targets=[Target(
expr='avg(s3_cloudserver_http_response_size_bytes{namespace="${namespace}", job=~"$job"}) by (quantile)', # noqa: E501 expr='avg(http_response_size_bytes{namespace="${namespace}", job=~"$job"}) by (quantile)', # noqa: E501
legendFormat='{{ quantile }}', legendFormat='{{ quantile }}',
)], )],
thresholds=[ thresholds=[
@ -547,174 +525,6 @@ top10Error5xxByBucket = top10_errors_by_bucket(
title="5xx : Top10 by Bucket", code='~"5.."' title="5xx : Top10 by Bucket", code='~"5.."'
) )
quotaHealth = TimeSeries(
title="Quota service uptime",
legendDisplayMode="hidden",
dataSource="${DS_PROMETHEUS}",
lineInterpolation="stepAfter",
fillOpacity=30,
unit=UNITS.PERCENT_FORMAT,
targets=[Target(
expr='avg(avg_over_time(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",job="${job}"}[$__rate_interval])) * 100', # noqa: E501
)],
thresholds=[
Threshold("green", 0, 95.0),
Threshold("orange", 1, 90.0),
Threshold("red", 2, 0.0),
],
)
quotaStatusCode = TimeSeries(
title="Quota evaluation status code over time",
dataSource="${DS_PROMETHEUS}",
fillOpacity=30,
lineInterpolation="smooth",
unit=UNITS.OPS_PER_SEC,
targets=[Target(
expr='sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace="${namespace}", code=~"2..", job="${job}"}[$__rate_interval]))', # noqa: E501
legendFormat="Success",
), Target(
expr='sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace="${namespace}", code="429", job="${job}"}[$__rate_interval]))', # noqa: E501
legendFormat="Quota Exceeded",
)],
)
quotaByAction = TimeSeries(
title="Quota evaluaton rate per S3 action",
dataSource="${DS_PROMETHEUS}",
legendDisplayMode="table",
legendPlacement="right",
legendValues=["min", "mean", "max"],
lineInterpolation="smooth",
unit=UNITS.OPS_PER_SEC,
targets=[
Target(
expr='sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace="${namespace}", job="${job}"}[$__rate_interval])) by(action)', # noqa: E501
legendFormat="{{action}}",
)
]
)
averageQuotaDuration = Heatmap(
title="Quota evaluation duration",
dataSource="${DS_PROMETHEUS}",
dataFormat="tsbuckets",
maxDataPoints=25,
tooltip=Tooltip(show=True, showHistogram=True),
yAxis=YAxis(format=UNITS.SECONDS),
color=HeatmapColor(mode="opacity"),
targets=[Target(
expr='sum by(le) (increase(s3_cloudserver_quota_evaluation_duration_seconds_bucket{namespace="${namespace}", job="${job}"}[$__rate_interval]))', # noqa: E501
format="heatmap",
legendFormat="{{ le }}",
)],
)
operationsWithUnavailableMetrics = TimeSeries(
title="Operations with unavailable metrics",
dataSource="${DS_PROMETHEUS}",
fillOpacity=30,
lineInterpolation="smooth",
unit=UNITS.OPS_PER_SEC,
legendDisplayMode="hidden",
targets=[Target(
expr='sum(rate(s3_cloudserver_quota_unavailable_count{namespace="${namespace}", job="${job}"}[$__rate_interval]))', # noqa: E501
)],
)
averageQuotaLatencies = TimeSeries(
title="Average quota evaluation latencies",
dataSource="${DS_PROMETHEUS}",
lineInterpolation="smooth",
spanNulls=3*60*1000,
legendDisplayMode="table",
legendPlacement="right",
legendValues=["min", "mean", "max"],
unit=UNITS.SECONDS,
targets=[
Target(
expr=average_quota_latency_target(code='~"2.."', by='type'),
legendFormat='{{ type }} (success)',
),
Target(
expr=average_quota_latency_target(code='"429"', by='type'),
legendFormat='{{ type }} (exceeded)',
),
],
)
averageMetricsRetrievalLatencies = TimeSeries(
title="Average utilization metrics retrieval latencies",
dataSource="${DS_PROMETHEUS}",
lineInterpolation="smooth",
spanNulls=3*60*1000,
unit=UNITS.SECONDS,
targets=[
Target(
expr=average_quota_retrieval_latency(code='~"2.."', by='class'),
legendFormat='{{ class }} (success)',
),
Target(
expr=average_quota_retrieval_latency(
code='~"4..|5.."',
by='class'
),
legendFormat='{{ class }} (error)',
),
],
)
bucketQuotaCounter = Stat(
title="Buckets with quota",
description=(
"Number of S3 buckets with quota enabled in the cluster.\n"
"This value is computed asynchronously, and update "
"may be delayed up to 1h."
),
dataSource="${DS_PROMETHEUS}",
colorMode="value",
format=UNITS.SHORT,
noValue="-",
reduceCalc="lastNotNull",
targets=[Target(
expr='max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job=~"${reportJob}"})', # noqa: E501
)],
thresholds=[
Threshold("#808080", 0, 0.0),
Threshold("blue", 1, 0.0),
],
)
accountQuotaCounter = Stat(
title="Accounts with quota",
description=(
"Number of accounts with quota enabled in the cluster.\n"
"This value is computed asynchronously, and update "
"may be delayed up to 1h."
),
dataSource="${DS_PROMETHEUS}",
colorMode="value",
format=UNITS.SHORT,
noValue="-",
reduceCalc="lastNotNull",
targets=[Target(
expr='max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job=~"${reportJob}"})', # noqa: E501
)],
thresholds=[
Threshold("#808080", 0, 0.0),
Threshold("blue", 1, 0.0),
],
)
dashboard = ( dashboard = (
Dashboard( Dashboard(
title="S3 service", title="S3 service",
@ -775,7 +585,7 @@ dashboard = (
label='Group', label='Group',
multi=True, multi=True,
name='job', name='job',
query='label_values(s3_cloudserver_http_active_requests{namespace="${namespace}", container="${container}"}, job)', # noqa: E501 query='label_values(http_active_requests{namespace="${namespace}", container="${container}"}, job)', # noqa: E501
regex='/(?<value>${zenkoName}-(?<text>\\w*).*)/', regex='/(?<value>${zenkoName}-(?<text>\\w*).*)/',
), ),
Template( Template(
@ -783,7 +593,7 @@ dashboard = (
hide=HIDE_VARIABLE, hide=HIDE_VARIABLE,
label='pod', label='pod',
name='pod', name='pod',
query='label_values(s3_cloudserver_http_active_requests{namespace="${namespace}", container="${container}", job=~"$job"}, pod)', # noqa: E501 query='label_values(http_active_requests{namespace="${namespace}", container="${container}", job=~"$job"}, pod)', # noqa: E501
) )
]), ]),
panels=layout.column([ panels=layout.column([
@ -820,24 +630,6 @@ dashboard = (
top10Error500ByBucket, top10Error500ByBucket,
top10Error5xxByBucket top10Error5xxByBucket
], height=8), ], height=8),
RowPanel(title="Quotas"),
layout.row([
layout.column([
layout.resize([bucketQuotaCounter], width=6, height=4),
layout.resize([accountQuotaCounter], width=6, height=4),
], height=8),
layout.resize([operationsWithUnavailableMetrics], width=6),
quotaByAction,
], height=8),
layout.row([
layout.resize([quotaHealth], width=6),
layout.resize([quotaStatusCode], width=6),
averageQuotaLatencies,
], height=8),
layout.row([
layout.resize([averageQuotaDuration], width=6),
averageMetricsRetrievalLatencies,
], height=8),
]), ]),
) )
.auto_panel_ids() .auto_panel_ids()

View File

@ -45,8 +45,8 @@ then
exit 1 exit 1
fi fi
REGISTRY=${REGISTRY:-"ghcr.io/scality"} REGISTRY=${REGISTRY:-"registry.scality.com"}
PROJECT=${PROJECT:-"cloudserver"} PROJECT=${PROJECT:-"cloudserver-dev"}
set -x set -x
${ORAS} push "${REGISTRY}/${PROJECT}/${NAME_TAG}" "${INPUT_FILE}:${MIME_TYPE}" ${ORAS} push "${REGISTRY}/${PROJECT}/${NAME_TAG}" "${INPUT_FILE}:${MIME_TYPE}"

View File

@ -1,6 +1,6 @@
{ {
"name": "@zenko/cloudserver", "name": "@zenko/cloudserver",
"version": "8.8.27", "version": "8.8.17",
"description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
"main": "index.js", "main": "index.js",
"engines": { "engines": {
@ -21,13 +21,14 @@
"dependencies": { "dependencies": {
"@azure/storage-blob": "^12.12.0", "@azure/storage-blob": "^12.12.0",
"@hapi/joi": "^17.1.0", "@hapi/joi": "^17.1.0",
"arsenal": "git+https://git.yourcmc.ru/vitalif/zenko-arsenal.git#development/8.1", "arsenal": "git+https://github.com/scality/arsenal#e265d0b80ee27b13c4012a90777b0c77399db1fb",
"async": "^2.5.0", "async": "~2.5.0",
"aws-sdk": "^2.905.0", "aws-sdk": "2.905.0",
"bucketclient": "scality/bucketclient#8.1.9",
"bufferutil": "^4.0.6", "bufferutil": "^4.0.6",
"commander": "^2.9.0", "commander": "^2.9.0",
"cron-parser": "^2.11.0", "cron-parser": "^2.11.0",
"diskusage": "^1.1.3", "diskusage": "1.1.3",
"google-auto-auth": "^0.9.1", "google-auto-auth": "^0.9.1",
"http-proxy": "^1.17.0", "http-proxy": "^1.17.0",
"http-proxy-agent": "^4.0.1", "http-proxy-agent": "^4.0.1",
@ -37,45 +38,37 @@
"mongodb": "^5.2.0", "mongodb": "^5.2.0",
"node-fetch": "^2.6.0", "node-fetch": "^2.6.0",
"node-forge": "^0.7.1", "node-forge": "^0.7.1",
"npm-run-all": "^4.1.5", "npm-run-all": "~4.1.5",
"prom-client": "14.2.0", "prom-client": "14.2.0",
"request": "^2.81.0", "request": "^2.81.0",
"scubaclient": "git+https://git.yourcmc.ru/vitalif/zenko-scubaclient.git", "sql-where-parser": "~2.2.1",
"sql-where-parser": "^2.2.1", "utapi": "github:scality/utapi#8.1.13",
"utapi": "git+https://git.yourcmc.ru/vitalif/zenko-utapi.git",
"utf-8-validate": "^5.0.8", "utf-8-validate": "^5.0.8",
"utf8": "^2.1.1", "utf8": "~2.1.1",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"werelogs": "git+https://git.yourcmc.ru/vitalif/zenko-werelogs.git#development/8.1", "vaultclient": "scality/vaultclient#8.3.11",
"werelogs": "scality/werelogs#8.1.2",
"ws": "^5.1.0", "ws": "^5.1.0",
"xml2js": "^0.4.16" "xml2js": "~0.4.16"
},
"overrides": {
"ltgt": "^2.2.0"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "^7.25.2",
"@babel/preset-env": "^7.25.3",
"babel-loader": "^9.1.3",
"bluebird": "^3.3.1", "bluebird": "^3.3.1",
"eslint": "^8.14.0", "eslint": "^8.14.0",
"eslint-config-airbnb-base": "^15.0.0", "eslint-config-airbnb-base": "^13.1.0",
"eslint-config-scality": "git+https://git.yourcmc.ru/vitalif/zenko-eslint-config-scality.git", "eslint-config-scality": "scality/Guidelines#8.2.0",
"eslint-plugin-import": "^2.14.0", "eslint-plugin-import": "^2.14.0",
"eslint-plugin-mocha": "^10.1.0", "eslint-plugin-mocha": "^10.1.0",
"express": "^4.17.1", "express": "^4.17.1",
"ioredis": "^4.9.5", "ioredis": "4.9.5",
"istanbul": "^1.0.0-alpha.2", "istanbul": "1.0.0-alpha.2",
"istanbul-api": "^1.0.0-alpha.13", "istanbul-api": "1.0.0-alpha.13",
"lolex": "^1.4.0", "lolex": "^1.4.0",
"mocha": ">=3.1.2", "mocha": "^2.3.4",
"mocha-junit-reporter": "^1.23.1", "mocha-junit-reporter": "^1.23.1",
"mocha-multi-reporters": "^1.1.7", "mocha-multi-reporters": "^1.1.7",
"node-mocks-http": "^1.5.2", "node-mocks-http": "1.5.2",
"sinon": "^13.0.1", "sinon": "^13.0.1",
"tv4": "^1.2.7", "tv4": "^1.2.7"
"webpack": "^5.93.0",
"webpack-cli": "^5.1.4"
}, },
"scripts": { "scripts": {
"cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server", "cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server",
@ -116,11 +109,10 @@
"utapi_replay": "node lib/utapi/utapiReplay.js", "utapi_replay": "node lib/utapi/utapiReplay.js",
"utapi_reindex": "node lib/utapi/utapiReindex.js", "utapi_reindex": "node lib/utapi/utapiReindex.js",
"management_agent": "node managementAgent.js", "management_agent": "node managementAgent.js",
"test": "CI=true S3BACKEND=mem S3QUOTA=scuba mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit", "test": "CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit",
"test_versionid_base62": "S3_VERSION_ID_ENCODING_TYPE=base62 CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api", "test_versionid_base62": "S3_VERSION_ID_ENCODING_TYPE=base62 CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api",
"test_legacy_location": "CI=true S3QUOTA=scuba S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit", "test_legacy_location": "CI=true S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit",
"test_utapi_v2": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/utapi", "test_utapi_v2": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/utapi",
"test_quota": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/quota",
"multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend", "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend",
"unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem istanbul cover --dir coverage/unit _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit", "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem istanbul cover --dir coverage/unit _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit",
"unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --reporter mocha-junit-reporter --recursive tests/unit" "unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --reporter mocha-junit-reporter --recursive tests/unit"

View File

@ -1,39 +0,0 @@
const AWS = require('aws-sdk');
const S3 = AWS.S3;
const assert = require('assert');
const getConfig = require('../support/config');
const sendRequest = require('../quota/tooling').sendRequest;
const bucket = 'deletequotatestbucket';
const nonExistantBucket = 'deletequotatestnonexistantbucket';
describe('Test delete bucket quota', () => {
let s3;
before(() => {
const config = getConfig('default', { signatureVersion: 'v4' });
s3 = new S3(config);
AWS.config.update(config);
});
beforeEach(done => s3.createBucket({ Bucket: bucket }, done));
afterEach(done => s3.deleteBucket({ Bucket: bucket }, done));
it('should delete the bucket quota', async () => {
try {
await sendRequest('DELETE', '127.0.0.1:8000', `/${bucket}/?quota=true`);
assert.ok(true);
} catch (err) {
assert.fail(`Expected no error, but got ${err}`);
}
});
it('should return no such bucket error', async () => {
try {
await sendRequest('DELETE', '127.0.0.1:8000', `/${nonExistantBucket}/?quota=true`);
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'NoSuchBucket');
}
});
});

View File

@ -1,77 +0,0 @@
const AWS = require('aws-sdk');
const S3 = AWS.S3;
const assert = require('assert');
const getConfig = require('../support/config');
const sendRequest = require('../quota/tooling').sendRequest;
const bucket = 'getquotatestbucket';
const quota = { quota: 1000 };
describe('Test get bucket quota', () => {
let s3;
before(() => {
const config = getConfig('default', { signatureVersion: 'v4' });
s3 = new S3(config);
AWS.config.update(config);
});
beforeEach(done => s3.createBucket({ Bucket: bucket }, done));
afterEach(done => s3.deleteBucket({ Bucket: bucket }, done));
it('should return the quota', async () => {
try {
await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota));
const data = await sendRequest('GET', '127.0.0.1:8000', `/${bucket}/?quota=true`);
assert.strictEqual(data.GetBucketQuota.Name[0], bucket);
assert.strictEqual(data.GetBucketQuota.Quota[0], '1000');
} catch (err) {
assert.fail(`Expected no error, but got ${err}`);
}
});
it('should return no such bucket error', async () => {
try {
await sendRequest('GET', '127.0.0.1:8000', '/test/?quota=true');
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'NoSuchBucket');
}
});
it('should return no such bucket quota', async () => {
try {
await sendRequest('DELETE', '127.0.0.1:8000', `/${bucket}/?quota=true`);
try {
await sendRequest('GET', '127.0.0.1:8000', `/${bucket}/?quota=true`);
assert.fail('Expected NoSuchQuota error');
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'NoSuchQuota');
}
} catch (err) {
assert.fail(`Expected no error, but got ${err}`);
}
});
it('should return no such bucket error', async () => {
try {
await sendRequest('GET', '127.0.0.1:8000', '/test/?quota=true');
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'NoSuchBucket');
}
});
it('should return no such bucket quota', async () => {
try {
await sendRequest('DELETE', '127.0.0.1:8000', `/${bucket}/?quota=true`);
try {
await sendRequest('GET', '127.0.0.1:8000', `/${bucket}/?quota=true`);
assert.fail('Expected NoSuchQuota error');
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'NoSuchQuota');
}
} catch (err) {
assert.fail(`Expected no error, but got ${err}`);
}
});
});

View File

@ -1,70 +0,0 @@
const AWS = require('aws-sdk');
const S3 = AWS.S3;
const assert = require('assert');
const getConfig = require('../support/config');
const sendRequest = require('../quota/tooling').sendRequest;
const bucket = 'updatequotatestbucket';
const nonExistantBucket = 'updatequotatestnonexistantbucket';
const quota = { quota: 2000 };
const negativeQuota = { quota: -1000 };
const wrongquotaFromat = '1000';
const largeQuota = { quota: 1000000000000 };
describe('Test update bucket quota', () => {
let s3;
before(() => {
const config = getConfig('default', { signatureVersion: 'v4' });
s3 = new S3(config);
AWS.config.update(config);
});
beforeEach(done => s3.createBucket({ Bucket: bucket }, done));
afterEach(done => s3.deleteBucket({ Bucket: bucket }, done));
it('should update the quota', async () => {
try {
await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota));
assert.ok(true);
} catch (err) {
assert.fail(`Expected no error, but got ${err}`);
}
});
it('should return no such bucket error', async () => {
try {
await sendRequest('PUT', '127.0.0.1:8000', `/${nonExistantBucket}/?quota=true`, JSON.stringify(quota));
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'NoSuchBucket');
}
});
it('should return error when quota is negative', async () => {
try {
await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(negativeQuota));
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'InvalidArgument');
assert.strictEqual(err.Error.Message[0], 'Quota value must be a positive number');
}
});
it('should return error when quota is not in correct format', async () => {
try {
await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, wrongquotaFromat);
} catch (err) {
assert.strictEqual(err.Error.Code[0], 'InvalidArgument');
assert.strictEqual(err.Error.Message[0], 'Request body must be a JSON object');
}
});
it('should handle large quota values', async () => {
try {
await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(largeQuota));
} catch (err) {
assert.fail(`Expected no error, but got ${err}`);
}
});
});

View File

@ -33,7 +33,7 @@ describe('aws-node-sdk v2auth query tests', function testSuite() {
let s3; let s3;
before(() => { before(() => {
const config = getConfig('default', { signatureVersion: 'v2' }); const config = getConfig('default');
s3 = new S3(config); s3 = new S3(config);
}); });

View File

@ -1,52 +0,0 @@
const fetch = require('node-fetch');
const AWS = require('aws-sdk');
const xml2js = require('xml2js');
const sendRequest = async (method, host, path, body = '', config = null) =>
new Promise(async (resolve, reject) => {
const service = 's3';
const endpoint = new AWS.Endpoint(host);
const request = new AWS.HttpRequest(endpoint);
request.method = method.toUpperCase();
request.path = path;
request.body = body;
request.headers.Host = host;
request.headers['X-Amz-Date'] = new Date().toISOString().replace(/[:\-]|\.\d{3}/g, '');
const sha256hash = AWS.util.crypto.sha256(request.body || '', 'hex');
request.headers['X-Amz-Content-SHA256'] = sha256hash;
request.region = 'us-east-1';
const signer = new AWS.Signers.V4(request, service);
const accessKeyId = config?.accessKey || AWS.config.credentials?.accessKeyId;
const secretAccessKey = config?.secretKey || AWS.config.credentials?.secretAccessKey;
const credentials = new AWS.Credentials(accessKeyId, secretAccessKey);
signer.addAuthorization(credentials, new Date());
const url = `http://${host}${path}`;
const options = {
method: request.method,
headers: request.headers,
};
if (method !== 'GET') {
options.body = request.body;
}
try {
const response = await fetch(url, options);
const text = await response.text();
const result = await xml2js.parseStringPromise(text);
if (result && result.Error) {
reject(result);
} else {
resolve(result);
}
} catch (error) {
reject(error);
}
});
module.exports = {
sendRequest,
};

View File

@ -41,9 +41,9 @@ function _decodeURI(uri) {
* @param {object} [params.headers] - headers and their string values * @param {object} [params.headers] - headers and their string values
* @param {string} [params.path] - URL-encoded request path * @param {string} [params.path] - URL-encoded request path
* @param {object} [params.authCredentials] - authentication credentials * @param {object} [params.authCredentials] - authentication credentials
* @param {string} params.authCredentials.accessKey - access key * @param {object} params.authCredentials.accessKey - access key
* @param {string} params.authCredentials.secretKey - secret key * @param {object} params.authCredentials.secretKey - secret key
* @param {boolean} params.GCP - flag to setup for GCP request * @param {object} params.GCP - flag to setup for GCP request
* @param {string} [params.requestBody] - request body contents * @param {string} [params.requestBody] - request body contents
* @param {string} [params.urlForSignature] - the url to use when signing the * @param {string} [params.urlForSignature] - the url to use when signing the
* request * request
@ -194,42 +194,8 @@ function makeGcpRequest(params, callback) {
makeRequest(options, callback); makeRequest(options, callback);
} }
/** makeBackbeatRequest - utility function to generate a request going
* through backbeat route
* @param {object} params - params for making request
* @param {string} params.method - request method
* @param {string} params.bucket - bucket name
* @param {string} params.objectKey - object key
* @param {string} params.subCommand - subcommand to backbeat
* @param {object} [params.headers] - headers and their string values
* @param {object} [params.authCredentials] - authentication credentials
* @param {object} params.authCredentials.accessKey - access key
* @param {object} params.authCredentials.secretKey - secret key
* @param {string} [params.requestBody] - request body contents
* @param {object} [params.queryObj] - query params
* @param {function} callback - with error and response parameters
* @return {undefined} - and call callback
*/
function makeBackbeatRequest(params, callback) {
const { method, headers, bucket, objectKey, resourceType,
authCredentials, requestBody, queryObj } = params;
const options = {
authCredentials,
hostname: ipAddress,
port: 8000,
method,
headers,
path: `/_/backbeat/${resourceType}/${bucket}/${objectKey}`,
requestBody,
jsonResponse: true,
queryObj,
};
makeRequest(options, callback);
}
module.exports = { module.exports = {
makeRequest, makeRequest,
makeS3Request, makeS3Request,
makeGcpRequest, makeGcpRequest,
makeBackbeatRequest,
}; };

View File

@ -3,11 +3,12 @@ const AWS = require('aws-sdk');
const async = require('async'); const async = require('async');
const crypto = require('crypto'); const crypto = require('crypto');
const { v4: uuidv4 } = require('uuid'); const { v4: uuidv4 } = require('uuid');
const { versioning } = require('arsenal'); const { models, versioning } = require('arsenal');
const versionIdUtils = versioning.VersionID; const versionIdUtils = versioning.VersionID;
const { ObjectMD } = models;
const { makeid } = require('../../unit/helpers'); const { makeid } = require('../../unit/helpers');
const { makeRequest, makeBackbeatRequest } = require('../../functional/raw-node/utils/makeRequest'); const { makeRequest } = require('../../functional/raw-node/utils/makeRequest');
const BucketUtility = const BucketUtility =
require('../../functional/aws-node-sdk/lib/utility/bucket-util'); require('../../functional/aws-node-sdk/lib/utility/bucket-util');
const { const {
@ -145,16 +146,50 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
}); });
} }
/** makeBackbeatRequest - utility function to generate a request going
* through backbeat route
* @param {object} params - params for making request
* @param {string} params.method - request method
* @param {string} params.bucket - bucket name
* @param {string} params.objectKey - object key
* @param {string} params.subCommand - subcommand to backbeat
* @param {object} [params.headers] - headers and their string values
* @param {object} [params.authCredentials] - authentication credentials
* @param {object} params.authCredentials.accessKey - access key
* @param {object} params.authCredentials.secretKey - secret key
* @param {string} [params.requestBody] - request body contents
* @param {function} callback - with error and response parameters
* @return {undefined} - and call callback
*/
function makeBackbeatRequest(params, callback) {
const { method, headers, bucket, objectKey, resourceType,
authCredentials, requestBody, queryObj } = params;
const options = {
authCredentials,
hostname: ipAddress,
port: 8000,
method,
headers,
path: `/_/backbeat/${resourceType}/${bucket}/${objectKey}`,
requestBody,
jsonResponse: true,
queryObj,
};
makeRequest(options, callback);
}
function updateStorageClass(data, storageClass) { function updateStorageClass(data, storageClass) {
let result; let parsedBody;
try { try {
const parsedBody = JSON.parse(JSON.parse(data.body).Body); parsedBody = JSON.parse(data.body);
parsedBody['x-amz-storage-class'] = storageClass;
result = JSON.stringify(parsedBody);
} catch (err) { } catch (err) {
return { error: err }; return { error: err };
} }
const { result, error } = ObjectMD.createFromBlob(parsedBody.Body);
if (error) {
return { error };
}
result.setAmzStorageClass(storageClass);
return { result }; return { result };
} }
@ -304,11 +339,11 @@ describe('backbeat routes', () => {
it('should update metadata of a current null version', done => { it('should update metadata of a current null version', done => {
let objMD; let objMD;
return async.series({ return async.series([
putObject: next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
enableVersioningSource: next => s3.putBucketVersioning( next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next), next),
getMetadata: next => makeBackbeatRequest({ next => makeBackbeatRequest({
method: 'GET', method: 'GET',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -328,7 +363,7 @@ describe('backbeat routes', () => {
objMD = result; objMD = result;
return next(); return next();
}), }),
putMetadata: next => makeBackbeatRequest({ next => makeBackbeatRequest({
method: 'PUT', method: 'PUT',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -337,37 +372,19 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
headObject: next => s3.headObject( next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
{ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
getMetadataAfter: next => makeBackbeatRequest({ ], (err, data) => {
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) { if (err) {
return done(err); return done(err);
} }
const headObjectRes = data[4];
const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null'); assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass); assert.strictEqual(headObjectRes.StorageClass, storageClass);
const getMetadataAfterRes = results.getMetadataAfter; const listObjectVersionsRes = data[5];
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
const expectedMd = JSON.parse(objMD);
expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed
assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd);
const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes; const { Versions } = listObjectVersionsRes;
assert.strictEqual(Versions.length, 1); assert.strictEqual(Versions.length, 1);
@ -381,20 +398,18 @@ describe('backbeat routes', () => {
it('should update metadata of a non-current null version', done => { it('should update metadata of a non-current null version', done => {
let objMD; let objMD;
let expectedVersionId; let expectedVersionId;
return async.series({ return async.series([
putObjectInitial: next => s3.putObject( next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
enableVersioning: next => s3.putBucketVersioning( next),
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next), next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
putObjectAgain: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
if (err) { if (err) {
return next(err); return next(err);
} }
expectedVersionId = data.VersionId; expectedVersionId = data.VersionId;
return next(); return next();
}), }),
getMetadata: next => makeBackbeatRequest({ next => makeBackbeatRequest({
method: 'GET', method: 'GET',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -414,7 +429,7 @@ describe('backbeat routes', () => {
objMD = result; objMD = result;
return next(); return next();
}), }),
putMetadata: next => makeBackbeatRequest({ next => makeBackbeatRequest({
method: 'PUT', method: 'PUT',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -423,36 +438,23 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({ next => s3.listObjectVersions({ Bucket: bucket }, next),
method: 'GET', ], (err, data) => {
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) { if (err) {
return done(err); return done(err);
} }
const headObjectRes = results.headObject; const headObjectRes = data[5];
assert.strictEqual(headObjectRes.VersionId, 'null'); assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass); assert.strictEqual(headObjectRes.StorageClass, storageClass);
const getMetadataAfterRes = results.getMetadataAfter; const listObjectVersionsRes = data[6];
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes; const { Versions } = listObjectVersionsRes;
assert.strictEqual(Versions.length, 2); assert.strictEqual(Versions.length, 2);
const currentVersion = Versions.find(v => v.IsLatest); const currentVersion = Versions.find(v => v.IsLatest);
assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId);
@ -462,160 +464,6 @@ describe('backbeat routes', () => {
}); });
}); });
it('should update metadata of a suspended null version', done => {
let objMD;
return async.series({
suspendVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
putObject: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
enableVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
getMetadata: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, (err, data) => {
if (err) {
return next(err);
}
const { error, result } = updateStorageClass(data, storageClass);
if (error) {
return next(error);
}
objMD = result;
return next();
}),
putUpdatedMetadata: next => makeBackbeatRequest({
method: 'PUT',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD,
}, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
return done(err);
}
const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);
const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;
assert.strictEqual(Versions.length, 1);
const [currentVersion] = Versions;
assertVersionIsNullAndUpdated(currentVersion);
return done();
});
});
it('should update metadata of a suspended null version with internal version id', done => {
let objMD;
return async.series({
suspendVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
putObject: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
enableVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
putObjectTagging: next => s3.putObjectTagging({
Bucket: bucket, Key: keyName, VersionId: 'null',
Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] },
}, next),
getMetadata: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, (err, data) => {
if (err) {
return next(err);
}
const { error, result } = updateStorageClass(data, storageClass);
if (error) {
return next(error);
}
objMD = result;
return next();
}),
putUpdatedMetadata: next => makeBackbeatRequest({
method: 'PUT',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD,
}, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
return done(err);
}
const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);
const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;
assert.strictEqual(Versions.length, 1);
const [currentVersion] = Versions;
assertVersionIsNullAndUpdated(currentVersion);
return done();
});
});
it('should update metadata of a non-version object', done => { it('should update metadata of a non-version object', done => {
let objMD; let objMD;
return async.series([ return async.series([
@ -649,7 +497,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -711,7 +559,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -774,7 +622,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -845,7 +693,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -910,7 +758,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -970,7 +818,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -1031,7 +879,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
@ -1094,7 +942,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next), next),
@ -1174,7 +1022,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -1249,7 +1097,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -1321,7 +1169,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
@ -1395,7 +1243,7 @@ describe('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD, requestBody: objMD.getSerialized(),
}, next), }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next), next),

File diff suppressed because it is too large Load Diff

View File

@ -1,785 +0,0 @@
const async = require('async');
const assert = require('assert');
const { S3 } = require('aws-sdk');
const getConfig = require('../functional/aws-node-sdk/test/support/config');
const { Scuba: MockScuba, inflightFlushFrequencyMS } = require('../utilities/mock/Scuba');
const sendRequest = require('../functional/aws-node-sdk/test/quota/tooling').sendRequest;
const memCredentials = require('../functional/aws-node-sdk/lib/json/mem_credentials.json');
const metadata = require('../../lib/metadata/wrapper');
const { fakeMetadataArchive } = require('../functional/aws-node-sdk/test/utils/init');
const { config: s3Config } = require('../../lib/Config');
let mockScuba = null;
let s3Client = null;
const quota = { quota: 1000 };
function wait(timeoutMs, cb) {
if (s3Config.isQuotaInflightEnabled()) {
return setTimeout(cb, timeoutMs);
}
return cb();
}
function createBucket(bucket, locked, cb) {
const config = {
Bucket: bucket,
};
if (locked) {
config.ObjectLockEnabledForBucket = true;
}
return s3Client.createBucket(config, (err, data) => {
assert.ifError(err);
return cb(err, data);
});
}
function configureBucketVersioning(bucket, cb) {
return s3Client.putBucketVersioning({
Bucket: bucket,
VersioningConfiguration: {
Status: 'Enabled',
},
}, (err, data) => {
assert.ifError(err);
return cb(err, data);
});
}
function putObjectLockConfiguration(bucket, cb) {
return s3Client.putObjectLockConfiguration({
Bucket: bucket,
ObjectLockConfiguration: {
ObjectLockEnabled: 'Enabled',
Rule: {
DefaultRetention: {
Mode: 'GOVERNANCE',
Days: 1,
},
},
},
}, (err, data) => {
assert.ifError(err);
return cb(err, data);
});
}
function deleteBucket(bucket, cb) {
return s3Client.deleteBucket({
Bucket: bucket,
}, err => {
assert.ifError(err);
return cb(err);
});
}
function putObject(bucket, key, size, cb) {
return s3Client.putObject({
Bucket: bucket,
Key: key,
Body: Buffer.alloc(size),
}, (err, data) => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, size);
}
return cb(err, data);
});
}
function putObjectWithCustomHeader(bucket, key, size, vID, cb) {
const request = s3Client.putObject({
Bucket: bucket,
Key: key,
Body: Buffer.alloc(size),
});
request.on('build', () => {
request.httpRequest.headers['x-scal-s3-version-id'] = vID;
});
return request.send((err, data) => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, 0);
}
return cb(err, data);
});
}
function copyObject(bucket, key, sourceSize, cb) {
return s3Client.copyObject({
Bucket: bucket,
CopySource: `/${bucket}/${key}`,
Key: `${key}-copy`,
}, (err, data) => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, sourceSize);
}
return cb(err, data);
});
}
function deleteObject(bucket, key, size, cb) {
return s3Client.deleteObject({
Bucket: bucket,
Key: key,
}, err => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, -size);
}
assert.ifError(err);
return cb(err);
});
}
function deleteVersionID(bucket, key, versionId, size, cb) {
return s3Client.deleteObject({
Bucket: bucket,
Key: key,
VersionId: versionId,
}, (err, data) => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, -size);
}
return cb(err, data);
});
}
function objectMPU(bucket, key, parts, partSize, callback) {
let ETags = [];
let uploadId = null;
const partNumbers = Array.from(Array(parts).keys());
const initiateMPUParams = {
Bucket: bucket,
Key: key,
};
return async.waterfall([
next => s3Client.createMultipartUpload(initiateMPUParams,
(err, data) => {
if (err) {
return next(err);
}
uploadId = data.UploadId;
return next();
}),
next =>
async.mapLimit(partNumbers, 1, (partNumber, callback) => {
const uploadPartParams = {
Bucket: bucket,
Key: key,
PartNumber: partNumber + 1,
UploadId: uploadId,
Body: Buffer.alloc(partSize),
};
return s3Client.uploadPart(uploadPartParams,
(err, data) => {
if (err) {
return callback(err);
}
return callback(null, data.ETag);
});
}, (err, results) => {
if (err) {
return next(err);
}
ETags = results;
return next();
}),
next => {
const params = {
Bucket: bucket,
Key: key,
MultipartUpload: {
Parts: partNumbers.map(n => ({
ETag: ETags[n],
PartNumber: n + 1,
})),
},
UploadId: uploadId,
};
return s3Client.completeMultipartUpload(params, next);
},
], err => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, parts * partSize);
}
return callback(err, uploadId);
});
}
function abortMPU(bucket, key, uploadId, size, callback) {
return s3Client.abortMultipartUpload({
Bucket: bucket,
Key: key,
UploadId: uploadId,
}, (err, data) => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, -size);
}
return callback(err, data);
});
}
function uploadPartCopy(bucket, key, partNumber, partSize, sleepDuration, keyToCopy, callback) {
const ETags = [];
let uploadId = null;
const parts = 5;
const partNumbers = Array.from(Array(parts).keys());
const initiateMPUParams = {
Bucket: bucket,
Key: key,
};
if (!s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, parts * partSize);
}
return async.waterfall([
next => s3Client.createMultipartUpload(initiateMPUParams,
(err, data) => {
if (err) {
return next(err);
}
uploadId = data.UploadId;
return next();
}),
next => {
const uploadPartParams = {
Bucket: bucket,
Key: key,
PartNumber: partNumber + 1,
UploadId: uploadId,
Body: Buffer.alloc(partSize),
};
return s3Client.uploadPart(uploadPartParams, (err, data) => {
if (err) {
return next(err);
}
ETags[partNumber] = data.ETag;
return next();
});
},
next => wait(sleepDuration, next),
next => {
const copyPartParams = {
Bucket: bucket,
CopySource: `/${bucket}/${keyToCopy}`,
Key: `${key}-copy`,
PartNumber: partNumber + 1,
UploadId: uploadId,
};
return s3Client.uploadPartCopy(copyPartParams, (err, data) => {
if (err) {
return next(err);
}
ETags[partNumber] = data.ETag;
return next(null, data.ETag);
});
},
next => {
const params = {
Bucket: bucket,
Key: key,
MultipartUpload: {
Parts: partNumbers.map(n => ({
ETag: ETags[n],
PartNumber: n + 1,
})),
},
UploadId: uploadId,
};
return s3Client.completeMultipartUpload(params, next);
},
], err => {
if (err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, -(parts * partSize));
}
return callback(err, uploadId);
});
}
function restoreObject(bucket, key, size, callback) {
return s3Client.restoreObject({
Bucket: bucket,
Key: key,
RestoreRequest: {
Days: 1,
},
}, (err, data) => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, size);
}
return callback(err, data);
});
}
function multiObjectDelete(bucket, keys, size, callback) {
if (!s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, -size);
}
return s3Client.deleteObjects({
Bucket: bucket,
Delete: {
Objects: keys.map(key => ({ Key: key })),
},
}, (err, data) => {
if (err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, size);
}
return callback(err, data);
});
}
(process.env.S3METADATA === 'mongodb' ? describe : describe.skip)('quota evaluation with scuba metrics',
function t() {
this.timeout(30000);
const scuba = new MockScuba();
const putQuotaVerb = 'PUT';
const config = {
accessKey: memCredentials.default.accessKey,
secretKey: memCredentials.default.secretKey,
};
mockScuba = scuba;
before(done => {
const config = getConfig('default', { signatureVersion: 'v4', maxRetries: 0 });
s3Client = new S3(config);
scuba.start();
return metadata.setup(err => wait(2000, () => done(err)));
});
afterEach(() => {
scuba.reset();
});
after(() => {
scuba.stop();
});
it('should return QuotaExceeded when trying to PutObject in a bucket with quota', done => {
const bucket = 'quota-test-bucket1';
const key = 'quota-test-object';
const size = 1024;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => deleteBucket(bucket, next),
], done);
});
it('should return QuotaExceeded when trying to copyObject in a versioned bucket with quota', done => {
const bucket = 'quota-test-bucket12';
const key = 'quota-test-object';
const size = 900;
let vID = null;
return async.series([
next => createBucket(bucket, false, next),
next => configureBucketVersioning(bucket, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, data) => {
assert.ifError(err);
vID = data.VersionId;
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => copyObject(bucket, key, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => deleteVersionID(bucket, key, vID, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should return QuotaExceeded when trying to CopyObject in a bucket with quota', done => {
const bucket = 'quota-test-bucket2';
const key = 'quota-test-object';
const size = 900;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => copyObject(bucket, key, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => deleteObject(bucket, key, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should return QuotaExceeded when trying to complete MPU in a bucket with quota', done => {
const bucket = 'quota-test-bucket3';
const key = 'quota-test-object';
const parts = 5;
const partSize = 1024 * 1024 * 6;
let uploadId = null;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => objectMPU(bucket, key, parts, partSize, (err, _uploadId) => {
uploadId = _uploadId;
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => abortMPU(bucket, key, uploadId, 0, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
return next();
},
next => deleteBucket(bucket, next),
], done);
});
it('should not return QuotaExceeded if the quota is not exceeded', done => {
const bucket = 'quota-test-bucket4';
const key = 'quota-test-object';
const size = 300;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, err => {
assert.ifError(err);
return next();
}),
next => deleteObject(bucket, key, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should not evaluate quotas if the backend is not available', done => {
scuba.stop();
const bucket = 'quota-test-bucket5';
const key = 'quota-test-object';
const size = 1024;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, err => {
assert.ifError(err);
return next();
}),
next => deleteObject(bucket, key, size, next),
next => deleteBucket(bucket, next),
], err => {
assert.ifError(err);
scuba.start();
return wait(2000, done);
});
});
it('should return QuotaExceeded when trying to copy a part in a bucket with quota', done => {
const bucket = 'quota-test-bucket6';
const key = 'quota-test-object-copy';
const keyToCopy = 'quota-test-existing';
const parts = 5;
const partSize = 1024 * 1024 * 6;
let uploadId = null;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify({ quota: Math.round(partSize * 2.5) }), config)
.then(() => next()).catch(err => next(err)),
next => putObject(bucket, keyToCopy, partSize, next),
next => uploadPartCopy(bucket, key, parts, partSize, inflightFlushFrequencyMS * 2, keyToCopy,
(err, _uploadId) => {
uploadId = _uploadId;
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => abortMPU(bucket, key, uploadId, parts * partSize, next),
next => deleteObject(bucket, keyToCopy, partSize, next),
next => deleteBucket(bucket, next),
], done);
});
it('should return QuotaExceeded when trying to restore an object in a bucket with quota', done => {
const bucket = 'quota-test-bucket7';
const key = 'quota-test-object';
const size = 900;
let vID = null;
return async.series([
next => createBucket(bucket, false, next),
next => configureBucketVersioning(bucket, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, data) => {
assert.ifError(err);
vID = data.VersionId;
return next();
}),
next => fakeMetadataArchive(bucket, key, vID, {
archiveInfo: {},
}, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => restoreObject(bucket, key, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => deleteVersionID(bucket, key, vID, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should not update the inflights if the quota check is passing but the object is already restored', done => {
const bucket = 'quota-test-bucket14';
const key = 'quota-test-object';
const size = 100;
let vID = null;
return async.series([
next => createBucket(bucket, false, next),
next => configureBucketVersioning(bucket, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, data) => {
assert.ifError(err);
vID = data.VersionId;
return next();
}),
next => fakeMetadataArchive(bucket, key, vID, {
archiveInfo: {},
restoreRequestedAt: new Date(0).toString(),
restoreCompletedAt: new Date(0).toString() + 1,
restoreRequestedDays: 5,
}, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => restoreObject(bucket, key, 0, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should allow writes after deleting data with quotas', done => {
const bucket = 'quota-test-bucket8';
const key = 'quota-test-object';
const size = 400;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, `${key}1`, size, err => {
assert.ifError(err);
return next();
}),
next => putObject(bucket, `${key}2`, size, err => {
assert.ifError(err);
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => putObject(bucket, `${key}3`, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size * 2);
return next();
},
next => wait(inflightFlushFrequencyMS * 2, next),
next => deleteObject(bucket, `${key}2`, size, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => putObject(bucket, `${key}4`, size, err => {
assert.ifError(err);
return next();
}),
next => deleteObject(bucket, `${key}1`, size, next),
next => deleteObject(bucket, `${key}3`, size, next),
next => deleteObject(bucket, `${key}4`, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should not increase the inflights when the object is being rewritten with a smaller object', done => {
const bucket = 'quota-test-bucket9';
const key = 'quota-test-object';
const size = 400;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, err => {
assert.ifError(err);
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => putObject(bucket, key, size - 100, err => {
assert.ifError(err);
if (!s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, -size);
}
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size - 100);
return next();
},
next => deleteObject(bucket, key, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should decrease the inflights when performing multi object delete', done => {
const bucket = 'quota-test-bucket10';
const key = 'quota-test-object';
const size = 400;
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, `${key}1`, size, err => {
assert.ifError(err);
return next();
}
),
next => putObject(bucket, `${key}2`, size, err => {
assert.ifError(err);
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 2, err => {
assert.ifError(err);
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
return next();
},
next => deleteBucket(bucket, next),
], done);
});
it('should not update the inflights if the API errored after evaluating quotas (deletion)', done => {
const bucket = 'quota-test-bucket11';
const key = 'quota-test-object';
const size = 100;
let vID = null;
return async.series([
next => createBucket(bucket, true, next),
next => putObjectLockConfiguration(bucket, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, val) => {
assert.ifError(err);
vID = val.VersionId;
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => deleteVersionID(bucket, key, vID, size, err => {
assert.strictEqual(err.code, 'AccessDenied');
next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
], done);
});
it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header',
done => {
const bucket = 'quota-test-bucket13';
const key = 'quota-test-object';
const size = 100;
let vID = null;
return async.series([
next => createBucket(bucket, true, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, val) => {
assert.ifError(err);
vID = val.VersionId;
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => fakeMetadataArchive(bucket, key, vID, {
archiveInfo: {},
restoreRequestedAt: new Date(0).toISOString(),
restoreRequestedDays: 7,
}, next),
// Simulate the real restore
next => putObjectWithCustomHeader(bucket, key, size, vID, err => {
assert.ifError(err);
return next();
}),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should allow a restore if the quota is full but the objet fits with its reserved storage space',
done => {
const bucket = 'quota-test-bucket15';
const key = 'quota-test-object';
const size = 1000;
let vID = null;
return async.series([
next => createBucket(bucket, true, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, val) => {
assert.ifError(err);
vID = val.VersionId;
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => fakeMetadataArchive(bucket, key, vID, {
archiveInfo: {},
restoreRequestedAt: new Date(0).toISOString(),
restoreRequestedDays: 7,
}, next),
// Put an object, the quota should be exceeded
next => putObject(bucket, `${key}-2`, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
// Simulate the real restore
next => putObjectWithCustomHeader(bucket, key, size, vID, err => {
assert.ifError(err);
return next();
}),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
next => deleteBucket(bucket, next),
], done);
});
});

View File

@ -350,108 +350,6 @@ describe('Config', () => {
}); });
}); });
describe('scuba option setup', () => {
let oldConfig;
before(() => {
oldConfig = process.env.S3_CONFIG_FILE;
process.env.S3_CONFIG_FILE =
'tests/unit/testConfigs/allOptsConfig/config.json';
});
after(() => {
process.env.S3_CONFIG_FILE = oldConfig;
});
it('should set up scuba', () => {
const { ConfigObject } = require('../../lib/Config');
const config = new ConfigObject();
assert.deepStrictEqual(
config.scuba,
{
host: 'localhost',
port: 8100,
},
);
});
it('should use environment variables for scuba', () => {
setEnv('SCUBA_HOST', 'scubahost');
setEnv('SCUBA_PORT', 1234);
const { ConfigObject } = require('../../lib/Config');
const config = new ConfigObject();
assert.deepStrictEqual(
config.scuba,
{
host: 'scubahost',
port: 1234,
},
);
});
});
describe('quota option setup', () => {
let oldConfig;
before(() => {
oldConfig = process.env.S3_CONFIG_FILE;
process.env.S3_CONFIG_FILE =
'tests/unit/testConfigs/allOptsConfig/config.json';
});
after(() => {
process.env.S3_CONFIG_FILE = oldConfig;
});
it('should set up quota', () => {
const { ConfigObject } = require('../../lib/Config');
const config = new ConfigObject();
assert.deepStrictEqual(
config.quota,
{
maxStaleness: 24 * 60 * 60 * 1000,
enableInflights: false,
},
);
});
it('should use environment variables for scuba', () => {
setEnv('QUOTA_MAX_STALENESS_MS', 1234);
setEnv('QUOTA_ENABLE_INFLIGHTS', 'true');
const { ConfigObject } = require('../../lib/Config');
const config = new ConfigObject();
assert.deepStrictEqual(
config.quota,
{
maxStaleness: 1234,
enableInflights: true,
},
);
});
it('should use the default if the maxStaleness is not a number', () => {
setEnv('QUOTA_MAX_STALENESS_MS', 'notanumber');
setEnv('QUOTA_ENABLE_INFLIGHTS', 'true');
const { ConfigObject } = require('../../lib/Config');
const config = new ConfigObject();
assert.deepStrictEqual(
config.quota,
{
maxStaleness: 24 * 60 * 60 * 1000,
enableInflights: true,
},
);
});
});
describe('utapi option setup', () => { describe('utapi option setup', () => {
let oldConfig; let oldConfig;

View File

@ -15,15 +15,13 @@ const sourceObject = 'objectsource';
const sourceVersionId = 'vid1'; const sourceVersionId = 'vid1';
describe('prepareRequestContexts', () => { describe('prepareRequestContexts', () => {
it('should return s3:DeleteObject if multiObjectDelete method', () => { it('should return null if multiObjectDelete method', () => {
const apiMethod = 'multiObjectDelete'; const apiMethod = 'multiObjectDelete';
const request = makeRequest(); const request = makeRequest();
const results = prepareRequestContexts(apiMethod, request, sourceBucket, const results = prepareRequestContexts(apiMethod, request, sourceBucket,
sourceObject, sourceVersionId); sourceObject, sourceVersionId);
assert.strictEqual(results.length, 1); assert.strictEqual(results, null);
const expectedAction = 's3:DeleteObject';
assert.strictEqual(results[0].getAction(), expectedAction);
}); });
it('should return s3:PutObjectVersion request context action for objectPut method with x-scal-s3-version-id' + it('should return s3:PutObjectVersion request context action for objectPut method with x-scal-s3-version-id' +

View File

@ -8,8 +8,6 @@ const {
} = require('../../../../lib/api/apiUtils/object/coldStorage'); } = require('../../../../lib/api/apiUtils/object/coldStorage');
const { DummyRequestLogger } = require('../../helpers'); const { DummyRequestLogger } = require('../../helpers');
const { ObjectMD, ObjectMDArchive } = require('arsenal/build/lib/models'); const { ObjectMD, ObjectMDArchive } = require('arsenal/build/lib/models');
const { config } = require('../../../../lib/Config');
const { scaledMsPerDay } = config.getTimeOptions();
const log = new DummyRequestLogger(); const log = new DummyRequestLogger();
const oneDay = 24 * 60 * 60 * 1000; const oneDay = 24 * 60 * 60 * 1000;
@ -229,7 +227,8 @@ describe('cold storage', () => {
assert.strictEqual(objectMd.archive.restoreCompletedAt, restoreCompletedAt); assert.strictEqual(objectMd.archive.restoreCompletedAt, restoreCompletedAt);
assert.strictEqual(objectMd.archive.restoreWillExpireAt.getTime(), assert.strictEqual(objectMd.archive.restoreWillExpireAt.getTime(),
objectMd.archive.restoreRequestedAt.getTime() + (5 * scaledMsPerDay)); objectMd.archive.restoreRequestedAt.getTime() + 5 * oneDay
);
assert.deepEqual(objectMd['x-amz-restore'], { assert.deepEqual(objectMd['x-amz-restore'], {
'ongoing-request': false, 'ongoing-request': false,
'expiry-date': objectMd.archive.restoreWillExpireAt, 'expiry-date': objectMd.archive.restoreWillExpireAt,

View File

@ -1,623 +0,0 @@
const sinon = require('sinon');
const assert = require('assert');
const { config } = require('../../../../../lib/Config');
const {
validateQuotas,
processBytesToWrite,
isMetricStale,
} = require('../../../../../lib/api/apiUtils/quotas/quotaUtils');
const QuotaService = require('../../../../../lib/quotas/quotas');
const mockLog = {
warn: sinon.stub(),
debug: sinon.stub(),
};
const mockBucket = {
getQuota: () => 100,
getName: () => 'bucketName',
getCreationDate: () => '2022-01-01T00:00:00.000Z',
};
const mockBucketNoQuota = {
getQuota: () => 0,
getName: () => 'bucketName',
getCreationDate: () => '2022-01-01T00:00:00.000Z',
};
describe('validateQuotas (buckets)', () => {
const request = {
getQuota: () => 100,
};
beforeEach(() => {
config.scuba = {
host: 'localhost',
port: 8080,
};
config.quota = {
maxStaleness: 24 * 60 * 60 * 1000,
enableInflights: true,
};
config.isQuotaEnabled = sinon.stub().returns(true);
QuotaService.enabled = true;
QuotaService._getLatestMetricsCallback = sinon.stub().resolves({});
request.finalizerHooks = [];
});
afterEach(() => {
sinon.restore();
});
it('should return null if quota is <= 0', done => {
validateQuotas(request, mockBucketNoQuota, {}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
});
});
it('should return null if scuba is disabled', done => {
QuotaService.enabled = false;
validateQuotas(request, mockBucket, {}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
});
});
it('should return null if metrics retrieval fails', done => {
QuotaService.enabled = true;
const error = new Error('Failed to get metrics');
QuotaService._getLatestMetricsCallback.yields(error);
validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectPut',
inflight: 1,
}
), true);
done();
});
});
it('should return errors.QuotaExceeded if quota is exceeded', done => {
const result1 = {
bytesTotal: 150,
};
const result2 = {
bytesTotal: 120,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.strictEqual(err.is.QuotaExceeded, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1);
assert.strictEqual(request.finalizerHooks.length, 1);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectPut',
inflight: 1,
}
), true);
done();
});
});
it('should not return QuotaExceeded if the quotas are exceeded but operation is a delete', done => {
const result1 = {
bytesTotal: 150,
};
const result2 = {
bytesTotal: 120,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectDelete',
inflight: -50,
}
), true);
done();
});
});
it('should return null if quota is not exceeded', done => {
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore',
true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectRestore',
inflight: true,
}
), true);
done();
});
});
it('should not include the inflights in the request if they are disabled', done => {
config.quota.enableInflights = false;
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore',
true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectRestore',
inflight: undefined,
}
), true);
done();
});
});
it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => {
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectPut'], 'objectPut',
true, true, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectPut',
inflight: 0,
}
), true);
done();
});
});
});
describe('validateQuotas (with accounts)', () => {
const request = {
getQuota: () => 100,
};
beforeEach(() => {
config.scuba = {
host: 'localhost',
port: 8080,
};
config.quota = {
maxStaleness: 24 * 60 * 60 * 1000,
enableInflights: true,
};
request.finalizerHooks = [];
config.isQuotaEnabled = sinon.stub().returns(true);
QuotaService.enabled = true;
QuotaService._getLatestMetricsCallback = sinon.stub().resolves({});
});
afterEach(() => {
sinon.restore();
});
it('should return null if quota is <= 0', done => {
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 0,
}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
});
});
it('should not return null if bucket quota is <= 0 but account quota is > 0', done => {
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 1000,
}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
});
});
it('should return null if scuba is disabled', done => {
QuotaService.enabled = false;
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
});
});
it('should return null if metrics retrieval fails', done => {
QuotaService.enabled = true;
const error = new Error('Failed to get metrics');
QuotaService._getLatestMetricsCallback.yields(error);
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectPut',
inflight: 1,
}
), true);
done();
});
});
it('should return errors.QuotaExceeded if quota is exceeded', done => {
const result1 = {
bytesTotal: 150,
};
const result2 = {
bytesTotal: 120,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 100,
}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.strictEqual(err.is.QuotaExceeded, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1);
assert.strictEqual(request.finalizerHooks.length, 1);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'account',
'test_1',
null,
{
action: 'objectPut',
inflight: 1,
}
), true);
done();
});
});
it('should not return QuotaExceeded if the quotas are exceeded but operation is a delete', done => {
const result1 = {
bytesTotal: 150,
};
const result2 = {
bytesTotal: 120,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 1000,
}, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'account',
'test_1',
null,
{
action: 'objectDelete',
inflight: -50,
}
), true);
done();
});
});
it('should return null if quota is not exceeded', done => {
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'account',
'test_1',
null,
{
action: 'objectRestore',
inflight: true,
}
), true);
done();
});
});
it('should return quota exceeded if account and bucket quotas are different', done => {
const result1 = {
bytesTotal: 150,
};
const result2 = {
bytesTotal: 120,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.strictEqual(err.is.QuotaExceeded, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 2);
assert.strictEqual(request.finalizerHooks.length, 1);
done();
});
});
it('should update the request with one function per action to clear quota updates', done => {
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'account',
'test_1',
null,
{
action: 'objectRestore',
inflight: true,
}
), true);
done();
});
});
it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => {
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectPut'], 'objectPut', true, true, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'account',
'test_1',
null,
{
action: 'objectPut',
inflight: 0,
}
), true);
done();
});
});
});
describe('processBytesToWrite', () => {
let bucket;
let versionId;
let contentLength;
let objMD;
beforeEach(() => {
bucket = {
isVersioningEnabled: sinon.stub(),
};
versionId = '';
contentLength = 0;
objMD = null;
});
it('should return a negative number if the operation is a delete and bucket is not versioned', () => {
bucket.isVersioningEnabled.returns(false);
objMD = { 'content-length': 100 };
const bytes = processBytesToWrite('objectDelete', bucket, versionId, contentLength, objMD);
assert.strictEqual(bytes, -100);
});
it('should return 0 if the operation is a delete and bucket is versioned', () => {
bucket.isVersioningEnabled.returns(true);
objMD = { 'content-length': 100 };
const bytes = processBytesToWrite('objectDelete', bucket, versionId, contentLength, objMD);
assert.strictEqual(bytes, 0);
});
it('should return a negative number for a versioned bucket with a versionid deletion', () => {
bucket.isVersioningEnabled.returns(true);
objMD = { 'content-length': 100 };
versionId = 'versionId';
const bytes = processBytesToWrite('objectDelete', bucket, versionId, contentLength, objMD);
assert.strictEqual(bytes, -100);
});
it('should return 0 for a delete operation if the object metadata is missing', () => {
bucket.isVersioningEnabled.returns(true);
objMD = null;
const bytes = processBytesToWrite('objectDelete', bucket, versionId, contentLength, objMD);
assert.strictEqual(bytes, 0);
});
it('should return the object metadata content length for a restore object operation', () => {
bucket.isVersioningEnabled.returns(true);
objMD = { 'content-length': 100 };
contentLength = 150;
const bytes = processBytesToWrite('objectRestore', bucket, versionId, contentLength, objMD);
assert.strictEqual(bytes, 100);
});
it('should return the difference of the content length if the object is being replaced', () => {
bucket.isVersioningEnabled.returns(false);
objMD = { 'content-length': 100 };
contentLength = 150;
const bytes = processBytesToWrite('objectPut', bucket, versionId, contentLength, objMD);
assert.strictEqual(bytes, 50);
});
it('should return content length if the object is being replaced and the bucket is versioned', () => {
bucket.isVersioningEnabled.returns(true);
objMD = { 'content-length': 100 };
contentLength = 150;
const bytes = processBytesToWrite('objectPut', bucket, versionId, contentLength, objMD);
assert.strictEqual(bytes, contentLength);
});
it('should return content length of the object metadata if the action is a copy (getObject authz)', () => {
bucket.isVersioningEnabled.returns(true);
objMD = { 'content-length': 100 };
const bytes = processBytesToWrite('objectCopy', bucket, versionId, 0, objMD);
assert.strictEqual(bytes, 100);
});
it('should return content length of the object metadata if the action is a copy part (getObject authz)', () => {
bucket.isVersioningEnabled.returns(true);
objMD = { 'content-length': 100 };
const bytes = processBytesToWrite('objectPutCopyPart', bucket, versionId, 0, objMD);
assert.strictEqual(bytes, 100);
});
it('should detect object replacement during copy object operation on a non versioned bucket', () => {
bucket.isVersioningEnabled.returns(false);
objMD = { 'content-length': 100 };
const destObjMD = { 'content-length': 20 };
const bytes = processBytesToWrite('objectCopy', bucket, versionId, 0, objMD, destObjMD);
assert.strictEqual(bytes, 80);
});
it('should not detect object replacement during copy object operation if the bucket is versioned', () => {
bucket.isVersioningEnabled.returns(true);
objMD = { 'content-length': 100 };
const destObjMD = { 'content-length': 20 };
const bytes = processBytesToWrite('objectCopy', bucket, versionId, 0, objMD, destObjMD);
assert.strictEqual(bytes, 100);
});
});
describe('isMetricStale', () => {
const metric = {
date: new Date().toISOString(),
};
const resourceType = 'bucket';
const resourceName = 'bucketName';
const action = 'objectPut';
const inflight = 1;
const log = {
warn: sinon.stub(),
};
it('should return false if the metric is not stale', () => {
const result = isMetricStale(metric, resourceType, resourceName, action, inflight, log);
assert.strictEqual(result, false);
assert.strictEqual(log.warn.called, false);
});
it('should return true and log a warning if the metric is stale', () => {
const staleDate = new Date(Date.now() - 24 * 60 * 60 * 1000 - 1);
metric.date = staleDate.toISOString();
const result = isMetricStale(metric, resourceType, resourceName, action, inflight, log);
assert.strictEqual(result, true);
assert.strictEqual(log.warn.calledOnce, true);
});
});

View File

@ -1,74 +0,0 @@
const assert = require('assert');
const { bucketPut } = require('../../../lib/api/bucketPut');
const { cleanup,
DummyRequestLogger,
makeAuthInfo,
TaggingConfigTester }
= require('../helpers');
const bucketPutTagging = require('../../../lib/api/bucketPutTagging');
const bucketGetTagging = require('../../../lib/api/bucketGetTagging');
const bucketDeleteTagging = require('../../../lib/api/bucketDeleteTagging');
const log = new DummyRequestLogger();
const authInfo = makeAuthInfo('accessKey1');
const bucketName = 'bucketGetTaggingTest';
const testBucketPutRequest = {
bucketName,
headers: { host: `${bucketName}.s3.amazonaws.com` },
url: '/',
actionImplicitDenies: false,
};
describe('deleteBucketTagging API', () => {
beforeEach(done => {
cleanup();
bucketPut(authInfo, testBucketPutRequest, log, done);
});
afterEach(() => cleanup());
it('should delete tags resource', done => {
const taggingUtil = new TaggingConfigTester();
const testBucketPutTaggingRequest = taggingUtil
.createBucketTaggingRequest('PUT', bucketName);
bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => {
assert.strictEqual(err, undefined);
const testBucketGetTaggingRequest = taggingUtil
.createBucketTaggingRequest('GET', bucketName);
return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log,
(err, xml) => {
assert.ifError(err);
assert.strictEqual(xml, taggingUtil.constructXml());
const testBucketDeleteTaggingRequest = taggingUtil
.createBucketTaggingRequest('DELETE', bucketName);
return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest,
log, err => {
assert.ifError(err);
return bucketGetTagging(authInfo, testBucketGetTaggingRequest,
log, err => {
assert(err.NoSuchTagSet);
return done();
});
});
});
});
});
it('should return access denied if the authorization check fails', done => {
const taggingUtil = new TaggingConfigTester();
const testBucketPutTaggingRequest = taggingUtil
.createBucketTaggingRequest('PUT', bucketName);
bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => {
assert.ifError(err);
const testBucketDeleteTaggingRequest = taggingUtil
.createBucketTaggingRequest('DELETE', bucketName, null, true);
return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest,
log, err => {
assert(err.AccessDenied);
return done();
});
});
});
});

View File

@ -1,67 +0,0 @@
const assert = require('assert');
const { bucketPut } = require('../../../lib/api/bucketPut');
const { cleanup,
DummyRequestLogger,
makeAuthInfo,
TaggingConfigTester,
} = require('../helpers');
const bucketPutTagging = require('../../../lib/api/bucketPutTagging');
const bucketGetTagging = require('../../../lib/api/bucketGetTagging');
const log = new DummyRequestLogger();
const authInfo = makeAuthInfo('accessKey1');
const bucketName = 'bucketGetTaggingTest';
const testBucketPutRequest = {
bucketName,
headers: { host: `${bucketName}.s3.amazonaws.com` },
url: '/',
actionImplicitDenies: false,
};
describe('getBucketTagging API', () => {
beforeEach(done => {
cleanup();
bucketPut(authInfo, testBucketPutRequest, log, done);
});
afterEach(() => cleanup());
it('should return tags resource', done => {
const taggingUtil = new TaggingConfigTester();
const testBucketPutTaggingRequest = taggingUtil
.createBucketTaggingRequest('PUT', bucketName);
bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => {
assert.strictEqual(err, undefined);
const testBucketGetTaggingRequest = taggingUtil
.createBucketTaggingRequest('GET', bucketName);
return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log,
(err, xml) => {
if (err) {
process.stdout.write(`Err getting object tagging ${err}`);
return done(err);
}
assert.strictEqual(xml, taggingUtil.constructXml());
return done();
});
});
});
it('should return access denied if the authorization check fails', done => {
const taggingUtil = new TaggingConfigTester();
const testBucketPutTaggingRequest = taggingUtil
.createBucketTaggingRequest('PUT', bucketName);
bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => {
assert.strictEqual(err, undefined);
const testBucketGetTaggingRequest = taggingUtil
.createBucketTaggingRequest('GET', bucketName, true);
const badAuthInfo = makeAuthInfo('accessKey2');
return bucketGetTagging(badAuthInfo, testBucketGetTaggingRequest, log,
err => {
assert.strictEqual(err.AccessDenied, true);
return done();
});
});
});
});

View File

@ -1,53 +0,0 @@
const assert = require('assert');
const { bucketPut } = require('../../../lib/api/bucketPut');
const { cleanup,
DummyRequestLogger,
makeAuthInfo,
TaggingConfigTester,
} = require('../helpers');
const bucketPutTagging = require('../../../lib/api/bucketPutTagging');
const log = new DummyRequestLogger();
const authInfo = makeAuthInfo('accessKey1');
const bucketName = 'bucketGetTaggingTest';
const testBucketPutRequest = {
bucketName,
headers: { host: `${bucketName}.s3.amazonaws.com` },
url: '/',
actionImplicitDenies: false,
};
describe('putBucketTagging API', () => {
beforeEach(done => {
cleanup();
bucketPut(authInfo, testBucketPutRequest, log, done);
});
afterEach(() => cleanup());
it('should set tags resource', done => {
const taggingUtil = new TaggingConfigTester();
const testBucketPutTaggingRequest = taggingUtil
.createBucketTaggingRequest('PUT', bucketName);
bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => {
if (err) {
process.stdout.write(`Err putting object tagging ${err}`);
return done(err);
}
assert.strictEqual(err, undefined);
return done();
});
});
it('should return access denied if the authorization check fails', done => {
const taggingUtil = new TaggingConfigTester();
const testBucketPutTaggingRequest = taggingUtil
.createBucketTaggingRequest('PUT', bucketName);
const authInfo = makeAuthInfo('accessKey2');
bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => {
assert(err.AccessDenied);
return done();
});
});
});

View File

@ -117,6 +117,7 @@ describe('deleted flag bucket handling', () => {
bucketMD.setLocationConstraint(locationConstraint); bucketMD.setLocationConstraint(locationConstraint);
metadata.createBucket(bucketName, bucketMD, log, () => { metadata.createBucket(bucketName, bucketMD, log, () => {
metadata.createBucket(usersBucketName, usersBucket, log, () => { metadata.createBucket(usersBucketName, usersBucket, log, () => {
console.log('bucket created!!!!');
done(); done();
}); });
}); });
@ -439,13 +440,15 @@ describe('deleted flag bucket handling', () => {
if (extraArgNeeded) { if (extraArgNeeded) {
return apiAction(authInfo, mpuRequest, undefined, return apiAction(authInfo, mpuRequest, undefined,
log, err => { log, err => {
console.log(err);
assert.strictEqual(err.is.NoSuchUpload, true); assert.strictEqual(err.is.NoSuchUpload, true);
return done(); return done();
}); });
} }
return apiAction(authInfo, mpuRequest, return apiAction(authInfo, mpuRequest,
log, err => { log, err => {
assert.strictEqual(err.is.NoSuchUpload, true); console.log(err);
assert.strictEqual(err.is.NoSuchUpload, true);
return done(); return done();
}); });
} }

View File

@ -1,4 +1,3 @@
const crypto = require('crypto');
const assert = require('assert'); const assert = require('assert');
const { errors, storage } = require('arsenal'); const { errors, storage } = require('arsenal');
@ -8,7 +7,6 @@ const multiObjectDelete = require('../../../lib/api/multiObjectDelete');
const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers');
const DummyRequest = require('../DummyRequest'); const DummyRequest = require('../DummyRequest');
const { bucketPut } = require('../../../lib/api/bucketPut'); const { bucketPut } = require('../../../lib/api/bucketPut');
const metadataWrapper = require('../../../lib/metadata/wrapper');
const objectPut = require('../../../lib/api/objectPut'); const objectPut = require('../../../lib/api/objectPut');
const log = new DummyRequestLogger(); const log = new DummyRequestLogger();
@ -27,7 +25,6 @@ const objectKey1 = 'objectName1';
const objectKey2 = 'objectName2'; const objectKey2 = 'objectName2';
const metadataUtils = require('../../../lib/metadata/metadataUtils'); const metadataUtils = require('../../../lib/metadata/metadataUtils');
const services = require('../../../lib/services'); const services = require('../../../lib/services');
const { BucketInfo } = require('arsenal/build/lib/models');
const testBucketPutRequest = new DummyRequest({ const testBucketPutRequest = new DummyRequest({
bucketName, bucketName,
namespace, namespace,
@ -42,7 +39,7 @@ describe('getObjMetadataAndDelete function for multiObjectDelete', () => {
headers: {}, headers: {},
parsedContentLength: contentLength, parsedContentLength: contentLength,
}, postBody); }, postBody);
const bucket = { getVersioningConfiguration: () => null, getQuota: () => 0 }; const bucket = { getVersioningConfiguration: () => null };
beforeEach(done => { beforeEach(done => {
cleanup(); cleanup();
@ -360,43 +357,3 @@ describe('decodeObjectVersion function helper', () => {
assert.deepStrictEqual(ret[1], undefined); assert.deepStrictEqual(ret[1], undefined);
}); });
}); });
describe('multiObjectDelete function', () => {
afterEach(() => {
sinon.restore();
});
it('should not authorize the bucket and initial IAM authorization results', done => {
const post = '<Delete><Object><Key>objectname</Key></Object></Delete>';
const request = new DummyRequest({
bucketName: 'bucketname',
objectKey: 'objectname',
parsedHost: 'localhost',
headers: {
'content-md5': crypto.createHash('md5').update(post, 'utf8').digest('base64')
},
post,
url: '/bucketname',
});
const authInfo = makeAuthInfo('123456');
sinon.stub(metadataWrapper, 'getBucket').callsFake((bucketName, log, cb) =>
cb(null, new BucketInfo(
'bucketname',
'123456',
'accountA',
new Date().toISOString(),
15,
)));
multiObjectDelete.multiObjectDelete(authInfo, request, log, (err, res) => {
// Expected result is an access denied on the object, and no error, as the API was authorized
assert.strictEqual(err, null);
assert.strictEqual(
res.includes('<Error><Key>objectname</Key><Code>AccessDenied</Code>'),
true
);
done();
});
});
});

Some files were not shown because too many files have changed in this diff Show More