Compare commits
15 Commits
developmen
...
w/8.8/impr
Author | SHA1 | Date |
---|---|---|
bert-e | d9faa219e3 | |
bert-e | 360f57e869 | |
Will Toozs | 970de50c69 | |
Will Toozs | 7b5ba98a57 | |
Will Toozs | 329dbcc4dc | |
Will Toozs | be51322c79 | |
Will Toozs | 718468f53b | |
Will Toozs | 452878deec | |
Will Toozs | 7ce2e6fc44 | |
Will Toozs | 40e4019ac3 | |
Will Toozs | b11f3cfc5d | |
Will Toozs | 4ea06f22ed | |
Will Toozs | 133c6f05f0 | |
Will Toozs | f8fe00e114 | |
Will Toozs | a1839e36ea |
|
@ -16,7 +16,7 @@ runs:
|
||||||
run: |-
|
run: |-
|
||||||
set -exu;
|
set -exu;
|
||||||
mkdir -p /tmp/artifacts/${JOB_NAME}/;
|
mkdir -p /tmp/artifacts/${JOB_NAME}/;
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@v2
|
||||||
with:
|
with:
|
||||||
node-version: '16'
|
node-version: '16'
|
||||||
cache: 'yarn'
|
cache: 'yarn'
|
||||||
|
|
|
@ -40,11 +40,6 @@ services:
|
||||||
- DEFAULT_BUCKET_KEY_FORMAT
|
- DEFAULT_BUCKET_KEY_FORMAT
|
||||||
- METADATA_MAX_CACHED_BUCKETS
|
- METADATA_MAX_CACHED_BUCKETS
|
||||||
- ENABLE_NULL_VERSION_COMPAT_MODE
|
- ENABLE_NULL_VERSION_COMPAT_MODE
|
||||||
- SCUBA_HOST
|
|
||||||
- SCUBA_PORT
|
|
||||||
- SCUBA_HEALTHCHECK_FREQUENCY
|
|
||||||
- S3QUOTA
|
|
||||||
- QUOTA_ENABLE_INFLIGHTS
|
|
||||||
env_file:
|
env_file:
|
||||||
- creds.env
|
- creds.env
|
||||||
depends_on:
|
depends_on:
|
||||||
|
@ -72,7 +67,7 @@ services:
|
||||||
pykmip:
|
pykmip:
|
||||||
network_mode: "host"
|
network_mode: "host"
|
||||||
profiles: ['pykmip']
|
profiles: ['pykmip']
|
||||||
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
|
image: registry.scality.com/cloudserver-dev/pykmip
|
||||||
volumes:
|
volumes:
|
||||||
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
||||||
mongo:
|
mongo:
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
FROM mongo:5.0.21
|
FROM mongo:4.4.21
|
||||||
|
|
||||||
ENV USER=scality \
|
ENV USER=scality \
|
||||||
HOME_DIR=/home/scality \
|
HOME_DIR=/home/scality \
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
|
FROM registry.scality.com/federation/sproxyd:7.10.6.8
|
||||||
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
|
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
|
||||||
RUN chown root:root /conf/sproxyd0.conf
|
RUN chown root:root /conf/sproxyd0.conf
|
||||||
|
|
|
@ -1,10 +1,7 @@
|
||||||
name: Test alerts
|
name: Test alerts
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push
|
||||||
branches-ignore:
|
|
||||||
- 'development/**'
|
|
||||||
- 'q/*/**'
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
run-alert-tests:
|
run-alert-tests:
|
||||||
|
@ -20,16 +17,13 @@ jobs:
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: Render and test ${{ matrix.tests.name }}
|
- name: Render and test ${{ matrix.tests.name }}
|
||||||
uses: scality/action-prom-render-test@1.0.3
|
uses: scality/action-prom-render-test@1.0.1
|
||||||
with:
|
with:
|
||||||
alert_file_path: monitoring/alerts.yaml
|
alert_file_path: monitoring/alerts.yaml
|
||||||
test_file_path: ${{ matrix.tests.file }}
|
test_file_path: ${{ matrix.tests.file }}
|
||||||
alert_inputs: |
|
alert_inputs: >-
|
||||||
namespace=zenko
|
namespace=zenko,service=artesca-data-connector-s3api-metrics,replicas=3
|
||||||
service=artesca-data-connector-s3api-metrics
|
|
||||||
reportJob=artesca-data-ops-report-handler
|
|
||||||
replicas=3
|
|
||||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
|
@ -3,7 +3,7 @@ name: codeQL
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [w/**, q/*]
|
branches: [development/*, stabilization/*, hotfix/*]
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [development/*, stabilization/*, hotfix/*]
|
branches: [development/*, stabilization/*, hotfix/*]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
@ -14,12 +14,12 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: Initialize CodeQL
|
- name: Initialize CodeQL
|
||||||
uses: github/codeql-action/init@v3
|
uses: github/codeql-action/init@v2
|
||||||
with:
|
with:
|
||||||
languages: javascript, python, ruby
|
languages: javascript, python, ruby
|
||||||
|
|
||||||
- name: Build and analyze
|
- name: Build and analyze
|
||||||
uses: github/codeql-action/analyze@v3
|
uses: github/codeql-action/analyze@v2
|
||||||
|
|
|
@ -10,7 +10,7 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: 'Checkout Repository'
|
- name: 'Checkout Repository'
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: 'Dependency Review'
|
- name: 'Dependency Review'
|
||||||
uses: actions/dependency-review-action@v4
|
uses: actions/dependency-review-action@v3
|
||||||
|
|
|
@ -10,69 +10,58 @@ on:
|
||||||
required: true
|
required: true
|
||||||
|
|
||||||
env:
|
env:
|
||||||
|
REGISTRY_NAME: registry.scality.com
|
||||||
PROJECT_NAME: ${{ github.event.repository.name }}
|
PROJECT_NAME: ${{ github.event.repository.name }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-federation-image:
|
build-federation-image:
|
||||||
runs-on: ubuntu-20.04
|
uses: scality/workflows/.github/workflows/docker-build.yaml@v1
|
||||||
steps:
|
secrets: inherit
|
||||||
- name: Checkout
|
with:
|
||||||
uses: actions/checkout@v4
|
push: true
|
||||||
- name: Set up Docker Buildx
|
registry: registry.scality.com
|
||||||
uses: docker/setup-buildx-action@v3
|
namespace: ${{ github.event.repository.name }}
|
||||||
- name: Login to GitHub Registry
|
name: ${{ github.event.repository.name }}
|
||||||
uses: docker/login-action@v3
|
context: .
|
||||||
with:
|
file: images/svc-base/Dockerfile
|
||||||
registry: ghcr.io
|
tag: ${{ github.event.inputs.tag }}-svc-base
|
||||||
username: ${{ github.repository_owner }}
|
|
||||||
password: ${{ github.token }}
|
|
||||||
- name: Build and push image for federation
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
context: .
|
|
||||||
file: images/svc-base/Dockerfile
|
|
||||||
tags: |
|
|
||||||
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
|
|
||||||
cache-from: type=gha,scope=federation
|
|
||||||
cache-to: type=gha,mode=max,scope=federation
|
|
||||||
|
|
||||||
release:
|
release:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: Set up Docker Buildk
|
- name: Set up Docker Buildk
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v2
|
||||||
|
|
||||||
- name: Login to Registry
|
- name: Login to Registry
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v2
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ${{ env.REGISTRY_NAME }}
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ secrets.REGISTRY_LOGIN }}
|
||||||
password: ${{ github.token }}
|
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
|
|
||||||
- name: Push dashboards into the production namespace
|
- name: Push dashboards into the production namespace
|
||||||
run: |
|
run: |
|
||||||
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
|
oras push ${{ env.REGISTRY_NAME }}/${{ env.PROJECT_NAME }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
|
||||||
dashboard.json:application/grafana-dashboard+json \
|
dashboard.json:application/grafana-dashboard+json \
|
||||||
alerts.yaml:application/prometheus-alerts+yaml
|
alerts.yaml:application/prometheus-alerts+yaml
|
||||||
working-directory: monitoring
|
working-directory: monitoring
|
||||||
|
|
||||||
- name: Build and push
|
- name: Build and push
|
||||||
uses: docker/build-push-action@v5
|
uses: docker/build-push-action@v4
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: true
|
push: true
|
||||||
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
|
tags: ${{ env.REGISTRY_NAME }}/${{ env.PROJECT_NAME }}/${{ env.PROJECT_NAME }}:${{ github.event.inputs.tag }}
|
||||||
cache-from: type=gha
|
cache-from: type=gha
|
||||||
cache-to: type=gha,mode=max
|
cache-to: type=gha,mode=max
|
||||||
|
|
||||||
- name: Create Release
|
- name: Create Release
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@v1
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ github.token }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
with:
|
with:
|
||||||
name: Release ${{ github.event.inputs.tag }}
|
name: Release ${{ github.event.inputs.tag }}
|
||||||
tag_name: ${{ github.event.inputs.tag }}
|
tag_name: ${{ github.event.inputs.tag }}
|
||||||
|
|
|
@ -67,24 +67,23 @@ env:
|
||||||
ENABLE_LOCAL_CACHE: "true"
|
ENABLE_LOCAL_CACHE: "true"
|
||||||
REPORT_TOKEN: "report-token-1"
|
REPORT_TOKEN: "report-token-1"
|
||||||
REMOTE_MANAGEMENT_DISABLE: "1"
|
REMOTE_MANAGEMENT_DISABLE: "1"
|
||||||
# https://github.com/git-lfs/git-lfs/issues/5749
|
|
||||||
GIT_CLONE_PROTECTION_ACTIVE: 'false'
|
|
||||||
jobs:
|
jobs:
|
||||||
linting-coverage:
|
linting-coverage:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@v2
|
||||||
with:
|
with:
|
||||||
node-version: '16'
|
node-version: '16'
|
||||||
cache: yarn
|
cache: yarn
|
||||||
- name: install dependencies
|
- name: install dependencies
|
||||||
run: yarn install --frozen-lockfile --network-concurrency 1
|
run: yarn install --frozen-lockfile --network-concurrency 1
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@v4
|
||||||
with:
|
with:
|
||||||
python-version: '3.9'
|
python-version: '3.9'
|
||||||
- uses: actions/cache@v4
|
- uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: ~/.cache/pip
|
path: ~/.cache/pip
|
||||||
key: ${{ runner.os }}-pip
|
key: ${{ runner.os }}-pip
|
||||||
|
@ -117,7 +116,7 @@ jobs:
|
||||||
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
|
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
|
||||||
if: always()
|
if: always()
|
||||||
- name: Upload files to artifacts
|
- name: Upload files to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v2
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -133,54 +132,46 @@ jobs:
|
||||||
packages: write
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v2
|
||||||
- name: Login to GitHub Registry
|
- name: Login to GitHub Registry
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v2
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ github.repository_owner }}
|
||||||
password: ${{ github.token }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
- name: Login to Registry
|
||||||
|
uses: docker/login-action@v2
|
||||||
|
with:
|
||||||
|
registry: registry.scality.com
|
||||||
|
username: ${{ secrets.REGISTRY_LOGIN }}
|
||||||
|
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
- name: Build and push cloudserver image
|
- name: Build and push cloudserver image
|
||||||
uses: docker/build-push-action@v5
|
uses: docker/build-push-action@v4
|
||||||
with:
|
with:
|
||||||
push: true
|
push: true
|
||||||
context: .
|
context: .
|
||||||
provenance: false
|
provenance: false
|
||||||
tags: |
|
tags: |
|
||||||
ghcr.io/${{ github.repository }}:${{ github.sha }}
|
ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
labels: |
|
registry.scality.com/cloudserver-dev/cloudserver:${{ github.sha }}
|
||||||
git.repository=${{ github.repository }}
|
|
||||||
git.commit-sha=${{ github.sha }}
|
|
||||||
cache-from: type=gha,scope=cloudserver
|
cache-from: type=gha,scope=cloudserver
|
||||||
cache-to: type=gha,mode=max,scope=cloudserver
|
cache-to: type=gha,mode=max,scope=cloudserver
|
||||||
- name: Build and push pykmip image
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
push: true
|
|
||||||
context: .github/pykmip
|
|
||||||
tags: |
|
|
||||||
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
|
||||||
labels: |
|
|
||||||
git.repository=${{ github.repository }}
|
|
||||||
git.commit-sha=${{ github.sha }}
|
|
||||||
cache-from: type=gha,scope=pykmip
|
|
||||||
cache-to: type=gha,mode=max,scope=pykmip
|
|
||||||
- name: Build and push MongoDB
|
- name: Build and push MongoDB
|
||||||
uses: docker/build-push-action@v5
|
uses: docker/build-push-action@v4
|
||||||
with:
|
with:
|
||||||
push: true
|
push: true
|
||||||
context: .github/docker/mongodb
|
context: .github/docker/mongodb
|
||||||
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
cache-from: type=gha,scope=mongodb
|
cache-from: type=gha
|
||||||
cache-to: type=gha,mode=max,scope=mongodb
|
cache-to: type=gha,mode=max
|
||||||
|
|
||||||
multiple-backend:
|
multiple-backend:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
env:
|
env:
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
S3BACKEND: mem
|
S3BACKEND: mem
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||||
|
@ -188,13 +179,13 @@ jobs:
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Login to Registry
|
- name: Login to Registry
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v2
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: registry.scality.com
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ secrets.REGISTRY_LOGIN }}
|
||||||
password: ${{ github.token }}
|
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- name: Setup CI services
|
- name: Setup CI services
|
||||||
|
@ -209,7 +200,7 @@ jobs:
|
||||||
env:
|
env:
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v3
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -229,11 +220,11 @@ jobs:
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||||
DEFAULT_BUCKET_KEY_FORMAT: v0
|
DEFAULT_BUCKET_KEY_FORMAT: v0
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- name: Setup CI services
|
- name: Setup CI services
|
||||||
|
@ -247,7 +238,7 @@ jobs:
|
||||||
env:
|
env:
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v3
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -268,11 +259,11 @@ jobs:
|
||||||
DEFAULT_BUCKET_KEY_FORMAT: v1
|
DEFAULT_BUCKET_KEY_FORMAT: v1
|
||||||
METADATA_MAX_CACHED_BUCKETS: 1
|
METADATA_MAX_CACHED_BUCKETS: 1
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- name: Setup CI services
|
- name: Setup CI services
|
||||||
|
@ -287,7 +278,7 @@ jobs:
|
||||||
env:
|
env:
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v3
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -307,13 +298,13 @@ jobs:
|
||||||
env:
|
env:
|
||||||
S3BACKEND: file
|
S3BACKEND: file
|
||||||
S3VAULT: mem
|
S3VAULT: mem
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
MPU_TESTING: "yes"
|
MPU_TESTING: "yes"
|
||||||
JOB_NAME: ${{ matrix.job-name }}
|
JOB_NAME: ${{ matrix.job-name }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- name: Setup matrix job artifacts directory
|
- name: Setup matrix job artifacts directory
|
||||||
|
@ -330,7 +321,7 @@ jobs:
|
||||||
bash wait_for_local_port.bash 8000 40
|
bash wait_for_local_port.bash 8000 40
|
||||||
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
|
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v3
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -346,12 +337,12 @@ jobs:
|
||||||
ENABLE_UTAPI_V2: t
|
ENABLE_UTAPI_V2: t
|
||||||
S3BACKEND: mem
|
S3BACKEND: mem
|
||||||
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- name: Setup CI services
|
- name: Setup CI services
|
||||||
|
@ -363,51 +354,7 @@ jobs:
|
||||||
bash wait_for_local_port.bash 8000 40
|
bash wait_for_local_port.bash 8000 40
|
||||||
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v3
|
||||||
with:
|
|
||||||
method: upload
|
|
||||||
url: https://artifacts.scality.net
|
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
|
||||||
source: /tmp/artifacts
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
quota-tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
inflights:
|
|
||||||
- name: "With Inflights"
|
|
||||||
value: "true"
|
|
||||||
- name: "Without Inflights"
|
|
||||||
value: "false"
|
|
||||||
env:
|
|
||||||
S3METADATA: mongodb
|
|
||||||
S3BACKEND: mem
|
|
||||||
S3QUOTA: scuba
|
|
||||||
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
|
|
||||||
SCUBA_HOST: localhost
|
|
||||||
SCUBA_PORT: 8100
|
|
||||||
SCUBA_HEALTHCHECK_FREQUENCY: 100
|
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
|
||||||
JOB_NAME: ${{ github.job }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Setup CI environment
|
|
||||||
uses: ./.github/actions/setup-ci
|
|
||||||
- name: Setup CI services
|
|
||||||
run: docker compose --profile mongo up -d
|
|
||||||
working-directory: .github/docker
|
|
||||||
- name: Run quota tests
|
|
||||||
run: |-
|
|
||||||
set -ex -o pipefail;
|
|
||||||
bash wait_for_local_port.bash 8000 40
|
|
||||||
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
|
|
||||||
- name: Upload logs to artifacts
|
|
||||||
uses: scality/action-artifacts@v4
|
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -423,13 +370,12 @@ jobs:
|
||||||
S3BACKEND: file
|
S3BACKEND: file
|
||||||
S3VAULT: mem
|
S3VAULT: mem
|
||||||
MPU_TESTING: "yes"
|
MPU_TESTING: "yes"
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- name: Copy KMIP certs
|
- name: Copy KMIP certs
|
||||||
|
@ -445,7 +391,7 @@ jobs:
|
||||||
bash wait_for_local_port.bash 5696 40
|
bash wait_for_local_port.bash 5696 40
|
||||||
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v3
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -465,25 +411,25 @@ jobs:
|
||||||
MPU_TESTING: "yes"
|
MPU_TESTING: "yes"
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
|
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
|
||||||
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v3
|
||||||
- name: Login to GitHub Registry
|
- name: Login to GitHub Registry
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v2
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ github.repository_owner }}
|
||||||
password: ${{ github.token }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- uses: ruby/setup-ruby@v1
|
- uses: ruby/setup-ruby@v1
|
||||||
with:
|
with:
|
||||||
ruby-version: '2.5.9'
|
ruby-version: '2.5.0'
|
||||||
- name: Install Ruby dependencies
|
- name: Install Ruby dependencies
|
||||||
run: |
|
run: |
|
||||||
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
|
gem install nokogiri:1.12.5 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
|
||||||
- name: Install Java dependencies
|
- name: Install Java dependencies
|
||||||
run: |
|
run: |
|
||||||
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
|
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
|
||||||
|
@ -510,7 +456,7 @@ jobs:
|
||||||
- name: Run Ruby tests
|
- name: Run Ruby tests
|
||||||
run: |-
|
run: |-
|
||||||
set -ex -o pipefail;
|
set -ex -o pipefail;
|
||||||
rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
|
rspec tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
|
||||||
working-directory: tests/functional/fog
|
working-directory: tests/functional/fog
|
||||||
- name: Run Javascript AWS SDK tests
|
- name: Run Javascript AWS SDK tests
|
||||||
run: |-
|
run: |-
|
||||||
|
@ -523,7 +469,7 @@ jobs:
|
||||||
S3VAULT: mem
|
S3VAULT: mem
|
||||||
S3METADATA: mongodb
|
S3METADATA: mongodb
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v4
|
uses: scality/action-artifacts@v3
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
|
|
@ -23,7 +23,6 @@ RUN apt-get update \
|
||||||
|
|
||||||
ENV PYTHON=python3
|
ENV PYTHON=python3
|
||||||
COPY package.json yarn.lock /usr/src/app/
|
COPY package.json yarn.lock /usr/src/app/
|
||||||
RUN npm install typescript -g
|
|
||||||
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
|
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
175
README.md
175
README.md
|
@ -1,7 +1,10 @@
|
||||||
# Zenko CloudServer with Vitastor Backend
|
# Zenko CloudServer
|
||||||
|
|
||||||
![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
|
![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
|
||||||
|
|
||||||
|
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/zenko/cloudserver)
|
||||||
|
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
|
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
|
||||||
|
@ -11,71 +14,137 @@ Scality’s Open Source Multi-Cloud Data Controller.
|
||||||
CloudServer provides a single AWS S3 API interface to access multiple
|
CloudServer provides a single AWS S3 API interface to access multiple
|
||||||
backend data storage both on-premise or public in the cloud.
|
backend data storage both on-premise or public in the cloud.
|
||||||
|
|
||||||
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
|
CloudServer is useful for Developers, either to run as part of a
|
||||||
backend support.
|
continous integration test environment to emulate the AWS S3 service locally
|
||||||
|
or as an abstraction layer to develop object storage enabled
|
||||||
|
application on the go.
|
||||||
|
|
||||||
## Quick Start with Vitastor
|
## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
|
||||||
|
|
||||||
Vitastor Backend is in experimental status, however you can already try to
|
## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
|
||||||
run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
|
|
||||||
it works too 😊.
|
|
||||||
|
|
||||||
Installation instructions:
|
## Docker
|
||||||
|
|
||||||
### Install Vitastor
|
[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/zenko/cloudserver/)
|
||||||
|
|
||||||
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
|
## Contributing
|
||||||
|
|
||||||
### Install Zenko with Vitastor Backend
|
In order to contribute, please follow the
|
||||||
|
[Contributing Guidelines](
|
||||||
|
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
|
||||||
|
|
||||||
- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
|
## Installation
|
||||||
- Install dependencies: `npm install --omit dev` or just `npm install`
|
|
||||||
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
|
|
||||||
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
|
|
||||||
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
|
|
||||||
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
|
|
||||||
|
|
||||||
### Install and Configure MongoDB
|
### Dependencies
|
||||||
|
|
||||||
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
|
Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
|
||||||
|
. Up-to-date versions can be found at
|
||||||
|
[Nodesource](https://github.com/nodesource/distributions).
|
||||||
|
|
||||||
### Setup Zenko
|
### Clone source code
|
||||||
|
|
||||||
- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
|
```shell
|
||||||
- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
|
git clone https://github.com/scality/S3.git
|
||||||
- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
|
|
||||||
- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
|
|
||||||
- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
|
|
||||||
and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
|
|
||||||
access keys, but it's not published, so let's use a file for now.
|
|
||||||
- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
|
|
||||||
You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
|
|
||||||
in this file.
|
|
||||||
|
|
||||||
Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
|
|
||||||
instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
|
|
||||||
|
|
||||||
### Start Zenko
|
|
||||||
|
|
||||||
Start the S3 server with: `node index.js`
|
|
||||||
|
|
||||||
If you use default settings, Zenko CloudServer starts on port 8000.
|
|
||||||
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
|
|
||||||
|
|
||||||
Now you can access your S3 with `s3cmd` or `geesefs`:
|
|
||||||
|
|
||||||
```
|
|
||||||
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
|
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
### Install js dependencies
|
||||||
AWS_ACCESS_KEY_ID=accessKey1 \
|
|
||||||
AWS_SECRET_ACCESS_KEY=verySecretKey1 \
|
Go to the ./S3 folder,
|
||||||
geesefs --endpoint http://localhost:8000 testbucket mountdir
|
|
||||||
|
```shell
|
||||||
|
yarn install --frozen-lockfile
|
||||||
```
|
```
|
||||||
|
|
||||||
# Author & License
|
If you get an error regarding installation of the diskUsage module,
|
||||||
|
please install g++.
|
||||||
|
|
||||||
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
If you get an error regarding level-down bindings, try clearing your yarn cache:
|
||||||
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
|
|
||||||
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way)
|
```shell
|
||||||
|
yarn cache clean
|
||||||
|
```
|
||||||
|
|
||||||
|
## Run it with a file backend
|
||||||
|
|
||||||
|
```shell
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
|
||||||
|
9991 are also open locally for internal transfer of metadata and data,
|
||||||
|
respectively.
|
||||||
|
|
||||||
|
The default access key is accessKey1 with
|
||||||
|
a secret key of verySecretKey1.
|
||||||
|
|
||||||
|
By default the metadata files will be saved in the
|
||||||
|
localMetadata directory and the data files will be saved
|
||||||
|
in the localData directory within the ./S3 directory on your
|
||||||
|
machine. These directories have been pre-created within the
|
||||||
|
repository. If you would like to save the data or metadata in
|
||||||
|
different locations of your choice, you must specify them with absolute paths.
|
||||||
|
So, when starting the server:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
mkdir -m 700 $(pwd)/myFavoriteDataPath
|
||||||
|
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
|
||||||
|
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
|
||||||
|
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
## Run it with multiple data backends
|
||||||
|
|
||||||
|
```shell
|
||||||
|
export S3DATA='multiple'
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer on port 8000.
|
||||||
|
The default access key is accessKey1 with
|
||||||
|
a secret key of verySecretKey1.
|
||||||
|
|
||||||
|
With multiple backends, you have the ability to
|
||||||
|
choose where each object will be saved by setting
|
||||||
|
the following header with a locationConstraint on
|
||||||
|
a PUT request:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
|
||||||
|
```
|
||||||
|
|
||||||
|
If no header is sent with a PUT object request, the
|
||||||
|
location constraint of the bucket will determine
|
||||||
|
where the data is saved. If the bucket has no location
|
||||||
|
constraint, the endpoint of the PUT request will be
|
||||||
|
used to determine location.
|
||||||
|
|
||||||
|
See the Configuration section in our documentation
|
||||||
|
[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
|
||||||
|
to learn how to set location constraints.
|
||||||
|
|
||||||
|
## Run it with an in-memory backend
|
||||||
|
|
||||||
|
```shell
|
||||||
|
yarn run mem_backend
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer on port 8000.
|
||||||
|
The default access key is accessKey1 with
|
||||||
|
a secret key of verySecretKey1.
|
||||||
|
|
||||||
|
## Run it with Vault user management
|
||||||
|
|
||||||
|
Note: Vault is proprietary and must be accessed separately.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
export S3VAULT=vault
|
||||||
|
yarn start
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts a Zenko CloudServer using Vault for user management.
|
||||||
|
|
||||||
|
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
|
||||||
|
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
|
||||||
|
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
|
||||||
|
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
'use strict'; // eslint-disable-line strict
|
||||||
|
|
||||||
|
const {
|
||||||
|
startWSManagementClient,
|
||||||
|
startPushConnectionHealthCheckServer,
|
||||||
|
} = require('../lib/management/push');
|
||||||
|
|
||||||
|
const logger = require('../lib/utilities/logger');
|
||||||
|
|
||||||
|
const {
|
||||||
|
PUSH_ENDPOINT: pushEndpoint,
|
||||||
|
INSTANCE_ID: instanceId,
|
||||||
|
MANAGEMENT_TOKEN: managementToken,
|
||||||
|
} = process.env;
|
||||||
|
|
||||||
|
if (!pushEndpoint) {
|
||||||
|
logger.error('missing push endpoint env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!instanceId) {
|
||||||
|
logger.error('missing instance id env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!managementToken) {
|
||||||
|
logger.error('missing management token env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
startPushConnectionHealthCheckServer(err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('could not start healthcheck server', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
const url = `${pushEndpoint}/${instanceId}/ws?metrics=1`;
|
||||||
|
startWSManagementClient(url, managementToken, err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('connection failed, exiting', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
logger.info('no more connection, exiting');
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
});
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
'use strict'; // eslint-disable-line strict
|
||||||
|
|
||||||
|
const {
|
||||||
|
startWSManagementClient,
|
||||||
|
startPushConnectionHealthCheckServer,
|
||||||
|
} = require('../lib/management/push');
|
||||||
|
|
||||||
|
const logger = require('../lib/utilities/logger');
|
||||||
|
|
||||||
|
const {
|
||||||
|
PUSH_ENDPOINT: pushEndpoint,
|
||||||
|
INSTANCE_ID: instanceId,
|
||||||
|
MANAGEMENT_TOKEN: managementToken,
|
||||||
|
} = process.env;
|
||||||
|
|
||||||
|
if (!pushEndpoint) {
|
||||||
|
logger.error('missing push endpoint env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!instanceId) {
|
||||||
|
logger.error('missing instance id env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!managementToken) {
|
||||||
|
logger.error('missing management token env var');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
startPushConnectionHealthCheckServer(err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('could not start healthcheck server', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
const url = `${pushEndpoint}/${instanceId}/ws?proxy=1`;
|
||||||
|
startWSManagementClient(url, managementToken, err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error('connection failed, exiting', { error: err });
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
logger.info('no more connection, exiting');
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
});
|
|
@ -4,7 +4,6 @@
|
||||||
"metricsPort": 8002,
|
"metricsPort": 8002,
|
||||||
"metricsListenOn": [],
|
"metricsListenOn": [],
|
||||||
"replicationGroupId": "RG001",
|
"replicationGroupId": "RG001",
|
||||||
"workers": 4,
|
|
||||||
"restEndpoints": {
|
"restEndpoints": {
|
||||||
"localhost": "us-east-1",
|
"localhost": "us-east-1",
|
||||||
"127.0.0.1": "us-east-1",
|
"127.0.0.1": "us-east-1",
|
||||||
|
@ -102,14 +101,6 @@
|
||||||
"readPreference": "primary",
|
"readPreference": "primary",
|
||||||
"database": "metadata"
|
"database": "metadata"
|
||||||
},
|
},
|
||||||
"authdata": "authdata.json",
|
|
||||||
"backends": {
|
|
||||||
"auth": "file",
|
|
||||||
"data": "file",
|
|
||||||
"metadata": "mongodb",
|
|
||||||
"kms": "file",
|
|
||||||
"quota": "none"
|
|
||||||
},
|
|
||||||
"externalBackends": {
|
"externalBackends": {
|
||||||
"aws_s3": {
|
"aws_s3": {
|
||||||
"httpAgent": {
|
"httpAgent": {
|
|
@ -1,71 +0,0 @@
|
||||||
{
|
|
||||||
"port": 8000,
|
|
||||||
"listenOn": [],
|
|
||||||
"metricsPort": 8002,
|
|
||||||
"metricsListenOn": [],
|
|
||||||
"replicationGroupId": "RG001",
|
|
||||||
"restEndpoints": {
|
|
||||||
"localhost": "STANDARD",
|
|
||||||
"127.0.0.1": "STANDARD",
|
|
||||||
"yourhostname.ru": "STANDARD"
|
|
||||||
},
|
|
||||||
"websiteEndpoints": [
|
|
||||||
"static.yourhostname.ru"
|
|
||||||
],
|
|
||||||
"replicationEndpoints": [ {
|
|
||||||
"site": "zenko",
|
|
||||||
"servers": ["127.0.0.1:8000"],
|
|
||||||
"default": true
|
|
||||||
} ],
|
|
||||||
"log": {
|
|
||||||
"logLevel": "info",
|
|
||||||
"dumpLevel": "error"
|
|
||||||
},
|
|
||||||
"healthChecks": {
|
|
||||||
"allowFrom": ["127.0.0.1/8", "::1"]
|
|
||||||
},
|
|
||||||
"backends": {
|
|
||||||
"metadata": "mongodb"
|
|
||||||
},
|
|
||||||
"mongodb": {
|
|
||||||
"replicaSetHosts": "127.0.0.1:27017",
|
|
||||||
"writeConcern": "majority",
|
|
||||||
"replicaSet": "rs0",
|
|
||||||
"readPreference": "primary",
|
|
||||||
"database": "s3",
|
|
||||||
"authCredentials": {
|
|
||||||
"username": "s3",
|
|
||||||
"password": ""
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"externalBackends": {
|
|
||||||
"aws_s3": {
|
|
||||||
"httpAgent": {
|
|
||||||
"keepAlive": false,
|
|
||||||
"keepAliveMsecs": 1000,
|
|
||||||
"maxFreeSockets": 256,
|
|
||||||
"maxSockets": null
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"gcp": {
|
|
||||||
"httpAgent": {
|
|
||||||
"keepAlive": true,
|
|
||||||
"keepAliveMsecs": 1000,
|
|
||||||
"maxFreeSockets": 256,
|
|
||||||
"maxSockets": null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"requests": {
|
|
||||||
"viaProxy": false,
|
|
||||||
"trustedProxyCIDRs": [],
|
|
||||||
"extractClientIPFromHeader": ""
|
|
||||||
},
|
|
||||||
"bucketNotificationDestinations": [
|
|
||||||
{
|
|
||||||
"resource": "target1",
|
|
||||||
"type": "dummy",
|
|
||||||
"host": "localhost:6000"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
20
constants.js
20
constants.js
|
@ -116,7 +116,7 @@ const constants = {
|
||||||
],
|
],
|
||||||
|
|
||||||
// user metadata header to set object locationConstraint
|
// user metadata header to set object locationConstraint
|
||||||
objectLocationConstraintHeader: 'x-amz-storage-class',
|
objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
|
||||||
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
|
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
|
||||||
legacyLocations: ['sproxyd', 'legacy'],
|
legacyLocations: ['sproxyd', 'legacy'],
|
||||||
// declare here all existing service accounts and their properties
|
// declare here all existing service accounts and their properties
|
||||||
|
@ -205,6 +205,9 @@ const constants = {
|
||||||
],
|
],
|
||||||
allowedUtapiEventFilterStates: ['allow', 'deny'],
|
allowedUtapiEventFilterStates: ['allow', 'deny'],
|
||||||
allowedRestoreObjectRequestTierValues: ['Standard'],
|
allowedRestoreObjectRequestTierValues: ['Standard'],
|
||||||
|
validStorageClasses: [
|
||||||
|
'STANDARD',
|
||||||
|
],
|
||||||
lifecycleListing: {
|
lifecycleListing: {
|
||||||
CURRENT_TYPE: 'current',
|
CURRENT_TYPE: 'current',
|
||||||
NON_CURRENT_TYPE: 'noncurrent',
|
NON_CURRENT_TYPE: 'noncurrent',
|
||||||
|
@ -217,7 +220,6 @@ const constants = {
|
||||||
'owner-id',
|
'owner-id',
|
||||||
'versionId',
|
'versionId',
|
||||||
'isNull',
|
'isNull',
|
||||||
'isDeleteMarker',
|
|
||||||
],
|
],
|
||||||
unsupportedSignatureChecksums: new Set([
|
unsupportedSignatureChecksums: new Set([
|
||||||
'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
|
'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
|
||||||
|
@ -229,20 +231,6 @@ const constants = {
|
||||||
'UNSIGNED-PAYLOAD',
|
'UNSIGNED-PAYLOAD',
|
||||||
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
|
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
|
||||||
]),
|
]),
|
||||||
ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
|
|
||||||
ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
|
|
||||||
// The AWS assumed Role resource type
|
|
||||||
assumedRoleArnResourceType: 'assumed-role',
|
|
||||||
// Session name of the backbeat lifecycle assumed role session.
|
|
||||||
backbeatLifecycleSessionName: 'backbeat-lifecycle',
|
|
||||||
actionsToConsiderAsObjectPut: [
|
|
||||||
'initiateMultipartUpload',
|
|
||||||
'objectPutPart',
|
|
||||||
'completeMultipartUpload',
|
|
||||||
],
|
|
||||||
// if requester is not bucket owner, bucket policy actions should be denied with
|
|
||||||
// MethodNotAllowed error
|
|
||||||
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = constants;
|
module.exports = constants;
|
||||||
|
|
|
@ -2,12 +2,11 @@
|
||||||
|
|
||||||
## Docker Image Generation
|
## Docker Image Generation
|
||||||
|
|
||||||
Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
|
Docker images are hosted on [registry.scality.com](registry.scality.com).
|
||||||
CloudServer has a few images there:
|
CloudServer has two namespaces there:
|
||||||
|
|
||||||
* Cloudserver container image: ghcr.io/scality/cloudserver
|
* Production Namespace: registry.scality.com/cloudserver
|
||||||
* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
|
* Dev Namespace: registry.scality.com/cloudserver-dev
|
||||||
* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
|
|
||||||
|
|
||||||
With every CI build, the CI will push images, tagging the
|
With every CI build, the CI will push images, tagging the
|
||||||
content with the developer branch's short SHA-1 commit hash.
|
content with the developer branch's short SHA-1 commit hash.
|
||||||
|
@ -19,8 +18,8 @@ Tagged versions of cloudserver will be stored in the production namespace.
|
||||||
## How to Pull Docker Images
|
## How to Pull Docker Images
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker pull ghcr.io/scality/cloudserver:<commit hash>
|
docker pull registry.scality.com/cloudserver-dev/cloudserver:<commit hash>
|
||||||
docker pull ghcr.io/scality/cloudserver:<tag>
|
docker pull registry.scality.com/cloudserver/cloudserver:<tag>
|
||||||
```
|
```
|
||||||
|
|
||||||
## Release Process
|
## Release Process
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
|
FROM registry.scality.com/federation/nodesvc-base:7.10.6.0
|
||||||
|
|
||||||
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
|
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
|
||||||
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
|
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
|
||||||
|
@ -14,10 +14,8 @@ RUN rm -f ~/.gitconfig && \
|
||||||
git config --global --add safe.directory . && \
|
git config --global --add safe.directory . && \
|
||||||
git lfs install && \
|
git lfs install && \
|
||||||
GIT_LFS_SKIP_SMUDGE=1 && \
|
GIT_LFS_SKIP_SMUDGE=1 && \
|
||||||
yarn global add typescript && \
|
|
||||||
yarn install --frozen-lockfile --production --network-concurrency 1 && \
|
yarn install --frozen-lockfile --production --network-concurrency 1 && \
|
||||||
yarn cache clean --all && \
|
yarn cache clean --all
|
||||||
yarn global remove typescript
|
|
||||||
|
|
||||||
# run symlinking separately to avoid yarn installation errors
|
# run symlinking separately to avoid yarn installation errors
|
||||||
# we might have to check if the symlinking is really needed!
|
# we might have to check if the symlinking is really needed!
|
||||||
|
|
12
index.js
12
index.js
|
@ -1,10 +1,10 @@
|
||||||
'use strict'; // eslint-disable-line strict
|
'use strict'; // eslint-disable-line strict
|
||||||
|
|
||||||
require('werelogs').stderrUtils.catchAndTimestampStderr(
|
/**
|
||||||
undefined,
|
* Catch uncaught exceptions and add timestamp to aid debugging
|
||||||
// Do not exit as workers have their own listener that will exit
|
*/
|
||||||
// But primary don't have another listener
|
process.on('uncaughtException', err => {
|
||||||
require('cluster').isPrimary ? 1 : null,
|
process.stderr.write(`${new Date().toISOString()}: Uncaught exception: \n${err.stack}`);
|
||||||
);
|
});
|
||||||
|
|
||||||
require('./lib/server.js')();
|
require('./lib/server.js')();
|
||||||
|
|
304
lib/Config.js
304
lib/Config.js
|
@ -107,47 +107,6 @@ function parseSproxydConfig(configSproxyd) {
|
||||||
return joi.attempt(configSproxyd, joiSchema, 'bad config');
|
return joi.attempt(configSproxyd, joiSchema, 'bad config');
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseRedisConfig(redisConfig) {
|
|
||||||
const joiSchema = joi.object({
|
|
||||||
password: joi.string().allow(''),
|
|
||||||
host: joi.string(),
|
|
||||||
port: joi.number(),
|
|
||||||
retry: joi.object({
|
|
||||||
connectBackoff: joi.object({
|
|
||||||
min: joi.number().required(),
|
|
||||||
max: joi.number().required(),
|
|
||||||
jitter: joi.number().required(),
|
|
||||||
factor: joi.number().required(),
|
|
||||||
deadline: joi.number().required(),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
// sentinel config
|
|
||||||
sentinels: joi.alternatives().try(
|
|
||||||
joi.string()
|
|
||||||
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
|
|
||||||
.custom(hosts => hosts.split(',').map(item => {
|
|
||||||
const [host, port] = item.split(':');
|
|
||||||
return { host, port: Number.parseInt(port, 10) };
|
|
||||||
})),
|
|
||||||
joi.array().items(
|
|
||||||
joi.object({
|
|
||||||
host: joi.string().required(),
|
|
||||||
port: joi.number().required(),
|
|
||||||
})
|
|
||||||
).min(1),
|
|
||||||
),
|
|
||||||
name: joi.string(),
|
|
||||||
sentinelPassword: joi.string().allow(''),
|
|
||||||
})
|
|
||||||
.and('host', 'port')
|
|
||||||
.and('sentinels', 'name')
|
|
||||||
.xor('host', 'sentinels')
|
|
||||||
.without('sentinels', ['host', 'port'])
|
|
||||||
.without('host', ['sentinels', 'sentinelPassword']);
|
|
||||||
|
|
||||||
return joi.attempt(redisConfig, joiSchema, 'bad config');
|
|
||||||
}
|
|
||||||
|
|
||||||
function restEndpointsAssert(restEndpoints, locationConstraints) {
|
function restEndpointsAssert(restEndpoints, locationConstraints) {
|
||||||
assert(typeof restEndpoints === 'object',
|
assert(typeof restEndpoints === 'object',
|
||||||
'bad config: restEndpoints must be an object of endpoints');
|
'bad config: restEndpoints must be an object of endpoints');
|
||||||
|
@ -377,7 +336,7 @@ function dmfLocationConstraintAssert(locationObj) {
|
||||||
function locationConstraintAssert(locationConstraints) {
|
function locationConstraintAssert(locationConstraints) {
|
||||||
const supportedBackends =
|
const supportedBackends =
|
||||||
['mem', 'file', 'scality',
|
['mem', 'file', 'scality',
|
||||||
'mongodb', 'dmf', 'azure_archive', 'vitastor'].concat(Object.keys(validExternalBackends));
|
'mongodb', 'dmf', 'azure_archive'].concat(Object.keys(validExternalBackends));
|
||||||
assert(typeof locationConstraints === 'object',
|
assert(typeof locationConstraints === 'object',
|
||||||
'bad config: locationConstraints must be an object');
|
'bad config: locationConstraints must be an object');
|
||||||
Object.keys(locationConstraints).forEach(l => {
|
Object.keys(locationConstraints).forEach(l => {
|
||||||
|
@ -502,23 +461,27 @@ function locationConstraintAssert(locationConstraints) {
|
||||||
locationConstraints[l].details.connector.hdclient);
|
locationConstraints[l].details.connector.hdclient);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
assert(Object.keys(locationConstraints)
|
||||||
|
.includes('us-east-1'), 'bad locationConfig: must ' +
|
||||||
|
'include us-east-1 as a locationConstraint');
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseUtapiReindex(config) {
|
function parseUtapiReindex(config) {
|
||||||
const {
|
const {
|
||||||
enabled,
|
enabled,
|
||||||
schedule,
|
schedule,
|
||||||
redis,
|
sentinel,
|
||||||
bucketd,
|
bucketd,
|
||||||
onlyCountLatestWhenObjectLocked,
|
onlyCountLatestWhenObjectLocked,
|
||||||
} = config;
|
} = config;
|
||||||
assert(typeof enabled === 'boolean',
|
assert(typeof enabled === 'boolean',
|
||||||
'bad config: utapi.reindex.enabled must be a boolean');
|
'bad config: utapi.reindex.enabled must be a boolean');
|
||||||
|
assert(typeof sentinel === 'object',
|
||||||
const parsedRedis = parseRedisConfig(redis);
|
'bad config: utapi.reindex.sentinel must be an object');
|
||||||
assert(Array.isArray(parsedRedis.sentinels),
|
assert(typeof sentinel.port === 'number',
|
||||||
'bad config: utapi reindex redis config requires a list of sentinels');
|
'bad config: utapi.reindex.sentinel.port must be a number');
|
||||||
|
assert(typeof sentinel.name === 'string',
|
||||||
|
'bad config: utapi.reindex.sentinel.name must be a string');
|
||||||
assert(typeof bucketd === 'object',
|
assert(typeof bucketd === 'object',
|
||||||
'bad config: utapi.reindex.bucketd must be an object');
|
'bad config: utapi.reindex.bucketd must be an object');
|
||||||
assert(typeof bucketd.port === 'number',
|
assert(typeof bucketd.port === 'number',
|
||||||
|
@ -536,13 +499,6 @@ function parseUtapiReindex(config) {
|
||||||
'bad config: utapi.reindex.schedule must be a valid ' +
|
'bad config: utapi.reindex.schedule must be a valid ' +
|
||||||
`cron schedule. ${e.message}.`);
|
`cron schedule. ${e.message}.`);
|
||||||
}
|
}
|
||||||
return {
|
|
||||||
enabled,
|
|
||||||
schedule,
|
|
||||||
redis: parsedRedis,
|
|
||||||
bucketd,
|
|
||||||
onlyCountLatestWhenObjectLocked,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function requestsConfigAssert(requestsConfig) {
|
function requestsConfigAssert(requestsConfig) {
|
||||||
|
@ -630,6 +586,7 @@ class Config extends EventEmitter {
|
||||||
// Read config automatically
|
// Read config automatically
|
||||||
this._getLocationConfig();
|
this._getLocationConfig();
|
||||||
this._getConfig();
|
this._getConfig();
|
||||||
|
this._configureBackends();
|
||||||
}
|
}
|
||||||
|
|
||||||
_getLocationConfig() {
|
_getLocationConfig() {
|
||||||
|
@ -841,11 +798,11 @@ class Config extends EventEmitter {
|
||||||
this.websiteEndpoints = config.websiteEndpoints;
|
this.websiteEndpoints = config.websiteEndpoints;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.workers = false;
|
this.clusters = false;
|
||||||
if (config.workers !== undefined) {
|
if (config.clusters !== undefined) {
|
||||||
assert(Number.isInteger(config.workers) && config.workers > 0,
|
assert(Number.isInteger(config.clusters) && config.clusters > 0,
|
||||||
'bad config: workers must be a positive integer');
|
'bad config: clusters must be a positive integer');
|
||||||
this.workers = config.workers;
|
this.clusters = config.clusters;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.usEastBehavior !== undefined) {
|
if (config.usEastBehavior !== undefined) {
|
||||||
|
@ -1083,7 +1040,8 @@ class Config extends EventEmitter {
|
||||||
assert(typeof config.localCache.port === 'number',
|
assert(typeof config.localCache.port === 'number',
|
||||||
'config: bad port for localCache. port must be a number');
|
'config: bad port for localCache. port must be a number');
|
||||||
if (config.localCache.password !== undefined) {
|
if (config.localCache.password !== undefined) {
|
||||||
assert(typeof config.localCache.password === 'string',
|
assert(
|
||||||
|
this._verifyRedisPassword(config.localCache.password),
|
||||||
'config: vad password for localCache. password must' +
|
'config: vad password for localCache. password must' +
|
||||||
' be a string');
|
' be a string');
|
||||||
}
|
}
|
||||||
|
@ -1109,46 +1067,56 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.redis) {
|
if (config.redis) {
|
||||||
this.redis = parseRedisConfig(config.redis);
|
if (config.redis.sentinels) {
|
||||||
}
|
this.redis = { sentinels: [], name: null };
|
||||||
if (config.scuba) {
|
|
||||||
this.scuba = {};
|
assert(typeof config.redis.name === 'string',
|
||||||
if (config.scuba.host) {
|
'bad config: redis sentinel name must be a string');
|
||||||
assert(typeof config.scuba.host === 'string',
|
this.redis.name = config.redis.name;
|
||||||
'bad config: scuba host must be a string');
|
assert(Array.isArray(config.redis.sentinels) ||
|
||||||
this.scuba.host = config.scuba.host;
|
typeof config.redis.sentinels === 'string',
|
||||||
|
'bad config: redis sentinels must be an array or string');
|
||||||
|
|
||||||
|
if (typeof config.redis.sentinels === 'string') {
|
||||||
|
config.redis.sentinels.split(',').forEach(item => {
|
||||||
|
const [host, port] = item.split(':');
|
||||||
|
this.redis.sentinels.push({ host,
|
||||||
|
port: Number.parseInt(port, 10) });
|
||||||
|
});
|
||||||
|
} else if (Array.isArray(config.redis.sentinels)) {
|
||||||
|
config.redis.sentinels.forEach(item => {
|
||||||
|
const { host, port } = item;
|
||||||
|
assert(typeof host === 'string',
|
||||||
|
'bad config: redis sentinel host must be a string');
|
||||||
|
assert(typeof port === 'number',
|
||||||
|
'bad config: redis sentinel port must be a number');
|
||||||
|
this.redis.sentinels.push({ host, port });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.redis.sentinelPassword !== undefined) {
|
||||||
|
assert(
|
||||||
|
this._verifyRedisPassword(config.redis.sentinelPassword));
|
||||||
|
this.redis.sentinelPassword = config.redis.sentinelPassword;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// check for standalone configuration
|
||||||
|
this.redis = {};
|
||||||
|
assert(typeof config.redis.host === 'string',
|
||||||
|
'bad config: redis.host must be a string');
|
||||||
|
assert(typeof config.redis.port === 'number',
|
||||||
|
'bad config: redis.port must be a number');
|
||||||
|
this.redis.host = config.redis.host;
|
||||||
|
this.redis.port = config.redis.port;
|
||||||
}
|
}
|
||||||
if (config.scuba.port) {
|
if (config.redis.password !== undefined) {
|
||||||
assert(Number.isInteger(config.scuba.port)
|
assert(
|
||||||
&& config.scuba.port > 0,
|
this._verifyRedisPassword(config.redis.password),
|
||||||
'bad config: scuba port must be a positive integer');
|
'bad config: invalid password for redis. password must ' +
|
||||||
this.scuba.port = config.scuba.port;
|
'be a string');
|
||||||
|
this.redis.password = config.redis.password;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (process.env.SCUBA_HOST && process.env.SCUBA_PORT) {
|
|
||||||
assert(typeof process.env.SCUBA_HOST === 'string',
|
|
||||||
'bad config: scuba host must be a string');
|
|
||||||
assert(Number.isInteger(Number(process.env.SCUBA_PORT))
|
|
||||||
&& Number(process.env.SCUBA_PORT) > 0,
|
|
||||||
'bad config: scuba port must be a positive integer');
|
|
||||||
this.scuba = {
|
|
||||||
host: process.env.SCUBA_HOST,
|
|
||||||
port: Number(process.env.SCUBA_PORT),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (this.scuba) {
|
|
||||||
this.quotaEnabled = true;
|
|
||||||
}
|
|
||||||
const maxStaleness = Number(process.env.QUOTA_MAX_STALENESS_MS) ||
|
|
||||||
config.quota?.maxStatenessMS ||
|
|
||||||
24 * 60 * 60 * 1000;
|
|
||||||
assert(Number.isInteger(maxStaleness), 'bad config: maxStalenessMS must be an integer');
|
|
||||||
const enableInflights = process.env.QUOTA_ENABLE_INFLIGHTS === 'true' ||
|
|
||||||
config.quota?.enableInflights || false;
|
|
||||||
this.quota = {
|
|
||||||
maxStaleness,
|
|
||||||
enableInflights,
|
|
||||||
};
|
|
||||||
if (config.utapi) {
|
if (config.utapi) {
|
||||||
this.utapi = { component: 's3' };
|
this.utapi = { component: 's3' };
|
||||||
if (config.utapi.host) {
|
if (config.utapi.host) {
|
||||||
|
@ -1177,8 +1145,50 @@ class Config extends EventEmitter {
|
||||||
assert(config.redis, 'missing required property of utapi ' +
|
assert(config.redis, 'missing required property of utapi ' +
|
||||||
'configuration: redis');
|
'configuration: redis');
|
||||||
if (config.utapi.redis) {
|
if (config.utapi.redis) {
|
||||||
this.utapi.redis = parseRedisConfig(config.utapi.redis);
|
if (config.utapi.redis.sentinels) {
|
||||||
if (this.utapi.redis.retry === undefined) {
|
this.utapi.redis = { sentinels: [], name: null };
|
||||||
|
|
||||||
|
assert(typeof config.utapi.redis.name === 'string',
|
||||||
|
'bad config: redis sentinel name must be a string');
|
||||||
|
this.utapi.redis.name = config.utapi.redis.name;
|
||||||
|
|
||||||
|
assert(Array.isArray(config.utapi.redis.sentinels),
|
||||||
|
'bad config: redis sentinels must be an array');
|
||||||
|
config.utapi.redis.sentinels.forEach(item => {
|
||||||
|
const { host, port } = item;
|
||||||
|
assert(typeof host === 'string',
|
||||||
|
'bad config: redis sentinel host must be a string');
|
||||||
|
assert(typeof port === 'number',
|
||||||
|
'bad config: redis sentinel port must be a number');
|
||||||
|
this.utapi.redis.sentinels.push({ host, port });
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// check for standalone configuration
|
||||||
|
this.utapi.redis = {};
|
||||||
|
assert(typeof config.utapi.redis.host === 'string',
|
||||||
|
'bad config: redis.host must be a string');
|
||||||
|
assert(typeof config.utapi.redis.port === 'number',
|
||||||
|
'bad config: redis.port must be a number');
|
||||||
|
this.utapi.redis.host = config.utapi.redis.host;
|
||||||
|
this.utapi.redis.port = config.utapi.redis.port;
|
||||||
|
}
|
||||||
|
if (config.utapi.redis.retry !== undefined) {
|
||||||
|
if (config.utapi.redis.retry.connectBackoff !== undefined) {
|
||||||
|
const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
|
||||||
|
assert.strictEqual(typeof min, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: min must be a number');
|
||||||
|
assert.strictEqual(typeof max, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: max must be a number');
|
||||||
|
assert.strictEqual(typeof jitter, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: jitter must be a number');
|
||||||
|
assert.strictEqual(typeof factor, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: factor must be a number');
|
||||||
|
assert.strictEqual(typeof deadline, 'number',
|
||||||
|
'utapi.redis.retry.connectBackoff: deadline must be a number');
|
||||||
|
}
|
||||||
|
|
||||||
|
this.utapi.redis.retry = config.utapi.redis.retry;
|
||||||
|
} else {
|
||||||
this.utapi.redis.retry = {
|
this.utapi.redis.retry = {
|
||||||
connectBackoff: {
|
connectBackoff: {
|
||||||
min: 10,
|
min: 10,
|
||||||
|
@ -1189,6 +1199,22 @@ class Config extends EventEmitter {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (config.utapi.redis.password !== undefined) {
|
||||||
|
assert(
|
||||||
|
this._verifyRedisPassword(config.utapi.redis.password),
|
||||||
|
'config: invalid password for utapi redis. password' +
|
||||||
|
' must be a string');
|
||||||
|
this.utapi.redis.password = config.utapi.redis.password;
|
||||||
|
}
|
||||||
|
if (config.utapi.redis.sentinelPassword !== undefined) {
|
||||||
|
assert(
|
||||||
|
this._verifyRedisPassword(
|
||||||
|
config.utapi.redis.sentinelPassword),
|
||||||
|
'config: invalid password for utapi redis. password' +
|
||||||
|
' must be a string');
|
||||||
|
this.utapi.redis.sentinelPassword =
|
||||||
|
config.utapi.redis.sentinelPassword;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (config.utapi.metrics) {
|
if (config.utapi.metrics) {
|
||||||
this.utapi.metrics = config.utapi.metrics;
|
this.utapi.metrics = config.utapi.metrics;
|
||||||
|
@ -1258,7 +1284,8 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.utapi && config.utapi.reindex) {
|
if (config.utapi && config.utapi.reindex) {
|
||||||
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
|
parseUtapiReindex(config.utapi.reindex);
|
||||||
|
this.utapi.reindex = config.utapi.reindex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1303,8 +1330,6 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.authdata = config.authdata || 'authdata.json';
|
|
||||||
|
|
||||||
this.kms = {};
|
this.kms = {};
|
||||||
if (config.kms) {
|
if (config.kms) {
|
||||||
assert(typeof config.kms.userName === 'string');
|
assert(typeof config.kms.userName === 'string');
|
||||||
|
@ -1524,6 +1549,25 @@ class Config extends EventEmitter {
|
||||||
this.outboundProxy.certs = certObj.certs;
|
this.outboundProxy.certs = certObj.certs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.managementAgent = {};
|
||||||
|
this.managementAgent.port = 8010;
|
||||||
|
this.managementAgent.host = 'localhost';
|
||||||
|
if (config.managementAgent !== undefined) {
|
||||||
|
if (config.managementAgent.port !== undefined) {
|
||||||
|
assert(Number.isInteger(config.managementAgent.port)
|
||||||
|
&& config.managementAgent.port > 0,
|
||||||
|
'bad config: managementAgent port must be a positive ' +
|
||||||
|
'integer');
|
||||||
|
this.managementAgent.port = config.managementAgent.port;
|
||||||
|
}
|
||||||
|
if (config.managementAgent.host !== undefined) {
|
||||||
|
assert.strictEqual(typeof config.managementAgent.host, 'string',
|
||||||
|
'bad config: management agent host must ' +
|
||||||
|
'be a string');
|
||||||
|
this.managementAgent.host = config.managementAgent.host;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Ephemeral token to protect the reporting endpoint:
|
// Ephemeral token to protect the reporting endpoint:
|
||||||
// try inherited from parent first, then hardcoded in conf file,
|
// try inherited from parent first, then hardcoded in conf file,
|
||||||
// then create a fresh one as last resort.
|
// then create a fresh one as last resort.
|
||||||
|
@ -1613,8 +1657,6 @@ class Config extends EventEmitter {
|
||||||
'bad config: maxScannedLifecycleListingEntries must be greater than 2');
|
'bad config: maxScannedLifecycleListingEntries must be greater than 2');
|
||||||
this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
|
this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
|
||||||
}
|
}
|
||||||
|
|
||||||
this._configureBackends(config);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_setTimeOptions() {
|
_setTimeOptions() {
|
||||||
|
@ -1653,43 +1695,40 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
_getAuthData() {
|
_getAuthData() {
|
||||||
return JSON.parse(fs.readFileSync(findConfigFile(process.env.S3AUTH_CONFIG || this.authdata), { encoding: 'utf-8' }));
|
return require(findConfigFile(process.env.S3AUTH_CONFIG || 'authdata.json'));
|
||||||
}
|
}
|
||||||
|
|
||||||
_configureBackends(config) {
|
_configureBackends() {
|
||||||
const backends = config.backends || {};
|
|
||||||
/**
|
/**
|
||||||
* Configure the backends for Authentication, Data and Metadata.
|
* Configure the backends for Authentication, Data and Metadata.
|
||||||
*/
|
*/
|
||||||
let auth = backends.auth || 'mem';
|
let auth = 'mem';
|
||||||
let data = backends.data || 'multiple';
|
let data = 'multiple';
|
||||||
let metadata = backends.metadata || 'file';
|
let metadata = 'file';
|
||||||
let kms = backends.kms || 'file';
|
let kms = 'file';
|
||||||
let quota = backends.quota || 'none';
|
|
||||||
if (process.env.S3BACKEND) {
|
if (process.env.S3BACKEND) {
|
||||||
const validBackends = ['mem', 'file', 'scality', 'cdmi'];
|
const validBackends = ['mem', 'file', 'scality', 'cdmi'];
|
||||||
assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
|
assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
|
||||||
'bad environment variable: S3BACKEND environment variable ' +
|
'bad environment variable: S3BACKEND environment variable ' +
|
||||||
'should be one of mem/file/scality/cdmi'
|
'should be one of mem/file/scality/cdmi'
|
||||||
);
|
);
|
||||||
auth = process.env.S3BACKEND == 'scality' ? 'scality' : 'mem';
|
auth = process.env.S3BACKEND;
|
||||||
data = process.env.S3BACKEND;
|
data = process.env.S3BACKEND;
|
||||||
metadata = process.env.S3BACKEND;
|
metadata = process.env.S3BACKEND;
|
||||||
kms = process.env.S3BACKEND;
|
kms = process.env.S3BACKEND;
|
||||||
}
|
}
|
||||||
if (process.env.S3VAULT) {
|
if (process.env.S3VAULT) {
|
||||||
auth = process.env.S3VAULT;
|
auth = process.env.S3VAULT;
|
||||||
auth = (auth === 'file' || auth === 'mem' || auth === 'cdmi' ? 'mem' : auth);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
|
if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
|
||||||
// Auth only checks for 'mem' since mem === file
|
// Auth only checks for 'mem' since mem === file
|
||||||
|
auth = 'mem';
|
||||||
let authData;
|
let authData;
|
||||||
if (process.env.SCALITY_ACCESS_KEY_ID &&
|
if (process.env.SCALITY_ACCESS_KEY_ID &&
|
||||||
process.env.SCALITY_SECRET_ACCESS_KEY) {
|
process.env.SCALITY_SECRET_ACCESS_KEY) {
|
||||||
authData = buildAuthDataAccount(
|
authData = buildAuthDataAccount(
|
||||||
process.env.SCALITY_ACCESS_KEY_ID,
|
process.env.SCALITY_ACCESS_KEY_ID,
|
||||||
process.env.SCALITY_SECRET_ACCESS_KEY);
|
process.env.SCALITY_SECRET_ACCESS_KEY);
|
||||||
} else {
|
} else {
|
||||||
authData = this._getAuthData();
|
authData = this._getAuthData();
|
||||||
}
|
}
|
||||||
|
@ -1697,7 +1736,7 @@ class Config extends EventEmitter {
|
||||||
throw new Error('bad config: invalid auth config file.');
|
throw new Error('bad config: invalid auth config file.');
|
||||||
}
|
}
|
||||||
this.authData = authData;
|
this.authData = authData;
|
||||||
} else if (auth === 'multiple') {
|
} else if (auth === 'multiple') {
|
||||||
const authData = this._getAuthData();
|
const authData = this._getAuthData();
|
||||||
if (validateAuthConfig(authData)) {
|
if (validateAuthConfig(authData)) {
|
||||||
throw new Error('bad config: invalid auth config file.');
|
throw new Error('bad config: invalid auth config file.');
|
||||||
|
@ -1712,9 +1751,9 @@ class Config extends EventEmitter {
|
||||||
'should be one of mem/file/scality/multiple'
|
'should be one of mem/file/scality/multiple'
|
||||||
);
|
);
|
||||||
data = process.env.S3DATA;
|
data = process.env.S3DATA;
|
||||||
if (data === 'scality' || data === 'multiple') {
|
}
|
||||||
data = 'multiple';
|
if (data === 'scality' || data === 'multiple') {
|
||||||
}
|
data = 'multiple';
|
||||||
}
|
}
|
||||||
assert(this.locationConstraints !== undefined &&
|
assert(this.locationConstraints !== undefined &&
|
||||||
this.restEndpoints !== undefined,
|
this.restEndpoints !== undefined,
|
||||||
|
@ -1727,18 +1766,18 @@ class Config extends EventEmitter {
|
||||||
if (process.env.S3KMS) {
|
if (process.env.S3KMS) {
|
||||||
kms = process.env.S3KMS;
|
kms = process.env.S3KMS;
|
||||||
}
|
}
|
||||||
if (process.env.S3QUOTA) {
|
|
||||||
quota = process.env.S3QUOTA;
|
|
||||||
}
|
|
||||||
this.backends = {
|
this.backends = {
|
||||||
auth,
|
auth,
|
||||||
data,
|
data,
|
||||||
metadata,
|
metadata,
|
||||||
kms,
|
kms,
|
||||||
quota,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_verifyRedisPassword(password) {
|
||||||
|
return typeof password === 'string';
|
||||||
|
}
|
||||||
|
|
||||||
setAuthDataAccounts(accounts) {
|
setAuthDataAccounts(accounts) {
|
||||||
this.authData.accounts = accounts;
|
this.authData.accounts = accounts;
|
||||||
this.emit('authdata-update');
|
this.emit('authdata-update');
|
||||||
|
@ -1861,19 +1900,10 @@ class Config extends EventEmitter {
|
||||||
.update(instanceId)
|
.update(instanceId)
|
||||||
.digest('hex');
|
.digest('hex');
|
||||||
}
|
}
|
||||||
|
|
||||||
isQuotaEnabled() {
|
|
||||||
return !!this.quotaEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
isQuotaInflightEnabled() {
|
|
||||||
return this.quota.enableInflights;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
parseSproxydConfig,
|
parseSproxydConfig,
|
||||||
parseRedisConfig,
|
|
||||||
locationConstraintAssert,
|
locationConstraintAssert,
|
||||||
ConfigObject: Config,
|
ConfigObject: Config,
|
||||||
config: new Config(),
|
config: new Config(),
|
||||||
|
|
|
@ -7,7 +7,6 @@ const bucketDeleteEncryption = require('./bucketDeleteEncryption');
|
||||||
const bucketDeleteWebsite = require('./bucketDeleteWebsite');
|
const bucketDeleteWebsite = require('./bucketDeleteWebsite');
|
||||||
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
|
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
|
||||||
const bucketDeletePolicy = require('./bucketDeletePolicy');
|
const bucketDeletePolicy = require('./bucketDeletePolicy');
|
||||||
const bucketDeleteQuota = require('./bucketDeleteQuota');
|
|
||||||
const { bucketGet } = require('./bucketGet');
|
const { bucketGet } = require('./bucketGet');
|
||||||
const bucketGetACL = require('./bucketGetACL');
|
const bucketGetACL = require('./bucketGetACL');
|
||||||
const bucketGetCors = require('./bucketGetCors');
|
const bucketGetCors = require('./bucketGetCors');
|
||||||
|
@ -18,7 +17,6 @@ const bucketGetLifecycle = require('./bucketGetLifecycle');
|
||||||
const bucketGetNotification = require('./bucketGetNotification');
|
const bucketGetNotification = require('./bucketGetNotification');
|
||||||
const bucketGetObjectLock = require('./bucketGetObjectLock');
|
const bucketGetObjectLock = require('./bucketGetObjectLock');
|
||||||
const bucketGetPolicy = require('./bucketGetPolicy');
|
const bucketGetPolicy = require('./bucketGetPolicy');
|
||||||
const bucketGetQuota = require('./bucketGetQuota');
|
|
||||||
const bucketGetEncryption = require('./bucketGetEncryption');
|
const bucketGetEncryption = require('./bucketGetEncryption');
|
||||||
const bucketHead = require('./bucketHead');
|
const bucketHead = require('./bucketHead');
|
||||||
const { bucketPut } = require('./bucketPut');
|
const { bucketPut } = require('./bucketPut');
|
||||||
|
@ -35,7 +33,6 @@ const bucketPutNotification = require('./bucketPutNotification');
|
||||||
const bucketPutEncryption = require('./bucketPutEncryption');
|
const bucketPutEncryption = require('./bucketPutEncryption');
|
||||||
const bucketPutPolicy = require('./bucketPutPolicy');
|
const bucketPutPolicy = require('./bucketPutPolicy');
|
||||||
const bucketPutObjectLock = require('./bucketPutObjectLock');
|
const bucketPutObjectLock = require('./bucketPutObjectLock');
|
||||||
const bucketUpdateQuota = require('./bucketUpdateQuota');
|
|
||||||
const bucketGetReplication = require('./bucketGetReplication');
|
const bucketGetReplication = require('./bucketGetReplication');
|
||||||
const bucketDeleteReplication = require('./bucketDeleteReplication');
|
const bucketDeleteReplication = require('./bucketDeleteReplication');
|
||||||
const corsPreflight = require('./corsPreflight');
|
const corsPreflight = require('./corsPreflight');
|
||||||
|
@ -67,7 +64,8 @@ const prepareRequestContexts
|
||||||
= require('./apiUtils/authorization/prepareRequestContexts');
|
= require('./apiUtils/authorization/prepareRequestContexts');
|
||||||
const serviceGet = require('./serviceGet');
|
const serviceGet = require('./serviceGet');
|
||||||
const vault = require('../auth/vault');
|
const vault = require('../auth/vault');
|
||||||
const website = require('./website');
|
const websiteGet = require('./websiteGet');
|
||||||
|
const websiteHead = require('./websiteHead');
|
||||||
const writeContinue = require('../utilities/writeContinue');
|
const writeContinue = require('../utilities/writeContinue');
|
||||||
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
|
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
|
||||||
const parseCopySource = require('./apiUtils/object/parseCopySource');
|
const parseCopySource = require('./apiUtils/object/parseCopySource');
|
||||||
|
@ -85,10 +83,6 @@ const api = {
|
||||||
// Attach the apiMethod method to the request, so it can used by monitoring in the server
|
// Attach the apiMethod method to the request, so it can used by monitoring in the server
|
||||||
// eslint-disable-next-line no-param-reassign
|
// eslint-disable-next-line no-param-reassign
|
||||||
request.apiMethod = apiMethod;
|
request.apiMethod = apiMethod;
|
||||||
// Array of end of API callbacks, used to perform some logic
|
|
||||||
// at the end of an API.
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
request.finalizerHooks = [];
|
|
||||||
|
|
||||||
const actionLog = monitoringMap[apiMethod];
|
const actionLog = monitoringMap[apiMethod];
|
||||||
if (!actionLog &&
|
if (!actionLog &&
|
||||||
|
@ -197,17 +191,14 @@ const api = {
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
next => auth.server.doAuth(
|
next => auth.server.doAuth(
|
||||||
request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
|
request, log, (err, userInfo, authorizationResults, streamingV4Params) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
// VaultClient returns standard errors, but the route requires
|
|
||||||
// Arsenal errors
|
|
||||||
const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
|
|
||||||
log.trace('authentication error', { error: err });
|
log.trace('authentication error', { error: err });
|
||||||
return next(arsenalError);
|
return next(err);
|
||||||
}
|
}
|
||||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
return next(null, userInfo, authorizationResults, streamingV4Params);
|
||||||
}, 's3', requestContexts),
|
}, 's3', requestContexts),
|
||||||
(userInfo, authorizationResults, streamingV4Params, infos, next) => {
|
(userInfo, authorizationResults, streamingV4Params, next) => {
|
||||||
const authNames = { accountName: userInfo.getAccountDisplayName() };
|
const authNames = { accountName: userInfo.getAccountDisplayName() };
|
||||||
if (userInfo.isRequesterAnIAMUser()) {
|
if (userInfo.isRequesterAnIAMUser()) {
|
||||||
authNames.userName = userInfo.getIAMdisplayName();
|
authNames.userName = userInfo.getIAMdisplayName();
|
||||||
|
@ -217,7 +208,7 @@ const api = {
|
||||||
}
|
}
|
||||||
log.addDefaultFields(authNames);
|
log.addDefaultFields(authNames);
|
||||||
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
||||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
return next(null, userInfo, authorizationResults, streamingV4Params);
|
||||||
}
|
}
|
||||||
// issue 100 Continue to the client
|
// issue 100 Continue to the client
|
||||||
writeContinue(request, response);
|
writeContinue(request, response);
|
||||||
|
@ -248,12 +239,12 @@ const api = {
|
||||||
}
|
}
|
||||||
// Convert array of post buffers into one string
|
// Convert array of post buffers into one string
|
||||||
request.post = Buffer.concat(post, postLength).toString();
|
request.post = Buffer.concat(post, postLength).toString();
|
||||||
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
|
return next(null, userInfo, authorizationResults, streamingV4Params);
|
||||||
});
|
});
|
||||||
return undefined;
|
return undefined;
|
||||||
},
|
},
|
||||||
// Tag condition keys require information from CloudServer for evaluation
|
// Tag condition keys require information from CloudServer for evaluation
|
||||||
(userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
|
(userInfo, authorizationResults, streamingV4Params, next) => tagConditionKeyAuth(
|
||||||
authorizationResults,
|
authorizationResults,
|
||||||
request,
|
request,
|
||||||
requestContexts,
|
requestContexts,
|
||||||
|
@ -264,14 +255,13 @@ const api = {
|
||||||
log.trace('tag authentication error', { error: err });
|
log.trace('tag authentication error', { error: err });
|
||||||
return next(err);
|
return next(err);
|
||||||
}
|
}
|
||||||
return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
|
return next(null, userInfo, authResultsWithTags, streamingV4Params);
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
|
], (err, userInfo, authorizationResults, streamingV4Params) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return callback(err);
|
return callback(err);
|
||||||
}
|
}
|
||||||
request.accountQuotas = infos?.accountQuota;
|
|
||||||
if (authorizationResults) {
|
if (authorizationResults) {
|
||||||
const checkedResults = checkAuthResults(authorizationResults);
|
const checkedResults = checkAuthResults(authorizationResults);
|
||||||
if (checkedResults instanceof Error) {
|
if (checkedResults instanceof Error) {
|
||||||
|
@ -288,23 +278,19 @@ const api = {
|
||||||
return acc;
|
return acc;
|
||||||
}, {});
|
}, {});
|
||||||
}
|
}
|
||||||
const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
|
|
||||||
(hook, done) => hook(err, done),
|
|
||||||
() => callback(err, ...results));
|
|
||||||
|
|
||||||
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
|
||||||
request._response = response;
|
request._response = response;
|
||||||
return this[apiMethod](userInfo, request, streamingV4Params,
|
return this[apiMethod](userInfo, request, streamingV4Params,
|
||||||
log, methodCallback, authorizationResults);
|
log, callback, authorizationResults);
|
||||||
}
|
}
|
||||||
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
|
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
|
||||||
return this[apiMethod](userInfo, request, sourceBucket,
|
return this[apiMethod](userInfo, request, sourceBucket,
|
||||||
sourceObject, sourceVersionId, log, methodCallback);
|
sourceObject, sourceVersionId, log, callback);
|
||||||
}
|
}
|
||||||
if (apiMethod === 'objectGet') {
|
if (apiMethod === 'objectGet') {
|
||||||
return this[apiMethod](userInfo, request, returnTagCount, log, callback);
|
return this[apiMethod](userInfo, request, returnTagCount, log, callback);
|
||||||
}
|
}
|
||||||
return this[apiMethod](userInfo, request, log, methodCallback);
|
return this[apiMethod](userInfo, request, log, callback);
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
bucketDelete,
|
bucketDelete,
|
||||||
|
@ -331,14 +317,11 @@ const api = {
|
||||||
bucketPutReplication,
|
bucketPutReplication,
|
||||||
bucketGetReplication,
|
bucketGetReplication,
|
||||||
bucketDeleteReplication,
|
bucketDeleteReplication,
|
||||||
bucketDeleteQuota,
|
|
||||||
bucketPutLifecycle,
|
bucketPutLifecycle,
|
||||||
bucketUpdateQuota,
|
|
||||||
bucketGetLifecycle,
|
bucketGetLifecycle,
|
||||||
bucketDeleteLifecycle,
|
bucketDeleteLifecycle,
|
||||||
bucketPutPolicy,
|
bucketPutPolicy,
|
||||||
bucketGetPolicy,
|
bucketGetPolicy,
|
||||||
bucketGetQuota,
|
|
||||||
bucketDeletePolicy,
|
bucketDeletePolicy,
|
||||||
bucketPutObjectLock,
|
bucketPutObjectLock,
|
||||||
bucketPutNotification,
|
bucketPutNotification,
|
||||||
|
@ -370,8 +353,8 @@ const api = {
|
||||||
objectPutRetention,
|
objectPutRetention,
|
||||||
objectRestore,
|
objectRestore,
|
||||||
serviceGet,
|
serviceGet,
|
||||||
websiteGet: website,
|
websiteGet,
|
||||||
websiteHead: website,
|
websiteHead,
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = api;
|
module.exports = api;
|
||||||
|
|
|
@ -1,19 +1,8 @@
|
||||||
const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
|
const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const { parseCIDR, isValid } = require('ipaddr.js');
|
|
||||||
const constants = require('../../../../constants');
|
const constants = require('../../../../constants');
|
||||||
const { config } = require('../../../Config');
|
const { config } = require('../../../Config');
|
||||||
|
|
||||||
const {
|
const { allAuthedUsersId, bucketOwnerActions, logId, publicId, arrayOfAllowed } = constants;
|
||||||
allAuthedUsersId,
|
|
||||||
bucketOwnerActions,
|
|
||||||
logId,
|
|
||||||
publicId,
|
|
||||||
arrayOfAllowed,
|
|
||||||
assumedRoleArnResourceType,
|
|
||||||
backbeatLifecycleSessionName,
|
|
||||||
actionsToConsiderAsObjectPut,
|
|
||||||
} = constants;
|
|
||||||
|
|
||||||
// whitelist buckets to allow public read on objects
|
// whitelist buckets to allow public read on objects
|
||||||
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
|
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
|
||||||
|
@ -51,21 +40,17 @@ function isRequesterNonAccountUser(authInfo) {
|
||||||
|
|
||||||
function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
||||||
// Same logic applies on the Versioned APIs, so let's simplify it.
|
// Same logic applies on the Versioned APIs, so let's simplify it.
|
||||||
let requestTypeParsed = requestType.endsWith('Version') ?
|
const requestTypeParsed = requestType.endsWith('Version') ?
|
||||||
requestType.slice(0, 'Version'.length * -1) : requestType;
|
requestType.slice(0, 'Version'.length * -1) : requestType;
|
||||||
requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
|
|
||||||
'objectPut' : requestTypeParsed;
|
|
||||||
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
|
|
||||||
'objectPut' : mainApiCall;
|
|
||||||
if (bucket.getOwner() === canonicalID) {
|
if (bucket.getOwner() === canonicalID) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (parsedMainApiCall === 'objectGet') {
|
if (mainApiCall === 'objectGet') {
|
||||||
if (requestTypeParsed === 'objectGetTagging') {
|
if (requestTypeParsed === 'objectGetTagging') {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (parsedMainApiCall === 'objectPut') {
|
if (mainApiCall === 'objectPut') {
|
||||||
if (arrayOfAllowed.includes(requestTypeParsed)) {
|
if (arrayOfAllowed.includes(requestTypeParsed)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -141,18 +126,14 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
|
||||||
// authorization check should just return true so can move on to check
|
// authorization check should just return true so can move on to check
|
||||||
// rights at the object level.
|
// rights at the object level.
|
||||||
return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
|
return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
|
||||||
|| requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
|
|| requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
|
function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
|
||||||
isUserUnauthenticated, mainApiCall) {
|
isUserUnauthenticated, mainApiCall) {
|
||||||
const bucketOwner = bucket.getOwner();
|
const bucketOwner = bucket.getOwner();
|
||||||
const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
|
|
||||||
'objectPut' : requestType;
|
|
||||||
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
|
|
||||||
'objectPut' : mainApiCall;
|
|
||||||
// acls don't distinguish between users and accounts, so both should be allowed
|
// acls don't distinguish between users and accounts, so both should be allowed
|
||||||
if (bucketOwnerActions.includes(requestTypeParsed)
|
if (bucketOwnerActions.includes(requestType)
|
||||||
&& (bucketOwner === canonicalID)) {
|
&& (bucketOwner === canonicalID)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -161,9 +142,9 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
}
|
}
|
||||||
|
|
||||||
// Backward compatibility
|
// Backward compatibility
|
||||||
if (parsedMainApiCall === 'objectGet') {
|
if (mainApiCall === 'objectGet') {
|
||||||
if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
|
if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
|
||||||
&& requestTypeParsed === 'objectGetTagging') {
|
&& requestType === 'objectGetTagging') {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -172,7 +153,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') {
|
if (requestType === 'objectGet' || requestType === 'objectHead') {
|
||||||
if (objectMD.acl.Canned === 'public-read'
|
if (objectMD.acl.Canned === 'public-read'
|
||||||
|| objectMD.acl.Canned === 'public-read-write'
|
|| objectMD.acl.Canned === 'public-read-write'
|
||||||
|| (objectMD.acl.Canned === 'authenticated-read'
|
|| (objectMD.acl.Canned === 'authenticated-read'
|
||||||
|
@ -198,11 +179,11 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
|
|
||||||
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
||||||
// bucket has canned ACL public-read-write
|
// bucket has canned ACL public-read-write
|
||||||
if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') {
|
if (requestType === 'objectPut' || requestType === 'objectDelete') {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'objectPutACL') {
|
if (requestType === 'objectPutACL') {
|
||||||
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
||||||
&& bucketOwner === canonicalID)
|
&& bucketOwner === canonicalID)
|
||||||
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
||||||
|
@ -218,7 +199,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (requestTypeParsed === 'objectGetACL') {
|
if (requestType === 'objectGetACL') {
|
||||||
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
|
||||||
&& bucketOwner === canonicalID)
|
&& bucketOwner === canonicalID)
|
||||||
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|
||||||
|
@ -239,7 +220,7 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs
|
||||||
const bucketAcl = bucket.getAcl();
|
const bucketAcl = bucket.getAcl();
|
||||||
const allowPublicReads = publicReadBuckets.includes(bucket.getName())
|
const allowPublicReads = publicReadBuckets.includes(bucket.getName())
|
||||||
&& bucketAcl.Canned === 'public-read'
|
&& bucketAcl.Canned === 'public-read'
|
||||||
&& (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
|
&& (requestType === 'objectGet' || requestType === 'objectHead');
|
||||||
if (allowPublicReads) {
|
if (allowPublicReads) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -373,7 +354,7 @@ function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner,
|
||||||
}
|
}
|
||||||
|
|
||||||
function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
|
function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
|
||||||
actionImplicitDeniesInput = {}, isWebsite = false) {
|
actionImplicitDeniesInput = {}) {
|
||||||
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
|
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
|
||||||
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
|
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
|
||||||
const mainApiCall = requestTypes[0];
|
const mainApiCall = requestTypes[0];
|
||||||
|
@ -397,15 +378,6 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo
|
||||||
return results[_requestType];
|
return results[_requestType];
|
||||||
}
|
}
|
||||||
const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
|
const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
|
||||||
// In case of error bucket access is checked with bucketGet
|
|
||||||
// For website, bucket policy only uses objectGet and ignores bucketGet
|
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
|
|
||||||
// bucketGet should be used to check acl but switched to objectGet for bucket policy
|
|
||||||
if (isWebsite && _requestType === 'bucketGet') {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
_requestType = 'objectGet';
|
|
||||||
actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
|
|
||||||
}
|
|
||||||
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
|
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
|
||||||
request, aclPermission, results, actionImplicitDenies);
|
request, aclPermission, results, actionImplicitDenies);
|
||||||
});
|
});
|
||||||
|
@ -425,12 +397,12 @@ function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, aut
|
||||||
arn = authInfo.getArn();
|
arn = authInfo.getArn();
|
||||||
}
|
}
|
||||||
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
|
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
|
||||||
request, true, results, actionImplicitDenies);
|
request, true, results, actionImplicitDenies);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
|
function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
|
||||||
actionImplicitDeniesInput = {}, isWebsite = false) {
|
actionImplicitDeniesInput = {}) {
|
||||||
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
|
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
|
||||||
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
|
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
|
||||||
const results = {};
|
const results = {};
|
||||||
|
@ -443,20 +415,16 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI
|
||||||
? _requestType.slice(0, -7) : _requestType;
|
? _requestType.slice(0, -7) : _requestType;
|
||||||
const bucketOwner = bucket.getOwner();
|
const bucketOwner = bucket.getOwner();
|
||||||
if (!objectMD) {
|
if (!objectMD) {
|
||||||
// check bucket has read access
|
|
||||||
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
|
|
||||||
let permission = 'bucketGet';
|
|
||||||
if (actionsToConsiderAsObjectPut.includes(_requestType)) {
|
|
||||||
permission = 'objectPut';
|
|
||||||
}
|
|
||||||
results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
|
|
||||||
actionImplicitDenies, isWebsite);
|
|
||||||
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
|
||||||
// bucket has canned ACL public-read-write
|
// bucket has canned ACL public-read-write
|
||||||
if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
|
if (parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete') {
|
||||||
&& results[_requestType] === false) {
|
|
||||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
||||||
|
return results[_requestType];
|
||||||
}
|
}
|
||||||
|
// check bucket has read access
|
||||||
|
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
|
||||||
|
results[_requestType] = isBucketAuthorized(bucket, 'bucketGet', canonicalID, authInfo, log, request,
|
||||||
|
actionImplicitDenies);
|
||||||
return results[_requestType];
|
return results[_requestType];
|
||||||
}
|
}
|
||||||
let requesterIsNotUser = true;
|
let requesterIsNotUser = true;
|
||||||
|
@ -476,8 +444,8 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI
|
||||||
// - account is the bucket owner
|
// - account is the bucket owner
|
||||||
// - requester is account, not user
|
// - requester is account, not user
|
||||||
if (bucketOwnerActions.includes(parsedMethodName)
|
if (bucketOwnerActions.includes(parsedMethodName)
|
||||||
&& (bucketOwner === canonicalID)
|
&& (bucketOwner === canonicalID)
|
||||||
&& requesterIsNotUser) {
|
&& requesterIsNotUser) {
|
||||||
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
results[_requestType] = actionImplicitDenies[_requestType] === false;
|
||||||
return results[_requestType];
|
return results[_requestType];
|
||||||
}
|
}
|
||||||
|
@ -515,46 +483,11 @@ function validatePolicyResource(bucketName, policy) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkIp(value) {
|
function checkIp(value) {
|
||||||
const errString = 'Invalid IP address in Conditions';
|
const isValid = /^(\d{1,3}\.){3}\d{1,3}(\/\d{1,2})?$/.test(value);
|
||||||
|
if (isValid) {
|
||||||
const values = Array.isArray(value) ? value : [value];
|
return null;
|
||||||
|
|
||||||
for (let i = 0; i < values.length; i++) {
|
|
||||||
// these preliminary checks are validating the provided
|
|
||||||
// ip address against ipaddr.js, the library we use when
|
|
||||||
// evaluating IP condition keys. It ensures compatibility,
|
|
||||||
// but additional checks are required to enforce the right
|
|
||||||
// notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
|
|
||||||
// we would accept different ip formats, which is not
|
|
||||||
// standard in an AWS use case.
|
|
||||||
try {
|
|
||||||
try {
|
|
||||||
parseCIDR(values[i]);
|
|
||||||
} catch (err) {
|
|
||||||
isValid(values[i]);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
return errString;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply the existing IP validation logic to each element
|
|
||||||
const validateIpRegex = ip => {
|
|
||||||
if (constants.ipv4Regex.test(ip)) {
|
|
||||||
return ip.split('.').every(part => parseInt(part, 10) <= 255);
|
|
||||||
}
|
|
||||||
if (constants.ipv6Regex.test(ip)) {
|
|
||||||
return ip.split(':').every(part => part.length <= 4);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
if (validateIpRegex(values[i]) !== true) {
|
|
||||||
return errString;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
return 'Invalid IP address in Conditions';
|
||||||
// If the function hasn't returned by now, all elements are valid
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// This function checks all bucket policy conditions if the values provided
|
// This function checks all bucket policy conditions if the values provided
|
||||||
|
@ -567,33 +500,29 @@ function validatePolicyConditions(policy) {
|
||||||
// keys where value type does not seem to be checked by AWS:
|
// keys where value type does not seem to be checked by AWS:
|
||||||
// - s3:object-lock-remaining-retention-days
|
// - s3:object-lock-remaining-retention-days
|
||||||
|
|
||||||
if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// there can be multiple statements in the policy, each with a Condition enclosure
|
// there can be multiple statements in the policy, each with a Condition enclosure
|
||||||
for (let i = 0; i < policy.Statement.length; i++) {
|
for (let i = 0; i < policy.Statement.length; i++) {
|
||||||
const s = policy.Statement[i];
|
const s = policy.Statement[i];
|
||||||
if (s.Condition) {
|
if (s.Condition) {
|
||||||
const conditionOperators = Object.keys(s.Condition);
|
const conditionOperators = Object.keys(s.Condition);
|
||||||
// there can be multiple condition operations in the Condition enclosure
|
// there can be multiple condition operations in the Condition enclosure
|
||||||
// eslint-disable-next-line no-restricted-syntax
|
for (let j = 0; j < conditionOperators.length; j++) {
|
||||||
for (const conditionOperator of conditionOperators) {
|
const conditionOperator = conditionOperators[j];
|
||||||
const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
|
const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
|
||||||
const conditionValue = s.Condition[conditionOperator][conditionKey];
|
const conditionValue = s.Condition[conditionOperator][conditionKey];
|
||||||
const validCondition = validConditions.find(validCondition =>
|
const validCondition = validConditions.find(validCondition =>
|
||||||
validCondition.conditionKey === conditionKey
|
validCondition.conditionKey === conditionKey
|
||||||
);
|
);
|
||||||
// AWS returns does not return an error if the condition starts with 'aws:'
|
// this is the seen behaviour on AWS... don't ask me why
|
||||||
// so we reproduce this behaviour
|
|
||||||
if (!validCondition && !conditionKey.startsWith('aws:')) {
|
if (!validCondition && !conditionKey.startsWith('aws:')) {
|
||||||
return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
|
return 'Policy has an invalid condition key';
|
||||||
}
|
}
|
||||||
|
let conditionValueTypeError;
|
||||||
if (validCondition && validCondition.conditionValueTypeChecker) {
|
if (validCondition && validCondition.conditionValueTypeChecker) {
|
||||||
const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
|
conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
|
||||||
if (conditionValueTypeError) {
|
}
|
||||||
return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
|
if (conditionValueTypeError) {
|
||||||
}
|
return conditionValueTypeError;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -52,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||||
apiMethod, 's3');
|
apiMethod, 's3');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (apiMethod === 'bucketPut') {
|
if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -65,17 +65,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||||
|
|
||||||
const requestContexts = [];
|
const requestContexts = [];
|
||||||
|
|
||||||
if (apiMethod === 'multiObjectDelete') {
|
if (apiMethodAfterVersionCheck === 'objectCopy'
|
||||||
// MultiObjectDelete does not require any authorization when evaluating
|
|
||||||
// the API. Instead, we authorize each object passed.
|
|
||||||
// But in order to get any relevant information from the authorization service
|
|
||||||
// for example, the account quota, we must send a request context object
|
|
||||||
// with no `specificResource`. We expect the result to be an implicit deny.
|
|
||||||
// In the API, we then ignore these authorization results, and we can use
|
|
||||||
// any information returned, e.g., the quota.
|
|
||||||
const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
|
|
||||||
requestContexts.push(requestContextMultiObjectDelete);
|
|
||||||
} else if (apiMethodAfterVersionCheck === 'objectCopy'
|
|
||||||
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') {
|
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') {
|
||||||
const objectGetAction = sourceVersionId ? 'objectGetVersion' :
|
const objectGetAction = sourceVersionId ? 'objectGetVersion' :
|
||||||
'objectGet';
|
'objectGet';
|
||||||
|
|
|
@ -30,9 +30,6 @@ function bucketShield(bucket, requestType) {
|
||||||
// Otherwise return an error to the client
|
// Otherwise return an error to the client
|
||||||
if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
|
if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
|
||||||
(requestType !== 'objectPut' &&
|
(requestType !== 'objectPut' &&
|
||||||
requestType !== 'initiateMultipartUpload' &&
|
|
||||||
requestType !== 'objectPutPart' &&
|
|
||||||
requestType !== 'completeMultipartUpload' &&
|
|
||||||
requestType !== 'bucketPutACL' &&
|
requestType !== 'bucketPutACL' &&
|
||||||
requestType !== 'bucketDelete')) {
|
requestType !== 'bucketDelete')) {
|
||||||
return true;
|
return true;
|
||||||
|
|
|
@ -14,7 +14,7 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
uploadId,
|
uploadId,
|
||||||
preciseRequestType: request.apiMethods || 'multipartDelete',
|
preciseRequestType: 'multipartDelete',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
// For validating the request at the destinationBucket level
|
// For validating the request at the destinationBucket level
|
||||||
|
|
|
@ -7,8 +7,6 @@ const errors = require('arsenal').errors;
|
||||||
const { config } = require('../../../Config');
|
const { config } = require('../../../Config');
|
||||||
const { locationConstraints } = config;
|
const { locationConstraints } = config;
|
||||||
|
|
||||||
const { scaledMsPerDay } = config.getTimeOptions();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get response header "x-amz-restore"
|
* Get response header "x-amz-restore"
|
||||||
* Be called by objectHead.js
|
* Be called by objectHead.js
|
||||||
|
@ -147,7 +145,7 @@ function _updateObjectExpirationDate(objectMD, log) {
|
||||||
});
|
});
|
||||||
if (isObjectAlreadyRestored) {
|
if (isObjectAlreadyRestored) {
|
||||||
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
|
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
|
||||||
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
|
expiryDate.setDate(expiryDate.getDate() + objectMD.archive.restoreRequestedDays);
|
||||||
|
|
||||||
/* eslint-disable no-param-reassign */
|
/* eslint-disable no-param-reassign */
|
||||||
objectMD.archive.restoreWillExpireAt = expiryDate;
|
objectMD.archive.restoreWillExpireAt = expiryDate;
|
||||||
|
|
|
@ -5,6 +5,7 @@ const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
|
||||||
const constants = require('../../../../constants');
|
const constants = require('../../../../constants');
|
||||||
const { data } = require('../../../data/wrapper');
|
const { data } = require('../../../data/wrapper');
|
||||||
const services = require('../../../services');
|
const services = require('../../../services');
|
||||||
|
const logger = require('../../../utilities/logger');
|
||||||
const { dataStore } = require('./storeObject');
|
const { dataStore } = require('./storeObject');
|
||||||
const locationConstraintCheck = require('./locationConstraintCheck');
|
const locationConstraintCheck = require('./locationConstraintCheck');
|
||||||
const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
|
const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
|
||||||
|
@ -20,7 +21,7 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' +
|
||||||
'a versioned object to a location-constraint of type Azure or GCP.';
|
'a versioned object to a location-constraint of type Azure or GCP.';
|
||||||
|
|
||||||
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
||||||
metadataStoreParams, dataToDelete, log, requestMethod, callback) {
|
metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
|
||||||
services.metadataStoreObject(bucketName, dataGetInfo,
|
services.metadataStoreObject(bucketName, dataGetInfo,
|
||||||
cipherBundle, metadataStoreParams, (err, result) => {
|
cipherBundle, metadataStoreParams, (err, result) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
|
@ -30,7 +31,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
|
||||||
const newDataStoreName = Array.isArray(dataGetInfo) ?
|
const newDataStoreName = Array.isArray(dataGetInfo) ?
|
||||||
dataGetInfo[0].dataStoreName : null;
|
dataGetInfo[0].dataStoreName : null;
|
||||||
return data.batchDelete(dataToDelete, requestMethod,
|
return data.batchDelete(dataToDelete, requestMethod,
|
||||||
newDataStoreName, log, err => callback(err, result));
|
newDataStoreName, deleteLog, err => callback(err, result));
|
||||||
}
|
}
|
||||||
return callback(null, result);
|
return callback(null, result);
|
||||||
});
|
});
|
||||||
|
@ -197,9 +198,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
const dontSkipBackend = externalBackends;
|
const dontSkipBackend = externalBackends;
|
||||||
/* eslint-enable camelcase */
|
/* eslint-enable camelcase */
|
||||||
|
|
||||||
|
const requestLogger =
|
||||||
|
logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
|
||||||
const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
|
const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
|
||||||
const mdOnlySize = request.headers['x-amz-meta-size'];
|
const mdOnlySize = request.headers['x-amz-meta-size'];
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
function storeData(next) {
|
function storeData(next) {
|
||||||
if (size === 0) {
|
if (size === 0) {
|
||||||
|
@ -294,7 +296,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
|
||||||
}
|
}
|
||||||
return _storeInMDandDeleteData(bucketName, infoArr,
|
return _storeInMDandDeleteData(bucketName, infoArr,
|
||||||
cipherBundle, metadataStoreParams,
|
cipherBundle, metadataStoreParams,
|
||||||
options.dataToDelete, log, requestMethod, next);
|
options.dataToDelete, requestLogger, requestMethod, next);
|
||||||
},
|
},
|
||||||
], callback);
|
], callback);
|
||||||
}
|
}
|
||||||
|
|
|
@ -306,7 +306,7 @@ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log,
|
||||||
return cb(err);
|
return cb(err);
|
||||||
}
|
}
|
||||||
const explicitDenyExists = authorizationResults.some(
|
const explicitDenyExists = authorizationResults.some(
|
||||||
authzResult => authzResult.isAllowed === false && !authzResult.isImplicit);
|
authzResult => authzResult.isAllowed === false && authzResult.isImplicit === false);
|
||||||
if (explicitDenyExists) {
|
if (explicitDenyExists) {
|
||||||
log.trace('authorization check failed for user',
|
log.trace('authorization check failed for user',
|
||||||
{
|
{
|
||||||
|
|
|
@ -8,7 +8,7 @@ const { pushMetric } = require('../../../utapi/utilities');
|
||||||
const { decodeVersionId } = require('./versioning');
|
const { decodeVersionId } = require('./versioning');
|
||||||
const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../../../utilities/collectCorsHeaders');
|
||||||
const { parseRestoreRequestXml } = s3middleware.objectRestore;
|
const { parseRestoreRequestXml } = s3middleware.objectRestore;
|
||||||
const { processBytesToWrite, validateQuotas } = require('../quotas/quotaUtils');
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if tier is supported
|
* Check if tier is supported
|
||||||
|
@ -58,15 +58,7 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId: decodedVidResult,
|
versionId: decodedVidResult,
|
||||||
requestType: request.apiMethods || 'restoreObject',
|
requestType: 'restoreObject',
|
||||||
/**
|
|
||||||
* Restoring an object might not cause any impact on
|
|
||||||
* the storage, if the object is already restored: in
|
|
||||||
* this case, the duration is extended. We disable the
|
|
||||||
* quota evaluation and trigger it manually.
|
|
||||||
*/
|
|
||||||
checkQuota: false,
|
|
||||||
request,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
|
@ -124,16 +116,6 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
|
||||||
return next(err, bucketMD, objectMD);
|
return next(err, bucketMD, objectMD);
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
function evaluateQuotas(bucketMD, objectMD, next) {
|
|
||||||
if (isObjectRestored) {
|
|
||||||
return next(null, bucketMD, objectMD);
|
|
||||||
}
|
|
||||||
const actions = Array.isArray(mdValueParams.requestType) ?
|
|
||||||
mdValueParams.requestType : [mdValueParams.requestType];
|
|
||||||
const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
|
|
||||||
return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
|
|
||||||
false, log, err => next(err, bucketMD, objectMD));
|
|
||||||
},
|
|
||||||
function updateObjectMD(bucketMD, objectMD, next) {
|
function updateObjectMD(bucketMD, objectMD, next) {
|
||||||
const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
|
const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};
|
||||||
metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
|
metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params,
|
||||||
|
|
|
@ -210,7 +210,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
|
||||||
// null keys are used, which is used as an optimization to
|
// null keys are used, which is used as an optimization to
|
||||||
// avoid having to check the versioned key since there can
|
// avoid having to check the versioned key since there can
|
||||||
// be no more versioned key to clean up
|
// be no more versioned key to clean up
|
||||||
if (mst.isNull && mst.versionId && !mst.isNull2) {
|
if (mst.isNull && !mst.isNull2) {
|
||||||
const delOptions = { versionId: mst.versionId };
|
const delOptions = { versionId: mst.versionId };
|
||||||
return { options, delOptions };
|
return { options, delOptions };
|
||||||
}
|
}
|
||||||
|
@ -241,7 +241,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
|
||||||
if (masterIsNull) {
|
if (masterIsNull) {
|
||||||
// if master is a null version or a non-versioned key,
|
// if master is a null version or a non-versioned key,
|
||||||
// copy it to a new null key
|
// copy it to a new null key
|
||||||
const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
|
const nullVersionId = mst.isNull ? mst.versionId : nonVersionedObjId;
|
||||||
if (nullVersionCompatMode) {
|
if (nullVersionCompatMode) {
|
||||||
options.extraMD = {
|
options.extraMD = {
|
||||||
nullVersionId,
|
nullVersionId,
|
||||||
|
|
|
@ -101,33 +101,8 @@ function validateWebsiteHeader(header) {
|
||||||
header.startsWith('http://') || header.startsWith('https://'));
|
header.startsWith('http://') || header.startsWith('https://'));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* appendWebsiteIndexDocument - append index to objectKey if necessary
|
|
||||||
* @param {object} request - normalized request object
|
|
||||||
* @param {string} indexDocumentSuffix - index document from website config
|
|
||||||
* @param {boolean} force - flag to force append index
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function appendWebsiteIndexDocument(request, indexDocumentSuffix, force = false) {
|
|
||||||
const reqObjectKey = request.objectKey ? request.objectKey : '';
|
|
||||||
/* eslint-disable no-param-reassign */
|
|
||||||
|
|
||||||
// find index document if "directory" sent in request
|
|
||||||
if (reqObjectKey.endsWith('/')) {
|
|
||||||
request.objectKey += indexDocumentSuffix;
|
|
||||||
// find index document if no key provided
|
|
||||||
} else if (reqObjectKey === '') {
|
|
||||||
request.objectKey = indexDocumentSuffix;
|
|
||||||
// force for redirect 302 on folder without trailing / that has an index
|
|
||||||
} else if (force) {
|
|
||||||
request.objectKey += `/${indexDocumentSuffix}`;
|
|
||||||
}
|
|
||||||
/* eslint-enable no-param-reassign */
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
findRoutingRule,
|
findRoutingRule,
|
||||||
extractRedirectInfo,
|
extractRedirectInfo,
|
||||||
validateWebsiteHeader,
|
validateWebsiteHeader,
|
||||||
appendWebsiteIndexDocument,
|
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,314 +0,0 @@
|
||||||
const async = require('async');
|
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const monitoring = require('../../../utilities/monitoringHandler');
|
|
||||||
const {
|
|
||||||
actionNeedQuotaCheckCopy,
|
|
||||||
actionNeedQuotaCheck,
|
|
||||||
actionWithDataDeletion,
|
|
||||||
} = require('arsenal').policies;
|
|
||||||
const { config } = require('../../../Config');
|
|
||||||
const QuotaService = require('../../../quotas/quotas');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Process the bytes to write based on the request and object metadata
|
|
||||||
* @param {string} apiMethod - api method
|
|
||||||
* @param {BucketInfo} bucket - bucket info
|
|
||||||
* @param {string} versionId - version id of the object
|
|
||||||
* @param {number} contentLength - content length of the object
|
|
||||||
* @param {object} objMD - object metadata
|
|
||||||
* @param {object} destObjMD - destination object metadata
|
|
||||||
* @return {number} processed content length
|
|
||||||
*/
|
|
||||||
function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, destObjMD = null) {
|
|
||||||
let bytes = contentLength;
|
|
||||||
if (apiMethod === 'objectRestore') {
|
|
||||||
// object is being restored
|
|
||||||
bytes = Number.parseInt(objMD['content-length'], 10);
|
|
||||||
} else if (!bytes && objMD?.['content-length']) {
|
|
||||||
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
|
|
||||||
if (!destObjMD || bucket.isVersioningEnabled()) {
|
|
||||||
// object is being copied
|
|
||||||
bytes = Number.parseInt(objMD['content-length'], 10);
|
|
||||||
} else if (!bucket.isVersioningEnabled()) {
|
|
||||||
// object is being copied and replaces the target
|
|
||||||
bytes = Number.parseInt(objMD['content-length'], 10) -
|
|
||||||
Number.parseInt(destObjMD['content-length'], 10);
|
|
||||||
}
|
|
||||||
} else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) {
|
|
||||||
// object is being deleted
|
|
||||||
bytes = -Number.parseInt(objMD['content-length'], 10);
|
|
||||||
}
|
|
||||||
} else if (bytes && objMD?.['content-length'] && !bucket.isVersioningEnabled()) {
|
|
||||||
// object is being replaced: store the diff, if the bucket is not versioned
|
|
||||||
bytes = bytes - Number.parseInt(objMD['content-length'], 10);
|
|
||||||
}
|
|
||||||
return bytes || 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if a metric is stale based on the provided parameters.
|
|
||||||
*
|
|
||||||
* @param {Object} metric - The metric object to check.
|
|
||||||
* @param {string} resourceType - The type of the resource.
|
|
||||||
* @param {string} resourceName - The name of the resource.
|
|
||||||
* @param {string} action - The action being performed.
|
|
||||||
* @param {number} inflight - The number of inflight requests.
|
|
||||||
* @param {Object} log - The logger object.
|
|
||||||
* @returns {boolean} Returns true if the metric is stale, false otherwise.
|
|
||||||
*/
|
|
||||||
function isMetricStale(metric, resourceType, resourceName, action, inflight, log) {
|
|
||||||
if (metric.date && Date.now() - new Date(metric.date).getTime() >
|
|
||||||
QuotaService.maxStaleness) {
|
|
||||||
log.warn('Stale metrics from the quota service, allowing the request', {
|
|
||||||
resourceType,
|
|
||||||
resourceName,
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
});
|
|
||||||
monitoring.requestWithQuotaMetricsUnavailable.inc();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Evaluates quotas for a bucket and an account and update inflight count.
|
|
||||||
*
|
|
||||||
* @param {number} bucketQuota - The quota limit for the bucket.
|
|
||||||
* @param {number} accountQuota - The quota limit for the account.
|
|
||||||
* @param {object} bucket - The bucket object.
|
|
||||||
* @param {object} account - The account object.
|
|
||||||
* @param {number} inflight - The number of inflight requests.
|
|
||||||
* @param {number} inflightForCheck - The number of inflight requests for checking quotas.
|
|
||||||
* @param {string} action - The action being performed.
|
|
||||||
* @param {object} log - The logger object.
|
|
||||||
* @param {function} callback - The callback function to be called when evaluation is complete.
|
|
||||||
* @returns {object} - The result of the evaluation.
|
|
||||||
*/
|
|
||||||
function _evaluateQuotas(
|
|
||||||
bucketQuota,
|
|
||||||
accountQuota,
|
|
||||||
bucket,
|
|
||||||
account,
|
|
||||||
inflight,
|
|
||||||
inflightForCheck,
|
|
||||||
action,
|
|
||||||
log,
|
|
||||||
callback,
|
|
||||||
) {
|
|
||||||
let bucketQuotaExceeded = false;
|
|
||||||
let accountQuotaExceeded = false;
|
|
||||||
const creationDate = new Date(bucket.getCreationDate()).getTime();
|
|
||||||
return async.parallel({
|
|
||||||
bucketQuota: parallelDone => {
|
|
||||||
if (bucketQuota > 0) {
|
|
||||||
return QuotaService.getUtilizationMetrics('bucket',
|
|
||||||
`${bucket.getName()}_${creationDate}`, null, {
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
}, (err, bucketMetrics) => {
|
|
||||||
if (err || inflight < 0) {
|
|
||||||
return parallelDone(err);
|
|
||||||
}
|
|
||||||
if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
|
|
||||||
bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
|
|
||||||
log.debug('Bucket quota exceeded', {
|
|
||||||
bucket: bucket.getName(),
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
quota: bucketQuota,
|
|
||||||
bytesTotal: bucketMetrics.bytesTotal,
|
|
||||||
});
|
|
||||||
bucketQuotaExceeded = true;
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
},
|
|
||||||
accountQuota: parallelDone => {
|
|
||||||
if (accountQuota > 0 && account?.account) {
|
|
||||||
return QuotaService.getUtilizationMetrics('account',
|
|
||||||
account.account, null, {
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
}, (err, accountMetrics) => {
|
|
||||||
if (err || inflight < 0) {
|
|
||||||
return parallelDone(err);
|
|
||||||
}
|
|
||||||
if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
|
|
||||||
accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
|
|
||||||
log.debug('Account quota exceeded', {
|
|
||||||
accountId: account.account,
|
|
||||||
action,
|
|
||||||
inflight,
|
|
||||||
quota: accountQuota,
|
|
||||||
bytesTotal: accountMetrics.bytesTotal,
|
|
||||||
});
|
|
||||||
accountQuotaExceeded = true;
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return parallelDone();
|
|
||||||
},
|
|
||||||
}, err => {
|
|
||||||
if (err) {
|
|
||||||
log.warn('Error evaluating quotas', {
|
|
||||||
error: err.name,
|
|
||||||
description: err.message,
|
|
||||||
isInflightDeletion: inflight < 0,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return callback(err, bucketQuotaExceeded, accountQuotaExceeded);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Monitors the duration of quota evaluation for a specific API method.
|
|
||||||
*
|
|
||||||
* @param {string} apiMethod - The name of the API method being monitored.
|
|
||||||
* @param {string} type - The type of quota being evaluated.
|
|
||||||
* @param {string} code - The code associated with the quota being evaluated.
|
|
||||||
* @param {number} duration - The duration of the quota evaluation in nanoseconds.
|
|
||||||
* @returns {undefined} - Returns nothing.
|
|
||||||
*/
|
|
||||||
function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
|
|
||||||
monitoring.quotaEvaluationDuration.labels({
|
|
||||||
action: apiMethod,
|
|
||||||
type,
|
|
||||||
code,
|
|
||||||
}).observe(duration / 1e9);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param {Request} request - request object
|
|
||||||
* @param {BucketInfo} bucket - bucket object
|
|
||||||
* @param {Account} account - account object
|
|
||||||
* @param {array} apiNames - action names: operations to authorize
|
|
||||||
* @param {string} apiMethod - the main API call
|
|
||||||
* @param {number} inflight - inflight bytes
|
|
||||||
* @param {boolean} isStorageReserved - Flag to check if the current quota, minus
|
|
||||||
* the incoming bytes, are under the limit.
|
|
||||||
* @param {Logger} log - logger
|
|
||||||
* @param {function} callback - callback function
|
|
||||||
* @returns {boolean} - true if the quota is valid, false otherwise
|
|
||||||
*/
|
|
||||||
function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
|
|
||||||
if (!config.isQuotaEnabled() || (!inflight && isStorageReserved)) {
|
|
||||||
return callback(null);
|
|
||||||
}
|
|
||||||
let type;
|
|
||||||
let bucketQuotaExceeded = false;
|
|
||||||
let accountQuotaExceeded = false;
|
|
||||||
let quotaEvaluationDuration;
|
|
||||||
const requestStartTime = process.hrtime.bigint();
|
|
||||||
const bucketQuota = bucket.getQuota();
|
|
||||||
const accountQuota = account?.quota || 0;
|
|
||||||
const shouldSendInflights = config.isQuotaInflightEnabled();
|
|
||||||
|
|
||||||
if (bucketQuota && accountQuota) {
|
|
||||||
type = 'bucket+account';
|
|
||||||
} else if (bucketQuota) {
|
|
||||||
type = 'bucket';
|
|
||||||
} else {
|
|
||||||
type = 'account';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (actionWithDataDeletion[apiMethod]) {
|
|
||||||
type = 'delete';
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((bucketQuota <= 0 && accountQuota <= 0) || !QuotaService?.enabled) {
|
|
||||||
if (bucketQuota > 0 || accountQuota > 0) {
|
|
||||||
log.warn('quota is set for a bucket, but the quota service is disabled', {
|
|
||||||
bucketName: bucket.getName(),
|
|
||||||
});
|
|
||||||
monitoring.requestWithQuotaMetricsUnavailable.inc();
|
|
||||||
}
|
|
||||||
return callback(null);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isStorageReserved) {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
inflight = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
return async.forEach(apiNames, (apiName, done) => {
|
|
||||||
// Object copy operations first check the target object,
|
|
||||||
// meaning the source object, containing the current bytes,
|
|
||||||
// is checked second. This logic handles these APIs calls by
|
|
||||||
// ensuring the bytes are positives (i.e., not an object
|
|
||||||
// replacement).
|
|
||||||
if (actionNeedQuotaCheckCopy(apiName, apiMethod)) {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
inflight = Math.abs(inflight);
|
|
||||||
} else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) {
|
|
||||||
return done();
|
|
||||||
}
|
|
||||||
// When inflights are disabled, the sum of the current utilization metrics
|
|
||||||
// and the current bytes are compared with the quota. The current bytes
|
|
||||||
// are not sent to the utilization service. When inflights are enabled,
|
|
||||||
// the sum of the current utilization metrics only are compared with the
|
|
||||||
// quota. They include the current inflight bytes sent in the request.
|
|
||||||
let _inflights = shouldSendInflights ? inflight : undefined;
|
|
||||||
const inflightForCheck = shouldSendInflights ? 0 : inflight;
|
|
||||||
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
|
|
||||||
inflightForCheck, apiName, log,
|
|
||||||
(err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
|
|
||||||
if (err) {
|
|
||||||
return done(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
bucketQuotaExceeded = _bucketQuotaExceeded;
|
|
||||||
accountQuotaExceeded = _accountQuotaExceeded;
|
|
||||||
|
|
||||||
// Inflights are inverted: in case of cleanup, we just re-issue
|
|
||||||
// the same API call.
|
|
||||||
if (_inflights) {
|
|
||||||
_inflights = -_inflights;
|
|
||||||
}
|
|
||||||
|
|
||||||
request.finalizerHooks.push((errorFromAPI, _done) => {
|
|
||||||
const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200;
|
|
||||||
const quotaCleanUpStartTime = process.hrtime.bigint();
|
|
||||||
// Quotas are cleaned only in case of error in the API
|
|
||||||
async.waterfall([
|
|
||||||
cb => {
|
|
||||||
if (errorFromAPI) {
|
|
||||||
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
|
|
||||||
null, apiName, log, cb);
|
|
||||||
}
|
|
||||||
return cb();
|
|
||||||
},
|
|
||||||
], () => {
|
|
||||||
monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration +
|
|
||||||
Number(process.hrtime.bigint() - quotaCleanUpStartTime));
|
|
||||||
return _done();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
return done();
|
|
||||||
});
|
|
||||||
}, err => {
|
|
||||||
quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime);
|
|
||||||
if (err) {
|
|
||||||
log.warn('Error getting metrics from the quota service, allowing the request', {
|
|
||||||
error: err.name,
|
|
||||||
description: err.message,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (!actionWithDataDeletion[apiMethod] &&
|
|
||||||
(bucketQuotaExceeded || accountQuotaExceeded)) {
|
|
||||||
return callback(errors.QuotaExceeded);
|
|
||||||
}
|
|
||||||
return callback();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
processBytesToWrite,
|
|
||||||
isMetricStale,
|
|
||||||
validateQuotas,
|
|
||||||
};
|
|
|
@ -38,8 +38,8 @@ function bucketDeleteCors(authInfo, request, log, callback) {
|
||||||
}
|
}
|
||||||
log.trace('found bucket in metadata');
|
log.trace('found bucket in metadata');
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request,
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketDeleteCors',
|
method: 'bucketDeleteCors',
|
||||||
|
|
|
@ -21,7 +21,7 @@ function bucketDeleteEncryption(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeleteEncryption',
|
requestType: 'bucketDeleteEncryption',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -18,7 +18,7 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeleteLifecycle',
|
requestType: 'bucketDeleteLifecycle',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
||||||
|
|
|
@ -16,7 +16,7 @@ function bucketDeletePolicy(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeletePolicy',
|
requestType: 'bucketDeletePolicy',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
||||||
|
|
|
@ -1,58 +0,0 @@
|
||||||
const { waterfall } = require('async');
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
|
||||||
const metadata = require('../metadata/wrapper');
|
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
|
||||||
|
|
||||||
const requestType = 'bucketDeleteQuota';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bucket Update Quota - Update bucket quota
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {object} log - Werelogs logger
|
|
||||||
* @param {function} callback - callback to server
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function bucketDeleteQuota(authInfo, request, log, callback) {
|
|
||||||
log.debug('processing request', { method: 'bucketDeleteQuota' });
|
|
||||||
|
|
||||||
const { bucketName } = request;
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: request.apiMethods || requestType,
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
return waterfall([
|
|
||||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
|
||||||
(err, bucket) => next(err, bucket)),
|
|
||||||
(bucket, next) => {
|
|
||||||
bucket.setQuota(0);
|
|
||||||
metadata.updateBucket(bucket.getName(), bucket, log, err =>
|
|
||||||
next(err, bucket));
|
|
||||||
},
|
|
||||||
], (err, bucket) => {
|
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
|
||||||
request.method, bucket);
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', {
|
|
||||||
error: err,
|
|
||||||
method: 'bucketDeleteQuota'
|
|
||||||
});
|
|
||||||
monitoring.promMetrics('DELETE', bucketName, err.code,
|
|
||||||
'bucketDeleteQuota');
|
|
||||||
return callback(err, err.code, corsHeaders);
|
|
||||||
}
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'DELETE', bucketName, '204', 'bucketDeleteQuota');
|
|
||||||
pushMetric('bucketDeleteQuota', log, {
|
|
||||||
authInfo,
|
|
||||||
bucket: bucketName,
|
|
||||||
});
|
|
||||||
return callback(null, 204, corsHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = bucketDeleteQuota;
|
|
|
@ -18,7 +18,7 @@ function bucketDeleteReplication(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeleteReplication',
|
requestType: 'bucketDeleteReplication',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
||||||
|
|
|
@ -21,19 +21,15 @@ function bucketDeleteTagging(authInfo, request, log, callback) {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketDeleteTagging',
|
requestType: request.apiMethods || 'bucketDeleteTagging',
|
||||||
request,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let bucket = null;
|
let bucket = null;
|
||||||
return waterfall([
|
return waterfall([
|
||||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
||||||
(err, b) => {
|
(err, b) => {
|
||||||
if (err) {
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
bucket = b;
|
bucket = b;
|
||||||
bucket.setTags([]);
|
bucket.setTags([]);
|
||||||
return next();
|
return next(err);
|
||||||
}),
|
}),
|
||||||
next => metadata.updateBucket(bucket.getName(), bucket, log, next),
|
next => metadata.updateBucket(bucket.getName(), bucket, log, next),
|
||||||
], err => {
|
], err => {
|
||||||
|
|
|
@ -30,8 +30,8 @@ function bucketDeleteWebsite(authInfo, request, log, callback) {
|
||||||
}
|
}
|
||||||
log.trace('found bucket in metadata');
|
log.trace('found bucket in metadata');
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request,
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketDeleteWebsite',
|
method: 'bucketDeleteWebsite',
|
||||||
|
|
|
@ -322,7 +322,7 @@ function bucketGet(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGet',
|
requestType: 'bucketGet',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const listParams = {
|
const listParams = {
|
||||||
|
|
|
@ -44,7 +44,7 @@ function bucketGetACL(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetACL',
|
requestType: 'bucketGetACL',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const grantInfo = {
|
const grantInfo = {
|
||||||
|
|
|
@ -39,8 +39,8 @@ function bucketGetCors(authInfo, request, log, callback) {
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucket);
|
request.method, bucket);
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log,
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
request, request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketGetCors',
|
method: 'bucketGetCors',
|
||||||
|
|
|
@ -22,7 +22,7 @@ function bucketGetEncryption(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetEncryption',
|
requestType: 'bucketGetEncryption',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -21,7 +21,7 @@ function bucketGetLifecycle(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetLifecycle',
|
requestType: 'bucketGetLifecycle',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
||||||
|
|
|
@ -41,8 +41,8 @@ function bucketGetLocation(authInfo, request, log, callback) {
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucket);
|
request.method, bucket);
|
||||||
|
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log, request,
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for account on bucket', {
|
log.debug('access denied for account on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketGetLocation',
|
method: 'bucketGetLocation',
|
||||||
|
|
|
@ -37,7 +37,7 @@ function bucketGetNotification(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetNotification',
|
requestType: 'bucketGetNotification',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -33,7 +33,7 @@ function bucketGetObjectLock(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetObjectLock',
|
requestType: 'bucketGetObjectLock',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
||||||
|
|
|
@ -17,7 +17,7 @@ function bucketGetPolicy(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetPolicy',
|
requestType: 'bucketGetPolicy',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,58 +0,0 @@
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* bucketGetQuota - Get the bucket quota
|
|
||||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
|
|
||||||
* @param {object} request - http request object
|
|
||||||
* @param {object} log - Werelogs logger
|
|
||||||
* @param {function} callback - callback to server
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function bucketGetQuota(authInfo, request, log, callback) {
|
|
||||||
log.debug('processing request', { method: 'bucketGetQuota' });
|
|
||||||
const { bucketName, headers, method } = request;
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: request.apiMethods || 'bucketGetQuota',
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
const xml = [];
|
|
||||||
|
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
|
||||||
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', {
|
|
||||||
error: err,
|
|
||||||
method: 'bucketGetQuota',
|
|
||||||
});
|
|
||||||
return callback(err, null, corsHeaders);
|
|
||||||
}
|
|
||||||
xml.push(
|
|
||||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
||||||
'<GetBucketQuota>',
|
|
||||||
'<Name>', bucket.getName(), '</Name>',
|
|
||||||
);
|
|
||||||
const bucketQuota = bucket.getQuota();
|
|
||||||
if (!bucketQuota) {
|
|
||||||
log.debug('bucket has no quota', {
|
|
||||||
method: 'bucketGetQuota',
|
|
||||||
});
|
|
||||||
return callback(errors.NoSuchQuota, null,
|
|
||||||
corsHeaders);
|
|
||||||
}
|
|
||||||
xml.push('<Quota>', bucketQuota, '</Quota>',
|
|
||||||
'</GetBucketQuota>');
|
|
||||||
|
|
||||||
pushMetric('getBucketQuota', log, {
|
|
||||||
authInfo,
|
|
||||||
bucket: bucketName,
|
|
||||||
});
|
|
||||||
return callback(null, xml.join(''), corsHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = bucketGetQuota;
|
|
|
@ -21,7 +21,7 @@ function bucketGetReplication(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetReplication',
|
requestType: 'bucketGetReplication',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
||||||
|
|
|
@ -37,7 +37,7 @@ const escapeForXml = s3middleware.escapeForXml;
|
||||||
function tagsToXml(tags) {
|
function tagsToXml(tags) {
|
||||||
const xml = [];
|
const xml = [];
|
||||||
|
|
||||||
xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Tagging> <TagSet>');
|
xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Tagging><TagSet>');
|
||||||
|
|
||||||
tags.forEach(tag => {
|
tags.forEach(tag => {
|
||||||
xml.push('<Tag>');
|
xml.push('<Tag>');
|
||||||
|
@ -46,7 +46,7 @@ function tagsToXml(tags) {
|
||||||
xml.push('</Tag>');
|
xml.push('</Tag>');
|
||||||
});
|
});
|
||||||
|
|
||||||
xml.push('</TagSet> </Tagging>');
|
xml.push('</TagSet></Tagging>');
|
||||||
|
|
||||||
return xml.join('');
|
return xml.join('');
|
||||||
}
|
}
|
||||||
|
|
|
@ -54,7 +54,7 @@ function bucketGetVersioning(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketGetVersioning',
|
requestType: 'bucketGetVersioning',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -39,8 +39,8 @@ function bucketGetWebsite(authInfo, request, log, callback) {
|
||||||
|
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
request.method, bucket);
|
request.method, bucket);
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, requestType, canonicalID, authInfo, log,
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
request, request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketGetWebsite',
|
method: 'bucketGetWebsite',
|
||||||
|
|
|
@ -19,7 +19,7 @@ function bucketHead(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketHead',
|
requestType: 'bucketHead',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
|
||||||
|
|
|
@ -45,8 +45,9 @@ function checkLocationConstraint(request, locationConstraint, log) {
|
||||||
} else if (parsedHost && restEndpoints[parsedHost]) {
|
} else if (parsedHost && restEndpoints[parsedHost]) {
|
||||||
locationConstraintChecked = restEndpoints[parsedHost];
|
locationConstraintChecked = restEndpoints[parsedHost];
|
||||||
} else {
|
} else {
|
||||||
locationConstraintChecked = Object.keys(locationConstrains)[0];
|
log.trace('no location constraint provided on bucket put;' +
|
||||||
log.trace('no location constraint provided on bucket put; setting '+locationConstraintChecked);
|
'setting us-east-1');
|
||||||
|
locationConstraintChecked = 'us-east-1';
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!locationConstraints[locationConstraintChecked]) {
|
if (!locationConstraints[locationConstraintChecked]) {
|
||||||
|
|
|
@ -70,8 +70,8 @@ function bucketPutCors(authInfo, request, log, callback) {
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
function validateBucketAuthorization(bucket, rules, corsHeaders, next) {
|
function validateBucketAuthorization(bucket, rules, corsHeaders, next) {
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo,
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
log, request, request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for account on bucket', {
|
log.debug('access denied for account on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
});
|
});
|
||||||
|
|
|
@ -26,7 +26,7 @@ function bucketPutObjectLock(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketPutObjectLock',
|
requestType: 'bucketPutObjectLock',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return waterfall([
|
return waterfall([
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const { errors, models } = require('arsenal');
|
const { errors, models } = require('arsenal');
|
||||||
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
||||||
|
@ -65,7 +66,17 @@ function bucketPutPolicy(authInfo, request, log, callback) {
|
||||||
return next(errors.MalformedPolicy.customizeDescription(
|
return next(errors.MalformedPolicy.customizeDescription(
|
||||||
'Policy has invalid resource'));
|
'Policy has invalid resource'));
|
||||||
}
|
}
|
||||||
return next(validatePolicyConditions(bucketPolicy), bucketPolicy);
|
return next(null, bucketPolicy);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
(bucketPolicy, next) => {
|
||||||
|
process.nextTick(() => {
|
||||||
|
// if policy contains conditions, validate them, and return relevant error if there is one
|
||||||
|
const conditionsError = validatePolicyConditions(bucketPolicy);
|
||||||
|
if (conditionsError) {
|
||||||
|
return next(errors.MalformedPolicy.customizeDescription(conditionsError));
|
||||||
|
}
|
||||||
|
return next(null, bucketPolicy);
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
(bucketPolicy, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
(bucketPolicy, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
||||||
|
|
|
@ -39,7 +39,6 @@ function bucketPutTagging(authInfo, request, log, callback) {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'bucketPutTagging',
|
requestType: request.apiMethods || 'bucketPutTagging',
|
||||||
request,
|
|
||||||
};
|
};
|
||||||
let bucket = null;
|
let bucket = null;
|
||||||
return waterfall([
|
return waterfall([
|
||||||
|
|
|
@ -49,8 +49,8 @@ function bucketPutWebsite(authInfo, request, log, callback) {
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
function validateBucketAuthorization(bucket, config, next) {
|
function validateBucketAuthorization(bucket, config, next) {
|
||||||
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID,
|
if (!isBucketAuthorized(bucket, request.apiMethods || requestType, canonicalID, authInfo,
|
||||||
authInfo, log, request, request.actionImplicitDenies)) {
|
log, request, request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for user on bucket', {
|
log.debug('access denied for user on bucket', {
|
||||||
requestType,
|
requestType,
|
||||||
method: 'bucketPutWebsite',
|
method: 'bucketPutWebsite',
|
||||||
|
|
|
@ -1,85 +0,0 @@
|
||||||
const { waterfall } = require('async');
|
|
||||||
const { errors } = require('arsenal');
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|
||||||
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
|
|
||||||
const metadata = require('../metadata/wrapper');
|
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
|
||||||
const { parseString } = require('xml2js');
|
|
||||||
|
|
||||||
function validateBucketQuotaProperty(requestBody, next) {
|
|
||||||
const quota = requestBody.quota;
|
|
||||||
const quotaValue = parseInt(quota, 10);
|
|
||||||
if (Number.isNaN(quotaValue)) {
|
|
||||||
return next(errors.InvalidArgument.customizeDescription('Quota Value should be a number'));
|
|
||||||
}
|
|
||||||
if (quotaValue <= 0) {
|
|
||||||
return next(errors.InvalidArgument.customizeDescription('Quota value must be a positive number'));
|
|
||||||
}
|
|
||||||
return next(null, quotaValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseRequestBody(requestBody, next) {
|
|
||||||
try {
|
|
||||||
const jsonData = JSON.parse(requestBody);
|
|
||||||
if (typeof jsonData !== 'object') {
|
|
||||||
throw new Error('Invalid JSON');
|
|
||||||
}
|
|
||||||
return next(null, jsonData);
|
|
||||||
} catch (jsonError) {
|
|
||||||
return parseString(requestBody, (xmlError, xmlData) => {
|
|
||||||
if (xmlError) {
|
|
||||||
return next(errors.InvalidArgument.customizeDescription('Request body must be a JSON object'));
|
|
||||||
}
|
|
||||||
return next(null, xmlData);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function bucketUpdateQuota(authInfo, request, log, callback) {
|
|
||||||
log.debug('processing request', { method: 'bucketUpdateQuota' });
|
|
||||||
|
|
||||||
const { bucketName } = request;
|
|
||||||
const metadataValParams = {
|
|
||||||
authInfo,
|
|
||||||
bucketName,
|
|
||||||
requestType: request.apiMethods || 'bucketUpdateQuota',
|
|
||||||
request,
|
|
||||||
};
|
|
||||||
let bucket = null;
|
|
||||||
return waterfall([
|
|
||||||
next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
|
|
||||||
(err, b) => {
|
|
||||||
bucket = b;
|
|
||||||
return next(err, bucket);
|
|
||||||
}),
|
|
||||||
(bucket, next) => parseRequestBody(request.post, (err, requestBody) => next(err, bucket, requestBody)),
|
|
||||||
(bucket, requestBody, next) => validateBucketQuotaProperty(requestBody, (err, quotaValue) =>
|
|
||||||
next(err, bucket, quotaValue)),
|
|
||||||
(bucket, quotaValue, next) => {
|
|
||||||
bucket.setQuota(quotaValue);
|
|
||||||
return metadata.updateBucket(bucket.getName(), bucket, log, next);
|
|
||||||
},
|
|
||||||
], (err, bucket) => {
|
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
|
||||||
request.method, bucket);
|
|
||||||
if (err) {
|
|
||||||
log.debug('error processing request', {
|
|
||||||
error: err,
|
|
||||||
method: 'bucketUpdateQuota'
|
|
||||||
});
|
|
||||||
monitoring.promMetrics('PUT', bucketName, err.code,
|
|
||||||
'updateBucketQuota');
|
|
||||||
return callback(err, err.code, corsHeaders);
|
|
||||||
}
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'PUT', bucketName, '200', 'updateBucketQuota');
|
|
||||||
pushMetric('updateBucketQuota', log, {
|
|
||||||
authInfo,
|
|
||||||
bucket: bucketName,
|
|
||||||
});
|
|
||||||
return callback(null, corsHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = bucketUpdateQuota;
|
|
|
@ -21,6 +21,8 @@ const { validateAndFilterMpuParts, generateMpuPartStorageInfo } =
|
||||||
const locationKeysHaveChanged
|
const locationKeysHaveChanged
|
||||||
= require('./apiUtils/object/locationKeysHaveChanged');
|
= require('./apiUtils/object/locationKeysHaveChanged');
|
||||||
const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
|
const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
|
||||||
|
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
const { validatePutVersionId } = require('./apiUtils/object/coldStorage');
|
const { validatePutVersionId } = require('./apiUtils/object/coldStorage');
|
||||||
|
|
||||||
const versionIdUtils = versioning.VersionID;
|
const versionIdUtils = versioning.VersionID;
|
||||||
|
@ -80,7 +82,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
|
||||||
uploadId,
|
uploadId,
|
||||||
// Note: permissions for completing a multipart upload are the
|
// Note: permissions for completing a multipart upload are the
|
||||||
// same as putting a part.
|
// same as putting a part.
|
||||||
requestType: request.apiMethods || 'putPart or complete',
|
requestType: 'putPart or complete',
|
||||||
log,
|
log,
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
@ -131,9 +133,8 @@ function completeMultipartUpload(authInfo, request, log, callback) {
|
||||||
bucketName,
|
bucketName,
|
||||||
// Required permissions for this action
|
// Required permissions for this action
|
||||||
// at the destinationBucket level are same as objectPut
|
// at the destinationBucket level are same as objectPut
|
||||||
requestType: request.apiMethods || 'completeMultipartUpload',
|
requestType: 'objectPut',
|
||||||
versionId,
|
versionId,
|
||||||
request,
|
|
||||||
};
|
};
|
||||||
standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, next);
|
standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, next);
|
||||||
},
|
},
|
||||||
|
@ -474,9 +475,12 @@ function completeMultipartUpload(authInfo, request, log, callback) {
|
||||||
const newDataStoreName =
|
const newDataStoreName =
|
||||||
Array.isArray(dataLocations) && dataLocations[0] ?
|
Array.isArray(dataLocations) && dataLocations[0] ?
|
||||||
dataLocations[0].dataStoreName : null;
|
dataLocations[0].dataStoreName : null;
|
||||||
|
const delLog =
|
||||||
|
logger.newRequestLoggerFromSerializedUids(log
|
||||||
|
.getSerializedUids());
|
||||||
return data.batchDelete(dataToDelete,
|
return data.batchDelete(dataToDelete,
|
||||||
request.method,
|
request.method,
|
||||||
newDataStoreName, log, err => {
|
newDataStoreName, delLog, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return next(err);
|
return next(err);
|
||||||
}
|
}
|
||||||
|
@ -499,8 +503,10 @@ function completeMultipartUpload(authInfo, request, log, callback) {
|
||||||
function batchDeleteExtraParts(extraPartLocations, destinationBucket,
|
function batchDeleteExtraParts(extraPartLocations, destinationBucket,
|
||||||
aggregateETag, generatedVersionId, next) {
|
aggregateETag, generatedVersionId, next) {
|
||||||
if (extraPartLocations && extraPartLocations.length > 0) {
|
if (extraPartLocations && extraPartLocations.length > 0) {
|
||||||
|
const delLog = logger.newRequestLoggerFromSerializedUids(
|
||||||
|
log.getSerializedUids());
|
||||||
return data.batchDelete(extraPartLocations, request.method,
|
return data.batchDelete(extraPartLocations, request.method,
|
||||||
null, log, err => {
|
null, delLog, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return next(err);
|
return next(err);
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,7 +6,6 @@ const convertToXml = s3middleware.convertToXml;
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
const { hasNonPrintables } = require('../utilities/stringChecks');
|
const { hasNonPrintables } = require('../utilities/stringChecks');
|
||||||
const { config } = require('../Config');
|
|
||||||
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
|
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
|
||||||
const constants = require('../../constants');
|
const constants = require('../../constants');
|
||||||
const services = require('../services');
|
const services = require('../services');
|
||||||
|
@ -66,7 +65,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
|
||||||
const websiteRedirectHeader =
|
const websiteRedirectHeader =
|
||||||
request.headers['x-amz-website-redirect-location'];
|
request.headers['x-amz-website-redirect-location'];
|
||||||
if (request.headers['x-amz-storage-class'] &&
|
if (request.headers['x-amz-storage-class'] &&
|
||||||
!config.locationConstraints[request.headers['x-amz-storage-class']]) {
|
!constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) {
|
||||||
log.trace('invalid storage-class header');
|
log.trace('invalid storage-class header');
|
||||||
monitoring.promMetrics('PUT', bucketName,
|
monitoring.promMetrics('PUT', bucketName,
|
||||||
errors.InvalidStorageClass.code, 'initiateMultipartUpload');
|
errors.InvalidStorageClass.code, 'initiateMultipartUpload');
|
||||||
|
@ -106,7 +105,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
// Required permissions for this action are same as objectPut
|
// Required permissions for this action are same as objectPut
|
||||||
requestType: request.apiMethods || 'initiateMultipartUpload',
|
requestType: 'objectPut',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const accountCanonicalID = authInfo.getCanonicalID();
|
const accountCanonicalID = authInfo.getCanonicalID();
|
||||||
|
|
|
@ -96,8 +96,8 @@ function listMultipartUploads(authInfo, request, log, callback) {
|
||||||
// to list the multipart uploads so we have provided here that
|
// to list the multipart uploads so we have provided here that
|
||||||
// the authorization to list multipart uploads is the same
|
// the authorization to list multipart uploads is the same
|
||||||
// as listing objects in a bucket.
|
// as listing objects in a bucket.
|
||||||
requestType: request.apiMethods || 'bucketGet',
|
requestType: 'bucketGet',
|
||||||
preciseRequestType: request.apiMethods || 'listMultipartUploads',
|
preciseRequestType: 'listMultipartUploads',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -97,7 +97,7 @@ function listParts(authInfo, request, log, callback) {
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
uploadId,
|
uploadId,
|
||||||
preciseRequestType: request.apiMethods || 'listParts',
|
preciseRequestType: 'listParts',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
// For validating the request at the destinationBucket level
|
// For validating the request at the destinationBucket level
|
||||||
|
|
|
@ -71,7 +71,7 @@ function metadataSearch(authInfo, request, log, callback) {
|
||||||
const metadataValParams = {
|
const metadataValParams = {
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName,
|
bucketName,
|
||||||
requestType: request.apiMethods || 'metadataSearch',
|
requestType: 'metadataSearch',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const listParams = {
|
const listParams = {
|
||||||
|
|
|
@ -31,7 +31,6 @@ const { overheadField } = require('../../constants');
|
||||||
const versionIdUtils = versioning.VersionID;
|
const versionIdUtils = versioning.VersionID;
|
||||||
const { data } = require('../data/wrapper');
|
const { data } = require('../data/wrapper');
|
||||||
const logger = require('../utilities/logger');
|
const logger = require('../utilities/logger');
|
||||||
const { validateQuotas } = require('./apiUtils/quotas/quotaUtils');
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Format of xml request:
|
Format of xml request:
|
||||||
|
@ -333,9 +332,6 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
|
||||||
|
|
||||||
return callback(null, objMD, versionId);
|
return callback(null, objMD, versionId);
|
||||||
},
|
},
|
||||||
(objMD, versionId, callback) => validateQuotas(
|
|
||||||
request, bucket, request.accountQuotas, ['objectDelete'], 'objectDelete',
|
|
||||||
-objMD?.['content-length'] || 0, false, log, err => callback(err, objMD, versionId)),
|
|
||||||
(objMD, versionId, callback) => {
|
(objMD, versionId, callback) => {
|
||||||
const options = preprocessingVersioningDelete(
|
const options = preprocessingVersioningDelete(
|
||||||
bucketName, bucket, objMD, versionId, config.nullVersionCompatMode);
|
bucketName, bucket, objMD, versionId, config.nullVersionCompatMode);
|
||||||
|
@ -508,9 +504,8 @@ function multiObjectDelete(authInfo, request, log, callback) {
|
||||||
if (bucketShield(bucketMD, 'objectDelete')) {
|
if (bucketShield(bucketMD, 'objectDelete')) {
|
||||||
return next(errors.NoSuchBucket);
|
return next(errors.NoSuchBucket);
|
||||||
}
|
}
|
||||||
// The implicit deny flag is ignored in the DeleteObjects API, as authorization only
|
if (!isBucketAuthorized(bucketMD, 'objectDelete', canonicalID, authInfo, log, request,
|
||||||
// affects the objects.
|
request.actionImplicitDenies)) {
|
||||||
if (!isBucketAuthorized(bucketMD, 'objectDelete', canonicalID, authInfo, log, request)) {
|
|
||||||
log.trace("access denied due to bucket acl's");
|
log.trace("access denied due to bucket acl's");
|
||||||
// if access denied at the bucket level, no access for
|
// if access denied at the bucket level, no access for
|
||||||
// any of the objects so all results will be error results
|
// any of the objects so all results will be error results
|
||||||
|
|
|
@ -12,6 +12,7 @@ const { checkQueryVersionId, versioningPreprocessing }
|
||||||
= require('./apiUtils/object/versioning');
|
= require('./apiUtils/object/versioning');
|
||||||
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
|
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
|
||||||
const { data } = require('../data/wrapper');
|
const { data } = require('../data/wrapper');
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
const services = require('../services');
|
const services = require('../services');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
const removeAWSChunked = require('./apiUtils/object/removeAWSChunked');
|
const removeAWSChunked = require('./apiUtils/object/removeAWSChunked');
|
||||||
|
@ -220,14 +221,6 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
versionId: sourceVersionId,
|
versionId: sourceVersionId,
|
||||||
getDeleteMarker: true,
|
getDeleteMarker: true,
|
||||||
requestType: 'objectGet',
|
requestType: 'objectGet',
|
||||||
/**
|
|
||||||
* Authorization will first check the target object, with an objectPut
|
|
||||||
* action. But in this context, the source object metadata is still
|
|
||||||
* unknown. In the context of quotas, to know the number of bytes that
|
|
||||||
* are being written, we explicitly enable the quota evaluation logic
|
|
||||||
* during the objectGet action instead.
|
|
||||||
*/
|
|
||||||
checkQuota: true,
|
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const valPutParams = {
|
const valPutParams = {
|
||||||
|
@ -235,7 +228,6 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
bucketName: destBucketName,
|
bucketName: destBucketName,
|
||||||
objectKey: destObjectKey,
|
objectKey: destObjectKey,
|
||||||
requestType: 'objectPut',
|
requestType: 'objectPut',
|
||||||
checkQuota: false,
|
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const dataStoreContext = {
|
const dataStoreContext = {
|
||||||
|
@ -249,7 +241,7 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
const responseHeaders = {};
|
const responseHeaders = {};
|
||||||
|
|
||||||
if (request.headers['x-amz-storage-class'] &&
|
if (request.headers['x-amz-storage-class'] &&
|
||||||
!config.locationConstraints[request.headers['x-amz-storage-class']]) {
|
!constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) {
|
||||||
log.trace('invalid storage-class header');
|
log.trace('invalid storage-class header');
|
||||||
monitoring.promMetrics('PUT', destBucketName,
|
monitoring.promMetrics('PUT', destBucketName,
|
||||||
errors.InvalidStorageClass.code, 'copyObject');
|
errors.InvalidStorageClass.code, 'copyObject');
|
||||||
|
@ -287,10 +279,7 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
function checkSourceAuthorization(destBucketMD, destObjMD, next) {
|
function checkSourceAuthorization(destBucketMD, destObjMD, next) {
|
||||||
return standardMetadataValidateBucketAndObj({
|
return standardMetadataValidateBucketAndObj(valGetParams, request.actionImplicitDenies, log,
|
||||||
...valGetParams,
|
|
||||||
destObjMD,
|
|
||||||
}, request.actionImplicitDenies, log,
|
|
||||||
(err, sourceBucketMD, sourceObjMD) => {
|
(err, sourceBucketMD, sourceObjMD) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log.debug('error validating get part of request',
|
log.debug('error validating get part of request',
|
||||||
|
@ -544,8 +533,10 @@ function objectCopy(authInfo, request, sourceBucket,
|
||||||
// the same as the destination
|
// the same as the destination
|
||||||
if (!sourceIsDestination && dataToDelete) {
|
if (!sourceIsDestination && dataToDelete) {
|
||||||
const newDataStoreName = storeMetadataParams.dataStoreName;
|
const newDataStoreName = storeMetadataParams.dataStoreName;
|
||||||
|
const delLog = logger.newRequestLoggerFromSerializedUids(
|
||||||
|
log.getSerializedUids());
|
||||||
return data.batchDelete(dataToDelete, request.method,
|
return data.batchDelete(dataToDelete, request.method,
|
||||||
newDataStoreName, log, err => {
|
newDataStoreName, delLog, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
// if error, log the error and move on as it is not
|
// if error, log the error and move on as it is not
|
||||||
// relevant to the client as the client's
|
// relevant to the client as the client's
|
||||||
|
|
|
@ -56,7 +56,7 @@ function objectDeleteInternal(authInfo, request, log, isExpiration, cb) {
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId: reqVersionId,
|
versionId: reqVersionId,
|
||||||
requestType: request.apiMethods || 'objectDelete',
|
requestType: 'objectDelete',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -44,7 +44,7 @@ function objectDeleteTagging(authInfo, request, log, callback) {
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId: reqVersionId,
|
versionId: reqVersionId,
|
||||||
getDeleteMarker: true,
|
getDeleteMarker: true,
|
||||||
requestType: request.apiMethods || 'objectDeleteTagging',
|
requestType: 'objectDeleteTagging',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -66,7 +66,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId,
|
versionId,
|
||||||
getDeleteMarker: true,
|
getDeleteMarker: true,
|
||||||
requestType: request.apiMethods || 'objectGet',
|
requestType: 'objectGet',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -61,7 +61,7 @@ function objectGetACL(authInfo, request, log, callback) {
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId,
|
versionId,
|
||||||
requestType: request.apiMethods || 'objectGetACL',
|
requestType: 'objectGetACL',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
const grantInfo = {
|
const grantInfo = {
|
||||||
|
|
|
@ -40,7 +40,7 @@ function objectGetLegalHold(authInfo, request, log, callback) {
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId,
|
versionId,
|
||||||
requestType: request.apiMethods || 'objectGetLegalHold',
|
requestType: 'objectGetLegalHold',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,7 @@ function objectGetRetention(authInfo, request, log, callback) {
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId: reqVersionId,
|
versionId: reqVersionId,
|
||||||
requestType: request.apiMethods || 'objectGetRetention',
|
requestType: 'objectGetRetention',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -41,7 +41,7 @@ function objectGetTagging(authInfo, request, log, callback) {
|
||||||
bucketName,
|
bucketName,
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId: reqVersionId,
|
versionId: reqVersionId,
|
||||||
requestType: request.apiMethods || 'objectGetTagging',
|
requestType: 'objectGetTagging',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -48,7 +48,7 @@ function objectHead(authInfo, request, log, callback) {
|
||||||
objectKey,
|
objectKey,
|
||||||
versionId,
|
versionId,
|
||||||
getDeleteMarker: true,
|
getDeleteMarker: true,
|
||||||
requestType: request.apiMethods || 'objectHead',
|
requestType: 'objectHead',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,6 @@ const { errors, versioning } = require('arsenal');
|
||||||
|
|
||||||
const constants = require('../../constants');
|
const constants = require('../../constants');
|
||||||
const aclUtils = require('../utilities/aclUtils');
|
const aclUtils = require('../utilities/aclUtils');
|
||||||
const { config } = require('../Config');
|
|
||||||
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
|
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
|
||||||
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
|
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
|
@ -72,7 +71,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
|
||||||
query,
|
query,
|
||||||
} = request;
|
} = request;
|
||||||
if (headers['x-amz-storage-class'] &&
|
if (headers['x-amz-storage-class'] &&
|
||||||
!config.locationConstraints[headers['x-amz-storage-class']]) {
|
!constants.validStorageClasses.includes(headers['x-amz-storage-class'])) {
|
||||||
log.trace('invalid storage-class header');
|
log.trace('invalid storage-class header');
|
||||||
monitoring.promMetrics('PUT', request.bucketName,
|
monitoring.promMetrics('PUT', request.bucketName,
|
||||||
errors.InvalidStorageClass.code, 'putObject');
|
errors.InvalidStorageClass.code, 'putObject');
|
||||||
|
@ -99,7 +98,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
|
||||||
'The encryption method specified is not supported');
|
'The encryption method specified is not supported');
|
||||||
const requestType = request.apiMethods || 'objectPut';
|
const requestType = request.apiMethods || 'objectPut';
|
||||||
const valParams = { authInfo, bucketName, objectKey, versionId,
|
const valParams = { authInfo, bucketName, objectKey, versionId,
|
||||||
requestType, request, withVersionId: isPutVersion };
|
requestType, request };
|
||||||
const canonicalID = authInfo.getCanonicalID();
|
const canonicalID = authInfo.getCanonicalID();
|
||||||
|
|
||||||
if (hasNonPrintables(objectKey)) {
|
if (hasNonPrintables(objectKey)) {
|
||||||
|
|
|
@ -88,7 +88,6 @@ function objectPutACL(authInfo, request, log, cb) {
|
||||||
versionId: reqVersionId,
|
versionId: reqVersionId,
|
||||||
getDeleteMarker: true,
|
getDeleteMarker: true,
|
||||||
requestType: request.apiMethods || 'objectPutACL',
|
requestType: request.apiMethods || 'objectPutACL',
|
||||||
request,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const possibleGrants = ['FULL_CONTROL', 'WRITE_ACP', 'READ', 'READ_ACP'];
|
const possibleGrants = ['FULL_CONTROL', 'WRITE_ACP', 'READ', 'READ_ACP'];
|
||||||
|
|
|
@ -9,12 +9,12 @@ const locationConstraintCheck =
|
||||||
require('./apiUtils/object/locationConstraintCheck');
|
require('./apiUtils/object/locationConstraintCheck');
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
const services = require('../services');
|
const services = require('../services');
|
||||||
const setUpCopyLocator = require('./apiUtils/object/setUpCopyLocator');
|
const setUpCopyLocator = require('./apiUtils/object/setUpCopyLocator');
|
||||||
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
|
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage');
|
const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage');
|
||||||
const { validateQuotas } = require('./apiUtils/quotas/quotaUtils');
|
|
||||||
|
|
||||||
const versionIdUtils = versioning.VersionID;
|
const versionIdUtils = versioning.VersionID;
|
||||||
|
|
||||||
|
@ -46,14 +46,6 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
|
||||||
versionId: reqVersionId,
|
versionId: reqVersionId,
|
||||||
getDeleteMarker: true,
|
getDeleteMarker: true,
|
||||||
requestType: 'objectGet',
|
requestType: 'objectGet',
|
||||||
/**
|
|
||||||
* Authorization will first check the target object, with an objectPut
|
|
||||||
* action. But in this context, the source object metadata is still
|
|
||||||
* unknown. In the context of quotas, to know the number of bytes that
|
|
||||||
* are being written, we explicitly enable the quota evaluation logic
|
|
||||||
* during the objectGet action instead.
|
|
||||||
*/
|
|
||||||
checkQuota: true,
|
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -76,8 +68,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
|
||||||
authInfo,
|
authInfo,
|
||||||
bucketName: destBucketName,
|
bucketName: destBucketName,
|
||||||
objectKey: destObjectKey,
|
objectKey: destObjectKey,
|
||||||
requestType: 'objectPutPart',
|
requestType: 'objectPut',
|
||||||
checkQuota: false,
|
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -98,7 +89,6 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
|
||||||
objectKey: destObjectKey,
|
objectKey: destObjectKey,
|
||||||
partNumber: paddedPartNumber,
|
partNumber: paddedPartNumber,
|
||||||
uploadId,
|
uploadId,
|
||||||
enableQuota: true,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
|
@ -192,16 +182,9 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
|
||||||
}
|
}
|
||||||
return next(null, copyLocator.dataLocator, destBucketMD,
|
return next(null, copyLocator.dataLocator, destBucketMD,
|
||||||
copyLocator.copyObjectSize, sourceVerId,
|
copyLocator.copyObjectSize, sourceVerId,
|
||||||
sourceLocationConstraintName, sourceObjMD);
|
sourceLocationConstraintName);
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
function _validateQuotas(dataLocator, destBucketMD,
|
|
||||||
copyObjectSize, sourceVerId,
|
|
||||||
sourceLocationConstraintName, sourceObjMD, next) {
|
|
||||||
return validateQuotas(request, destBucketMD, request.accountQuotas, valPutParams.requestType,
|
|
||||||
request.apiMethod, sourceObjMD?.['content-length'] || 0, false, log, err =>
|
|
||||||
next(err, dataLocator, destBucketMD, copyObjectSize, sourceVerId, sourceLocationConstraintName));
|
|
||||||
},
|
|
||||||
// get MPU shadow bucket to get splitter based on MD version
|
// get MPU shadow bucket to get splitter based on MD version
|
||||||
function getMpuShadowBucket(dataLocator, destBucketMD,
|
function getMpuShadowBucket(dataLocator, destBucketMD,
|
||||||
copyObjectSize, sourceVerId,
|
copyObjectSize, sourceVerId,
|
||||||
|
@ -399,8 +382,10 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
|
||||||
// Clean up the old data now that new metadata (with new
|
// Clean up the old data now that new metadata (with new
|
||||||
// data locations) has been stored
|
// data locations) has been stored
|
||||||
if (oldLocationsToDelete) {
|
if (oldLocationsToDelete) {
|
||||||
|
const delLog = logger.newRequestLoggerFromSerializedUids(
|
||||||
|
log.getSerializedUids());
|
||||||
return data.batchDelete(oldLocationsToDelete, request.method, null,
|
return data.batchDelete(oldLocationsToDelete, request.method, null,
|
||||||
log, err => {
|
delLog, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
// if error, log the error and move on as it is not
|
// if error, log the error and move on as it is not
|
||||||
// relevant to the client as the client's
|
// relevant to the client as the client's
|
||||||
|
|
|
@ -11,6 +11,7 @@ const { isBucketAuthorized } =
|
||||||
const kms = require('../kms/wrapper');
|
const kms = require('../kms/wrapper');
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
const services = require('../services');
|
const services = require('../services');
|
||||||
const locationConstraintCheck
|
const locationConstraintCheck
|
||||||
= require('./apiUtils/object/locationConstraintCheck');
|
= require('./apiUtils/object/locationConstraintCheck');
|
||||||
|
@ -21,7 +22,6 @@ const { BackendInfo } = models;
|
||||||
const writeContinue = require('../utilities/writeContinue');
|
const writeContinue = require('../utilities/writeContinue');
|
||||||
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
|
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
|
||||||
const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders');
|
const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders');
|
||||||
const { validateQuotas } = require('./apiUtils/quotas/quotaUtils');
|
|
||||||
|
|
||||||
const skipError = new Error('skip');
|
const skipError = new Error('skip');
|
||||||
|
|
||||||
|
@ -61,9 +61,6 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
|
||||||
log.debug('processing request', { method: 'objectPutPart' });
|
log.debug('processing request', { method: 'objectPutPart' });
|
||||||
const size = request.parsedContentLength;
|
const size = request.parsedContentLength;
|
||||||
|
|
||||||
const putVersionId = request.headers['x-scal-s3-version-id'];
|
|
||||||
const isPutVersion = putVersionId || putVersionId === '';
|
|
||||||
|
|
||||||
if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) {
|
if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) {
|
||||||
log.debug('put part size too large', { size });
|
log.debug('put part size too large', { size });
|
||||||
monitoring.promMetrics('PUT', request.bucketName, 400,
|
monitoring.promMetrics('PUT', request.bucketName, 400,
|
||||||
|
@ -107,9 +104,6 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
|
||||||
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
|
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
|
||||||
const { objectKey } = request;
|
const { objectKey } = request;
|
||||||
const originalIdentityAuthzResults = request.actionImplicitDenies;
|
const originalIdentityAuthzResults = request.actionImplicitDenies;
|
||||||
// For validating the request at the destinationBucket level the
|
|
||||||
// `requestType` is the general 'objectPut'.
|
|
||||||
const requestType = request.apiMethods || 'objectPutPart';
|
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
// Get the destination bucket.
|
// Get the destination bucket.
|
||||||
|
@ -129,15 +123,16 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
|
||||||
}),
|
}),
|
||||||
// Check the bucket authorization.
|
// Check the bucket authorization.
|
||||||
(destinationBucket, next) => {
|
(destinationBucket, next) => {
|
||||||
if (!isBucketAuthorized(destinationBucket, requestType, canonicalID, authInfo,
|
// For validating the request at the destinationBucket level the
|
||||||
|
// `requestType` is the general 'objectPut'.
|
||||||
|
const requestType = 'objectPut';
|
||||||
|
if (!isBucketAuthorized(destinationBucket, request.apiMethods || requestType, canonicalID, authInfo,
|
||||||
log, request, request.actionImplicitDenies)) {
|
log, request, request.actionImplicitDenies)) {
|
||||||
log.debug('access denied for user on bucket', { requestType });
|
log.debug('access denied for user on bucket', { requestType });
|
||||||
return next(errors.AccessDenied, destinationBucket);
|
return next(errors.AccessDenied, destinationBucket);
|
||||||
}
|
}
|
||||||
return next(null, destinationBucket);
|
return next(null, destinationBucket);
|
||||||
},
|
},
|
||||||
(destinationBucket, next) => validateQuotas(request, destinationBucket, request.accountQuotas,
|
|
||||||
requestType, request.apiMethod, size, isPutVersion, log, err => next(err, destinationBucket)),
|
|
||||||
// Get bucket server-side encryption, if it exists.
|
// Get bucket server-side encryption, if it exists.
|
||||||
(destinationBucket, next) => getObjectSSEConfiguration(
|
(destinationBucket, next) => getObjectSSEConfiguration(
|
||||||
request.headers, destinationBucket, log,
|
request.headers, destinationBucket, log,
|
||||||
|
@ -385,8 +380,10 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
|
||||||
prevObjectSize, next) => {
|
prevObjectSize, next) => {
|
||||||
if (oldLocationsToDelete) {
|
if (oldLocationsToDelete) {
|
||||||
log.trace('overwriting mpu part, deleting data');
|
log.trace('overwriting mpu part, deleting data');
|
||||||
|
const delLog = logger.newRequestLoggerFromSerializedUids(
|
||||||
|
log.getSerializedUids());
|
||||||
return data.batchDelete(oldLocationsToDelete, request.method,
|
return data.batchDelete(oldLocationsToDelete, request.method,
|
||||||
objectLocationConstraint, log, err => {
|
objectLocationConstraint, delLog, err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
// if error, log the error and move on as it is not
|
// if error, log the error and move on as it is not
|
||||||
// relevant to the client as the client's
|
// relevant to the client as the client's
|
||||||
|
|
|
@ -50,7 +50,37 @@ function objectPutRetention(authInfo, request, log, callback) {
|
||||||
};
|
};
|
||||||
|
|
||||||
return async.waterfall([
|
return async.waterfall([
|
||||||
next => {
|
next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log,
|
||||||
|
(err, bucket, objectMD) => {
|
||||||
|
if (err) {
|
||||||
|
log.trace('request authorization failed',
|
||||||
|
{ method: 'objectPutRetention', error: err });
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
if (!objectMD) {
|
||||||
|
const err = reqVersionId ? errors.NoSuchVersion :
|
||||||
|
errors.NoSuchKey;
|
||||||
|
log.trace('error no object metadata found',
|
||||||
|
{ method: 'objectPutRetention', error: err });
|
||||||
|
return next(err, bucket);
|
||||||
|
}
|
||||||
|
if (objectMD.isDeleteMarker) {
|
||||||
|
log.trace('version is a delete marker',
|
||||||
|
{ method: 'objectPutRetention' });
|
||||||
|
// FIXME we should return a `x-amz-delete-marker: true` header,
|
||||||
|
// see S3C-7592
|
||||||
|
return next(errors.MethodNotAllowed, bucket);
|
||||||
|
}
|
||||||
|
if (!bucket.isObjectLockEnabled()) {
|
||||||
|
log.trace('object lock not enabled on bucket',
|
||||||
|
{ method: 'objectPutRetention' });
|
||||||
|
return next(errors.InvalidRequest.customizeDescription(
|
||||||
|
'Bucket is missing Object Lock Configuration'
|
||||||
|
), bucket);
|
||||||
|
}
|
||||||
|
return next(null, bucket, objectMD);
|
||||||
|
}),
|
||||||
|
(bucket, objectMD, next) => {
|
||||||
log.trace('parsing retention information');
|
log.trace('parsing retention information');
|
||||||
parseRetentionXml(request.post, log,
|
parseRetentionXml(request.post, log,
|
||||||
(err, retentionInfo) => {
|
(err, retentionInfo) => {
|
||||||
|
|
|
@ -1,313 +0,0 @@
|
||||||
const { errors, s3middleware } = require('arsenal');
|
|
||||||
const validateHeaders = s3middleware.validateConditionalHeaders;
|
|
||||||
|
|
||||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
|
||||||
const constants = require('../../constants');
|
|
||||||
const metadata = require('../metadata/wrapper');
|
|
||||||
const bucketShield = require('./apiUtils/bucket/bucketShield');
|
|
||||||
const { appendWebsiteIndexDocument, findRoutingRule, extractRedirectInfo } =
|
|
||||||
require('./apiUtils/object/websiteServing');
|
|
||||||
const { isObjAuthorized, isBucketAuthorized } =
|
|
||||||
require('./apiUtils/authorization/permissionChecks');
|
|
||||||
const collectResponseHeaders = require('../utilities/collectResponseHeaders');
|
|
||||||
const { pushMetric } = require('../utapi/utilities');
|
|
||||||
const monitoring = require('../utilities/monitoringHandler');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* _errorActions - take a number of actions once have error getting obj
|
|
||||||
* @param {object} err - arsenal errors object
|
|
||||||
* @param {string} errorDocument - key to get error document
|
|
||||||
* @param {object []} routingRules - array of routingRule objects
|
|
||||||
* @param {object} bucket - bucket metadata
|
|
||||||
* @param {string} objectKey - object key from request (or as translated in
|
|
||||||
* website)
|
|
||||||
* @param {object} corsHeaders - CORS-related response headers
|
|
||||||
* @param {object} request - normalized request object
|
|
||||||
* @param {object} log - Werelogs instance
|
|
||||||
* @param {function} callback - callback to function in route
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function _errorActions(err, errorDocument, routingRules,
|
|
||||||
bucket, objectKey, corsHeaders, request, log, callback) {
|
|
||||||
const bucketName = bucket.getName();
|
|
||||||
const errRoutingRule = findRoutingRule(routingRules,
|
|
||||||
objectKey, err.code);
|
|
||||||
if (errRoutingRule) {
|
|
||||||
// route will redirect
|
|
||||||
const action = request.method === 'HEAD' ? 'headObject' : 'getObject';
|
|
||||||
monitoring.promMetrics(
|
|
||||||
request.method, bucketName, err.code, action);
|
|
||||||
return callback(err, false, null, corsHeaders, errRoutingRule,
|
|
||||||
objectKey);
|
|
||||||
}
|
|
||||||
if (request.method === 'HEAD') {
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'HEAD', bucketName, err.code, 'headObject');
|
|
||||||
return callback(err, false, null, corsHeaders);
|
|
||||||
}
|
|
||||||
if (errorDocument) {
|
|
||||||
return metadata.getObjectMD(bucketName, errorDocument, {}, log,
|
|
||||||
(errObjErr, errObjMD) => {
|
|
||||||
if (errObjErr) {
|
|
||||||
// error retrieving error document so return original error
|
|
||||||
// and set boolean of error retrieving user's error document
|
|
||||||
// to true
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'getObject');
|
|
||||||
return callback(err, true, null, corsHeaders);
|
|
||||||
}
|
|
||||||
// return the default error message if the object is private
|
|
||||||
// rather than sending a stored error file
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
request.objectKey = errorDocument;
|
|
||||||
if (!isObjAuthorized(bucket, errObjMD, request.apiMethods || 'objectGet',
|
|
||||||
constants.publicId, null, log, request, request.actionImplicitDenies, true)) {
|
|
||||||
log.trace('errorObj not authorized', { error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'getObject');
|
|
||||||
return callback(err, true, null, corsHeaders);
|
|
||||||
}
|
|
||||||
const dataLocator = errObjMD.location;
|
|
||||||
if (errObjMD['x-amz-server-side-encryption']) {
|
|
||||||
for (let i = 0; i < dataLocator.length; i++) {
|
|
||||||
dataLocator[i].masterKeyId =
|
|
||||||
errObjMD['x-amz-server-side-encryption-aws-' +
|
|
||||||
'kms-key-id'];
|
|
||||||
dataLocator[i].algorithm =
|
|
||||||
errObjMD['x-amz-server-side-encryption'];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (errObjMD['x-amz-website-redirect-location']) {
|
|
||||||
const redirectLocation =
|
|
||||||
errObjMD['x-amz-website-redirect-location'];
|
|
||||||
const redirectInfo = { withError: true,
|
|
||||||
location: redirectLocation };
|
|
||||||
log.trace('redirecting to x-amz-website-redirect-location',
|
|
||||||
{ location: redirectLocation });
|
|
||||||
return callback(err, false, dataLocator, corsHeaders,
|
|
||||||
redirectInfo, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
const responseMetaHeaders = collectResponseHeaders(errObjMD,
|
|
||||||
corsHeaders);
|
|
||||||
pushMetric('getObject', log, {
|
|
||||||
bucket: bucketName,
|
|
||||||
newByteLength: responseMetaHeaders['Content-Length'],
|
|
||||||
});
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'getObject');
|
|
||||||
return callback(err, false, dataLocator, responseMetaHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
monitoring.promMetrics(
|
|
||||||
'GET', bucketName, err.code, 'getObject');
|
|
||||||
return callback(err, false, null, corsHeaders);
|
|
||||||
}
|
|
||||||
|
|
||||||
function capitalize(str) {
|
|
||||||
if (!str || typeof str !== 'string') {
|
|
||||||
return str;
|
|
||||||
}
|
|
||||||
return str.charAt(0).toUpperCase() + str.slice(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Callbacks have different signature for GET and HEAD
|
|
||||||
* The website function uses GET callback signature
|
|
||||||
* This encapsulate HEAD callback to match GET signature
|
|
||||||
* @param {function} callback - HEAD callback
|
|
||||||
* @returns {function} HEAD callback with GET signature
|
|
||||||
*/
|
|
||||||
function callbackGetToHead(callback) {
|
|
||||||
return (err, userErrorPageFailure, dataGetInfo,
|
|
||||||
resMetaHeaders, redirectInfo, key) =>
|
|
||||||
callback(err, resMetaHeaders, redirectInfo, key);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Website - Common website function for GET and HEAD
|
|
||||||
* Gets metadata and object for website or redirects
|
|
||||||
* @param {object} request - normalized request object
|
|
||||||
* @param {object} log - Werelogs instance
|
|
||||||
* @param {function} callback - callback to function in route
|
|
||||||
* @return {undefined}
|
|
||||||
*/
|
|
||||||
function website(request, log, callback) {
|
|
||||||
if (request.method === 'HEAD') {
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
|
||||||
callback = callbackGetToHead(callback);
|
|
||||||
}
|
|
||||||
const methodCapitalized = capitalize(request.method);
|
|
||||||
const action = request.method === 'HEAD' ? 'headObject' : 'getObject';
|
|
||||||
log.debug('processing request', { method: `website${methodCapitalized}` });
|
|
||||||
const bucketName = request.bucketName;
|
|
||||||
const reqObjectKey = request.objectKey ? request.objectKey : '';
|
|
||||||
|
|
||||||
return metadata.getBucket(bucketName, log, (err, bucket) => {
|
|
||||||
if (err) {
|
|
||||||
log.trace('error retrieving bucket metadata', { error: err });
|
|
||||||
monitoring.promMetrics(
|
|
||||||
request.method, bucketName, err.code, action);
|
|
||||||
return callback(err, false);
|
|
||||||
}
|
|
||||||
if (bucketShield(bucket, `object${methodCapitalized}`)) {
|
|
||||||
log.trace('bucket in transient/deleted state so shielding');
|
|
||||||
monitoring.promMetrics(
|
|
||||||
request.method, bucketName, 404, action);
|
|
||||||
return callback(errors.NoSuchBucket, false);
|
|
||||||
}
|
|
||||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
|
||||||
request.method, bucket);
|
|
||||||
// bucket ACL's do not matter for website head since it is always the
|
|
||||||
// head of an object. object ACL's are what matter
|
|
||||||
const websiteConfig = bucket.getWebsiteConfiguration();
|
|
||||||
if (!websiteConfig) {
|
|
||||||
monitoring.promMetrics(
|
|
||||||
request.method, bucketName, 404, action);
|
|
||||||
return callback(errors.NoSuchWebsiteConfiguration, false, null,
|
|
||||||
corsHeaders);
|
|
||||||
}
|
|
||||||
// any errors above would be our own created generic error html
|
|
||||||
// if have a website config, error going forward would be user's
|
|
||||||
// redirect or error page if they set either in the config
|
|
||||||
|
|
||||||
// handle redirect all
|
|
||||||
if (websiteConfig.getRedirectAllRequestsTo()) {
|
|
||||||
return callback(null, false, null, corsHeaders,
|
|
||||||
websiteConfig.getRedirectAllRequestsTo(), reqObjectKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
// check whether need to redirect based on key
|
|
||||||
const routingRules = websiteConfig.getRoutingRules();
|
|
||||||
const keyRoutingRule = findRoutingRule(routingRules, reqObjectKey);
|
|
||||||
|
|
||||||
if (keyRoutingRule) {
|
|
||||||
// TODO: optimize by not rerouting if only routing
|
|
||||||
// rule is to change out key
|
|
||||||
return callback(null, false, null, corsHeaders,
|
|
||||||
keyRoutingRule, reqObjectKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument());
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Recursive function with 1 recursive call to look for index
|
|
||||||
* in case of error for potential redirect to folder notation
|
|
||||||
* if there is not already an index appended
|
|
||||||
* @param {Error} [originalError] - presence of this argument
|
|
||||||
* differentiates original user request from recursive call to /index.
|
|
||||||
* This error is returned if /index is not found
|
|
||||||
* @returns {undefined}
|
|
||||||
*/
|
|
||||||
function runWebsite(originalError) {
|
|
||||||
// get object metadata and check authorization and header
|
|
||||||
// validation
|
|
||||||
return metadata.getObjectMD(bucketName, request.objectKey, {}, log,
|
|
||||||
(err, objMD) => {
|
|
||||||
// Note: In case of error, we intentionally send the original
|
|
||||||
// object key to _errorActions as in case of a redirect, we do
|
|
||||||
// not want to append index key to redirect location
|
|
||||||
if (err) {
|
|
||||||
log.trace('error retrieving object metadata',
|
|
||||||
{ error: err });
|
|
||||||
let returnErr = err;
|
|
||||||
const bucketAuthorized = isBucketAuthorized(bucket, request.apiMethods || 'bucketGet',
|
|
||||||
constants.publicId, null, log, request, request.actionImplicitDenies, true);
|
|
||||||
// if index object does not exist and bucket is private AWS
|
|
||||||
// returns 403 - AccessDenied error.
|
|
||||||
if (err.is.NoSuchKey && !bucketAuthorized) {
|
|
||||||
returnErr = errors.AccessDenied;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if key is a folder containing index for redirect 302
|
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html
|
|
||||||
if (!originalError && reqObjectKey && !reqObjectKey.endsWith('/')) {
|
|
||||||
appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument(), true);
|
|
||||||
// propagate returnErr as originalError to be used if index is not found
|
|
||||||
return runWebsite(returnErr);
|
|
||||||
}
|
|
||||||
|
|
||||||
return _errorActions(originalError || returnErr,
|
|
||||||
websiteConfig.getErrorDocument(), routingRules,
|
|
||||||
bucket, reqObjectKey, corsHeaders, request, log,
|
|
||||||
callback);
|
|
||||||
}
|
|
||||||
if (!isObjAuthorized(bucket, objMD, request.apiMethods || 'objectGet',
|
|
||||||
constants.publicId, null, log, request, request.actionImplicitDenies, true)) {
|
|
||||||
const err = errors.AccessDenied;
|
|
||||||
log.trace('request not authorized', { error: err });
|
|
||||||
return _errorActions(err, websiteConfig.getErrorDocument(),
|
|
||||||
routingRules, bucket,
|
|
||||||
reqObjectKey, corsHeaders, request, log, callback);
|
|
||||||
}
|
|
||||||
|
|
||||||
// access granted to index document, needs a redirect 302
|
|
||||||
// to the original key with trailing /
|
|
||||||
if (originalError) {
|
|
||||||
const redirectInfo = { withError: true,
|
|
||||||
location: `/${reqObjectKey}/` };
|
|
||||||
return callback(errors.Found, false, null, corsHeaders,
|
|
||||||
redirectInfo, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
const headerValResult = validateHeaders(request.headers,
|
|
||||||
objMD['last-modified'], objMD['content-md5']);
|
|
||||||
if (headerValResult.error) {
|
|
||||||
const err = headerValResult.error;
|
|
||||||
log.trace('header validation error', { error: err });
|
|
||||||
return _errorActions(err, websiteConfig.getErrorDocument(),
|
|
||||||
routingRules, bucket, reqObjectKey,
|
|
||||||
corsHeaders, request, log, callback);
|
|
||||||
}
|
|
||||||
// check if object to serve has website redirect header
|
|
||||||
// Note: AWS prioritizes website configuration rules over
|
|
||||||
// object key's website redirect header, so we make the
|
|
||||||
// check at the end.
|
|
||||||
if (objMD['x-amz-website-redirect-location']) {
|
|
||||||
const redirectLocation =
|
|
||||||
objMD['x-amz-website-redirect-location'];
|
|
||||||
const redirectInfo =
|
|
||||||
extractRedirectInfo(redirectLocation);
|
|
||||||
log.trace('redirecting to x-amz-website-redirect-location',
|
|
||||||
{ location: redirectLocation });
|
|
||||||
return callback(null, false, null, corsHeaders,
|
|
||||||
redirectInfo, '');
|
|
||||||
}
|
|
||||||
// got obj metadata, authorized and headers validated,
|
|
||||||
// good to go
|
|
||||||
const responseMetaHeaders = collectResponseHeaders(objMD,
|
|
||||||
corsHeaders);
|
|
||||||
|
|
||||||
if (request.method === 'HEAD') {
|
|
||||||
pushMetric('headObject', log, { bucket: bucketName });
|
|
||||||
monitoring.promMetrics('HEAD',
|
|
||||||
bucketName, '200', 'headObject');
|
|
||||||
return callback(null, false, null, responseMetaHeaders);
|
|
||||||
}
|
|
||||||
|
|
||||||
const dataLocator = objMD.location;
|
|
||||||
if (objMD['x-amz-server-side-encryption']) {
|
|
||||||
for (let i = 0; i < dataLocator.length; i++) {
|
|
||||||
dataLocator[i].masterKeyId =
|
|
||||||
objMD['x-amz-server-side-encryption-aws-' +
|
|
||||||
'kms-key-id'];
|
|
||||||
dataLocator[i].algorithm =
|
|
||||||
objMD['x-amz-server-side-encryption'];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pushMetric('getObject', log, {
|
|
||||||
bucket: bucketName,
|
|
||||||
newByteLength: responseMetaHeaders['Content-Length'],
|
|
||||||
});
|
|
||||||
monitoring.promMetrics('GET', bucketName, '200',
|
|
||||||
'getObject', responseMetaHeaders['Content-Length']);
|
|
||||||
return callback(null, false, dataLocator, responseMetaHeaders);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return runWebsite();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = website;
|
|
|
@ -0,0 +1,235 @@
|
||||||
|
const { errors, s3middleware } = require('arsenal');
|
||||||
|
const validateHeaders = s3middleware.validateConditionalHeaders;
|
||||||
|
|
||||||
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
|
const constants = require('../../constants');
|
||||||
|
const metadata = require('../metadata/wrapper');
|
||||||
|
const bucketShield = require('./apiUtils/bucket/bucketShield');
|
||||||
|
const { findRoutingRule, extractRedirectInfo } =
|
||||||
|
require('./apiUtils/object/websiteServing');
|
||||||
|
const { isObjAuthorized, isBucketAuthorized } =
|
||||||
|
require('./apiUtils/authorization/permissionChecks');
|
||||||
|
const collectResponseHeaders = require('../utilities/collectResponseHeaders');
|
||||||
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* _errorActions - take a number of actions once have error getting obj
|
||||||
|
* @param {object} err - arsenal errors object
|
||||||
|
* @param {string} errorDocument - key to get error document
|
||||||
|
* @param {object []} routingRules - array of routingRule objects
|
||||||
|
* @param {object} bucket - bucket metadata
|
||||||
|
* @param {string} objectKey - object key from request (or as translated in
|
||||||
|
* websiteGet)
|
||||||
|
* @param {object} corsHeaders - CORS-related response headers
|
||||||
|
* @param {object} request - normalized request object
|
||||||
|
* @param {object} log - Werelogs instance
|
||||||
|
* @param {function} callback - callback to function in route
|
||||||
|
* @return {undefined}
|
||||||
|
*/
|
||||||
|
function _errorActions(err, errorDocument, routingRules,
|
||||||
|
bucket, objectKey, corsHeaders, request, log, callback) {
|
||||||
|
const bucketName = bucket.getName();
|
||||||
|
const errRoutingRule = findRoutingRule(routingRules,
|
||||||
|
objectKey, err.code);
|
||||||
|
if (errRoutingRule) {
|
||||||
|
// route will redirect
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, err.code, 'getObject');
|
||||||
|
return callback(err, false, null, corsHeaders, errRoutingRule,
|
||||||
|
objectKey);
|
||||||
|
}
|
||||||
|
if (errorDocument) {
|
||||||
|
return metadata.getObjectMD(bucketName, errorDocument, {}, log,
|
||||||
|
(errObjErr, errObjMD) => {
|
||||||
|
if (errObjErr) {
|
||||||
|
// error retrieving error document so return original error
|
||||||
|
// and set boolean of error retrieving user's error document
|
||||||
|
// to true
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, err.code, 'getObject');
|
||||||
|
return callback(err, true, null, corsHeaders);
|
||||||
|
}
|
||||||
|
// return the default error message if the object is private
|
||||||
|
// rather than sending a stored error file
|
||||||
|
if (!isObjAuthorized(bucket, errObjMD, request.apiMethods || 'objectGet',
|
||||||
|
constants.publicId, null, log, null, request.actionImplicitDenies)) {
|
||||||
|
log.trace('errorObj not authorized', { error: err });
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, err.code, 'getObject');
|
||||||
|
return callback(err, true, null, corsHeaders);
|
||||||
|
}
|
||||||
|
const dataLocator = errObjMD.location;
|
||||||
|
if (errObjMD['x-amz-server-side-encryption']) {
|
||||||
|
for (let i = 0; i < dataLocator.length; i++) {
|
||||||
|
dataLocator[i].masterKeyId =
|
||||||
|
errObjMD['x-amz-server-side-encryption-aws-' +
|
||||||
|
'kms-key-id'];
|
||||||
|
dataLocator[i].algorithm =
|
||||||
|
errObjMD['x-amz-server-side-encryption'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const responseMetaHeaders = collectResponseHeaders(errObjMD,
|
||||||
|
corsHeaders);
|
||||||
|
pushMetric('getObject', log, {
|
||||||
|
bucket: bucketName,
|
||||||
|
newByteLength: responseMetaHeaders['Content-Length'],
|
||||||
|
});
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, err.code, 'getObject');
|
||||||
|
return callback(err, false, dataLocator, responseMetaHeaders);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, err.code, 'getObject');
|
||||||
|
return callback(err, false, null, corsHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET Website - Gets object for website or redirects
|
||||||
|
* @param {object} request - normalized request object
|
||||||
|
* @param {object} log - Werelogs instance
|
||||||
|
* @param {function} callback - callback to function in route
|
||||||
|
* @return {undefined}
|
||||||
|
*/
|
||||||
|
function websiteGet(request, log, callback) {
|
||||||
|
log.debug('processing request', { method: 'websiteGet' });
|
||||||
|
const bucketName = request.bucketName;
|
||||||
|
const reqObjectKey = request.objectKey ? request.objectKey : '';
|
||||||
|
let objectKey = reqObjectKey;
|
||||||
|
|
||||||
|
return metadata.getBucket(bucketName, log, (err, bucket) => {
|
||||||
|
if (err) {
|
||||||
|
log.trace('error retrieving bucket metadata', { error: err });
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, err.code, 'getObject');
|
||||||
|
return callback(err, false);
|
||||||
|
}
|
||||||
|
if (bucketShield(bucket, 'objectGet')) {
|
||||||
|
log.trace('bucket in transient/deleted state so shielding');
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, 404, 'getObject');
|
||||||
|
return callback(errors.NoSuchBucket, false);
|
||||||
|
}
|
||||||
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
|
request.method, bucket);
|
||||||
|
const websiteConfig = bucket.getWebsiteConfiguration();
|
||||||
|
if (!websiteConfig) {
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, 404, 'getObject');
|
||||||
|
return callback(errors.NoSuchWebsiteConfiguration, false, null,
|
||||||
|
corsHeaders);
|
||||||
|
}
|
||||||
|
// any errors above would be our own created generic error html
|
||||||
|
// if have a website config, error going forward would be user's
|
||||||
|
// redirect or error page if they set either in the config
|
||||||
|
|
||||||
|
// handle redirect all
|
||||||
|
if (websiteConfig.getRedirectAllRequestsTo()) {
|
||||||
|
return callback(null, false, null, corsHeaders,
|
||||||
|
websiteConfig.getRedirectAllRequestsTo(), objectKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
// check whether need to redirect based on key
|
||||||
|
const routingRules = websiteConfig.getRoutingRules();
|
||||||
|
const keyRoutingRule = findRoutingRule(routingRules, objectKey);
|
||||||
|
|
||||||
|
if (keyRoutingRule) {
|
||||||
|
// TODO: optimize by not rerouting if only routing
|
||||||
|
// rule is to change out key
|
||||||
|
return callback(null, false, null, corsHeaders,
|
||||||
|
keyRoutingRule, objectKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
// find index document if "directory" sent in request
|
||||||
|
if (reqObjectKey.endsWith('/')) {
|
||||||
|
objectKey += websiteConfig.getIndexDocument();
|
||||||
|
}
|
||||||
|
// find index document if no key provided
|
||||||
|
if (reqObjectKey === '') {
|
||||||
|
objectKey = websiteConfig.getIndexDocument();
|
||||||
|
}
|
||||||
|
|
||||||
|
// get object metadata and check authorization and header
|
||||||
|
// validation
|
||||||
|
return metadata.getObjectMD(bucketName, objectKey, {}, log,
|
||||||
|
(err, objMD) => {
|
||||||
|
// Note: In case of error, we intentionally send the original
|
||||||
|
// object key to _errorActions as in case of a redirect, we do
|
||||||
|
// not want to append index key to redirect location
|
||||||
|
if (err) {
|
||||||
|
log.trace('error retrieving object metadata',
|
||||||
|
{ error: err });
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'GET', bucketName, err.code, 'getObject');
|
||||||
|
let returnErr = err;
|
||||||
|
const bucketAuthorized = isBucketAuthorized(bucket, request.apiMethods || 'bucketGet',
|
||||||
|
constants.publicId, null, log, request, request.actionImplicitDenies);
|
||||||
|
// if index object does not exist and bucket is private AWS
|
||||||
|
// returns 403 - AccessDenied error.
|
||||||
|
if (err.is.NoSuchKey && !bucketAuthorized) {
|
||||||
|
returnErr = errors.AccessDenied;
|
||||||
|
}
|
||||||
|
return _errorActions(returnErr,
|
||||||
|
websiteConfig.getErrorDocument(), routingRules,
|
||||||
|
bucket, reqObjectKey, corsHeaders, request, log,
|
||||||
|
callback);
|
||||||
|
}
|
||||||
|
if (!isObjAuthorized(bucket, objMD, request.apiMethods || 'objectGet',
|
||||||
|
constants.publicId, null, log, request, request.actionImplicitDenies)) {
|
||||||
|
const err = errors.AccessDenied;
|
||||||
|
log.trace('request not authorized', { error: err });
|
||||||
|
return _errorActions(err, websiteConfig.getErrorDocument(),
|
||||||
|
routingRules, bucket,
|
||||||
|
reqObjectKey, corsHeaders, request, log, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headerValResult = validateHeaders(request.headers,
|
||||||
|
objMD['last-modified'], objMD['content-md5']);
|
||||||
|
if (headerValResult.error) {
|
||||||
|
const err = headerValResult.error;
|
||||||
|
log.trace('header validation error', { error: err });
|
||||||
|
return _errorActions(err, websiteConfig.getErrorDocument(),
|
||||||
|
routingRules, bucket, reqObjectKey,
|
||||||
|
corsHeaders, request, log, callback);
|
||||||
|
}
|
||||||
|
// check if object to serve has website redirect header
|
||||||
|
// Note: AWS prioritizes website configuration rules over
|
||||||
|
// object key's website redirect header, so we make the
|
||||||
|
// check at the end.
|
||||||
|
if (objMD['x-amz-website-redirect-location']) {
|
||||||
|
const redirectLocation =
|
||||||
|
objMD['x-amz-website-redirect-location'];
|
||||||
|
const redirectInfo =
|
||||||
|
extractRedirectInfo(redirectLocation);
|
||||||
|
log.trace('redirecting to x-amz-website-redirect-location',
|
||||||
|
{ location: redirectLocation });
|
||||||
|
return callback(null, false, null, corsHeaders,
|
||||||
|
redirectInfo, '');
|
||||||
|
}
|
||||||
|
// got obj metadata, authorized and headers validated,
|
||||||
|
// good to go
|
||||||
|
const responseMetaHeaders = collectResponseHeaders(objMD,
|
||||||
|
corsHeaders);
|
||||||
|
const dataLocator = objMD.location;
|
||||||
|
if (objMD['x-amz-server-side-encryption']) {
|
||||||
|
for (let i = 0; i < dataLocator.length; i++) {
|
||||||
|
dataLocator[i].masterKeyId =
|
||||||
|
objMD['x-amz-server-side-encryption-aws-' +
|
||||||
|
'kms-key-id'];
|
||||||
|
dataLocator[i].algorithm =
|
||||||
|
objMD['x-amz-server-side-encryption'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pushMetric('getObject', log, {
|
||||||
|
bucket: bucketName,
|
||||||
|
newByteLength: responseMetaHeaders['Content-Length'],
|
||||||
|
});
|
||||||
|
monitoring.promMetrics('GET', bucketName, '200',
|
||||||
|
'getObject', responseMetaHeaders['Content-Length']);
|
||||||
|
return callback(null, false, dataLocator, responseMetaHeaders);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = websiteGet;
|
|
@ -0,0 +1,168 @@
|
||||||
|
const { errors, s3middleware } = require('arsenal');
|
||||||
|
const validateHeaders = s3middleware.validateConditionalHeaders;
|
||||||
|
|
||||||
|
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||||
|
const constants = require('../../constants');
|
||||||
|
const metadata = require('../metadata/wrapper');
|
||||||
|
const bucketShield = require('./apiUtils/bucket/bucketShield');
|
||||||
|
const { findRoutingRule, extractRedirectInfo } =
|
||||||
|
require('./apiUtils/object/websiteServing');
|
||||||
|
const collectResponseHeaders = require('../utilities/collectResponseHeaders');
|
||||||
|
const { pushMetric } = require('../utapi/utilities');
|
||||||
|
const monitoring = require('../utilities/monitoringHandler');
|
||||||
|
const { isBucketAuthorized, isObjAuthorized } =
|
||||||
|
require('./apiUtils/authorization/permissionChecks');
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* _errorActions - take a number of actions once have error getting obj
|
||||||
|
* @param {object} err - arsenal errors object
|
||||||
|
* @param {object []} routingRules - array of routingRule objects
|
||||||
|
* @param {string} objectKey - object key from request (or as translated in
|
||||||
|
* websiteGet)
|
||||||
|
* @param {object} corsHeaders - CORS-related response headers
|
||||||
|
* @param {object} log - Werelogs instance
|
||||||
|
* @param {function} callback - callback to function in route
|
||||||
|
* @return {undefined}
|
||||||
|
*/
|
||||||
|
function _errorActions(err, routingRules, objectKey, corsHeaders, log,
|
||||||
|
callback) {
|
||||||
|
const errRoutingRule = findRoutingRule(routingRules, objectKey, err.code);
|
||||||
|
if (errRoutingRule) {
|
||||||
|
// route will redirect
|
||||||
|
return callback(err, corsHeaders, errRoutingRule, objectKey);
|
||||||
|
}
|
||||||
|
return callback(err, corsHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* HEAD Website - Gets metadata for object for website or redirects
|
||||||
|
* @param {object} request - normalized request object
|
||||||
|
* @param {object} log - Werelogs instance
|
||||||
|
* @param {function} callback - callback to function in route
|
||||||
|
* @return {undefined}
|
||||||
|
*/
|
||||||
|
function websiteHead(request, log, callback) {
|
||||||
|
log.debug('processing request', { method: 'websiteHead' });
|
||||||
|
const bucketName = request.bucketName;
|
||||||
|
const reqObjectKey = request.objectKey ? request.objectKey : '';
|
||||||
|
let objectKey = reqObjectKey;
|
||||||
|
|
||||||
|
return metadata.getBucket(bucketName, log, (err, bucket) => {
|
||||||
|
if (err) {
|
||||||
|
log.trace('error retrieving bucket metadata', { error: err });
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'HEAD', bucketName, err.code, 'headObject');
|
||||||
|
return callback(err);
|
||||||
|
}
|
||||||
|
if (bucketShield(bucket, 'objectHead')) {
|
||||||
|
log.trace('bucket in transient/deleted state so shielding');
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'HEAD', bucketName, 404, 'headObject');
|
||||||
|
return callback(errors.NoSuchBucket);
|
||||||
|
}
|
||||||
|
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||||
|
request.method, bucket);
|
||||||
|
// bucket ACL's do not matter for website head since it is always the
|
||||||
|
// head of an object. object ACL's are what matter
|
||||||
|
const websiteConfig = bucket.getWebsiteConfiguration();
|
||||||
|
if (!websiteConfig) {
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'HEAD', bucketName, 404, 'headObject');
|
||||||
|
return callback(errors.NoSuchWebsiteConfiguration);
|
||||||
|
}
|
||||||
|
// any errors above would be generic header error response
|
||||||
|
// if have a website config, error going forward could be redirect
|
||||||
|
// if a redirect rule for error is in config
|
||||||
|
|
||||||
|
// handle redirect all
|
||||||
|
if (websiteConfig.getRedirectAllRequestsTo()) {
|
||||||
|
return callback(null, corsHeaders,
|
||||||
|
websiteConfig.getRedirectAllRequestsTo(), objectKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
// find index document if "directory" sent in request
|
||||||
|
if (reqObjectKey.endsWith('/')) {
|
||||||
|
objectKey += websiteConfig.getIndexDocument();
|
||||||
|
}
|
||||||
|
// find index document if no key provided
|
||||||
|
if (reqObjectKey === '') {
|
||||||
|
objectKey = websiteConfig.getIndexDocument();
|
||||||
|
}
|
||||||
|
// check whether need to redirect based on key
|
||||||
|
const routingRules = websiteConfig.getRoutingRules();
|
||||||
|
|
||||||
|
const keyRoutingRule = findRoutingRule(routingRules, objectKey);
|
||||||
|
|
||||||
|
if (keyRoutingRule) {
|
||||||
|
return callback(null, corsHeaders, keyRoutingRule, reqObjectKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
// get object metadata and check authorization and header
|
||||||
|
// validation
|
||||||
|
return metadata.getObjectMD(bucketName, objectKey, {}, log,
|
||||||
|
(err, objMD) => {
|
||||||
|
// Note: In case of error, we intentionally send the original
|
||||||
|
// object key to _errorActions as in case of a redirect, we do
|
||||||
|
// not want to append index key to redirect location
|
||||||
|
if (err) {
|
||||||
|
log.trace('error retrieving object metadata',
|
||||||
|
{ error: err });
|
||||||
|
let returnErr = err;
|
||||||
|
const bucketAuthorized = isBucketAuthorized(bucket,
|
||||||
|
'bucketGet', constants.publicId, null, log, request, request.actionImplicitDenies);
|
||||||
|
// if index object does not exist and bucket is private AWS
|
||||||
|
// returns 403 - AccessDenied error.
|
||||||
|
if (err.is.NoSuchKey && !bucketAuthorized) {
|
||||||
|
returnErr = errors.AccessDenied;
|
||||||
|
}
|
||||||
|
return _errorActions(returnErr, routingRules,
|
||||||
|
reqObjectKey, corsHeaders, log, callback);
|
||||||
|
}
|
||||||
|
if (!isObjAuthorized(bucket, objMD, 'objectGet',
|
||||||
|
constants.publicId, null, log, request, request.actionImplicitDenies)) {
|
||||||
|
const err = errors.AccessDenied;
|
||||||
|
log.trace('request not authorized', { error: err });
|
||||||
|
return _errorActions(err, routingRules, reqObjectKey,
|
||||||
|
corsHeaders, log, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headerValResult = validateHeaders(request.headers,
|
||||||
|
objMD['last-modified'], objMD['content-md5']);
|
||||||
|
if (headerValResult.error) {
|
||||||
|
const err = headerValResult.error;
|
||||||
|
log.trace('header validation error', { error: err });
|
||||||
|
return _errorActions(err, routingRules, reqObjectKey,
|
||||||
|
corsHeaders, log, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
// check if object to serve has website redirect header
|
||||||
|
// Note: AWS prioritizes website configuration rules over
|
||||||
|
// object key's website redirect header, so we make the
|
||||||
|
// check at the end.
|
||||||
|
if (objMD['x-amz-website-redirect-location']) {
|
||||||
|
const redirectLocation =
|
||||||
|
objMD['x-amz-website-redirect-location'];
|
||||||
|
const redirectInfo =
|
||||||
|
extractRedirectInfo(redirectLocation);
|
||||||
|
log.trace('redirecting to x-amz-website-redirect-location',
|
||||||
|
{ location: redirectLocation });
|
||||||
|
return callback(null, corsHeaders, redirectInfo, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
// got obj metadata, authorized and headers validated,
|
||||||
|
// good to go
|
||||||
|
const responseMetaHeaders = collectResponseHeaders(objMD,
|
||||||
|
corsHeaders);
|
||||||
|
pushMetric('headObject', log, {
|
||||||
|
bucket: bucketName,
|
||||||
|
});
|
||||||
|
monitoring.promMetrics(
|
||||||
|
'HEAD', bucketName, '200', 'headObject');
|
||||||
|
return callback(null, responseMetaHeaders);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = websiteHead;
|
|
@ -1,3 +1,4 @@
|
||||||
|
const vaultclient = require('vaultclient');
|
||||||
const { auth } = require('arsenal');
|
const { auth } = require('arsenal');
|
||||||
|
|
||||||
const { config } = require('../Config');
|
const { config } = require('../Config');
|
||||||
|
@ -20,7 +21,6 @@ function getVaultClient(config) {
|
||||||
port,
|
port,
|
||||||
https: true,
|
https: true,
|
||||||
});
|
});
|
||||||
const vaultclient = require('vaultclient');
|
|
||||||
vaultClient = new vaultclient.Client(host, port, true, key, cert, ca);
|
vaultClient = new vaultclient.Client(host, port, true, key, cert, ca);
|
||||||
} else {
|
} else {
|
||||||
logger.info('vaultclient configuration', {
|
logger.info('vaultclient configuration', {
|
||||||
|
@ -28,7 +28,6 @@ function getVaultClient(config) {
|
||||||
port,
|
port,
|
||||||
https: false,
|
https: false,
|
||||||
});
|
});
|
||||||
const vaultclient = require('vaultclient');
|
|
||||||
vaultClient = new vaultclient.Client(host, port);
|
vaultClient = new vaultclient.Client(host, port);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,6 +49,10 @@ function getMemBackend(config) {
|
||||||
}
|
}
|
||||||
|
|
||||||
switch (config.backends.auth) {
|
switch (config.backends.auth) {
|
||||||
|
case 'mem':
|
||||||
|
implName = 'vaultMem';
|
||||||
|
client = getMemBackend(config);
|
||||||
|
break;
|
||||||
case 'multiple':
|
case 'multiple':
|
||||||
implName = 'vaultChain';
|
implName = 'vaultChain';
|
||||||
client = new ChainBackend('s3', [
|
client = new ChainBackend('s3', [
|
||||||
|
@ -57,14 +60,9 @@ case 'multiple':
|
||||||
getVaultClient(config),
|
getVaultClient(config),
|
||||||
]);
|
]);
|
||||||
break;
|
break;
|
||||||
case 'vault':
|
default: // vault
|
||||||
implName = 'vault';
|
implName = 'vault';
|
||||||
client = getVaultClient(config);
|
client = getVaultClient(config);
|
||||||
break;
|
|
||||||
default: // mem
|
|
||||||
implName = 'vaultMem';
|
|
||||||
client = getMemBackend(config);
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = new Vault(client, implName);
|
module.exports = new Vault(client, implName);
|
||||||
|
|
|
@ -8,6 +8,20 @@ const inMemory = require('./in_memory/backend').backend;
|
||||||
const file = require('./file/backend');
|
const file = require('./file/backend');
|
||||||
const KMIPClient = require('arsenal').network.kmipClient;
|
const KMIPClient = require('arsenal').network.kmipClient;
|
||||||
const Common = require('./common');
|
const Common = require('./common');
|
||||||
|
let scalityKMS;
|
||||||
|
let scalityKMSImpl;
|
||||||
|
try {
|
||||||
|
// eslint-disable-next-line import/no-unresolved
|
||||||
|
const ScalityKMS = require('scality-kms');
|
||||||
|
scalityKMS = new ScalityKMS(config.kms);
|
||||||
|
scalityKMSImpl = 'scalityKms';
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn('scality kms unavailable. ' +
|
||||||
|
'Using file kms backend unless mem specified.',
|
||||||
|
{ error });
|
||||||
|
scalityKMS = file;
|
||||||
|
scalityKMSImpl = 'fileKms';
|
||||||
|
}
|
||||||
|
|
||||||
let client;
|
let client;
|
||||||
let implName;
|
let implName;
|
||||||
|
@ -19,9 +33,8 @@ if (config.backends.kms === 'mem') {
|
||||||
client = file;
|
client = file;
|
||||||
implName = 'fileKms';
|
implName = 'fileKms';
|
||||||
} else if (config.backends.kms === 'scality') {
|
} else if (config.backends.kms === 'scality') {
|
||||||
const ScalityKMS = require('scality-kms');
|
client = scalityKMS;
|
||||||
client = new ScalityKMS(config.kms);
|
implName = scalityKMSImpl;
|
||||||
implName = 'scalityKms';
|
|
||||||
} else if (config.backends.kms === 'kmip') {
|
} else if (config.backends.kms === 'kmip') {
|
||||||
const kmipConfig = { kmip: config.kmip };
|
const kmipConfig = { kmip: config.kmip };
|
||||||
if (!kmipConfig.kmip) {
|
if (!kmipConfig.kmip) {
|
||||||
|
|
|
@ -0,0 +1,131 @@
|
||||||
|
/**
|
||||||
|
* Target service that should handle a message
|
||||||
|
* @readonly
|
||||||
|
* @enum {number}
|
||||||
|
*/
|
||||||
|
const MessageType = {
|
||||||
|
/** Message that contains a configuration overlay */
|
||||||
|
CONFIG_OVERLAY_MESSAGE: 1,
|
||||||
|
/** Message that requests a metrics report */
|
||||||
|
METRICS_REQUEST_MESSAGE: 2,
|
||||||
|
/** Message that contains a metrics report */
|
||||||
|
METRICS_REPORT_MESSAGE: 3,
|
||||||
|
/** Close the virtual TCP socket associated to the channel */
|
||||||
|
CHANNEL_CLOSE_MESSAGE: 4,
|
||||||
|
/** Write data to the virtual TCP socket associated to the channel */
|
||||||
|
CHANNEL_PAYLOAD_MESSAGE: 5,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Target service that should handle a message
|
||||||
|
* @readonly
|
||||||
|
* @enum {number}
|
||||||
|
*/
|
||||||
|
const TargetType = {
|
||||||
|
/** Let the dispatcher choose the most appropriate message */
|
||||||
|
TARGET_ANY: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
const headerSize = 3;
|
||||||
|
|
||||||
|
class ChannelMessageV0 {
|
||||||
|
/**
|
||||||
|
* @param {Buffer} buffer Message bytes
|
||||||
|
*/
|
||||||
|
constructor(buffer) {
|
||||||
|
this.messageType = buffer.readUInt8(0);
|
||||||
|
this.channelNumber = buffer.readUInt8(1);
|
||||||
|
this.target = buffer.readUInt8(2);
|
||||||
|
this.payload = buffer.slice(headerSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {number} Message type
|
||||||
|
*/
|
||||||
|
getType() {
|
||||||
|
return this.messageType;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {number} Channel number if applicable
|
||||||
|
*/
|
||||||
|
getChannelNumber() {
|
||||||
|
return this.channelNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {number} Target service, or 0 to choose automatically
|
||||||
|
*/
|
||||||
|
getTarget() {
|
||||||
|
return this.target;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {Buffer} Message payload if applicable
|
||||||
|
*/
|
||||||
|
getPayload() {
|
||||||
|
return this.payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a wire representation of a channel close message
|
||||||
|
*
|
||||||
|
* @param {number} channelId Channel number
|
||||||
|
*
|
||||||
|
* @returns {Buffer} wire representation
|
||||||
|
*/
|
||||||
|
static encodeChannelCloseMessage(channelId) {
|
||||||
|
const buf = Buffer.alloc(headerSize);
|
||||||
|
buf.writeUInt8(MessageType.CHANNEL_CLOSE_MESSAGE, 0);
|
||||||
|
buf.writeUInt8(channelId, 1);
|
||||||
|
buf.writeUInt8(TargetType.TARGET_ANY, 2);
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a wire representation of a channel data message
|
||||||
|
*
|
||||||
|
* @param {number} channelId Channel number
|
||||||
|
* @param {Buffer} data Payload
|
||||||
|
*
|
||||||
|
* @returns {Buffer} wire representation
|
||||||
|
*/
|
||||||
|
static encodeChannelDataMessage(channelId, data) {
|
||||||
|
const buf = Buffer.alloc(data.length + headerSize);
|
||||||
|
buf.writeUInt8(MessageType.CHANNEL_PAYLOAD_MESSAGE, 0);
|
||||||
|
buf.writeUInt8(channelId, 1);
|
||||||
|
buf.writeUInt8(TargetType.TARGET_ANY, 2);
|
||||||
|
data.copy(buf, headerSize);
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a wire representation of a metrics message
|
||||||
|
*
|
||||||
|
* @param {object} body Metrics report
|
||||||
|
*
|
||||||
|
* @returns {Buffer} wire representation
|
||||||
|
*/
|
||||||
|
static encodeMetricsReportMessage(body) {
|
||||||
|
const report = JSON.stringify(body);
|
||||||
|
const buf = Buffer.alloc(report.length + headerSize);
|
||||||
|
buf.writeUInt8(MessageType.METRICS_REPORT_MESSAGE, 0);
|
||||||
|
buf.writeUInt8(0, 1);
|
||||||
|
buf.writeUInt8(TargetType.TARGET_ANY, 2);
|
||||||
|
buf.write(report, headerSize);
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Protocol name used for subprotocol negociation
|
||||||
|
*/
|
||||||
|
static get protocolName() {
|
||||||
|
return 'zenko-secure-channel-v0';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
ChannelMessageV0,
|
||||||
|
MessageType,
|
||||||
|
TargetType,
|
||||||
|
};
|
|
@ -0,0 +1,94 @@
|
||||||
|
const WebSocket = require('ws');
|
||||||
|
const arsenal = require('arsenal');
|
||||||
|
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
|
const _config = require('../Config').config;
|
||||||
|
const { patchConfiguration } = require('./configuration');
|
||||||
|
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||||
|
|
||||||
|
|
||||||
|
const managementAgentMessageType = {
|
||||||
|
/** Message that contains the loaded overlay */
|
||||||
|
NEW_OVERLAY: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const CONNECTION_RETRY_TIMEOUT_MS = 5000;
|
||||||
|
|
||||||
|
|
||||||
|
function initManagementClient() {
|
||||||
|
const { host, port } = _config.managementAgent;
|
||||||
|
|
||||||
|
const ws = new WebSocket(`ws://${host}:${port}/watch`);
|
||||||
|
|
||||||
|
ws.on('open', () => {
|
||||||
|
logger.info('connected with management agent');
|
||||||
|
});
|
||||||
|
|
||||||
|
ws.on('close', (code, reason) => {
|
||||||
|
logger.info('disconnected from management agent', { reason });
|
||||||
|
setTimeout(initManagementClient, CONNECTION_RETRY_TIMEOUT_MS);
|
||||||
|
});
|
||||||
|
|
||||||
|
ws.on('error', error => {
|
||||||
|
logger.error('error on connection with management agent', { error });
|
||||||
|
});
|
||||||
|
|
||||||
|
ws.on('message', data => {
|
||||||
|
const method = 'initManagementclient::onMessage';
|
||||||
|
const log = logger.newRequestLogger();
|
||||||
|
let msg;
|
||||||
|
|
||||||
|
if (!data) {
|
||||||
|
log.error('message without data', { method });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
msg = JSON.parse(data);
|
||||||
|
} catch (err) {
|
||||||
|
log.error('data is an invalid json', { method, err, data });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (msg.payload === undefined) {
|
||||||
|
log.error('message without payload', { method });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (typeof msg.messageType !== 'number') {
|
||||||
|
log.error('messageType is not an integer', {
|
||||||
|
type: typeof msg.messageType,
|
||||||
|
method,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (msg.messageType) {
|
||||||
|
case managementAgentMessageType.NEW_OVERLAY:
|
||||||
|
patchConfiguration(msg.payload, log, err => {
|
||||||
|
if (err) {
|
||||||
|
log.error('failed to patch overlay', {
|
||||||
|
error: reshapeExceptionError(err),
|
||||||
|
method,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
default:
|
||||||
|
log.error('new overlay message with unmanaged message type', {
|
||||||
|
method,
|
||||||
|
type: msg.messageType,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isManagementAgentUsed() {
|
||||||
|
return process.env.MANAGEMENT_USE_AGENT === '1';
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
managementAgentMessageType,
|
||||||
|
initManagementClient,
|
||||||
|
isManagementAgentUsed,
|
||||||
|
};
|
|
@ -0,0 +1,240 @@
|
||||||
|
const arsenal = require('arsenal');
|
||||||
|
|
||||||
|
const { buildAuthDataAccount } = require('../auth/in_memory/builder');
|
||||||
|
const _config = require('../Config').config;
|
||||||
|
const metadata = require('../metadata/wrapper');
|
||||||
|
|
||||||
|
const { getStoredCredentials } = require('./credentials');
|
||||||
|
|
||||||
|
const latestOverlayVersionKey = 'configuration/overlay-version';
|
||||||
|
const managementDatabaseName = 'PENSIEVE';
|
||||||
|
const replicatorEndpoint = 'zenko-cloudserver-replicator';
|
||||||
|
const { decryptSecret } = arsenal.pensieve.credentialUtils;
|
||||||
|
const { patchLocations } = arsenal.patches.locationConstraints;
|
||||||
|
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||||
|
const { replicationBackends } = require('arsenal').constants;
|
||||||
|
|
||||||
|
function overlayHasVersion(overlay) {
|
||||||
|
return overlay && overlay.version !== undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function remoteOverlayIsNewer(cachedOverlay, remoteOverlay) {
|
||||||
|
return (overlayHasVersion(remoteOverlay) &&
|
||||||
|
(!overlayHasVersion(cachedOverlay) ||
|
||||||
|
remoteOverlay.version > cachedOverlay.version));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the live {Config} object with the new overlay configuration.
|
||||||
|
*
|
||||||
|
* No-op if this version was already applied to the live {Config}.
|
||||||
|
*
|
||||||
|
* @param {object} newConf Overlay configuration to apply
|
||||||
|
* @param {werelogs~Logger} log Request-scoped logger
|
||||||
|
* @param {function} cb Function to call with (error, newConf)
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function patchConfiguration(newConf, log, cb) {
|
||||||
|
if (newConf.version === undefined) {
|
||||||
|
log.debug('no remote configuration created yet');
|
||||||
|
return process.nextTick(cb, null, newConf);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_config.overlayVersion !== undefined &&
|
||||||
|
newConf.version <= _config.overlayVersion) {
|
||||||
|
log.debug('configuration version already applied',
|
||||||
|
{ configurationVersion: newConf.version });
|
||||||
|
return process.nextTick(cb, null, newConf);
|
||||||
|
}
|
||||||
|
return getStoredCredentials(log, (err, creds) => {
|
||||||
|
if (err) {
|
||||||
|
return cb(err);
|
||||||
|
}
|
||||||
|
const accounts = [];
|
||||||
|
if (newConf.users) {
|
||||||
|
newConf.users.forEach(u => {
|
||||||
|
if (u.secretKey && u.secretKey.length > 0) {
|
||||||
|
const secretKey = decryptSecret(creds, u.secretKey);
|
||||||
|
// accountType will be service-replication or service-clueso
|
||||||
|
let serviceName;
|
||||||
|
if (u.accountType && u.accountType.startsWith('service-')) {
|
||||||
|
serviceName = u.accountType.split('-')[1];
|
||||||
|
}
|
||||||
|
const newAccount = buildAuthDataAccount(
|
||||||
|
u.accessKey, secretKey, u.canonicalId, serviceName,
|
||||||
|
u.userName);
|
||||||
|
accounts.push(newAccount.accounts[0]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const restEndpoints = Object.assign({}, _config.restEndpoints);
|
||||||
|
if (newConf.endpoints) {
|
||||||
|
newConf.endpoints.forEach(e => {
|
||||||
|
restEndpoints[e.hostname] = e.locationName;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!restEndpoints[replicatorEndpoint]) {
|
||||||
|
restEndpoints[replicatorEndpoint] = 'us-east-1';
|
||||||
|
}
|
||||||
|
|
||||||
|
const locations = patchLocations(newConf.locations, creds, log);
|
||||||
|
if (Object.keys(locations).length !== 0) {
|
||||||
|
try {
|
||||||
|
_config.setLocationConstraints(locations);
|
||||||
|
} catch (error) {
|
||||||
|
const exceptionError = reshapeExceptionError(error);
|
||||||
|
log.error('could not apply configuration version location ' +
|
||||||
|
'constraints', { error: exceptionError,
|
||||||
|
method: 'getStoredCredentials' });
|
||||||
|
return cb(exceptionError);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const locationsWithReplicationBackend = Object.keys(locations)
|
||||||
|
// NOTE: In Orbit, we don't need to have Scality location in our
|
||||||
|
// replication endpoind config, since we do not replicate to
|
||||||
|
// any Scality Instance yet.
|
||||||
|
.filter(key => replicationBackends
|
||||||
|
[locations[key].type])
|
||||||
|
.reduce((obj, key) => {
|
||||||
|
/* eslint no-param-reassign:0 */
|
||||||
|
obj[key] = locations[key];
|
||||||
|
return obj;
|
||||||
|
}, {});
|
||||||
|
_config.setReplicationEndpoints(
|
||||||
|
locationsWithReplicationBackend);
|
||||||
|
} catch (error) {
|
||||||
|
const exceptionError = reshapeExceptionError(error);
|
||||||
|
log.error('could not apply replication endpoints',
|
||||||
|
{ error: exceptionError, method: 'getStoredCredentials' });
|
||||||
|
return cb(exceptionError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_config.setAuthDataAccounts(accounts);
|
||||||
|
_config.setRestEndpoints(restEndpoints);
|
||||||
|
_config.setPublicInstanceId(newConf.instanceId);
|
||||||
|
|
||||||
|
if (newConf.browserAccess) {
|
||||||
|
if (Boolean(_config.browserAccessEnabled) !==
|
||||||
|
Boolean(newConf.browserAccess.enabled)) {
|
||||||
|
_config.browserAccessEnabled =
|
||||||
|
Boolean(newConf.browserAccess.enabled);
|
||||||
|
_config.emit('browser-access-enabled-change');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_config.overlayVersion = newConf.version;
|
||||||
|
|
||||||
|
log.info('applied configuration version',
|
||||||
|
{ configurationVersion: _config.overlayVersion });
|
||||||
|
|
||||||
|
return cb(null, newConf);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes configuration version to the management database
|
||||||
|
*
|
||||||
|
* @param {object} cachedOverlay Latest stored configuration version
|
||||||
|
* for freshness comparison purposes
|
||||||
|
* @param {object} remoteOverlay New configuration version
|
||||||
|
* @param {werelogs~Logger} log Request-scoped logger
|
||||||
|
* @param {function} cb Function to call with (error, remoteOverlay)
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function saveConfigurationVersion(cachedOverlay, remoteOverlay, log, cb) {
|
||||||
|
if (remoteOverlayIsNewer(cachedOverlay, remoteOverlay)) {
|
||||||
|
const objName = `configuration/overlay/${remoteOverlay.version}`;
|
||||||
|
metadata.putObjectMD(managementDatabaseName, objName, remoteOverlay,
|
||||||
|
{}, log, error => {
|
||||||
|
if (error) {
|
||||||
|
const exceptionError = reshapeExceptionError(error);
|
||||||
|
log.error('could not save configuration',
|
||||||
|
{ error: exceptionError,
|
||||||
|
method: 'saveConfigurationVersion',
|
||||||
|
configurationVersion: remoteOverlay.version });
|
||||||
|
cb(exceptionError);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
metadata.putObjectMD(managementDatabaseName,
|
||||||
|
latestOverlayVersionKey, remoteOverlay.version, {}, log,
|
||||||
|
error => {
|
||||||
|
if (error) {
|
||||||
|
log.error('could not save configuration version', {
|
||||||
|
configurationVersion: remoteOverlay.version,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
cb(error, remoteOverlay);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
log.debug('no remote configuration to cache yet');
|
||||||
|
process.nextTick(cb, null, remoteOverlay);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads the latest cached configuration overlay from the management
|
||||||
|
* database, without contacting the Orbit API.
|
||||||
|
*
|
||||||
|
* @param {werelogs~Logger} log Request-scoped logger
|
||||||
|
* @param {function} callback Function called with (error, cachedOverlay)
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function loadCachedOverlay(log, callback) {
|
||||||
|
return metadata.getObjectMD(managementDatabaseName,
|
||||||
|
latestOverlayVersionKey, {}, log, (err, version) => {
|
||||||
|
if (err) {
|
||||||
|
if (err.is.NoSuchKey) {
|
||||||
|
return process.nextTick(callback, null, {});
|
||||||
|
}
|
||||||
|
return callback(err);
|
||||||
|
}
|
||||||
|
return metadata.getObjectMD(managementDatabaseName,
|
||||||
|
`configuration/overlay/${version}`, {}, log, (err, conf) => {
|
||||||
|
if (err) {
|
||||||
|
if (err.is.NoSuchKey) {
|
||||||
|
return process.nextTick(callback, null, {});
|
||||||
|
}
|
||||||
|
return callback(err);
|
||||||
|
}
|
||||||
|
return callback(null, conf);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyAndSaveOverlay(overlay, log) {
|
||||||
|
patchConfiguration(overlay, log, err => {
|
||||||
|
if (err) {
|
||||||
|
log.error('could not apply pushed overlay', {
|
||||||
|
error: reshapeExceptionError(err),
|
||||||
|
method: 'applyAndSaveOverlay',
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
saveConfigurationVersion(null, overlay, log, err => {
|
||||||
|
if (err) {
|
||||||
|
log.error('could not cache overlay version', {
|
||||||
|
error: reshapeExceptionError(err),
|
||||||
|
method: 'applyAndSaveOverlay',
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info('overlay push processed');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
loadCachedOverlay,
|
||||||
|
managementDatabaseName,
|
||||||
|
patchConfiguration,
|
||||||
|
saveConfigurationVersion,
|
||||||
|
remoteOverlayIsNewer,
|
||||||
|
applyAndSaveOverlay,
|
||||||
|
};
|
|
@ -0,0 +1,145 @@
|
||||||
|
const arsenal = require('arsenal');
|
||||||
|
const forge = require('node-forge');
|
||||||
|
const request = require('../utilities/request');
|
||||||
|
|
||||||
|
const metadata = require('../metadata/wrapper');
|
||||||
|
|
||||||
|
const managementDatabaseName = 'PENSIEVE';
|
||||||
|
const tokenConfigurationKey = 'auth/zenko/remote-management-token';
|
||||||
|
const tokenRotationDelay = 3600 * 24 * 7 * 1000; // 7 days
|
||||||
|
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Retrieves Orbit API token from the management database.
|
||||||
|
*
|
||||||
|
* The token is used to authenticate stat posting and
|
||||||
|
*
|
||||||
|
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
|
||||||
|
* initialization process
|
||||||
|
* @param {function} callback Function called with (error, result)
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function getStoredCredentials(log, callback) {
|
||||||
|
metadata.getObjectMD(managementDatabaseName, tokenConfigurationKey, {},
|
||||||
|
log, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
function issueCredentials(managementEndpoint, instanceId, log, callback) {
|
||||||
|
log.info('registering with API to get token');
|
||||||
|
|
||||||
|
const keyPair = forge.pki.rsa.generateKeyPair({ bits: 2048, e: 0x10001 });
|
||||||
|
const privateKey = forge.pki.privateKeyToPem(keyPair.privateKey);
|
||||||
|
const publicKey = forge.pki.publicKeyToPem(keyPair.publicKey);
|
||||||
|
|
||||||
|
const postData = {
|
||||||
|
publicKey,
|
||||||
|
};
|
||||||
|
|
||||||
|
request.post(`${managementEndpoint}/${instanceId}/register`,
|
||||||
|
{ body: postData, json: true }, (error, response, body) => {
|
||||||
|
if (error) {
|
||||||
|
return callback(error);
|
||||||
|
}
|
||||||
|
if (response.statusCode !== 201) {
|
||||||
|
log.error('could not register instance', {
|
||||||
|
statusCode: response.statusCode,
|
||||||
|
});
|
||||||
|
return callback(arsenal.errors.InternalError);
|
||||||
|
}
|
||||||
|
/* eslint-disable no-param-reassign */
|
||||||
|
body.privateKey = privateKey;
|
||||||
|
/* eslint-enable no-param-reassign */
|
||||||
|
return callback(null, body);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function confirmInstanceCredentials(
|
||||||
|
managementEndpoint, instanceId, creds, log, callback) {
|
||||||
|
const postData = {
|
||||||
|
serial: creds.serial || 0,
|
||||||
|
publicKey: creds.publicKey,
|
||||||
|
};
|
||||||
|
|
||||||
|
const opts = {
|
||||||
|
headers: {
|
||||||
|
'x-instance-authentication-token': creds.token,
|
||||||
|
},
|
||||||
|
body: postData,
|
||||||
|
};
|
||||||
|
|
||||||
|
request.post(`${managementEndpoint}/${instanceId}/confirm`,
|
||||||
|
opts, (error, response) => {
|
||||||
|
if (error) {
|
||||||
|
return callback(error);
|
||||||
|
}
|
||||||
|
if (response.statusCode === 200) {
|
||||||
|
return callback(null, instanceId, creds.token);
|
||||||
|
}
|
||||||
|
return callback(arsenal.errors.InternalError);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initializes credentials and PKI in the management database.
|
||||||
|
*
|
||||||
|
* In case the management database is new and empty, the instance
|
||||||
|
* is registered as new against the Orbit API with newly-generated
|
||||||
|
* RSA key pair.
|
||||||
|
*
|
||||||
|
* @param {string} managementEndpoint API endpoint
|
||||||
|
* @param {string} instanceId UUID of this deployment
|
||||||
|
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
|
||||||
|
* initialization process
|
||||||
|
* @param {function} callback Function called with (error, result)
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function initManagementCredentials(
|
||||||
|
managementEndpoint, instanceId, log, callback) {
|
||||||
|
getStoredCredentials(log, (error, value) => {
|
||||||
|
if (error) {
|
||||||
|
if (error.is.NoSuchKey) {
|
||||||
|
return issueCredentials(managementEndpoint, instanceId, log,
|
||||||
|
(error, value) => {
|
||||||
|
if (error) {
|
||||||
|
log.error('could not issue token',
|
||||||
|
{ error: reshapeExceptionError(error),
|
||||||
|
method: 'initManagementCredentials' });
|
||||||
|
return callback(error);
|
||||||
|
}
|
||||||
|
log.debug('saving token');
|
||||||
|
return metadata.putObjectMD(managementDatabaseName,
|
||||||
|
tokenConfigurationKey, value, {}, log, error => {
|
||||||
|
if (error) {
|
||||||
|
log.error('could not save token',
|
||||||
|
{ error: reshapeExceptionError(error),
|
||||||
|
method: 'initManagementCredentials',
|
||||||
|
});
|
||||||
|
return callback(error);
|
||||||
|
}
|
||||||
|
log.info('saved token locally, ' +
|
||||||
|
'confirming instance');
|
||||||
|
return confirmInstanceCredentials(
|
||||||
|
managementEndpoint, instanceId, value, log,
|
||||||
|
callback);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
log.debug('could not get token', { error });
|
||||||
|
return callback(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info('returning existing token');
|
||||||
|
if (Date.now() - value.issueDate > tokenRotationDelay) {
|
||||||
|
log.warn('management API token is too old, should re-issue');
|
||||||
|
}
|
||||||
|
|
||||||
|
return callback(null, instanceId, value.token);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
getStoredCredentials,
|
||||||
|
initManagementCredentials,
|
||||||
|
};
|
|
@ -0,0 +1,138 @@
|
||||||
|
const arsenal = require('arsenal');
|
||||||
|
const async = require('async');
|
||||||
|
|
||||||
|
const metadata = require('../metadata/wrapper');
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
|
|
||||||
|
const {
|
||||||
|
loadCachedOverlay,
|
||||||
|
managementDatabaseName,
|
||||||
|
patchConfiguration,
|
||||||
|
} = require('./configuration');
|
||||||
|
const { initManagementCredentials } = require('./credentials');
|
||||||
|
const { startWSManagementClient } = require('./push');
|
||||||
|
const { startPollingManagementClient } = require('./poll');
|
||||||
|
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||||
|
const { isManagementAgentUsed } = require('./agentClient');
|
||||||
|
|
||||||
|
const initRemoteManagementRetryDelay = 10000;
|
||||||
|
|
||||||
|
const managementEndpointRoot =
|
||||||
|
process.env.MANAGEMENT_ENDPOINT ||
|
||||||
|
'https://api.zenko.io';
|
||||||
|
const managementEndpoint = `${managementEndpointRoot}/api/v1/instance`;
|
||||||
|
|
||||||
|
const pushEndpointRoot =
|
||||||
|
process.env.PUSH_ENDPOINT ||
|
||||||
|
'https://push.api.zenko.io';
|
||||||
|
const pushEndpoint = `${pushEndpointRoot}/api/v1/instance`;
|
||||||
|
|
||||||
|
function initManagementDatabase(log, callback) {
|
||||||
|
// XXX choose proper owner names
|
||||||
|
const md = new arsenal.models.BucketInfo(managementDatabaseName, 'owner',
|
||||||
|
'owner display name', new Date().toJSON());
|
||||||
|
|
||||||
|
metadata.createBucket(managementDatabaseName, md, log, error => {
|
||||||
|
if (error) {
|
||||||
|
if (error.is.BucketAlreadyExists) {
|
||||||
|
log.info('created management database');
|
||||||
|
return callback();
|
||||||
|
}
|
||||||
|
log.error('could not initialize management database',
|
||||||
|
{ error: reshapeExceptionError(error),
|
||||||
|
method: 'initManagementDatabase' });
|
||||||
|
return callback(error);
|
||||||
|
}
|
||||||
|
log.info('initialized management database');
|
||||||
|
return callback();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function startManagementListeners(instanceId, token) {
|
||||||
|
const mode = process.env.MANAGEMENT_MODE || 'push';
|
||||||
|
if (mode === 'push') {
|
||||||
|
const url = `${pushEndpoint}/${instanceId}/ws`;
|
||||||
|
startWSManagementClient(url, token);
|
||||||
|
} else {
|
||||||
|
startPollingManagementClient(managementEndpoint, instanceId, token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initializes Orbit-based management by:
|
||||||
|
* - creating the management database in metadata
|
||||||
|
* - generating a key pair for credentials encryption
|
||||||
|
* - generating an instance-unique ID
|
||||||
|
* - getting an authentication token for the API
|
||||||
|
* - loading and applying the latest cached overlay configuration
|
||||||
|
* - starting a configuration update and metrics push background task
|
||||||
|
*
|
||||||
|
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
|
||||||
|
* initialization process
|
||||||
|
* @param {function} callback Function to call once the overlay is loaded
|
||||||
|
* (overlay)
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function initManagement(log, callback) {
|
||||||
|
if ((process.env.REMOTE_MANAGEMENT_DISABLE &&
|
||||||
|
process.env.REMOTE_MANAGEMENT_DISABLE !== '0')
|
||||||
|
|| process.env.S3BACKEND === 'mem') {
|
||||||
|
log.info('remote management disabled');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Temporary check before to fully move to the process management agent. */
|
||||||
|
if (isManagementAgentUsed() ^ typeof callback === 'function') {
|
||||||
|
let msg = 'misuse of initManagement function: ';
|
||||||
|
msg += `MANAGEMENT_USE_AGENT: ${process.env.MANAGEMENT_USE_AGENT}`;
|
||||||
|
msg += `, callback type: ${typeof callback}`;
|
||||||
|
throw new Error(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
async.waterfall([
|
||||||
|
// eslint-disable-next-line arrow-body-style
|
||||||
|
cb => { return isManagementAgentUsed() ? metadata.setup(cb) : cb(); },
|
||||||
|
cb => initManagementDatabase(log, cb),
|
||||||
|
cb => metadata.getUUID(log, cb),
|
||||||
|
(instanceId, cb) => initManagementCredentials(
|
||||||
|
managementEndpoint, instanceId, log, cb),
|
||||||
|
(instanceId, token, cb) => {
|
||||||
|
if (!isManagementAgentUsed()) {
|
||||||
|
cb(null, instanceId, token, {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
loadCachedOverlay(log, (err, overlay) => cb(err, instanceId,
|
||||||
|
token, overlay));
|
||||||
|
},
|
||||||
|
(instanceId, token, overlay, cb) => {
|
||||||
|
if (!isManagementAgentUsed()) {
|
||||||
|
cb(null, instanceId, token, overlay);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
patchConfiguration(overlay, log,
|
||||||
|
err => cb(err, instanceId, token, overlay));
|
||||||
|
},
|
||||||
|
], (error, instanceId, token, overlay) => {
|
||||||
|
if (error) {
|
||||||
|
log.error('could not initialize remote management, retrying later',
|
||||||
|
{ error: reshapeExceptionError(error),
|
||||||
|
method: 'initManagement' });
|
||||||
|
setTimeout(initManagement,
|
||||||
|
initRemoteManagementRetryDelay,
|
||||||
|
logger.newRequestLogger());
|
||||||
|
} else {
|
||||||
|
log.info(`this deployment's Instance ID is ${instanceId}`);
|
||||||
|
log.end('management init done');
|
||||||
|
startManagementListeners(instanceId, token);
|
||||||
|
if (callback) {
|
||||||
|
callback(overlay);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
initManagement,
|
||||||
|
initManagementDatabase,
|
||||||
|
};
|
|
@ -0,0 +1,157 @@
|
||||||
|
const arsenal = require('arsenal');
|
||||||
|
const async = require('async');
|
||||||
|
const request = require('../utilities/request');
|
||||||
|
|
||||||
|
const _config = require('../Config').config;
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
|
const metadata = require('../metadata/wrapper');
|
||||||
|
const {
|
||||||
|
loadCachedOverlay,
|
||||||
|
patchConfiguration,
|
||||||
|
saveConfigurationVersion,
|
||||||
|
} = require('./configuration');
|
||||||
|
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||||
|
|
||||||
|
const pushReportDelay = 30000;
|
||||||
|
const pullConfigurationOverlayDelay = 60000;
|
||||||
|
|
||||||
|
function loadRemoteOverlay(
|
||||||
|
managementEndpoint, instanceId, remoteToken, cachedOverlay, log, cb) {
|
||||||
|
log.debug('loading remote overlay');
|
||||||
|
const opts = {
|
||||||
|
headers: {
|
||||||
|
'x-instance-authentication-token': remoteToken,
|
||||||
|
'x-scal-request-id': log.getSerializedUids(),
|
||||||
|
},
|
||||||
|
json: true,
|
||||||
|
};
|
||||||
|
request.get(`${managementEndpoint}/${instanceId}/config/overlay`, opts,
|
||||||
|
(error, response, body) => {
|
||||||
|
if (error) {
|
||||||
|
return cb(error);
|
||||||
|
}
|
||||||
|
if (response.statusCode === 200) {
|
||||||
|
return cb(null, cachedOverlay, body);
|
||||||
|
}
|
||||||
|
if (response.statusCode === 404) {
|
||||||
|
return cb(null, cachedOverlay, {});
|
||||||
|
}
|
||||||
|
return cb(arsenal.errors.AccessForbidden, cachedOverlay, {});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO save only after successful patch
|
||||||
|
function applyConfigurationOverlay(
|
||||||
|
managementEndpoint, instanceId, remoteToken, log) {
|
||||||
|
async.waterfall([
|
||||||
|
wcb => loadCachedOverlay(log, wcb),
|
||||||
|
(cachedOverlay, wcb) => patchConfiguration(cachedOverlay,
|
||||||
|
log, wcb),
|
||||||
|
(cachedOverlay, wcb) =>
|
||||||
|
loadRemoteOverlay(managementEndpoint, instanceId, remoteToken,
|
||||||
|
cachedOverlay, log, wcb),
|
||||||
|
(cachedOverlay, remoteOverlay, wcb) =>
|
||||||
|
saveConfigurationVersion(cachedOverlay, remoteOverlay, log, wcb),
|
||||||
|
(remoteOverlay, wcb) => patchConfiguration(remoteOverlay,
|
||||||
|
log, wcb),
|
||||||
|
], error => {
|
||||||
|
if (error) {
|
||||||
|
log.error('could not apply managed configuration',
|
||||||
|
{ error: reshapeExceptionError(error),
|
||||||
|
method: 'applyConfigurationOverlay' });
|
||||||
|
}
|
||||||
|
setTimeout(applyConfigurationOverlay, pullConfigurationOverlayDelay,
|
||||||
|
managementEndpoint, instanceId, remoteToken,
|
||||||
|
logger.newRequestLogger());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function postStats(managementEndpoint, instanceId, remoteToken, report, next) {
|
||||||
|
const toURL = `${managementEndpoint}/${instanceId}/stats`;
|
||||||
|
const toOptions = {
|
||||||
|
json: true,
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/json',
|
||||||
|
'x-instance-authentication-token': remoteToken,
|
||||||
|
},
|
||||||
|
body: report,
|
||||||
|
};
|
||||||
|
const toCallback = (err, response, body) => {
|
||||||
|
if (err) {
|
||||||
|
logger.info('could not post stats', { error: err });
|
||||||
|
}
|
||||||
|
if (response && response.statusCode !== 201) {
|
||||||
|
logger.info('could not post stats', {
|
||||||
|
body,
|
||||||
|
statusCode: response.statusCode,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (next) {
|
||||||
|
next(null, instanceId, remoteToken);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return request.post(toURL, toOptions, toCallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getStats(next) {
|
||||||
|
const fromURL = `http://localhost:${_config.port}/_/report`;
|
||||||
|
const fromOptions = {
|
||||||
|
headers: {
|
||||||
|
'x-scal-report-token': process.env.REPORT_TOKEN,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return request.get(fromURL, fromOptions, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pushStats(managementEndpoint, instanceId, remoteToken, next) {
|
||||||
|
if (process.env.PUSH_STATS === 'false') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
getStats((err, res, report) => {
|
||||||
|
if (err) {
|
||||||
|
logger.info('could not retrieve stats', { error: err });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug('report', { report });
|
||||||
|
postStats(
|
||||||
|
managementEndpoint,
|
||||||
|
instanceId,
|
||||||
|
remoteToken,
|
||||||
|
report,
|
||||||
|
next
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
});
|
||||||
|
|
||||||
|
setTimeout(pushStats, pushReportDelay,
|
||||||
|
managementEndpoint, instanceId, remoteToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Starts background task that updates configuration and pushes stats.
|
||||||
|
*
|
||||||
|
* Periodically polls for configuration updates, and pushes stats at
|
||||||
|
* a fixed interval.
|
||||||
|
*
|
||||||
|
* @param {string} managementEndpoint API endpoint
|
||||||
|
* @param {string} instanceId UUID of this deployment
|
||||||
|
* @param {string} remoteToken API authentication token
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function startPollingManagementClient(
|
||||||
|
managementEndpoint, instanceId, remoteToken) {
|
||||||
|
metadata.notifyBucketChange(() => {
|
||||||
|
pushStats(managementEndpoint, instanceId, remoteToken);
|
||||||
|
});
|
||||||
|
|
||||||
|
pushStats(managementEndpoint, instanceId, remoteToken);
|
||||||
|
applyConfigurationOverlay(managementEndpoint, instanceId, remoteToken,
|
||||||
|
logger.newRequestLogger());
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
startPollingManagementClient,
|
||||||
|
};
|
|
@ -0,0 +1,301 @@
|
||||||
|
const arsenal = require('arsenal');
|
||||||
|
const HttpsProxyAgent = require('https-proxy-agent');
|
||||||
|
const net = require('net');
|
||||||
|
const request = require('../utilities/request');
|
||||||
|
const { URL } = require('url');
|
||||||
|
const WebSocket = require('ws');
|
||||||
|
const assert = require('assert');
|
||||||
|
const http = require('http');
|
||||||
|
|
||||||
|
const _config = require('../Config').config;
|
||||||
|
const logger = require('../utilities/logger');
|
||||||
|
const metadata = require('../metadata/wrapper');
|
||||||
|
|
||||||
|
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||||
|
const { isManagementAgentUsed } = require('./agentClient');
|
||||||
|
const { applyAndSaveOverlay } = require('./configuration');
|
||||||
|
const {
|
||||||
|
ChannelMessageV0,
|
||||||
|
MessageType,
|
||||||
|
} = require('./ChannelMessageV0');
|
||||||
|
|
||||||
|
const {
|
||||||
|
CONFIG_OVERLAY_MESSAGE,
|
||||||
|
METRICS_REQUEST_MESSAGE,
|
||||||
|
CHANNEL_CLOSE_MESSAGE,
|
||||||
|
CHANNEL_PAYLOAD_MESSAGE,
|
||||||
|
} = MessageType;
|
||||||
|
|
||||||
|
const PING_INTERVAL_MS = 10000;
|
||||||
|
const subprotocols = [ChannelMessageV0.protocolName];
|
||||||
|
|
||||||
|
const cloudServerHost = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_HOST
|
||||||
|
|| 'localhost';
|
||||||
|
const cloudServerPort = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_PORT
|
||||||
|
|| _config.port;
|
||||||
|
|
||||||
|
let overlayMessageListener = null;
|
||||||
|
let connected = false;
|
||||||
|
|
||||||
|
// No wildcard nor cidr/mask match for now
|
||||||
|
function createWSAgent(pushEndpoint, env, log) {
|
||||||
|
const url = new URL(pushEndpoint);
|
||||||
|
const noProxy = (env.NO_PROXY || env.no_proxy
|
||||||
|
|| '').split(',');
|
||||||
|
|
||||||
|
if (noProxy.includes(url.hostname)) {
|
||||||
|
log.info('push server ws has proxy exclusion', { noProxy });
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (url.protocol === 'https:' || url.protocol === 'wss:') {
|
||||||
|
const httpsProxy = (env.HTTPS_PROXY || env.https_proxy);
|
||||||
|
if (httpsProxy) {
|
||||||
|
log.info('push server ws using https proxy', { httpsProxy });
|
||||||
|
return new HttpsProxyAgent(httpsProxy);
|
||||||
|
}
|
||||||
|
} else if (url.protocol === 'http:' || url.protocol === 'ws:') {
|
||||||
|
const httpProxy = (env.HTTP_PROXY || env.http_proxy);
|
||||||
|
if (httpProxy) {
|
||||||
|
log.info('push server ws using http proxy', { httpProxy });
|
||||||
|
return new HttpsProxyAgent(httpProxy);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const allProxy = (env.ALL_PROXY || env.all_proxy);
|
||||||
|
if (allProxy) {
|
||||||
|
log.info('push server ws using wildcard proxy', { allProxy });
|
||||||
|
return new HttpsProxyAgent(allProxy);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info('push server ws not using proxy');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Starts background task that updates configuration and pushes stats.
|
||||||
|
*
|
||||||
|
* Receives pushed Websocket messages on configuration updates, and
|
||||||
|
* sends stat messages in response to API sollicitations.
|
||||||
|
*
|
||||||
|
* @param {string} url API endpoint
|
||||||
|
* @param {string} token API authentication token
|
||||||
|
* @param {function} cb end-of-connection callback
|
||||||
|
*
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
function startWSManagementClient(url, token, cb) {
|
||||||
|
logger.info('connecting to push server', { url });
|
||||||
|
function _logError(error, errorMessage, method) {
|
||||||
|
if (error) {
|
||||||
|
logger.error(`management client error: ${errorMessage}`,
|
||||||
|
{ error: reshapeExceptionError(error), method });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const socketsByChannelId = [];
|
||||||
|
const headers = {
|
||||||
|
'x-instance-authentication-token': token,
|
||||||
|
};
|
||||||
|
const agent = createWSAgent(url, process.env, logger);
|
||||||
|
|
||||||
|
const ws = new WebSocket(url, subprotocols, { headers, agent });
|
||||||
|
let pingTimeout = null;
|
||||||
|
|
||||||
|
function sendPing() {
|
||||||
|
if (ws.readyState === ws.OPEN) {
|
||||||
|
ws.ping(err => _logError(err, 'failed to send a ping', 'sendPing'));
|
||||||
|
}
|
||||||
|
pingTimeout = setTimeout(() => ws.terminate(), PING_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
function initiatePing() {
|
||||||
|
clearTimeout(pingTimeout);
|
||||||
|
setTimeout(sendPing, PING_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pushStats(options) {
|
||||||
|
if (process.env.PUSH_STATS === 'false') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const fromURL = `http://${cloudServerHost}:${cloudServerPort}/_/report`;
|
||||||
|
const fromOptions = {
|
||||||
|
json: true,
|
||||||
|
headers: {
|
||||||
|
'x-scal-report-token': process.env.REPORT_TOKEN,
|
||||||
|
'x-scal-report-skip-cache': Boolean(options && options.noCache),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
request.get(fromURL, fromOptions, (err, response, body) => {
|
||||||
|
if (err) {
|
||||||
|
_logError(err, 'failed to get metrics report', 'pushStats');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ws.send(ChannelMessageV0.encodeMetricsReportMessage(body),
|
||||||
|
err => _logError(err, 'failed to send metrics report message',
|
||||||
|
'pushStats'));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeChannel(channelId) {
|
||||||
|
const socket = socketsByChannelId[channelId];
|
||||||
|
if (socket) {
|
||||||
|
socket.destroy();
|
||||||
|
delete socketsByChannelId[channelId];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function receiveChannelData(channelId, payload) {
|
||||||
|
let socket = socketsByChannelId[channelId];
|
||||||
|
if (!socket) {
|
||||||
|
socket = net.createConnection(cloudServerPort, cloudServerHost);
|
||||||
|
|
||||||
|
socket.on('data', data => {
|
||||||
|
ws.send(ChannelMessageV0.
|
||||||
|
encodeChannelDataMessage(channelId, data), err =>
|
||||||
|
_logError(err, 'failed to send channel data message',
|
||||||
|
'receiveChannelData'));
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('connect', () => {
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('drain', () => {
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('error', error => {
|
||||||
|
logger.error('failed to connect to S3', {
|
||||||
|
code: error.code,
|
||||||
|
host: error.address,
|
||||||
|
port: error.port,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('end', () => {
|
||||||
|
socket.destroy();
|
||||||
|
socketsByChannelId[channelId] = null;
|
||||||
|
ws.send(ChannelMessageV0.encodeChannelCloseMessage(channelId),
|
||||||
|
err => _logError(err,
|
||||||
|
'failed to send channel close message',
|
||||||
|
'receiveChannelData'));
|
||||||
|
});
|
||||||
|
|
||||||
|
socketsByChannelId[channelId] = socket;
|
||||||
|
}
|
||||||
|
socket.write(payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
function browserAccessChangeHandler() {
|
||||||
|
if (!_config.browserAccessEnabled) {
|
||||||
|
socketsByChannelId.forEach(s => s.close());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ws.on('open', () => {
|
||||||
|
connected = true;
|
||||||
|
logger.info('connected to push server');
|
||||||
|
|
||||||
|
metadata.notifyBucketChange(() => {
|
||||||
|
pushStats({ noCache: true });
|
||||||
|
});
|
||||||
|
_config.on('browser-access-enabled-change', browserAccessChangeHandler);
|
||||||
|
|
||||||
|
initiatePing();
|
||||||
|
});
|
||||||
|
|
||||||
|
const cbOnce = cb ? arsenal.jsutil.once(cb) : null;
|
||||||
|
|
||||||
|
ws.on('close', () => {
|
||||||
|
logger.info('disconnected from push server, reconnecting in 10s');
|
||||||
|
metadata.notifyBucketChange(null);
|
||||||
|
_config.removeListener('browser-access-enabled-change',
|
||||||
|
browserAccessChangeHandler);
|
||||||
|
setTimeout(startWSManagementClient, 10000, url, token);
|
||||||
|
connected = false;
|
||||||
|
|
||||||
|
if (cbOnce) {
|
||||||
|
process.nextTick(cbOnce);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ws.on('error', err => {
|
||||||
|
connected = false;
|
||||||
|
logger.error('error from push server connection', {
|
||||||
|
error: err,
|
||||||
|
errorMessage: err.message,
|
||||||
|
});
|
||||||
|
if (cbOnce) {
|
||||||
|
process.nextTick(cbOnce, err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ws.on('ping', () => {
|
||||||
|
ws.pong(err => _logError(err, 'failed to send a pong'));
|
||||||
|
});
|
||||||
|
|
||||||
|
ws.on('pong', () => {
|
||||||
|
initiatePing();
|
||||||
|
});
|
||||||
|
|
||||||
|
ws.on('message', data => {
|
||||||
|
const log = logger.newRequestLogger();
|
||||||
|
const message = new ChannelMessageV0(data);
|
||||||
|
switch (message.getType()) {
|
||||||
|
case CONFIG_OVERLAY_MESSAGE:
|
||||||
|
if (!isManagementAgentUsed()) {
|
||||||
|
applyAndSaveOverlay(JSON.parse(message.getPayload()), log);
|
||||||
|
} else {
|
||||||
|
if (overlayMessageListener) {
|
||||||
|
overlayMessageListener(message.getPayload().toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case METRICS_REQUEST_MESSAGE:
|
||||||
|
pushStats();
|
||||||
|
break;
|
||||||
|
case CHANNEL_CLOSE_MESSAGE:
|
||||||
|
closeChannel(message.getChannelNumber());
|
||||||
|
break;
|
||||||
|
case CHANNEL_PAYLOAD_MESSAGE:
|
||||||
|
// browserAccessEnabled defaults to true unless explicitly false
|
||||||
|
if (_config.browserAccessEnabled !== false) {
|
||||||
|
receiveChannelData(
|
||||||
|
message.getChannelNumber(), message.getPayload());
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
logger.error('unknown message type from push server',
|
||||||
|
{ messageType: message.getType() });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function addOverlayMessageListener(callback) {
|
||||||
|
assert(typeof callback === 'function');
|
||||||
|
overlayMessageListener = callback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function startPushConnectionHealthCheckServer(cb) {
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
if (req.url !== '/_/healthcheck') {
|
||||||
|
res.writeHead(404);
|
||||||
|
res.write('Not Found');
|
||||||
|
} else if (connected) {
|
||||||
|
res.writeHead(200);
|
||||||
|
res.write('Connected');
|
||||||
|
} else {
|
||||||
|
res.writeHead(503);
|
||||||
|
res.write('Not Connected');
|
||||||
|
}
|
||||||
|
res.end();
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(_config.port, cb);
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
createWSAgent,
|
||||||
|
startWSManagementClient,
|
||||||
|
startPushConnectionHealthCheckServer,
|
||||||
|
addOverlayMessageListener,
|
||||||
|
};
|
|
@ -6,9 +6,6 @@ const BucketInfo = require('arsenal').models.BucketInfo;
|
||||||
const { isBucketAuthorized, isObjAuthorized } =
|
const { isBucketAuthorized, isObjAuthorized } =
|
||||||
require('../api/apiUtils/authorization/permissionChecks');
|
require('../api/apiUtils/authorization/permissionChecks');
|
||||||
const bucketShield = require('../api/apiUtils/bucket/bucketShield');
|
const bucketShield = require('../api/apiUtils/bucket/bucketShield');
|
||||||
const { onlyOwnerAllowed } = require('../../constants');
|
|
||||||
const { actionNeedQuotaCheck, actionWithDataDeletion } = require('arsenal/build/lib/policyEvaluator/RequestContext');
|
|
||||||
const { processBytesToWrite, validateQuotas } = require('../api/apiUtils/quotas/quotaUtils');
|
|
||||||
|
|
||||||
/** getNullVersionFromMaster - retrieves the null version
|
/** getNullVersionFromMaster - retrieves the null version
|
||||||
* metadata via retrieving the master key
|
* metadata via retrieving the master key
|
||||||
|
@ -155,6 +152,9 @@ function validateBucket(bucket, params, log, actionImplicitDenies = {}) {
|
||||||
});
|
});
|
||||||
return errors.NoSuchBucket;
|
return errors.NoSuchBucket;
|
||||||
}
|
}
|
||||||
|
// if requester is not bucket owner, bucket policy actions should be denied with
|
||||||
|
// MethodNotAllowed error
|
||||||
|
const onlyOwnerAllowed = ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'];
|
||||||
const canonicalID = authInfo.getCanonicalID();
|
const canonicalID = authInfo.getCanonicalID();
|
||||||
if (!Array.isArray(requestType)) {
|
if (!Array.isArray(requestType)) {
|
||||||
requestType = [requestType];
|
requestType = [requestType];
|
||||||
|
@ -184,7 +184,7 @@ function validateBucket(bucket, params, log, actionImplicitDenies = {}) {
|
||||||
* @return {undefined} - and call callback with params err, bucket md
|
* @return {undefined} - and call callback with params err, bucket md
|
||||||
*/
|
*/
|
||||||
function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log, callback) {
|
function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log, callback) {
|
||||||
const { authInfo, bucketName, objectKey, versionId, getDeleteMarker, request, withVersionId } = params;
|
const { authInfo, bucketName, objectKey, versionId, getDeleteMarker, request } = params;
|
||||||
let requestType = params.requestType;
|
let requestType = params.requestType;
|
||||||
if (!Array.isArray(requestType)) {
|
if (!Array.isArray(requestType)) {
|
||||||
requestType = [requestType];
|
requestType = [requestType];
|
||||||
|
@ -238,21 +238,6 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
|
||||||
}
|
}
|
||||||
return next(null, bucket, objMD);
|
return next(null, bucket, objMD);
|
||||||
},
|
},
|
||||||
(bucket, objMD, next) => {
|
|
||||||
const needQuotaCheck = requestType => requestType.some(type => actionNeedQuotaCheck[type] ||
|
|
||||||
actionWithDataDeletion[type]);
|
|
||||||
const checkQuota = params.checkQuota === undefined ? needQuotaCheck(requestType) : params.checkQuota;
|
|
||||||
// withVersionId cover cases when an object is being restored with a specific version ID.
|
|
||||||
// In this case, the storage space was already accounted for when the RestoreObject API call
|
|
||||||
// was made, so we don't need to add any inflight, but quota must be evaluated.
|
|
||||||
if (!checkQuota) {
|
|
||||||
return next(null, bucket, objMD);
|
|
||||||
}
|
|
||||||
const contentLength = processBytesToWrite(request.apiMethod, bucket, versionId,
|
|
||||||
request?.parsedContentLength || 0, objMD, params.destObjMD);
|
|
||||||
return validateQuotas(request, bucket, request.accountQuotas, requestType, request.apiMethod,
|
|
||||||
contentLength, withVersionId, log, err => next(err, bucket, objMD));
|
|
||||||
},
|
|
||||||
], (err, bucket, objMD) => {
|
], (err, bucket, objMD) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
// still return bucket for cors headers
|
// still return bucket for cors headers
|
||||||
|
@ -294,7 +279,6 @@ module.exports = {
|
||||||
validateBucket,
|
validateBucket,
|
||||||
metadataGetObject,
|
metadataGetObject,
|
||||||
metadataGetObjects,
|
metadataGetObjects,
|
||||||
processBytesToWrite,
|
|
||||||
standardMetadataValidateBucketAndObj,
|
standardMetadataValidateBucketAndObj,
|
||||||
standardMetadataValidateBucket,
|
standardMetadataValidateBucket,
|
||||||
};
|
};
|
||||||
|
|
|
@ -2,9 +2,9 @@ const MetadataWrapper = require('arsenal').storage.metadata.MetadataWrapper;
|
||||||
const { config } = require('../Config');
|
const { config } = require('../Config');
|
||||||
const logger = require('../utilities/logger');
|
const logger = require('../utilities/logger');
|
||||||
const constants = require('../../constants');
|
const constants = require('../../constants');
|
||||||
|
const bucketclient = require('bucketclient');
|
||||||
|
|
||||||
const clientName = config.backends.metadata;
|
const clientName = config.backends.metadata;
|
||||||
let bucketclient;
|
|
||||||
let params;
|
let params;
|
||||||
if (clientName === 'mem') {
|
if (clientName === 'mem') {
|
||||||
params = {};
|
params = {};
|
||||||
|
@ -21,7 +21,6 @@ if (clientName === 'mem') {
|
||||||
noDbOpen: null,
|
noDbOpen: null,
|
||||||
};
|
};
|
||||||
} else if (clientName === 'scality') {
|
} else if (clientName === 'scality') {
|
||||||
bucketclient = require('bucketclient');
|
|
||||||
params = {
|
params = {
|
||||||
bucketdBootstrap: config.bucketd.bootstrap,
|
bucketdBootstrap: config.bucketd.bootstrap,
|
||||||
bucketdLog: config.bucketd.log,
|
bucketdLog: config.bucketd.log,
|
||||||
|
|
|
@ -1,17 +0,0 @@
|
||||||
const { config } = require('../Config');
|
|
||||||
const { ScubaClientImpl } = require('./scuba/wrapper');
|
|
||||||
|
|
||||||
let instance = null;
|
|
||||||
|
|
||||||
switch (config.backends.quota) {
|
|
||||||
case 'scuba':
|
|
||||||
instance = new ScubaClientImpl(config);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
instance = {
|
|
||||||
enabled: false,
|
|
||||||
};
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = instance;
|
|
|
@ -1,80 +0,0 @@
|
||||||
const util = require('util');
|
|
||||||
const { default: ScubaClient } = require('scubaclient');
|
|
||||||
const { externalBackendHealthCheckInterval } = require('../../../constants');
|
|
||||||
const monitoring = require('../../utilities/monitoringHandler');
|
|
||||||
|
|
||||||
class ScubaClientImpl extends ScubaClient {
|
|
||||||
constructor(config) {
|
|
||||||
super(config.scuba);
|
|
||||||
this.enabled = false;
|
|
||||||
this.maxStaleness = config.quota.maxStaleness;
|
|
||||||
this._healthCheckTimer = null;
|
|
||||||
this._log = null;
|
|
||||||
this._getLatestMetricsCallback = util.callbackify(this.getLatestMetrics);
|
|
||||||
|
|
||||||
if (config.scuba) {
|
|
||||||
this.enabled = true;
|
|
||||||
} else {
|
|
||||||
this.enabled = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
setup(log) {
|
|
||||||
this._log = log;
|
|
||||||
if (this.enabled) {
|
|
||||||
this.periodicHealthCheck();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_healthCheck() {
|
|
||||||
return this.healthCheck().then(data => {
|
|
||||||
if (data?.date) {
|
|
||||||
const date = new Date(data.date);
|
|
||||||
if (Date.now() - date.getTime() > this.maxStaleness) {
|
|
||||||
throw new Error('Data is stale, disabling quotas');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!this.enabled) {
|
|
||||||
this._log.info('Scuba health check passed, enabling quotas');
|
|
||||||
}
|
|
||||||
monitoring.utilizationServiceAvailable.set(1);
|
|
||||||
this.enabled = true;
|
|
||||||
}).catch(err => {
|
|
||||||
if (this.enabled) {
|
|
||||||
this._log.warn('Scuba health check failed, disabling quotas', {
|
|
||||||
err: err.name,
|
|
||||||
description: err.message,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
monitoring.utilizationServiceAvailable.set(0);
|
|
||||||
this.enabled = false;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
periodicHealthCheck() {
|
|
||||||
if (this._healthCheckTimer) {
|
|
||||||
clearInterval(this._healthCheckTimer);
|
|
||||||
}
|
|
||||||
this._healthCheck();
|
|
||||||
this._healthCheckTimer = setInterval(async () => {
|
|
||||||
this._healthCheck();
|
|
||||||
}, Number(process.env.SCUBA_HEALTHCHECK_FREQUENCY)
|
|
||||||
|| externalBackendHealthCheckInterval);
|
|
||||||
}
|
|
||||||
|
|
||||||
getUtilizationMetrics(metricsClass, resourceName, options, body, callback) {
|
|
||||||
const requestStartTime = process.hrtime.bigint();
|
|
||||||
return this._getLatestMetricsCallback(metricsClass, resourceName, options, body, (err, data) => {
|
|
||||||
const responseTimeInNs = Number(process.hrtime.bigint() - requestStartTime);
|
|
||||||
monitoring.utilizationMetricsRetrievalDuration.labels({
|
|
||||||
code: err ? (err.statusCode || 500) : 200,
|
|
||||||
class: metricsClass,
|
|
||||||
}).observe(responseTimeInNs / 1e9);
|
|
||||||
return callback(err, data);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
ScubaClientImpl,
|
|
||||||
};
|
|
|
@ -52,7 +52,6 @@ const NAMESPACE = 'default';
|
||||||
const CIPHER = null; // replication/lifecycle does not work on encrypted objects
|
const CIPHER = null; // replication/lifecycle does not work on encrypted objects
|
||||||
|
|
||||||
let { locationConstraints } = config;
|
let { locationConstraints } = config;
|
||||||
const { nullVersionCompatMode } = config;
|
|
||||||
const { implName } = dataWrapper;
|
const { implName } = dataWrapper;
|
||||||
let dataClient = dataWrapper.client;
|
let dataClient = dataWrapper.client;
|
||||||
config.on('location-constraints-update', () => {
|
config.on('location-constraints-update', () => {
|
||||||
|
@ -505,7 +504,9 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (versionId === 'null') {
|
if (versionId === 'null') {
|
||||||
isNull = true;
|
if (!config.nullVersionCompatMode) {
|
||||||
|
isNull = true;
|
||||||
|
}
|
||||||
// Retrieve the null version id from the object metadata.
|
// Retrieve the null version id from the object metadata.
|
||||||
versionId = objMd && objMd.versionId;
|
versionId = objMd && objMd.versionId;
|
||||||
if (!versionId) {
|
if (!versionId) {
|
||||||
|
@ -514,16 +515,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
// the flag is needed to allow cloudserver to know that the version
|
// the flag is needed to allow cloudserver to know that the version
|
||||||
// is a null version and allow access to it using the "null" versionId.
|
// is a null version and allow access to it using the "null" versionId.
|
||||||
omVal.isNull = true;
|
omVal.isNull = true;
|
||||||
// If the new null keys logic (S3C-7352) is supported (not compatibility mode),
|
|
||||||
// create a null key with the isNull2 flag.
|
|
||||||
if (!nullVersionCompatMode) {
|
|
||||||
omVal.isNull2 = true;
|
|
||||||
}
|
|
||||||
// Delete the version id from the version metadata payload to prevent issues
|
|
||||||
// with creating a non-version object (versioning set to false) that includes a version id.
|
|
||||||
// For example, this version ID might come from a null version of a suspended bucket being
|
|
||||||
// replicated to this bucket.
|
|
||||||
delete omVal.versionId;
|
|
||||||
if (versioning) {
|
if (versioning) {
|
||||||
// If the null version does not have a version id, it is a current null version.
|
// If the null version does not have a version id, it is a current null version.
|
||||||
// To update the metadata of a current version, versioning is set to false.
|
// To update the metadata of a current version, versioning is set to false.
|
||||||
|
@ -558,6 +549,8 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const options = {
|
const options = {
|
||||||
|
versionId,
|
||||||
|
isNull,
|
||||||
overheadField: constants.overheadField,
|
overheadField: constants.overheadField,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -570,19 +563,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
options.versioning = true;
|
options.versioning = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// NOTE: When options fields are sent to Metadata through the query string,
|
|
||||||
// they are converted to strings. As a result, Metadata interprets the value undefined
|
|
||||||
// in the versionId field as an empty string ('').
|
|
||||||
// To prevent this, the versionId field is only included in options when it is defined.
|
|
||||||
if (versionId !== undefined) {
|
|
||||||
options.versionId = versionId;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the new null keys logic (S3C-7352) is not supported (compatibility mode), 'isNull' remains undefined.
|
|
||||||
if (!nullVersionCompatMode) {
|
|
||||||
options.isNull = isNull;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.trace('putting object version', {
|
log.trace('putting object version', {
|
||||||
objectKey: request.objectKey, omVal, options });
|
objectKey: request.objectKey, omVal, options });
|
||||||
return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,
|
return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,
|
||||||
|
@ -1501,7 +1481,7 @@ function routeBackbeat(clientIP, request, response, log) {
|
||||||
objectKey: request.objectKey,
|
objectKey: request.objectKey,
|
||||||
authInfo: userInfo,
|
authInfo: userInfo,
|
||||||
versionId,
|
versionId,
|
||||||
requestType: request.apiMethods || 'ReplicateObject',
|
requestType: 'ReplicateObject',
|
||||||
request,
|
request,
|
||||||
};
|
};
|
||||||
return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, next);
|
return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, next);
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue