Compare commits
17 Commits
developmen
...
hotfix/7.7
Author | SHA1 | Date |
---|---|---|
Mickael Bourgois | 3d7225b5d2 | |
Mickael Bourgois | 907e9eebee | |
Mickael Bourgois | 7f09ee5236 | |
Jonathan Gramain | 3f8dcf9747 | |
Jonathan Gramain | 608492154c | |
Nicolas Humbert | 6bb2fa59cd | |
Nicolas Humbert | 7ae4876cba | |
Nicolas Humbert | 606e9b68be | |
Nicolas Humbert | f158c635ab | |
Nicolas Humbert | 961bfd925b | |
Francois Ferrand | e3c5b99d3c | |
Francois Ferrand | 4cbe3adfe0 | |
Francois Ferrand | 778c29ec89 | |
Francois Ferrand | 98531b5454 | |
Taylor McKinnon | 10c0f78a87 | |
Taylor McKinnon | f410e2dce6 | |
Taylor McKinnon | 6b7f8cd892 |
|
@ -16,7 +16,7 @@ runs:
|
||||||
run: |-
|
run: |-
|
||||||
set -exu;
|
set -exu;
|
||||||
mkdir -p /tmp/artifacts/${{ github.job }}/;
|
mkdir -p /tmp/artifacts/${{ github.job }}/;
|
||||||
- uses: actions/setup-node@v2
|
- uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '16'
|
node-version: '16'
|
||||||
cache: 'yarn'
|
cache: 'yarn'
|
||||||
|
|
|
@ -62,6 +62,6 @@ services:
|
||||||
pykmip:
|
pykmip:
|
||||||
network_mode: "host"
|
network_mode: "host"
|
||||||
profiles: ['pykmip']
|
profiles: ['pykmip']
|
||||||
image: registry.scality.com/cloudserver-dev/pykmip
|
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
|
||||||
volumes:
|
volumes:
|
||||||
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
||||||
|
|
|
@ -14,12 +14,12 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Initialize CodeQL
|
- name: Initialize CodeQL
|
||||||
uses: github/codeql-action/init@v2
|
uses: github/codeql-action/init@v3
|
||||||
with:
|
with:
|
||||||
languages: javascript, python, ruby
|
languages: javascript, python, ruby
|
||||||
|
|
||||||
- name: Build and analyze
|
- name: Build and analyze
|
||||||
uses: github/codeql-action/analyze@v2
|
uses: github/codeql-action/analyze@v3
|
||||||
|
|
|
@ -10,7 +10,7 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: 'Checkout Repository'
|
- name: 'Checkout Repository'
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: 'Dependency Review'
|
- name: 'Dependency Review'
|
||||||
uses: actions/dependency-review-action@v3
|
uses: actions/dependency-review-action@v4
|
||||||
|
|
|
@ -11,36 +11,59 @@ on:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-federation-image:
|
build-federation-image:
|
||||||
uses: scality/workflows/.github/workflows/docker-build.yaml@v1
|
runs-on: ubuntu-20.04
|
||||||
secrets: inherit
|
steps:
|
||||||
with:
|
- name: Checkout
|
||||||
push: true
|
uses: actions/checkout@v4
|
||||||
registry: registry.scality.com
|
- name: Set up Docker Buildx
|
||||||
namespace: ${{ github.event.repository.name }}
|
uses: docker/setup-buildx-action@v3
|
||||||
name: ${{ github.event.repository.name }}
|
- name: Login to GitHub Registry
|
||||||
context: .
|
uses: docker/login-action@v3
|
||||||
file: images/svc-base/Dockerfile
|
with:
|
||||||
tag: ${{ github.event.inputs.tag }}-svc-base
|
registry: ghcr.io
|
||||||
|
username: ${{ github.repository_owner }}
|
||||||
|
password: ${{ github.token }}
|
||||||
|
- name: Build and push image for federation
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
push: true
|
||||||
|
context: .
|
||||||
|
file: images/svc-base/Dockerfile
|
||||||
|
tags: |
|
||||||
|
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
|
||||||
|
cache-from: type=gha,scope=federation
|
||||||
|
cache-to: type=gha,mode=max,scope=federation
|
||||||
|
|
||||||
build-image:
|
build-image:
|
||||||
uses: scality/workflows/.github/workflows/docker-build.yaml@v1
|
runs-on: ubuntu-20.04
|
||||||
secrets: inherit
|
steps:
|
||||||
with:
|
- name: Checkout
|
||||||
push: true
|
uses: actions/checkout@v4
|
||||||
registry: registry.scality.com
|
- name: Set up Docker Buildx
|
||||||
namespace: ${{ github.event.repository.name }}
|
uses: docker/setup-buildx-action@v3
|
||||||
name: ${{ github.event.repository.name }}
|
- name: Login to GitHub Registry
|
||||||
context: .
|
uses: docker/login-action@v3
|
||||||
file: Dockerfile
|
with:
|
||||||
tag: ${{ github.event.inputs.tag }}
|
registry: ghcr.io
|
||||||
|
username: ${{ github.repository_owner }}
|
||||||
|
password: ${{ github.token }}
|
||||||
|
- name: Build and push image
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
push: true
|
||||||
|
context: .
|
||||||
|
tags: |
|
||||||
|
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
|
||||||
|
cache-from: type=gha
|
||||||
|
cache-to: type=gha,mode=max
|
||||||
|
|
||||||
github-release:
|
github-release:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Create Release
|
- name: Create Release
|
||||||
uses: softprops/action-gh-release@v1
|
uses: softprops/action-gh-release@v2
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
with:
|
with:
|
||||||
name: Release ${{ github.event.inputs.tag }}
|
name: Release ${{ github.event.inputs.tag }}
|
||||||
tag_name: ${{ github.event.inputs.tag }}
|
tag_name: ${{ github.event.inputs.tag }}
|
||||||
|
|
|
@ -65,23 +65,24 @@ env:
|
||||||
ENABLE_LOCAL_CACHE: "true"
|
ENABLE_LOCAL_CACHE: "true"
|
||||||
REPORT_TOKEN: "report-token-1"
|
REPORT_TOKEN: "report-token-1"
|
||||||
REMOTE_MANAGEMENT_DISABLE: "1"
|
REMOTE_MANAGEMENT_DISABLE: "1"
|
||||||
|
# https://github.com/git-lfs/git-lfs/issues/5749
|
||||||
|
GIT_CLONE_PROTECTION_ACTIVE: 'false'
|
||||||
jobs:
|
jobs:
|
||||||
linting-coverage:
|
linting-coverage:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v4
|
||||||
- uses: actions/setup-node@v2
|
- uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '16'
|
node-version: '16'
|
||||||
cache: yarn
|
cache: yarn
|
||||||
- name: install dependencies
|
- name: install dependencies
|
||||||
run: yarn install --frozen-lockfile --network-concurrency 1
|
run: yarn install --frozen-lockfile --network-concurrency 1
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.9'
|
python-version: '3.9'
|
||||||
- uses: actions/cache@v2
|
- uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.cache/pip
|
path: ~/.cache/pip
|
||||||
key: ${{ runner.os }}-pip
|
key: ${{ runner.os }}-pip
|
||||||
|
@ -114,7 +115,7 @@ jobs:
|
||||||
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
|
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
|
||||||
if: always()
|
if: always()
|
||||||
- name: Upload files to artifacts
|
- name: Upload files to artifacts
|
||||||
uses: scality/action-artifacts@v2
|
uses: scality/action-artifacts@v4
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -127,58 +128,72 @@ jobs:
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v4
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v1.6.0
|
uses: docker/setup-buildx-action@v3
|
||||||
- name: Login to GitHub Registry
|
- name: Login to GitHub Registry
|
||||||
uses: docker/login-action@v1.10.0
|
uses: docker/login-action@v3
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ github.repository_owner }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ github.token }}
|
||||||
- name: Login to Registry
|
|
||||||
uses: docker/login-action@v1
|
|
||||||
with:
|
|
||||||
registry: registry.scality.com
|
|
||||||
username: ${{ secrets.REGISTRY_LOGIN }}
|
|
||||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
|
||||||
- name: Build and push cloudserver image
|
- name: Build and push cloudserver image
|
||||||
uses: docker/build-push-action@v3
|
uses: docker/build-push-action@v5
|
||||||
with:
|
with:
|
||||||
push: true
|
push: true
|
||||||
context: .
|
context: .
|
||||||
provenance: false
|
provenance: false
|
||||||
tags: |
|
tags: |
|
||||||
ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||||
registry.scality.com/cloudserver-dev/cloudserver:${{ github.sha }}
|
|
||||||
cache-from: type=gha,scope=cloudserver
|
cache-from: type=gha,scope=cloudserver
|
||||||
cache-to: type=gha,mode=max,scope=cloudserver
|
cache-to: type=gha,mode=max,scope=cloudserver
|
||||||
|
- name: Build and push pykmip image
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
push: true
|
||||||
|
context: .github/pykmip
|
||||||
|
tags: |
|
||||||
|
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
||||||
|
cache-from: type=gha,scope=pykmip
|
||||||
|
cache-to: type=gha,mode=max,scope=pykmip
|
||||||
|
|
||||||
build-federation-image:
|
build-federation-image:
|
||||||
uses: scality/workflows/.github/workflows/docker-build.yaml@v1
|
runs-on: ubuntu-20.04
|
||||||
secrets: inherit
|
steps:
|
||||||
with:
|
- name: Checkout
|
||||||
push: true
|
uses: actions/checkout@v4
|
||||||
registry: registry.scality.com
|
- name: Set up Docker Buildx
|
||||||
namespace: cloudserver-dev
|
uses: docker/setup-buildx-action@v3
|
||||||
name: cloudserver
|
- name: Login to GitHub Registry
|
||||||
context: .
|
uses: docker/login-action@v3
|
||||||
file: images/svc-base/Dockerfile
|
with:
|
||||||
tag: ${{ github.sha }}-svc-base
|
registry: ghcr.io
|
||||||
|
username: ${{ github.repository_owner }}
|
||||||
|
password: ${{ github.token }}
|
||||||
|
- name: Build and push image for federation
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
push: true
|
||||||
|
context: .
|
||||||
|
file: images/svc-base/Dockerfile
|
||||||
|
tags: |
|
||||||
|
ghcr.io/${{ github.repository }}:${{ github.sha }}-svc-base
|
||||||
|
cache-from: type=gha,scope=federation
|
||||||
|
cache-to: type=gha,mode=max,scope=federation
|
||||||
|
|
||||||
multiple-backend:
|
multiple-backend:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
env:
|
env:
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||||
S3BACKEND: mem
|
S3BACKEND: mem
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||||
S3DATA: multiple
|
S3DATA: multiple
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: 3.9
|
python-version: 3.9
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
|
@ -194,7 +209,7 @@ jobs:
|
||||||
env:
|
env:
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v3
|
uses: scality/action-artifacts@v4
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -217,14 +232,14 @@ jobs:
|
||||||
env:
|
env:
|
||||||
S3BACKEND: file
|
S3BACKEND: file
|
||||||
S3VAULT: mem
|
S3VAULT: mem
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||||
MPU_TESTING: "yes"
|
MPU_TESTING: "yes"
|
||||||
ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
|
ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
|
||||||
JOB_NAME: ${{ matrix.job-name }}
|
JOB_NAME: ${{ matrix.job-name }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: 3.9
|
python-version: 3.9
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
|
@ -247,7 +262,7 @@ jobs:
|
||||||
bash wait_for_local_port.bash 8000 40
|
bash wait_for_local_port.bash 8000 40
|
||||||
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
|
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v3
|
uses: scality/action-artifacts@v4
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -261,14 +276,14 @@ jobs:
|
||||||
needs: build
|
needs: build
|
||||||
env:
|
env:
|
||||||
ENABLE_UTAPI_V2: t
|
ENABLE_UTAPI_V2: t
|
||||||
S3BACKEND: mem
|
S3BACKEND: mem
|
||||||
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: 3.9
|
python-version: 3.9
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
|
@ -282,7 +297,7 @@ jobs:
|
||||||
bash wait_for_local_port.bash 8000 40
|
bash wait_for_local_port.bash 8000 40
|
||||||
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v3
|
uses: scality/action-artifacts@v4
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
@ -298,12 +313,13 @@ jobs:
|
||||||
S3BACKEND: file
|
S3BACKEND: file
|
||||||
S3VAULT: mem
|
S3VAULT: mem
|
||||||
MPU_TESTING: true
|
MPU_TESTING: true
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||||
|
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: 3.9
|
python-version: 3.9
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
|
@ -321,7 +337,7 @@ jobs:
|
||||||
bash wait_for_local_port.bash 5696 40
|
bash wait_for_local_port.bash 5696 40
|
||||||
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v3
|
uses: scality/action-artifacts@v4
|
||||||
with:
|
with:
|
||||||
method: upload
|
method: upload
|
||||||
url: https://artifacts.scality.net
|
url: https://artifacts.scality.net
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
FROM registry.scality.com/federation/nodesvc-base:7.10.6.0
|
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
|
||||||
|
|
||||||
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
|
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
|
||||||
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
|
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
|
||||||
|
|
12
index.js
12
index.js
|
@ -1,10 +1,10 @@
|
||||||
'use strict'; // eslint-disable-line strict
|
'use strict'; // eslint-disable-line strict
|
||||||
|
|
||||||
/**
|
require('werelogs').stderrUtils.catchAndTimestampStderr(
|
||||||
* Catch uncaught exceptions and add timestamp to aid debugging
|
undefined,
|
||||||
*/
|
// Do not exit as workers have their own listener that will exit
|
||||||
process.on('uncaughtException', err => {
|
// But primary don't have another listener
|
||||||
process.stderr.write(`${new Date().toISOString()}: Uncaught exception: \n${err.stack}`);
|
require('cluster').isPrimary ? 1 : null,
|
||||||
});
|
);
|
||||||
|
|
||||||
require('./lib/server.js')();
|
require('./lib/server.js')();
|
||||||
|
|
186
lib/Config.js
186
lib/Config.js
|
@ -88,6 +88,47 @@ function parseSproxydConfig(configSproxyd) {
|
||||||
return joi.attempt(configSproxyd, joiSchema, 'bad config');
|
return joi.attempt(configSproxyd, joiSchema, 'bad config');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseRedisConfig(redisConfig) {
|
||||||
|
const joiSchema = joi.object({
|
||||||
|
password: joi.string().allow(''),
|
||||||
|
host: joi.string(),
|
||||||
|
port: joi.number(),
|
||||||
|
retry: joi.object({
|
||||||
|
connectBackoff: joi.object({
|
||||||
|
min: joi.number().required(),
|
||||||
|
max: joi.number().required(),
|
||||||
|
jitter: joi.number().required(),
|
||||||
|
factor: joi.number().required(),
|
||||||
|
deadline: joi.number().required(),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
// sentinel config
|
||||||
|
sentinels: joi.alternatives().try(
|
||||||
|
joi.string()
|
||||||
|
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
|
||||||
|
.custom(hosts => hosts.split(',').map(item => {
|
||||||
|
const [host, port] = item.split(':');
|
||||||
|
return { host, port: Number.parseInt(port, 10) };
|
||||||
|
})),
|
||||||
|
joi.array().items(
|
||||||
|
joi.object({
|
||||||
|
host: joi.string().required(),
|
||||||
|
port: joi.number().required(),
|
||||||
|
})
|
||||||
|
).min(1),
|
||||||
|
),
|
||||||
|
name: joi.string(),
|
||||||
|
sentinelPassword: joi.string().allow(''),
|
||||||
|
})
|
||||||
|
.and('host', 'port')
|
||||||
|
.and('sentinels', 'name')
|
||||||
|
.xor('host', 'sentinels')
|
||||||
|
.without('sentinels', ['host', 'port'])
|
||||||
|
.without('host', ['sentinels', 'sentinelPassword']);
|
||||||
|
|
||||||
|
return joi.attempt(redisConfig, joiSchema, 'bad config');
|
||||||
|
}
|
||||||
|
|
||||||
function restEndpointsAssert(restEndpoints, locationConstraints) {
|
function restEndpointsAssert(restEndpoints, locationConstraints) {
|
||||||
assert(typeof restEndpoints === 'object',
|
assert(typeof restEndpoints === 'object',
|
||||||
'bad config: restEndpoints must be an object of endpoints');
|
'bad config: restEndpoints must be an object of endpoints');
|
||||||
|
@ -294,18 +335,17 @@ function parseUtapiReindex(config) {
|
||||||
const {
|
const {
|
||||||
enabled,
|
enabled,
|
||||||
schedule,
|
schedule,
|
||||||
sentinel,
|
redis,
|
||||||
bucketd,
|
bucketd,
|
||||||
onlyCountLatestWhenObjectLocked,
|
onlyCountLatestWhenObjectLocked,
|
||||||
} = config;
|
} = config;
|
||||||
assert(typeof enabled === 'boolean',
|
assert(typeof enabled === 'boolean',
|
||||||
'bad config: utapi.reindex.enabled must be a boolean');
|
'bad config: utapi.reindex.enabled must be a boolean');
|
||||||
assert(typeof sentinel === 'object',
|
|
||||||
'bad config: utapi.reindex.sentinel must be an object');
|
const parsedRedis = parseRedisConfig(redis);
|
||||||
assert(typeof sentinel.port === 'number',
|
assert(Array.isArray(parsedRedis.sentinels),
|
||||||
'bad config: utapi.reindex.sentinel.port must be a number');
|
'bad config: utapi reindex redis config requires a list of sentinels');
|
||||||
assert(typeof sentinel.name === 'string',
|
|
||||||
'bad config: utapi.reindex.sentinel.name must be a string');
|
|
||||||
assert(typeof bucketd === 'object',
|
assert(typeof bucketd === 'object',
|
||||||
'bad config: utapi.reindex.bucketd must be an object');
|
'bad config: utapi.reindex.bucketd must be an object');
|
||||||
assert(typeof bucketd.port === 'number',
|
assert(typeof bucketd.port === 'number',
|
||||||
|
@ -323,6 +363,13 @@ function parseUtapiReindex(config) {
|
||||||
'bad config: utapi.reindex.schedule must be a valid ' +
|
'bad config: utapi.reindex.schedule must be a valid ' +
|
||||||
`cron schedule. ${e.message}.`);
|
`cron schedule. ${e.message}.`);
|
||||||
}
|
}
|
||||||
|
return {
|
||||||
|
enabled,
|
||||||
|
schedule,
|
||||||
|
redis: parsedRedis,
|
||||||
|
bucketd,
|
||||||
|
onlyCountLatestWhenObjectLocked,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function requestsConfigAssert(requestsConfig) {
|
function requestsConfigAssert(requestsConfig) {
|
||||||
|
@ -761,8 +808,7 @@ class Config extends EventEmitter {
|
||||||
assert(typeof config.localCache.port === 'number',
|
assert(typeof config.localCache.port === 'number',
|
||||||
'config: invalid port for localCache. port must be a number');
|
'config: invalid port for localCache. port must be a number');
|
||||||
if (config.localCache.password !== undefined) {
|
if (config.localCache.password !== undefined) {
|
||||||
assert(
|
assert(typeof config.localCache.password === 'string',
|
||||||
this._verifyRedisPassword(config.localCache.password),
|
|
||||||
'config: invalid password for localCache. password must' +
|
'config: invalid password for localCache. password must' +
|
||||||
' be a string');
|
' be a string');
|
||||||
}
|
}
|
||||||
|
@ -774,55 +820,7 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.redis) {
|
if (config.redis) {
|
||||||
if (config.redis.sentinels) {
|
this.redis = parseRedisConfig(config.redis);
|
||||||
this.redis = { sentinels: [], name: null };
|
|
||||||
|
|
||||||
assert(typeof config.redis.name === 'string',
|
|
||||||
'bad config: redis sentinel name must be a string');
|
|
||||||
this.redis.name = config.redis.name;
|
|
||||||
assert(Array.isArray(config.redis.sentinels) ||
|
|
||||||
typeof config.redis.sentinels === 'string',
|
|
||||||
'bad config: redis sentinels must be an array or string');
|
|
||||||
|
|
||||||
if (typeof config.redis.sentinels === 'string') {
|
|
||||||
config.redis.sentinels.split(',').forEach(item => {
|
|
||||||
const [host, port] = item.split(':');
|
|
||||||
this.redis.sentinels.push({ host,
|
|
||||||
port: Number.parseInt(port, 10) });
|
|
||||||
});
|
|
||||||
} else if (Array.isArray(config.redis.sentinels)) {
|
|
||||||
config.redis.sentinels.forEach(item => {
|
|
||||||
const { host, port } = item;
|
|
||||||
assert(typeof host === 'string',
|
|
||||||
'bad config: redis sentinel host must be a string');
|
|
||||||
assert(typeof port === 'number',
|
|
||||||
'bad config: redis sentinel port must be a number');
|
|
||||||
this.redis.sentinels.push({ host, port });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.redis.sentinelPassword !== undefined) {
|
|
||||||
assert(
|
|
||||||
this._verifyRedisPassword(config.redis.sentinelPassword));
|
|
||||||
this.redis.sentinelPassword = config.redis.sentinelPassword;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// check for standalone configuration
|
|
||||||
this.redis = {};
|
|
||||||
assert(typeof config.redis.host === 'string',
|
|
||||||
'bad config: redis.host must be a string');
|
|
||||||
assert(typeof config.redis.port === 'number',
|
|
||||||
'bad config: redis.port must be a number');
|
|
||||||
this.redis.host = config.redis.host;
|
|
||||||
this.redis.port = config.redis.port;
|
|
||||||
}
|
|
||||||
if (config.redis.password !== undefined) {
|
|
||||||
assert(
|
|
||||||
this._verifyRedisPassword(config.redis.password),
|
|
||||||
'bad config: invalid password for redis. password must ' +
|
|
||||||
'be a string');
|
|
||||||
this.redis.password = config.redis.password;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (config.utapi) {
|
if (config.utapi) {
|
||||||
this.utapi = { component: 's3' };
|
this.utapi = { component: 's3' };
|
||||||
|
@ -851,65 +849,8 @@ class Config extends EventEmitter {
|
||||||
this.utapi.localCache = config.localCache;
|
this.utapi.localCache = config.localCache;
|
||||||
assert(config.utapi.redis, 'missing required property of utapi ' +
|
assert(config.utapi.redis, 'missing required property of utapi ' +
|
||||||
'configuration: redis');
|
'configuration: redis');
|
||||||
if (config.utapi.redis.sentinels) {
|
this.utapi.redis = parseRedisConfig(config.utapi.redis);
|
||||||
this.utapi.redis = { sentinels: [], name: null };
|
if (this.utapi.redis.retry === undefined) {
|
||||||
|
|
||||||
assert(typeof config.utapi.redis.name === 'string',
|
|
||||||
'bad config: redis sentinel name must be a string');
|
|
||||||
this.utapi.redis.name = config.utapi.redis.name;
|
|
||||||
|
|
||||||
assert(Array.isArray(config.utapi.redis.sentinels),
|
|
||||||
'bad config: redis sentinels must be an array');
|
|
||||||
config.utapi.redis.sentinels.forEach(item => {
|
|
||||||
const { host, port } = item;
|
|
||||||
assert(typeof host === 'string',
|
|
||||||
'bad config: redis sentinel host must be a string');
|
|
||||||
assert(typeof port === 'number',
|
|
||||||
'bad config: redis sentinel port must be a number');
|
|
||||||
this.utapi.redis.sentinels.push({ host, port });
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
// check for standalone configuration
|
|
||||||
this.utapi.redis = {};
|
|
||||||
assert(typeof config.utapi.redis.host === 'string',
|
|
||||||
'bad config: redis.host must be a string');
|
|
||||||
assert(typeof config.utapi.redis.port === 'number',
|
|
||||||
'bad config: redis.port must be a number');
|
|
||||||
this.utapi.redis.host = config.utapi.redis.host;
|
|
||||||
this.utapi.redis.port = config.utapi.redis.port;
|
|
||||||
}
|
|
||||||
if (config.utapi.redis.password !== undefined) {
|
|
||||||
assert(
|
|
||||||
this._verifyRedisPassword(config.utapi.redis.password),
|
|
||||||
'config: invalid password for utapi redis. password' +
|
|
||||||
' must be a string');
|
|
||||||
this.utapi.redis.password = config.utapi.redis.password;
|
|
||||||
}
|
|
||||||
if (config.utapi.redis.sentinelPassword !== undefined) {
|
|
||||||
assert(
|
|
||||||
this._verifyRedisPassword(config.utapi.redis.sentinelPassword),
|
|
||||||
'config: invalid password for utapi redis. password' +
|
|
||||||
' must be a string');
|
|
||||||
this.utapi.redis.sentinelPassword =
|
|
||||||
config.utapi.redis.sentinelPassword;
|
|
||||||
}
|
|
||||||
if (config.utapi.redis.retry !== undefined) {
|
|
||||||
if (config.utapi.redis.retry.connectBackoff !== undefined) {
|
|
||||||
const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
|
|
||||||
assert.strictEqual(typeof min, 'number',
|
|
||||||
'utapi.redis.retry.connectBackoff: min must be a number');
|
|
||||||
assert.strictEqual(typeof max, 'number',
|
|
||||||
'utapi.redis.retry.connectBackoff: max must be a number');
|
|
||||||
assert.strictEqual(typeof jitter, 'number',
|
|
||||||
'utapi.redis.retry.connectBackoff: jitter must be a number');
|
|
||||||
assert.strictEqual(typeof factor, 'number',
|
|
||||||
'utapi.redis.retry.connectBackoff: factor must be a number');
|
|
||||||
assert.strictEqual(typeof deadline, 'number',
|
|
||||||
'utapi.redis.retry.connectBackoff: deadline must be a number');
|
|
||||||
}
|
|
||||||
|
|
||||||
this.utapi.redis.retry = config.utapi.redis.retry;
|
|
||||||
} else {
|
|
||||||
this.utapi.redis.retry = {
|
this.utapi.redis.retry = {
|
||||||
connectBackoff: {
|
connectBackoff: {
|
||||||
min: 10,
|
min: 10,
|
||||||
|
@ -920,6 +861,7 @@ class Config extends EventEmitter {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.utapi.metrics) {
|
if (config.utapi.metrics) {
|
||||||
this.utapi.metrics = config.utapi.metrics;
|
this.utapi.metrics = config.utapi.metrics;
|
||||||
}
|
}
|
||||||
|
@ -988,8 +930,7 @@ class Config extends EventEmitter {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.utapi && config.utapi.reindex) {
|
if (config.utapi && config.utapi.reindex) {
|
||||||
parseUtapiReindex(config.utapi.reindex);
|
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
|
||||||
this.utapi.reindex = config.utapi.reindex;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1430,10 +1371,6 @@ class Config extends EventEmitter {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
_verifyRedisPassword(password) {
|
|
||||||
return typeof password === 'string';
|
|
||||||
}
|
|
||||||
|
|
||||||
setAuthDataAccounts(accounts) {
|
setAuthDataAccounts(accounts) {
|
||||||
this.authData.accounts = accounts;
|
this.authData.accounts = accounts;
|
||||||
this.emit('authdata-update');
|
this.emit('authdata-update');
|
||||||
|
@ -1547,6 +1484,7 @@ class Config extends EventEmitter {
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
parseSproxydConfig,
|
parseSproxydConfig,
|
||||||
|
parseRedisConfig,
|
||||||
locationConstraintAssert,
|
locationConstraintAssert,
|
||||||
ConfigObject: Config,
|
ConfigObject: Config,
|
||||||
config: new Config(),
|
config: new Config(),
|
||||||
|
|
|
@ -193,7 +193,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
|
||||||
// null keys are used, which is used as an optimization to
|
// null keys are used, which is used as an optimization to
|
||||||
// avoid having to check the versioned key since there can
|
// avoid having to check the versioned key since there can
|
||||||
// be no more versioned key to clean up
|
// be no more versioned key to clean up
|
||||||
if (mst.isNull && !mst.isNull2) {
|
if (mst.isNull && mst.versionId && !mst.isNull2) {
|
||||||
const delOptions = { versionId: mst.versionId };
|
const delOptions = { versionId: mst.versionId };
|
||||||
return { options, delOptions };
|
return { options, delOptions };
|
||||||
}
|
}
|
||||||
|
@ -224,7 +224,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
|
||||||
if (masterIsNull) {
|
if (masterIsNull) {
|
||||||
// if master is a null version or a non-versioned key,
|
// if master is a null version or a non-versioned key,
|
||||||
// copy it to a new null key
|
// copy it to a new null key
|
||||||
const nullVersionId = mst.isNull ? mst.versionId : nonVersionedObjId;
|
const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
|
||||||
if (nullVersionCompatMode) {
|
if (nullVersionCompatMode) {
|
||||||
options.extraMD = {
|
options.extraMD = {
|
||||||
nullVersionId,
|
nullVersionId,
|
||||||
|
|
|
@ -49,6 +49,7 @@ const NAMESPACE = 'default';
|
||||||
const CIPHER = null; // replication/lifecycle does not work on encrypted objects
|
const CIPHER = null; // replication/lifecycle does not work on encrypted objects
|
||||||
|
|
||||||
let { locationConstraints } = config;
|
let { locationConstraints } = config;
|
||||||
|
const { nullVersionCompatMode } = config;
|
||||||
const { implName } = dataWrapper;
|
const { implName } = dataWrapper;
|
||||||
let dataClient = dataWrapper.client;
|
let dataClient = dataWrapper.client;
|
||||||
config.on('location-constraints-update', () => {
|
config.on('location-constraints-update', () => {
|
||||||
|
@ -492,9 +493,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
let isNull = false;
|
let isNull = false;
|
||||||
|
|
||||||
if (versionId === 'null') {
|
if (versionId === 'null') {
|
||||||
if (!config.nullVersionCompatMode) {
|
isNull = true;
|
||||||
isNull = true;
|
|
||||||
}
|
|
||||||
// Retrieve the null version id from the object metadata.
|
// Retrieve the null version id from the object metadata.
|
||||||
versionId = objMd && objMd.versionId;
|
versionId = objMd && objMd.versionId;
|
||||||
if (!versionId) {
|
if (!versionId) {
|
||||||
|
@ -503,6 +502,16 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
// the flag is needed to allow cloudserver to know that the version
|
// the flag is needed to allow cloudserver to know that the version
|
||||||
// is a null version and allow access to it using the "null" versionId.
|
// is a null version and allow access to it using the "null" versionId.
|
||||||
omVal.isNull = true;
|
omVal.isNull = true;
|
||||||
|
// If the new null keys logic (S3C-7352) is supported (not compatibility mode),
|
||||||
|
// create a null key with the isNull2 flag.
|
||||||
|
if (!nullVersionCompatMode) {
|
||||||
|
omVal.isNull2 = true;
|
||||||
|
}
|
||||||
|
// Delete the version id from the version metadata payload to prevent issues
|
||||||
|
// with creating a non-version object (versioning set to false) that includes a version id.
|
||||||
|
// For example, this version ID might come from a null version of a suspended bucket being
|
||||||
|
// replicated to this bucket.
|
||||||
|
delete omVal.versionId;
|
||||||
if (versioning) {
|
if (versioning) {
|
||||||
// If the null version does not have a version id, it is a current null version.
|
// If the null version does not have a version id, it is a current null version.
|
||||||
// To update the metadata of a current version, versioning is set to false.
|
// To update the metadata of a current version, versioning is set to false.
|
||||||
|
@ -537,7 +546,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const options = {
|
const options = {
|
||||||
isNull,
|
|
||||||
overheadField: constants.overheadField,
|
overheadField: constants.overheadField,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -558,6 +566,11 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
||||||
options.versionId = versionId;
|
options.versionId = versionId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If the new null keys logic (S3C-7352) is not supported (compatibility mode), 'isNull' remains undefined.
|
||||||
|
if (!nullVersionCompatMode) {
|
||||||
|
options.isNull = isNull;
|
||||||
|
}
|
||||||
|
|
||||||
log.trace('putting object version', {
|
log.trace('putting object version', {
|
||||||
objectKey: request.objectKey, omVal, options });
|
objectKey: request.objectKey, omVal, options });
|
||||||
return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,
|
return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
require('werelogs').stderrUtils.catchAndTimestampStderr();
|
||||||
const _config = require('../Config').config;
|
const _config = require('../Config').config;
|
||||||
const { utapiVersion, UtapiServer: utapiServer } = require('utapi');
|
const { utapiVersion, UtapiServer: utapiServer } = require('utapi');
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
require('werelogs').stderrUtils.catchAndTimestampStderr();
|
||||||
const UtapiReindex = require('utapi').UtapiReindex;
|
const UtapiReindex = require('utapi').UtapiReindex;
|
||||||
const { config } = require('../Config');
|
const { config } = require('../Config');
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
require('werelogs').stderrUtils.catchAndTimestampStderr();
|
||||||
const UtapiReplay = require('utapi').UtapiReplay;
|
const UtapiReplay = require('utapi').UtapiReplay;
|
||||||
const _config = require('../Config').config;
|
const _config = require('../Config').config;
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "s3",
|
"name": "s3",
|
||||||
"version": "7.70.45",
|
"version": "7.70.45-4",
|
||||||
"description": "S3 connector",
|
"description": "S3 connector",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"engines": {
|
"engines": {
|
||||||
|
@ -20,7 +20,7 @@
|
||||||
"homepage": "https://github.com/scality/S3#readme",
|
"homepage": "https://github.com/scality/S3#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@hapi/joi": "^17.1.0",
|
"@hapi/joi": "^17.1.0",
|
||||||
"arsenal": "git+https://github.com/scality/arsenal#7.70.25",
|
"arsenal": "git+https://github.com/scality/arsenal#7.70.25-1",
|
||||||
"async": "~2.5.0",
|
"async": "~2.5.0",
|
||||||
"aws-sdk": "2.905.0",
|
"aws-sdk": "2.905.0",
|
||||||
"azure-storage": "^2.1.0",
|
"azure-storage": "^2.1.0",
|
||||||
|
@ -35,11 +35,11 @@
|
||||||
"moment": "^2.26.0",
|
"moment": "^2.26.0",
|
||||||
"npm-run-all": "~4.1.5",
|
"npm-run-all": "~4.1.5",
|
||||||
"prom-client": "14.2.0",
|
"prom-client": "14.2.0",
|
||||||
"utapi": "git+https://github.com/scality/utapi#7.70.3",
|
"utapi": "git+https://github.com/scality/utapi#7.70.5",
|
||||||
"utf8": "~2.1.1",
|
"utf8": "~2.1.1",
|
||||||
"uuid": "^3.0.1",
|
"uuid": "^3.0.1",
|
||||||
"vaultclient": "scality/vaultclient#7.10.13",
|
"vaultclient": "scality/vaultclient#7.10.13",
|
||||||
"werelogs": "scality/werelogs#8.1.0",
|
"werelogs": "scality/werelogs#8.1.0-1",
|
||||||
"xml2js": "~0.4.16"
|
"xml2js": "~0.4.16"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
|
@ -1,15 +1,15 @@
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const async = require('async');
|
const async = require('async');
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const { models, versioning } = require('arsenal');
|
const { versioning } = require('arsenal');
|
||||||
const versionIdUtils = versioning.VersionID;
|
const versionIdUtils = versioning.VersionID;
|
||||||
const { ObjectMD } = models;
|
|
||||||
|
|
||||||
const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest');
|
const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest');
|
||||||
const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
|
const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
|
||||||
|
|
||||||
const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1';
|
const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1';
|
||||||
const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe;
|
const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe;
|
||||||
|
const isNullVersionCompatMode = process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true';
|
||||||
|
|
||||||
const backbeatAuthCredentials = {
|
const backbeatAuthCredentials = {
|
||||||
accessKey: 'accessKey1',
|
accessKey: 'accessKey1',
|
||||||
|
@ -87,17 +87,15 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateStorageClass(data, storageClass) {
|
function updateStorageClass(data, storageClass) {
|
||||||
let parsedBody;
|
let result;
|
||||||
try {
|
try {
|
||||||
parsedBody = JSON.parse(data.body);
|
const parsedBody = JSON.parse(JSON.parse(data.body).Body);
|
||||||
|
parsedBody['x-amz-storage-class'] = storageClass;
|
||||||
|
result = JSON.stringify(parsedBody);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return { error: err };
|
return { error: err };
|
||||||
}
|
}
|
||||||
const { result, error } = ObjectMD.createFromBlob(parsedBody.Body);
|
|
||||||
if (error) {
|
|
||||||
return { error };
|
|
||||||
}
|
|
||||||
result.setAmzStorageClass(storageClass);
|
|
||||||
return { result };
|
return { result };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -202,11 +200,11 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
|
|
||||||
it('should update metadata of a current null version', done => {
|
it('should update metadata of a current null version', done => {
|
||||||
let objMD;
|
let objMD;
|
||||||
return async.series([
|
return async.series({
|
||||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
putObject: next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
next),
|
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
next => makeBackbeatRequest({
|
getMetadata: next => makeBackbeatRequest({
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
resourceType: 'metadata',
|
resourceType: 'metadata',
|
||||||
bucket,
|
bucket,
|
||||||
|
@ -226,7 +224,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
objMD = result;
|
objMD = result;
|
||||||
return next();
|
return next();
|
||||||
}),
|
}),
|
||||||
next => makeBackbeatRequest({
|
putMetadata: next => makeBackbeatRequest({
|
||||||
method: 'PUT',
|
method: 'PUT',
|
||||||
resourceType: 'metadata',
|
resourceType: 'metadata',
|
||||||
bucket,
|
bucket,
|
||||||
|
@ -235,19 +233,40 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
headObject: next => s3.headObject(
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
{ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
], (err, data) => {
|
getMetadataAfter: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
}, (err, results) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return done(err);
|
return done(err);
|
||||||
}
|
}
|
||||||
const headObjectRes = data[4];
|
|
||||||
|
const headObjectRes = results.headObject;
|
||||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||||
|
|
||||||
const listObjectVersionsRes = data[5];
|
const getMetadataAfterRes = results.getMetadataAfter;
|
||||||
|
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||||
|
const expectedMd = JSON.parse(objMD);
|
||||||
|
expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed
|
||||||
|
if (!isNullVersionCompatMode) {
|
||||||
|
expectedMd.isNull2 = true; // TODO remove the line once CLDSRV-509 is fixed
|
||||||
|
}
|
||||||
|
assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd);
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
const { Versions } = listObjectVersionsRes;
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
assert.strictEqual(Versions.length, 1);
|
assert.strictEqual(Versions.length, 1);
|
||||||
|
@ -261,18 +280,20 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
it('should update metadata of a non-current null version', done => {
|
it('should update metadata of a non-current null version', done => {
|
||||||
let objMD;
|
let objMD;
|
||||||
let expectedVersionId;
|
let expectedVersionId;
|
||||||
return async.series([
|
return async.series({
|
||||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
putObjectInitial: next => s3.putObject(
|
||||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||||
next),
|
enableVersioning: next => s3.putBucketVersioning(
|
||||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
|
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectAgain: next => s3.putObject(
|
||||||
|
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return next(err);
|
return next(err);
|
||||||
}
|
}
|
||||||
expectedVersionId = data.VersionId;
|
expectedVersionId = data.VersionId;
|
||||||
return next();
|
return next();
|
||||||
}),
|
}),
|
||||||
next => makeBackbeatRequest({
|
getMetadata: next => makeBackbeatRequest({
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
resourceType: 'metadata',
|
resourceType: 'metadata',
|
||||||
bucket,
|
bucket,
|
||||||
|
@ -292,7 +313,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
objMD = result;
|
objMD = result;
|
||||||
return next();
|
return next();
|
||||||
}),
|
}),
|
||||||
next => makeBackbeatRequest({
|
putMetadata: next => makeBackbeatRequest({
|
||||||
method: 'PUT',
|
method: 'PUT',
|
||||||
resourceType: 'metadata',
|
resourceType: 'metadata',
|
||||||
bucket,
|
bucket,
|
||||||
|
@ -301,23 +322,36 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
getMetadataAfter: next => makeBackbeatRequest({
|
||||||
], (err, data) => {
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
}, (err, results) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return done(err);
|
return done(err);
|
||||||
}
|
}
|
||||||
const headObjectRes = data[5];
|
const headObjectRes = results.headObject;
|
||||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||||
|
|
||||||
const listObjectVersionsRes = data[6];
|
const getMetadataAfterRes = results.getMetadataAfter;
|
||||||
|
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||||
|
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
const { Versions } = listObjectVersionsRes;
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
assert.strictEqual(Versions.length, 2);
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
|
||||||
const currentVersion = Versions.find(v => v.IsLatest);
|
const currentVersion = Versions.find(v => v.IsLatest);
|
||||||
assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId);
|
assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId);
|
||||||
|
|
||||||
|
@ -327,6 +361,160 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should update metadata of a suspended null version', done => {
|
||||||
|
let objMD;
|
||||||
|
return async.series({
|
||||||
|
suspendVersioning: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||||
|
putObject: next => s3.putObject(
|
||||||
|
{ Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
enableVersioning: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
const { error, result } = updateStorageClass(data, storageClass);
|
||||||
|
if (error) {
|
||||||
|
return next(error);
|
||||||
|
}
|
||||||
|
objMD = result;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
putUpdatedMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
getMetadataAfter: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
const headObjectRes = results.headObject;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||||
|
|
||||||
|
const getMetadataAfterRes = results.getMetadataAfter;
|
||||||
|
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||||
|
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 1);
|
||||||
|
|
||||||
|
const [currentVersion] = Versions;
|
||||||
|
assertVersionIsNullAndUpdated(currentVersion);
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should update metadata of a suspended null version with internal version id', done => {
|
||||||
|
let objMD;
|
||||||
|
return async.series({
|
||||||
|
suspendVersioning: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||||
|
putObject: next => s3.putObject(
|
||||||
|
{ Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
enableVersioning: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectTagging: next => s3.putObjectTagging({
|
||||||
|
Bucket: bucket, Key: keyName, VersionId: 'null',
|
||||||
|
Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] },
|
||||||
|
}, next),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
const { error, result } = updateStorageClass(data, storageClass);
|
||||||
|
if (error) {
|
||||||
|
return next(error);
|
||||||
|
}
|
||||||
|
objMD = result;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
putUpdatedMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
getMetadataAfter: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
const headObjectRes = results.headObject;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||||
|
|
||||||
|
const getMetadataAfterRes = results.getMetadataAfter;
|
||||||
|
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||||
|
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 1);
|
||||||
|
|
||||||
|
const [currentVersion] = Versions;
|
||||||
|
assertVersionIsNullAndUpdated(currentVersion);
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// Skipping is necessary because non-versioned buckets are not supported by S3C backbeat routes.
|
// Skipping is necessary because non-versioned buckets are not supported by S3C backbeat routes.
|
||||||
it.skip('should update metadata of a non-version object', done => {
|
it.skip('should update metadata of a non-version object', done => {
|
||||||
let objMD;
|
let objMD;
|
||||||
|
@ -361,7 +549,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -423,7 +611,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -486,7 +674,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -557,7 +745,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -622,7 +810,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -682,7 +870,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -743,7 +931,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
@ -806,7 +994,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
||||||
next),
|
next),
|
||||||
|
@ -886,7 +1074,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -961,7 +1149,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||||
|
@ -1033,7 +1221,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
@ -1107,7 +1295,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
||||||
versionId: 'null',
|
versionId: 'null',
|
||||||
},
|
},
|
||||||
authCredentials: backbeatAuthCredentials,
|
authCredentials: backbeatAuthCredentials,
|
||||||
requestBody: objMD.getSerialized(),
|
requestBody: objMD,
|
||||||
}, next),
|
}, next),
|
||||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
||||||
next),
|
next),
|
||||||
|
|
|
@ -100,6 +100,67 @@ describeSkipIfAWS('backbeat routes for replication', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should successfully replicate a suspended null version', done => {
|
||||||
|
let objMD;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
suspendVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||||
|
putObject: next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObject: next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectRes = results.headObject;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 1);
|
||||||
|
|
||||||
|
const [currentVersion] = Versions;
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('should successfully replicate a null version and update it', done => {
|
it('should successfully replicate a null version and update it', done => {
|
||||||
let objMD;
|
let objMD;
|
||||||
|
|
||||||
|
@ -178,4 +239,930 @@ describeSkipIfAWS('backbeat routes for replication', () => {
|
||||||
return done();
|
return done();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should successfully put object after replicating a null version', done => {
|
||||||
|
let objMD;
|
||||||
|
let expectedVersionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
putObjectSource: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
putObjectDestination: next => s3.putObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
expectedVersionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
headObject: next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectRes = results.headObject;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
assert.strictEqual(currentVersion.VersionId, expectedVersionId);
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put metadata to a destination that has a version', done => {
|
||||||
|
let objMD;
|
||||||
|
let firstVersionId;
|
||||||
|
let secondVersionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectDestination: next => s3.putObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
firstVersionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectSource: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
secondVersionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: secondVersionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: secondVersionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObjectFirstVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: firstVersionId }, next),
|
||||||
|
headObjectSecondVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: secondVersionId }, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstHeadObjectRes = results.headObjectFirstVersion;
|
||||||
|
assert.strictEqual(firstHeadObjectRes.VersionId, firstVersionId);
|
||||||
|
|
||||||
|
const secondHeadObjectRes = results.headObjectSecondVersion;
|
||||||
|
assert.strictEqual(secondHeadObjectRes.VersionId, secondVersionId);
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, secondVersionId);
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, firstVersionId);
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put metadata to a destination that has a null version', done => {
|
||||||
|
let objMD;
|
||||||
|
let versionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
putObjectDestinationInitial: next => s3.putObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectSource: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
versionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObjectNullVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions(
|
||||||
|
{ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectRes = results.headObjectNullVersion;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put metadata to a destination that has a suspended null version', done => {
|
||||||
|
let objMD;
|
||||||
|
let versionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
suspendVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||||
|
putObjectDestinationInitial: next => s3.putObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectSource: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
versionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObjectNullVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectRes = results.headObjectNullVersion;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put metadata to a destination that has a previously updated null version', done => {
|
||||||
|
let objMD;
|
||||||
|
let objMDNull;
|
||||||
|
let versionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
putObjectDestinationInitial: next => s3.putObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
getMetadataNullVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMDNull = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
updateMetadataNullVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMDNull,
|
||||||
|
}, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectSource: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
versionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObjectNullVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectRes = results.headObjectNullVersion;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put metadata to a destination that has a suspended null version with internal version',
|
||||||
|
done => {
|
||||||
|
const tagSet = [
|
||||||
|
{
|
||||||
|
Key: 'key1',
|
||||||
|
Value: 'value1',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let objMD;
|
||||||
|
let versionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
suspendVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||||
|
putObjectDestinationInitial: next => s3.putObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
putObjectTagging: next => s3.putObjectTagging(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, Tagging: { TagSet: tagSet } }, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
putObjectSource: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
versionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObjectNullVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
getObjectTaggingNullVersion: next => s3.getObjectTagging(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectRes = results.headObjectNullVersion;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const getObjectTaggingRes = results.getObjectTaggingNullVersion;
|
||||||
|
assert.deepStrictEqual(getObjectTaggingRes.TagSet, tagSet);
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should mimic null version replication by crrExistingObjects, then replicate version', done => {
|
||||||
|
let objMDNull;
|
||||||
|
let objMDNullReplicated;
|
||||||
|
let objMDVersion;
|
||||||
|
let versionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
createNullSoloMasterKey: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
simulateCrrExistingObjectsGetMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMDNull = JSON.parse(data.body).Body;
|
||||||
|
assert.strictEqual(JSON.parse(objMDNull).versionId, undefined);
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
simulateCrrExistingObjectsPutMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMDNull,
|
||||||
|
}, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||||
|
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||||
|
replicateNullVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMDNullReplicated = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
putReplicatedNullVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId: 'null',
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMDNullReplicated,
|
||||||
|
}, next),
|
||||||
|
putNewVersionSource: next => s3.putObject(
|
||||||
|
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
versionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
simulateMetadataReplicationVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMDVersion = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
listObjectVersionsBeforeReplicate: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
putReplicatedVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: {
|
||||||
|
versionId,
|
||||||
|
},
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMDVersion,
|
||||||
|
}, next),
|
||||||
|
checkReplicatedNullVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||||
|
checkReplicatedVersion: next => s3.headObject(
|
||||||
|
{ Bucket: bucketDestination, Key: keyName, VersionId: versionId }, next),
|
||||||
|
listObjectVersionsAfterReplicate: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectNullVersionRes = results.checkReplicatedNullVersion;
|
||||||
|
assert.strictEqual(headObjectNullVersionRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const headObjectVersionRes = results.checkReplicatedVersion;
|
||||||
|
assert.strictEqual(headObjectVersionRes.VersionId, versionId);
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersionsAfterReplicate;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put NULL metadata to a destination that has a version', done => {
|
||||||
|
let objMD;
|
||||||
|
let versionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
VersioningConfiguration: { Status: 'Enabled' },
|
||||||
|
}, next),
|
||||||
|
putObjectDestination: next => s3.putObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
Body: Buffer.from(testData),
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
versionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
putObjectSource: next => s3.putObject({
|
||||||
|
Bucket: bucketSource,
|
||||||
|
Key: keyName,
|
||||||
|
Body: Buffer.from(testData),
|
||||||
|
}, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning({
|
||||||
|
Bucket: bucketSource,
|
||||||
|
VersioningConfiguration: { Status: 'Enabled' },
|
||||||
|
}, next),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObjectByVersionId: next => s3.headObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
VersionId: versionId,
|
||||||
|
}, next),
|
||||||
|
headObjectByNullVersionId: next => s3.headObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
VersionId: 'null',
|
||||||
|
}, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
}, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstHeadObjectRes = results.headObjectByVersionId;
|
||||||
|
assert.strictEqual(firstHeadObjectRes.VersionId, versionId);
|
||||||
|
|
||||||
|
const secondHeadObjectRes = results.headObjectByNullVersionId;
|
||||||
|
assert.strictEqual(secondHeadObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, versionId);
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put NULL metadata to a destination that has a null version', done => {
|
||||||
|
let objMD;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
putObjectDestinationInitial: next => s3.putObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
Body: Buffer.from(testData),
|
||||||
|
}, next),
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
VersioningConfiguration: { Status: 'Enabled' },
|
||||||
|
}, next),
|
||||||
|
putObjectSource: next => s3.putObject({
|
||||||
|
Bucket: bucketSource,
|
||||||
|
Key: keyName,
|
||||||
|
Body: Buffer.from(testData),
|
||||||
|
}, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning({
|
||||||
|
Bucket: bucketSource,
|
||||||
|
VersioningConfiguration: { Status: 'Enabled' },
|
||||||
|
}, next),
|
||||||
|
putObjectTaggingSource: next => s3.putObjectTagging({
|
||||||
|
Bucket: bucketSource,
|
||||||
|
Key: keyName,
|
||||||
|
VersionId: 'null',
|
||||||
|
Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] },
|
||||||
|
}, next),
|
||||||
|
getMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMD = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
replicateMetadata: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMD,
|
||||||
|
}, next),
|
||||||
|
headObjectNullVersion: next => s3.headObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
VersionId: 'null',
|
||||||
|
}, next),
|
||||||
|
getObjectTaggingNullVersion: next => s3.getObjectTagging({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
VersionId: 'null',
|
||||||
|
}, next),
|
||||||
|
listObjectVersions: next => s3.listObjectVersions({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
}, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headObjectRes = results.headObjectNullVersion;
|
||||||
|
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const getObjectTaggingRes = results.getObjectTaggingNullVersion;
|
||||||
|
assert.deepStrictEqual(getObjectTaggingRes.TagSet, [{ Key: 'key1', Value: 'value1' }]);
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersions;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 1);
|
||||||
|
const [currentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should replicate/put a lifecycled NULL metadata to a destination that has a version', done => {
|
||||||
|
let objMDUpdated;
|
||||||
|
let objMDReplicated;
|
||||||
|
let versionId;
|
||||||
|
|
||||||
|
async.series({
|
||||||
|
enableVersioningDestination: next => s3.putBucketVersioning({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
VersioningConfiguration: { Status: 'Enabled' },
|
||||||
|
}, next),
|
||||||
|
putObjectDestination: next => s3.putObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
Body: Buffer.from(testData),
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
versionId = data.VersionId;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
putObjectSource: next => s3.putObject({
|
||||||
|
Bucket: bucketSource,
|
||||||
|
Key: keyName,
|
||||||
|
Body: Buffer.from(testData),
|
||||||
|
}, next),
|
||||||
|
enableVersioningSource: next => s3.putBucketVersioning({
|
||||||
|
Bucket: bucketSource,
|
||||||
|
VersioningConfiguration: { Status: 'Enabled' },
|
||||||
|
}, next),
|
||||||
|
simulateLifecycleNullVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMDUpdated = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
updateMetadataSource: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMDUpdated,
|
||||||
|
}, next),
|
||||||
|
getReplicatedNullVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'GET',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketSource,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
}, (err, data) => {
|
||||||
|
if (err) {
|
||||||
|
return next(err);
|
||||||
|
}
|
||||||
|
objMDReplicated = JSON.parse(data.body).Body;
|
||||||
|
return next();
|
||||||
|
}),
|
||||||
|
putReplicatedNullVersion: next => makeBackbeatRequest({
|
||||||
|
method: 'PUT',
|
||||||
|
resourceType: 'metadata',
|
||||||
|
bucket: bucketDestination,
|
||||||
|
objectKey: keyName,
|
||||||
|
queryObj: { versionId: 'null' },
|
||||||
|
authCredentials: backbeatAuthCredentials,
|
||||||
|
requestBody: objMDReplicated,
|
||||||
|
}, next),
|
||||||
|
headObjectByVersionId: next => s3.headObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
VersionId: versionId,
|
||||||
|
}, next),
|
||||||
|
headObjectByNullVersion: next => s3.headObject({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
Key: keyName,
|
||||||
|
VersionId: 'null',
|
||||||
|
}, next),
|
||||||
|
listObjectVersionsDestination: next => s3.listObjectVersions({
|
||||||
|
Bucket: bucketDestination,
|
||||||
|
}, next),
|
||||||
|
}, (err, results) => {
|
||||||
|
if (err) {
|
||||||
|
return done(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstHeadObjectRes = results.headObjectByVersionId;
|
||||||
|
assert.strictEqual(firstHeadObjectRes.VersionId, versionId);
|
||||||
|
|
||||||
|
const secondHeadObjectRes = results.headObjectByNullVersion;
|
||||||
|
assert.strictEqual(secondHeadObjectRes.VersionId, 'null');
|
||||||
|
|
||||||
|
const listObjectVersionsRes = results.listObjectVersionsDestination;
|
||||||
|
const { Versions } = listObjectVersionsRes;
|
||||||
|
|
||||||
|
assert.strictEqual(Versions.length, 2);
|
||||||
|
const [currentVersion, nonCurrentVersion] = Versions;
|
||||||
|
|
||||||
|
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||||
|
assert.strictEqual(currentVersion.IsLatest, true);
|
||||||
|
|
||||||
|
assert.strictEqual(nonCurrentVersion.VersionId, versionId);
|
||||||
|
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||||
|
|
||||||
|
return done();
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -0,0 +1,254 @@
|
||||||
|
const assert = require('assert');
|
||||||
|
const { parseRedisConfig } = require('../../../lib/Config');
|
||||||
|
|
||||||
|
describe('parseRedisConfig', () => {
|
||||||
|
[
|
||||||
|
{
|
||||||
|
desc: 'with host and port',
|
||||||
|
input: {
|
||||||
|
host: 'localhost',
|
||||||
|
port: 6479,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with host, port and password',
|
||||||
|
input: {
|
||||||
|
host: 'localhost',
|
||||||
|
port: 6479,
|
||||||
|
password: 'mypass',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with host, port and an empty password',
|
||||||
|
input: {
|
||||||
|
host: 'localhost',
|
||||||
|
port: 6479,
|
||||||
|
password: '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with host, port and an empty retry config',
|
||||||
|
input: {
|
||||||
|
host: 'localhost',
|
||||||
|
port: 6479,
|
||||||
|
retry: {
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with host, port and a custom retry config',
|
||||||
|
input: {
|
||||||
|
host: 'localhost',
|
||||||
|
port: 6479,
|
||||||
|
retry: {
|
||||||
|
connectBackoff: {
|
||||||
|
min: 10,
|
||||||
|
max: 1000,
|
||||||
|
jitter: 0.1,
|
||||||
|
factor: 1.5,
|
||||||
|
deadline: 10000,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a single sentinel and no sentinel password',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: 'localhost',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with two sentinels and a sentinel password',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: '10.20.30.40',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
host: '10.20.30.41',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
sentinelPassword: 'mypass',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a sentinel and an empty sentinel password',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: '10.20.30.40',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
sentinelPassword: '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a basic production-like config with sentinels',
|
||||||
|
input: {
|
||||||
|
name: 'scality-s3',
|
||||||
|
password: '',
|
||||||
|
sentinelPassword: '',
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: 'storage-1',
|
||||||
|
port: 16379,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
host: 'storage-2',
|
||||||
|
port: 16379,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
host: 'storage-3',
|
||||||
|
port: 16379,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
host: 'storage-4',
|
||||||
|
port: 16379,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
host: 'storage-5',
|
||||||
|
port: 16379,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a single sentinel passed as a string',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: '10.20.30.40:16479',
|
||||||
|
},
|
||||||
|
output: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: '10.20.30.40',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a list of sentinels passed as a string',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: '10.20.30.40:16479,another-host:16480,10.20.30.42:16481',
|
||||||
|
sentinelPassword: 'mypass',
|
||||||
|
},
|
||||||
|
output: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: '10.20.30.40',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
host: 'another-host',
|
||||||
|
port: 16480,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
host: '10.20.30.42',
|
||||||
|
port: 16481,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
sentinelPassword: 'mypass',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
].forEach(testCase => {
|
||||||
|
it(`should parse a valid config ${testCase.desc}`, () => {
|
||||||
|
const redisConfig = parseRedisConfig(testCase.input);
|
||||||
|
assert.deepStrictEqual(redisConfig, testCase.output || testCase.input);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
desc: 'that is empty',
|
||||||
|
input: {},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with only a host',
|
||||||
|
input: {
|
||||||
|
host: 'localhost',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with only a port',
|
||||||
|
input: {
|
||||||
|
port: 6479,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a custom retry config with missing values',
|
||||||
|
input: {
|
||||||
|
host: 'localhost',
|
||||||
|
port: 6479,
|
||||||
|
retry: {
|
||||||
|
connectBackoff: {
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a sentinel but no name',
|
||||||
|
input: {
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: 'localhost',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with a sentinel but an empty name',
|
||||||
|
input: {
|
||||||
|
name: '',
|
||||||
|
sentinels: [
|
||||||
|
{
|
||||||
|
host: 'localhost',
|
||||||
|
port: 16479,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with an empty list of sentinels',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with an empty list of sentinels passed as a string',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: 'with an invalid list of sentinels passed as a string (missing port)',
|
||||||
|
input: {
|
||||||
|
name: 'myname',
|
||||||
|
sentinels: '10.20.30.40:16479,10.20.30.50',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
].forEach(testCase => {
|
||||||
|
it(`should fail to parse an invalid config ${testCase.desc}`, () => {
|
||||||
|
assert.throws(() => {
|
||||||
|
parseRedisConfig(testCase.input);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
18
yarn.lock
18
yarn.lock
|
@ -499,9 +499,9 @@ arraybuffer.slice@~0.0.7:
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
ioctl "^2.0.2"
|
ioctl "^2.0.2"
|
||||||
|
|
||||||
"arsenal@git+https://github.com/scality/arsenal#7.70.25":
|
"arsenal@git+https://github.com/scality/arsenal#7.70.25-1":
|
||||||
version "7.70.25"
|
version "7.70.25-1"
|
||||||
resolved "git+https://github.com/scality/arsenal#7423fac674529efb69fe81bab4404bfa9737382e"
|
resolved "git+https://github.com/scality/arsenal#9ba0900ff857d50c14bf3c5adf3ae3bb643d7fe6"
|
||||||
dependencies:
|
dependencies:
|
||||||
"@js-sdsl/ordered-set" "^4.4.2"
|
"@js-sdsl/ordered-set" "^4.4.2"
|
||||||
"@types/async" "^3.2.12"
|
"@types/async" "^3.2.12"
|
||||||
|
@ -5476,9 +5476,9 @@ user-home@^2.0.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
os-homedir "^1.0.0"
|
os-homedir "^1.0.0"
|
||||||
|
|
||||||
"utapi@git+https://github.com/scality/utapi#7.70.3":
|
"utapi@git+https://github.com/scality/utapi#7.70.5":
|
||||||
version "7.70.3"
|
version "7.70.5"
|
||||||
resolved "git+https://github.com/scality/utapi#e8882a28cc888b2a96479f0301a16f45ce2b0603"
|
resolved "git+https://github.com/scality/utapi#20667ff74176879da22d745fc6db0896270dd610"
|
||||||
dependencies:
|
dependencies:
|
||||||
"@hapi/joi" "^17.1.1"
|
"@hapi/joi" "^17.1.1"
|
||||||
"@senx/warp10" "^1.0.14"
|
"@senx/warp10" "^1.0.14"
|
||||||
|
@ -5606,6 +5606,12 @@ werelogs@scality/werelogs#8.1.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
safe-json-stringify "1.0.3"
|
safe-json-stringify "1.0.3"
|
||||||
|
|
||||||
|
werelogs@scality/werelogs#8.1.0-1:
|
||||||
|
version "8.1.0-1"
|
||||||
|
resolved "https://codeload.github.com/scality/werelogs/tar.gz/1a3a7b12aa15e9b72f7fa3231b7c3adbd74f75b3"
|
||||||
|
dependencies:
|
||||||
|
safe-json-stringify "1.0.3"
|
||||||
|
|
||||||
werelogs@scality/werelogs#GA7.2.0.5:
|
werelogs@scality/werelogs#GA7.2.0.5:
|
||||||
version "7.2.0"
|
version "7.2.0"
|
||||||
resolved "https://codeload.github.com/scality/werelogs/tar.gz/bc034589ebf7810d6e6d61932f94327976de6eef"
|
resolved "https://codeload.github.com/scality/werelogs/tar.gz/bc034589ebf7810d6e6d61932f94327976de6eef"
|
||||||
|
|
Loading…
Reference in New Issue