Compare commits
17 Commits
developmen
...
hotfix/7.7
Author | SHA1 | Date |
---|---|---|
Mickael Bourgois | 3d7225b5d2 | |
Mickael Bourgois | 907e9eebee | |
Mickael Bourgois | 7f09ee5236 | |
Jonathan Gramain | 3f8dcf9747 | |
Jonathan Gramain | 608492154c | |
Nicolas Humbert | 6bb2fa59cd | |
Nicolas Humbert | 7ae4876cba | |
Nicolas Humbert | 606e9b68be | |
Nicolas Humbert | f158c635ab | |
Nicolas Humbert | 961bfd925b | |
Francois Ferrand | e3c5b99d3c | |
Francois Ferrand | 4cbe3adfe0 | |
Francois Ferrand | 778c29ec89 | |
Francois Ferrand | 98531b5454 | |
Taylor McKinnon | 10c0f78a87 | |
Taylor McKinnon | f410e2dce6 | |
Taylor McKinnon | 6b7f8cd892 |
|
@ -16,7 +16,7 @@ runs:
|
|||
run: |-
|
||||
set -exu;
|
||||
mkdir -p /tmp/artifacts/${{ github.job }}/;
|
||||
- uses: actions/setup-node@v2
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '16'
|
||||
cache: 'yarn'
|
||||
|
|
|
@ -62,6 +62,6 @@ services:
|
|||
pykmip:
|
||||
network_mode: "host"
|
||||
profiles: ['pykmip']
|
||||
image: registry.scality.com/cloudserver-dev/pykmip
|
||||
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
|
||||
volumes:
|
||||
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
||||
|
|
|
@ -14,12 +14,12 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: javascript, python, ruby
|
||||
|
||||
- name: Build and analyze
|
||||
uses: github/codeql-action/analyze@v2
|
||||
uses: github/codeql-action/analyze@v3
|
||||
|
|
|
@ -10,7 +10,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@v3
|
||||
uses: actions/dependency-review-action@v4
|
||||
|
|
|
@ -11,36 +11,59 @@ on:
|
|||
|
||||
jobs:
|
||||
build-federation-image:
|
||||
uses: scality/workflows/.github/workflows/docker-build.yaml@v1
|
||||
secrets: inherit
|
||||
with:
|
||||
push: true
|
||||
registry: registry.scality.com
|
||||
namespace: ${{ github.event.repository.name }}
|
||||
name: ${{ github.event.repository.name }}
|
||||
context: .
|
||||
file: images/svc-base/Dockerfile
|
||||
tag: ${{ github.event.inputs.tag }}-svc-base
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to GitHub Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ github.token }}
|
||||
- name: Build and push image for federation
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
context: .
|
||||
file: images/svc-base/Dockerfile
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
|
||||
cache-from: type=gha,scope=federation
|
||||
cache-to: type=gha,mode=max,scope=federation
|
||||
|
||||
build-image:
|
||||
uses: scality/workflows/.github/workflows/docker-build.yaml@v1
|
||||
secrets: inherit
|
||||
with:
|
||||
push: true
|
||||
registry: registry.scality.com
|
||||
namespace: ${{ github.event.repository.name }}
|
||||
name: ${{ github.event.repository.name }}
|
||||
context: .
|
||||
file: Dockerfile
|
||||
tag: ${{ github.event.inputs.tag }}
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to GitHub Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ github.token }}
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
context: .
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
github-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
uses: softprops/action-gh-release@v2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
name: Release ${{ github.event.inputs.tag }}
|
||||
tag_name: ${{ github.event.inputs.tag }}
|
||||
|
|
|
@ -65,23 +65,24 @@ env:
|
|||
ENABLE_LOCAL_CACHE: "true"
|
||||
REPORT_TOKEN: "report-token-1"
|
||||
REMOTE_MANAGEMENT_DISABLE: "1"
|
||||
|
||||
# https://github.com/git-lfs/git-lfs/issues/5749
|
||||
GIT_CLONE_PROTECTION_ACTIVE: 'false'
|
||||
jobs:
|
||||
linting-coverage:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- uses: actions/setup-node@v2
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '16'
|
||||
cache: yarn
|
||||
- name: install dependencies
|
||||
run: yarn install --frozen-lockfile --network-concurrency 1
|
||||
- uses: actions/setup-python@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.9'
|
||||
- uses: actions/cache@v2
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip
|
||||
|
@ -114,7 +115,7 @@ jobs:
|
|||
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
|
||||
if: always()
|
||||
- name: Upload files to artifacts
|
||||
uses: scality/action-artifacts@v2
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
|
@ -127,58 +128,72 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1.6.0
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to GitHub Registry
|
||||
uses: docker/login-action@v1.10.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Login to Registry
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
registry: registry.scality.com
|
||||
username: ${{ secrets.REGISTRY_LOGIN }}
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
password: ${{ github.token }}
|
||||
- name: Build and push cloudserver image
|
||||
uses: docker/build-push-action@v3
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
context: .
|
||||
provenance: false
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||
registry.scality.com/cloudserver-dev/cloudserver:${{ github.sha }}
|
||||
ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
cache-from: type=gha,scope=cloudserver
|
||||
cache-to: type=gha,mode=max,scope=cloudserver
|
||||
- name: Build and push pykmip image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
context: .github/pykmip
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
||||
cache-from: type=gha,scope=pykmip
|
||||
cache-to: type=gha,mode=max,scope=pykmip
|
||||
|
||||
build-federation-image:
|
||||
uses: scality/workflows/.github/workflows/docker-build.yaml@v1
|
||||
secrets: inherit
|
||||
with:
|
||||
push: true
|
||||
registry: registry.scality.com
|
||||
namespace: cloudserver-dev
|
||||
name: cloudserver
|
||||
context: .
|
||||
file: images/svc-base/Dockerfile
|
||||
tag: ${{ github.sha }}-svc-base
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to GitHub Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ github.token }}
|
||||
- name: Build and push image for federation
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
context: .
|
||||
file: images/svc-base/Dockerfile
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository }}:${{ github.sha }}-svc-base
|
||||
cache-from: type=gha,scope=federation
|
||||
cache-to: type=gha,mode=max,scope=federation
|
||||
|
||||
multiple-backend:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
env:
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
S3BACKEND: mem
|
||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||
S3DATA: multiple
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
|
@ -194,7 +209,7 @@ jobs:
|
|||
env:
|
||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v3
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
|
@ -217,14 +232,14 @@ jobs:
|
|||
env:
|
||||
S3BACKEND: file
|
||||
S3VAULT: mem
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
MPU_TESTING: "yes"
|
||||
ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
|
||||
JOB_NAME: ${{ matrix.job-name }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
|
@ -247,7 +262,7 @@ jobs:
|
|||
bash wait_for_local_port.bash 8000 40
|
||||
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v3
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
|
@ -261,14 +276,14 @@ jobs:
|
|||
needs: build
|
||||
env:
|
||||
ENABLE_UTAPI_V2: t
|
||||
S3BACKEND: mem
|
||||
S3BACKEND: mem
|
||||
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
|
@ -282,7 +297,7 @@ jobs:
|
|||
bash wait_for_local_port.bash 8000 40
|
||||
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v3
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
|
@ -298,12 +313,13 @@ jobs:
|
|||
S3BACKEND: file
|
||||
S3VAULT: mem
|
||||
MPU_TESTING: true
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
|
||||
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
|
||||
JOB_NAME: ${{ github.job }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
|
@ -321,7 +337,7 @@ jobs:
|
|||
bash wait_for_local_port.bash 5696 40
|
||||
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v3
|
||||
uses: scality/action-artifacts@v4
|
||||
with:
|
||||
method: upload
|
||||
url: https://artifacts.scality.net
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
FROM registry.scality.com/federation/nodesvc-base:7.10.6.0
|
||||
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
|
||||
|
||||
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
|
||||
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
|
||||
|
|
12
index.js
12
index.js
|
@ -1,10 +1,10 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
/**
|
||||
* Catch uncaught exceptions and add timestamp to aid debugging
|
||||
*/
|
||||
process.on('uncaughtException', err => {
|
||||
process.stderr.write(`${new Date().toISOString()}: Uncaught exception: \n${err.stack}`);
|
||||
});
|
||||
require('werelogs').stderrUtils.catchAndTimestampStderr(
|
||||
undefined,
|
||||
// Do not exit as workers have their own listener that will exit
|
||||
// But primary don't have another listener
|
||||
require('cluster').isPrimary ? 1 : null,
|
||||
);
|
||||
|
||||
require('./lib/server.js')();
|
||||
|
|
186
lib/Config.js
186
lib/Config.js
|
@ -88,6 +88,47 @@ function parseSproxydConfig(configSproxyd) {
|
|||
return joi.attempt(configSproxyd, joiSchema, 'bad config');
|
||||
}
|
||||
|
||||
function parseRedisConfig(redisConfig) {
|
||||
const joiSchema = joi.object({
|
||||
password: joi.string().allow(''),
|
||||
host: joi.string(),
|
||||
port: joi.number(),
|
||||
retry: joi.object({
|
||||
connectBackoff: joi.object({
|
||||
min: joi.number().required(),
|
||||
max: joi.number().required(),
|
||||
jitter: joi.number().required(),
|
||||
factor: joi.number().required(),
|
||||
deadline: joi.number().required(),
|
||||
}),
|
||||
}),
|
||||
// sentinel config
|
||||
sentinels: joi.alternatives().try(
|
||||
joi.string()
|
||||
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
|
||||
.custom(hosts => hosts.split(',').map(item => {
|
||||
const [host, port] = item.split(':');
|
||||
return { host, port: Number.parseInt(port, 10) };
|
||||
})),
|
||||
joi.array().items(
|
||||
joi.object({
|
||||
host: joi.string().required(),
|
||||
port: joi.number().required(),
|
||||
})
|
||||
).min(1),
|
||||
),
|
||||
name: joi.string(),
|
||||
sentinelPassword: joi.string().allow(''),
|
||||
})
|
||||
.and('host', 'port')
|
||||
.and('sentinels', 'name')
|
||||
.xor('host', 'sentinels')
|
||||
.without('sentinels', ['host', 'port'])
|
||||
.without('host', ['sentinels', 'sentinelPassword']);
|
||||
|
||||
return joi.attempt(redisConfig, joiSchema, 'bad config');
|
||||
}
|
||||
|
||||
function restEndpointsAssert(restEndpoints, locationConstraints) {
|
||||
assert(typeof restEndpoints === 'object',
|
||||
'bad config: restEndpoints must be an object of endpoints');
|
||||
|
@ -294,18 +335,17 @@ function parseUtapiReindex(config) {
|
|||
const {
|
||||
enabled,
|
||||
schedule,
|
||||
sentinel,
|
||||
redis,
|
||||
bucketd,
|
||||
onlyCountLatestWhenObjectLocked,
|
||||
} = config;
|
||||
assert(typeof enabled === 'boolean',
|
||||
'bad config: utapi.reindex.enabled must be a boolean');
|
||||
assert(typeof sentinel === 'object',
|
||||
'bad config: utapi.reindex.sentinel must be an object');
|
||||
assert(typeof sentinel.port === 'number',
|
||||
'bad config: utapi.reindex.sentinel.port must be a number');
|
||||
assert(typeof sentinel.name === 'string',
|
||||
'bad config: utapi.reindex.sentinel.name must be a string');
|
||||
'bad config: utapi.reindex.enabled must be a boolean');
|
||||
|
||||
const parsedRedis = parseRedisConfig(redis);
|
||||
assert(Array.isArray(parsedRedis.sentinels),
|
||||
'bad config: utapi reindex redis config requires a list of sentinels');
|
||||
|
||||
assert(typeof bucketd === 'object',
|
||||
'bad config: utapi.reindex.bucketd must be an object');
|
||||
assert(typeof bucketd.port === 'number',
|
||||
|
@ -323,6 +363,13 @@ function parseUtapiReindex(config) {
|
|||
'bad config: utapi.reindex.schedule must be a valid ' +
|
||||
`cron schedule. ${e.message}.`);
|
||||
}
|
||||
return {
|
||||
enabled,
|
||||
schedule,
|
||||
redis: parsedRedis,
|
||||
bucketd,
|
||||
onlyCountLatestWhenObjectLocked,
|
||||
};
|
||||
}
|
||||
|
||||
function requestsConfigAssert(requestsConfig) {
|
||||
|
@ -761,8 +808,7 @@ class Config extends EventEmitter {
|
|||
assert(typeof config.localCache.port === 'number',
|
||||
'config: invalid port for localCache. port must be a number');
|
||||
if (config.localCache.password !== undefined) {
|
||||
assert(
|
||||
this._verifyRedisPassword(config.localCache.password),
|
||||
assert(typeof config.localCache.password === 'string',
|
||||
'config: invalid password for localCache. password must' +
|
||||
' be a string');
|
||||
}
|
||||
|
@ -774,55 +820,7 @@ class Config extends EventEmitter {
|
|||
}
|
||||
|
||||
if (config.redis) {
|
||||
if (config.redis.sentinels) {
|
||||
this.redis = { sentinels: [], name: null };
|
||||
|
||||
assert(typeof config.redis.name === 'string',
|
||||
'bad config: redis sentinel name must be a string');
|
||||
this.redis.name = config.redis.name;
|
||||
assert(Array.isArray(config.redis.sentinels) ||
|
||||
typeof config.redis.sentinels === 'string',
|
||||
'bad config: redis sentinels must be an array or string');
|
||||
|
||||
if (typeof config.redis.sentinels === 'string') {
|
||||
config.redis.sentinels.split(',').forEach(item => {
|
||||
const [host, port] = item.split(':');
|
||||
this.redis.sentinels.push({ host,
|
||||
port: Number.parseInt(port, 10) });
|
||||
});
|
||||
} else if (Array.isArray(config.redis.sentinels)) {
|
||||
config.redis.sentinels.forEach(item => {
|
||||
const { host, port } = item;
|
||||
assert(typeof host === 'string',
|
||||
'bad config: redis sentinel host must be a string');
|
||||
assert(typeof port === 'number',
|
||||
'bad config: redis sentinel port must be a number');
|
||||
this.redis.sentinels.push({ host, port });
|
||||
});
|
||||
}
|
||||
|
||||
if (config.redis.sentinelPassword !== undefined) {
|
||||
assert(
|
||||
this._verifyRedisPassword(config.redis.sentinelPassword));
|
||||
this.redis.sentinelPassword = config.redis.sentinelPassword;
|
||||
}
|
||||
} else {
|
||||
// check for standalone configuration
|
||||
this.redis = {};
|
||||
assert(typeof config.redis.host === 'string',
|
||||
'bad config: redis.host must be a string');
|
||||
assert(typeof config.redis.port === 'number',
|
||||
'bad config: redis.port must be a number');
|
||||
this.redis.host = config.redis.host;
|
||||
this.redis.port = config.redis.port;
|
||||
}
|
||||
if (config.redis.password !== undefined) {
|
||||
assert(
|
||||
this._verifyRedisPassword(config.redis.password),
|
||||
'bad config: invalid password for redis. password must ' +
|
||||
'be a string');
|
||||
this.redis.password = config.redis.password;
|
||||
}
|
||||
this.redis = parseRedisConfig(config.redis);
|
||||
}
|
||||
if (config.utapi) {
|
||||
this.utapi = { component: 's3' };
|
||||
|
@ -851,65 +849,8 @@ class Config extends EventEmitter {
|
|||
this.utapi.localCache = config.localCache;
|
||||
assert(config.utapi.redis, 'missing required property of utapi ' +
|
||||
'configuration: redis');
|
||||
if (config.utapi.redis.sentinels) {
|
||||
this.utapi.redis = { sentinels: [], name: null };
|
||||
|
||||
assert(typeof config.utapi.redis.name === 'string',
|
||||
'bad config: redis sentinel name must be a string');
|
||||
this.utapi.redis.name = config.utapi.redis.name;
|
||||
|
||||
assert(Array.isArray(config.utapi.redis.sentinels),
|
||||
'bad config: redis sentinels must be an array');
|
||||
config.utapi.redis.sentinels.forEach(item => {
|
||||
const { host, port } = item;
|
||||
assert(typeof host === 'string',
|
||||
'bad config: redis sentinel host must be a string');
|
||||
assert(typeof port === 'number',
|
||||
'bad config: redis sentinel port must be a number');
|
||||
this.utapi.redis.sentinels.push({ host, port });
|
||||
});
|
||||
} else {
|
||||
// check for standalone configuration
|
||||
this.utapi.redis = {};
|
||||
assert(typeof config.utapi.redis.host === 'string',
|
||||
'bad config: redis.host must be a string');
|
||||
assert(typeof config.utapi.redis.port === 'number',
|
||||
'bad config: redis.port must be a number');
|
||||
this.utapi.redis.host = config.utapi.redis.host;
|
||||
this.utapi.redis.port = config.utapi.redis.port;
|
||||
}
|
||||
if (config.utapi.redis.password !== undefined) {
|
||||
assert(
|
||||
this._verifyRedisPassword(config.utapi.redis.password),
|
||||
'config: invalid password for utapi redis. password' +
|
||||
' must be a string');
|
||||
this.utapi.redis.password = config.utapi.redis.password;
|
||||
}
|
||||
if (config.utapi.redis.sentinelPassword !== undefined) {
|
||||
assert(
|
||||
this._verifyRedisPassword(config.utapi.redis.sentinelPassword),
|
||||
'config: invalid password for utapi redis. password' +
|
||||
' must be a string');
|
||||
this.utapi.redis.sentinelPassword =
|
||||
config.utapi.redis.sentinelPassword;
|
||||
}
|
||||
if (config.utapi.redis.retry !== undefined) {
|
||||
if (config.utapi.redis.retry.connectBackoff !== undefined) {
|
||||
const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
|
||||
assert.strictEqual(typeof min, 'number',
|
||||
'utapi.redis.retry.connectBackoff: min must be a number');
|
||||
assert.strictEqual(typeof max, 'number',
|
||||
'utapi.redis.retry.connectBackoff: max must be a number');
|
||||
assert.strictEqual(typeof jitter, 'number',
|
||||
'utapi.redis.retry.connectBackoff: jitter must be a number');
|
||||
assert.strictEqual(typeof factor, 'number',
|
||||
'utapi.redis.retry.connectBackoff: factor must be a number');
|
||||
assert.strictEqual(typeof deadline, 'number',
|
||||
'utapi.redis.retry.connectBackoff: deadline must be a number');
|
||||
}
|
||||
|
||||
this.utapi.redis.retry = config.utapi.redis.retry;
|
||||
} else {
|
||||
this.utapi.redis = parseRedisConfig(config.utapi.redis);
|
||||
if (this.utapi.redis.retry === undefined) {
|
||||
this.utapi.redis.retry = {
|
||||
connectBackoff: {
|
||||
min: 10,
|
||||
|
@ -920,6 +861,7 @@ class Config extends EventEmitter {
|
|||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (config.utapi.metrics) {
|
||||
this.utapi.metrics = config.utapi.metrics;
|
||||
}
|
||||
|
@ -988,8 +930,7 @@ class Config extends EventEmitter {
|
|||
}
|
||||
|
||||
if (config.utapi && config.utapi.reindex) {
|
||||
parseUtapiReindex(config.utapi.reindex);
|
||||
this.utapi.reindex = config.utapi.reindex;
|
||||
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1430,10 +1371,6 @@ class Config extends EventEmitter {
|
|||
};
|
||||
}
|
||||
|
||||
_verifyRedisPassword(password) {
|
||||
return typeof password === 'string';
|
||||
}
|
||||
|
||||
setAuthDataAccounts(accounts) {
|
||||
this.authData.accounts = accounts;
|
||||
this.emit('authdata-update');
|
||||
|
@ -1547,6 +1484,7 @@ class Config extends EventEmitter {
|
|||
|
||||
module.exports = {
|
||||
parseSproxydConfig,
|
||||
parseRedisConfig,
|
||||
locationConstraintAssert,
|
||||
ConfigObject: Config,
|
||||
config: new Config(),
|
||||
|
|
|
@ -193,7 +193,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
|
|||
// null keys are used, which is used as an optimization to
|
||||
// avoid having to check the versioned key since there can
|
||||
// be no more versioned key to clean up
|
||||
if (mst.isNull && !mst.isNull2) {
|
||||
if (mst.isNull && mst.versionId && !mst.isNull2) {
|
||||
const delOptions = { versionId: mst.versionId };
|
||||
return { options, delOptions };
|
||||
}
|
||||
|
@ -224,7 +224,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
|
|||
if (masterIsNull) {
|
||||
// if master is a null version or a non-versioned key,
|
||||
// copy it to a new null key
|
||||
const nullVersionId = mst.isNull ? mst.versionId : nonVersionedObjId;
|
||||
const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
|
||||
if (nullVersionCompatMode) {
|
||||
options.extraMD = {
|
||||
nullVersionId,
|
||||
|
|
|
@ -49,6 +49,7 @@ const NAMESPACE = 'default';
|
|||
const CIPHER = null; // replication/lifecycle does not work on encrypted objects
|
||||
|
||||
let { locationConstraints } = config;
|
||||
const { nullVersionCompatMode } = config;
|
||||
const { implName } = dataWrapper;
|
||||
let dataClient = dataWrapper.client;
|
||||
config.on('location-constraints-update', () => {
|
||||
|
@ -492,9 +493,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
|||
let isNull = false;
|
||||
|
||||
if (versionId === 'null') {
|
||||
if (!config.nullVersionCompatMode) {
|
||||
isNull = true;
|
||||
}
|
||||
isNull = true;
|
||||
// Retrieve the null version id from the object metadata.
|
||||
versionId = objMd && objMd.versionId;
|
||||
if (!versionId) {
|
||||
|
@ -503,6 +502,16 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
|||
// the flag is needed to allow cloudserver to know that the version
|
||||
// is a null version and allow access to it using the "null" versionId.
|
||||
omVal.isNull = true;
|
||||
// If the new null keys logic (S3C-7352) is supported (not compatibility mode),
|
||||
// create a null key with the isNull2 flag.
|
||||
if (!nullVersionCompatMode) {
|
||||
omVal.isNull2 = true;
|
||||
}
|
||||
// Delete the version id from the version metadata payload to prevent issues
|
||||
// with creating a non-version object (versioning set to false) that includes a version id.
|
||||
// For example, this version ID might come from a null version of a suspended bucket being
|
||||
// replicated to this bucket.
|
||||
delete omVal.versionId;
|
||||
if (versioning) {
|
||||
// If the null version does not have a version id, it is a current null version.
|
||||
// To update the metadata of a current version, versioning is set to false.
|
||||
|
@ -537,7 +546,6 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
|||
}
|
||||
|
||||
const options = {
|
||||
isNull,
|
||||
overheadField: constants.overheadField,
|
||||
};
|
||||
|
||||
|
@ -558,6 +566,11 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
|||
options.versionId = versionId;
|
||||
}
|
||||
|
||||
// If the new null keys logic (S3C-7352) is not supported (compatibility mode), 'isNull' remains undefined.
|
||||
if (!nullVersionCompatMode) {
|
||||
options.isNull = isNull;
|
||||
}
|
||||
|
||||
log.trace('putting object version', {
|
||||
objectKey: request.objectKey, omVal, options });
|
||||
return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
require('werelogs').stderrUtils.catchAndTimestampStderr();
|
||||
const _config = require('../Config').config;
|
||||
const { utapiVersion, UtapiServer: utapiServer } = require('utapi');
|
||||
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
require('werelogs').stderrUtils.catchAndTimestampStderr();
|
||||
const UtapiReindex = require('utapi').UtapiReindex;
|
||||
const { config } = require('../Config');
|
||||
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
require('werelogs').stderrUtils.catchAndTimestampStderr();
|
||||
const UtapiReplay = require('utapi').UtapiReplay;
|
||||
const _config = require('../Config').config;
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "s3",
|
||||
"version": "7.70.45",
|
||||
"version": "7.70.45-4",
|
||||
"description": "S3 connector",
|
||||
"main": "index.js",
|
||||
"engines": {
|
||||
|
@ -20,7 +20,7 @@
|
|||
"homepage": "https://github.com/scality/S3#readme",
|
||||
"dependencies": {
|
||||
"@hapi/joi": "^17.1.0",
|
||||
"arsenal": "git+https://github.com/scality/arsenal#7.70.25",
|
||||
"arsenal": "git+https://github.com/scality/arsenal#7.70.25-1",
|
||||
"async": "~2.5.0",
|
||||
"aws-sdk": "2.905.0",
|
||||
"azure-storage": "^2.1.0",
|
||||
|
@ -35,11 +35,11 @@
|
|||
"moment": "^2.26.0",
|
||||
"npm-run-all": "~4.1.5",
|
||||
"prom-client": "14.2.0",
|
||||
"utapi": "git+https://github.com/scality/utapi#7.70.3",
|
||||
"utapi": "git+https://github.com/scality/utapi#7.70.5",
|
||||
"utf8": "~2.1.1",
|
||||
"uuid": "^3.0.1",
|
||||
"vaultclient": "scality/vaultclient#7.10.13",
|
||||
"werelogs": "scality/werelogs#8.1.0",
|
||||
"werelogs": "scality/werelogs#8.1.0-1",
|
||||
"xml2js": "~0.4.16"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
@ -1,15 +1,15 @@
|
|||
const assert = require('assert');
|
||||
const async = require('async');
|
||||
const crypto = require('crypto');
|
||||
const { models, versioning } = require('arsenal');
|
||||
const { versioning } = require('arsenal');
|
||||
const versionIdUtils = versioning.VersionID;
|
||||
const { ObjectMD } = models;
|
||||
|
||||
const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest');
|
||||
const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
|
||||
|
||||
const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1';
|
||||
const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe;
|
||||
const isNullVersionCompatMode = process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true';
|
||||
|
||||
const backbeatAuthCredentials = {
|
||||
accessKey: 'accessKey1',
|
||||
|
@ -87,17 +87,15 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
|
|||
}
|
||||
|
||||
function updateStorageClass(data, storageClass) {
|
||||
let parsedBody;
|
||||
let result;
|
||||
try {
|
||||
parsedBody = JSON.parse(data.body);
|
||||
const parsedBody = JSON.parse(JSON.parse(data.body).Body);
|
||||
parsedBody['x-amz-storage-class'] = storageClass;
|
||||
result = JSON.stringify(parsedBody);
|
||||
} catch (err) {
|
||||
return { error: err };
|
||||
}
|
||||
const { result, error } = ObjectMD.createFromBlob(parsedBody.Body);
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
result.setAmzStorageClass(storageClass);
|
||||
|
||||
return { result };
|
||||
}
|
||||
|
||||
|
@ -202,11 +200,11 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
|
||||
it('should update metadata of a current null version', done => {
|
||||
let objMD;
|
||||
return async.series([
|
||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
||||
next),
|
||||
next => makeBackbeatRequest({
|
||||
return async.series({
|
||||
putObject: next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
|
@ -226,7 +224,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
objMD = result;
|
||||
return next();
|
||||
}),
|
||||
next => makeBackbeatRequest({
|
||||
putMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
|
@ -235,19 +233,40 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
], (err, data) => {
|
||||
headObject: next => s3.headObject(
|
||||
{ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
getMetadataAfter: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const headObjectRes = data[4];
|
||||
|
||||
const headObjectRes = results.headObject;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||
|
||||
const listObjectVersionsRes = data[5];
|
||||
const getMetadataAfterRes = results.getMetadataAfter;
|
||||
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||
const expectedMd = JSON.parse(objMD);
|
||||
expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed
|
||||
if (!isNullVersionCompatMode) {
|
||||
expectedMd.isNull2 = true; // TODO remove the line once CLDSRV-509 is fixed
|
||||
}
|
||||
assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd);
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 1);
|
||||
|
@ -261,18 +280,20 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
it('should update metadata of a non-current null version', done => {
|
||||
let objMD;
|
||||
let expectedVersionId;
|
||||
return async.series([
|
||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
||||
next),
|
||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
|
||||
return async.series({
|
||||
putObjectInitial: next => s3.putObject(
|
||||
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||
enableVersioning: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectAgain: next => s3.putObject(
|
||||
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
expectedVersionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
next => makeBackbeatRequest({
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
|
@ -292,7 +313,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
objMD = result;
|
||||
return next();
|
||||
}),
|
||||
next => makeBackbeatRequest({
|
||||
putMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
|
@ -301,23 +322,36 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
], (err, data) => {
|
||||
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
getMetadataAfter: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const headObjectRes = data[5];
|
||||
const headObjectRes = results.headObject;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||
|
||||
const listObjectVersionsRes = data[6];
|
||||
const getMetadataAfterRes = results.getMetadataAfter;
|
||||
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
|
||||
const currentVersion = Versions.find(v => v.IsLatest);
|
||||
assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId);
|
||||
|
||||
|
@ -327,6 +361,160 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should update metadata of a suspended null version', done => {
|
||||
let objMD;
|
||||
return async.series({
|
||||
suspendVersioning: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||
putObject: next => s3.putObject(
|
||||
{ Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
enableVersioning: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
const { error, result } = updateStorageClass(data, storageClass);
|
||||
if (error) {
|
||||
return next(error);
|
||||
}
|
||||
objMD = result;
|
||||
return next();
|
||||
}),
|
||||
putUpdatedMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
getMetadataAfter: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const headObjectRes = results.headObject;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||
|
||||
const getMetadataAfterRes = results.getMetadataAfter;
|
||||
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 1);
|
||||
|
||||
const [currentVersion] = Versions;
|
||||
assertVersionIsNullAndUpdated(currentVersion);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should update metadata of a suspended null version with internal version id', done => {
|
||||
let objMD;
|
||||
return async.series({
|
||||
suspendVersioning: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||
putObject: next => s3.putObject(
|
||||
{ Bucket: bucket, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
enableVersioning: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectTagging: next => s3.putObjectTagging({
|
||||
Bucket: bucket, Key: keyName, VersionId: 'null',
|
||||
Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] },
|
||||
}, next),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
const { error, result } = updateStorageClass(data, storageClass);
|
||||
if (error) {
|
||||
return next(error);
|
||||
}
|
||||
objMD = result;
|
||||
return next();
|
||||
}),
|
||||
putUpdatedMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
getMetadataAfter: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const headObjectRes = results.headObject;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
assert.strictEqual(headObjectRes.StorageClass, storageClass);
|
||||
|
||||
const getMetadataAfterRes = results.getMetadataAfter;
|
||||
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
|
||||
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 1);
|
||||
|
||||
const [currentVersion] = Versions;
|
||||
assertVersionIsNullAndUpdated(currentVersion);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
// Skipping is necessary because non-versioned buckets are not supported by S3C backbeat routes.
|
||||
it.skip('should update metadata of a non-version object', done => {
|
||||
let objMD;
|
||||
|
@ -361,7 +549,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -423,7 +611,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -486,7 +674,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -557,7 +745,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -622,7 +810,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -682,7 +870,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -743,7 +931,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
|
@ -806,7 +994,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
||||
next),
|
||||
|
@ -886,7 +1074,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -961,7 +1149,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
next => s3.listObjectVersions({ Bucket: bucket }, next),
|
||||
|
@ -1033,7 +1221,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
|
||||
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
|
||||
|
@ -1107,7 +1295,7 @@ describeSkipIfAWS('backbeat routes', () => {
|
|||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD.getSerialized(),
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
|
||||
next),
|
||||
|
|
|
@ -100,6 +100,67 @@ describeSkipIfAWS('backbeat routes for replication', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should successfully replicate a suspended null version', done => {
|
||||
let objMD;
|
||||
|
||||
async.series({
|
||||
suspendVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||
putObject: next => s3.putObject({ Bucket: bucketSource, Key: keyName, Body: new Buffer(testData) }, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObject: next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectRes = results.headObject;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 1);
|
||||
|
||||
const [currentVersion] = Versions;
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should successfully replicate a null version and update it', done => {
|
||||
let objMD;
|
||||
|
||||
|
@ -178,4 +239,930 @@ describeSkipIfAWS('backbeat routes for replication', () => {
|
|||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should successfully put object after replicating a null version', done => {
|
||||
let objMD;
|
||||
let expectedVersionId;
|
||||
|
||||
async.series({
|
||||
putObjectSource: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
putObjectDestination: next => s3.putObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
expectedVersionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
headObject: next => s3.headObject({ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectRes = results.headObject;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
assert.strictEqual(currentVersion.VersionId, expectedVersionId);
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put metadata to a destination that has a version', done => {
|
||||
let objMD;
|
||||
let firstVersionId;
|
||||
let secondVersionId;
|
||||
|
||||
async.series({
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectDestination: next => s3.putObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
firstVersionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectSource: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
secondVersionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: secondVersionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: secondVersionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObjectFirstVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: firstVersionId }, next),
|
||||
headObjectSecondVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: secondVersionId }, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const firstHeadObjectRes = results.headObjectFirstVersion;
|
||||
assert.strictEqual(firstHeadObjectRes.VersionId, firstVersionId);
|
||||
|
||||
const secondHeadObjectRes = results.headObjectSecondVersion;
|
||||
assert.strictEqual(secondHeadObjectRes.VersionId, secondVersionId);
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, secondVersionId);
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, firstVersionId);
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put metadata to a destination that has a null version', done => {
|
||||
let objMD;
|
||||
let versionId;
|
||||
|
||||
async.series({
|
||||
putObjectDestinationInitial: next => s3.putObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectSource: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
versionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObjectNullVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
listObjectVersions: next => s3.listObjectVersions(
|
||||
{ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectRes = results.headObjectNullVersion;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put metadata to a destination that has a suspended null version', done => {
|
||||
let objMD;
|
||||
let versionId;
|
||||
|
||||
async.series({
|
||||
suspendVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||
putObjectDestinationInitial: next => s3.putObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectSource: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
versionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObjectNullVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectRes = results.headObjectNullVersion;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put metadata to a destination that has a previously updated null version', done => {
|
||||
let objMD;
|
||||
let objMDNull;
|
||||
let versionId;
|
||||
|
||||
async.series({
|
||||
putObjectDestinationInitial: next => s3.putObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
getMetadataNullVersion: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMDNull = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
updateMetadataNullVersion: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMDNull,
|
||||
}, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectSource: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
versionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObjectNullVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectRes = results.headObjectNullVersion;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put metadata to a destination that has a suspended null version with internal version',
|
||||
done => {
|
||||
const tagSet = [
|
||||
{
|
||||
Key: 'key1',
|
||||
Value: 'value1',
|
||||
},
|
||||
];
|
||||
let objMD;
|
||||
let versionId;
|
||||
|
||||
async.series({
|
||||
suspendVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Suspended' } }, next),
|
||||
putObjectDestinationInitial: next => s3.putObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
putObjectTagging: next => s3.putObjectTagging(
|
||||
{ Bucket: bucketDestination, Key: keyName, Tagging: { TagSet: tagSet } }, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
putObjectSource: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
versionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObjectNullVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
getObjectTaggingNullVersion: next => s3.getObjectTagging(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectRes = results.headObjectNullVersion;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
|
||||
const getObjectTaggingRes = results.getObjectTaggingNullVersion;
|
||||
assert.deepStrictEqual(getObjectTaggingRes.TagSet, tagSet);
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should mimic null version replication by crrExistingObjects, then replicate version', done => {
|
||||
let objMDNull;
|
||||
let objMDNullReplicated;
|
||||
let objMDVersion;
|
||||
let versionId;
|
||||
|
||||
async.series({
|
||||
createNullSoloMasterKey: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketSource, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
simulateCrrExistingObjectsGetMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMDNull = JSON.parse(data.body).Body;
|
||||
assert.strictEqual(JSON.parse(objMDNull).versionId, undefined);
|
||||
return next();
|
||||
}),
|
||||
simulateCrrExistingObjectsPutMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMDNull,
|
||||
}, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning(
|
||||
{ Bucket: bucketDestination, VersioningConfiguration: { Status: 'Enabled' } }, next),
|
||||
replicateNullVersion: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMDNullReplicated = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
putReplicatedNullVersion: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId: 'null',
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMDNullReplicated,
|
||||
}, next),
|
||||
putNewVersionSource: next => s3.putObject(
|
||||
{ Bucket: bucketSource, Key: keyName, Body: Buffer.from(testData) }, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
versionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
simulateMetadataReplicationVersion: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMDVersion = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
listObjectVersionsBeforeReplicate: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
putReplicatedVersion: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: {
|
||||
versionId,
|
||||
},
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMDVersion,
|
||||
}, next),
|
||||
checkReplicatedNullVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: 'null' }, next),
|
||||
checkReplicatedVersion: next => s3.headObject(
|
||||
{ Bucket: bucketDestination, Key: keyName, VersionId: versionId }, next),
|
||||
listObjectVersionsAfterReplicate: next => s3.listObjectVersions({ Bucket: bucketDestination }, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectNullVersionRes = results.checkReplicatedNullVersion;
|
||||
assert.strictEqual(headObjectNullVersionRes.VersionId, 'null');
|
||||
|
||||
const headObjectVersionRes = results.checkReplicatedVersion;
|
||||
assert.strictEqual(headObjectVersionRes.VersionId, versionId);
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersionsAfterReplicate;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, versionId);
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, 'null');
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put NULL metadata to a destination that has a version', done => {
|
||||
let objMD;
|
||||
let versionId;
|
||||
|
||||
async.series({
|
||||
enableVersioningDestination: next => s3.putBucketVersioning({
|
||||
Bucket: bucketDestination,
|
||||
VersioningConfiguration: { Status: 'Enabled' },
|
||||
}, next),
|
||||
putObjectDestination: next => s3.putObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
Body: Buffer.from(testData),
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
versionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
putObjectSource: next => s3.putObject({
|
||||
Bucket: bucketSource,
|
||||
Key: keyName,
|
||||
Body: Buffer.from(testData),
|
||||
}, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning({
|
||||
Bucket: bucketSource,
|
||||
VersioningConfiguration: { Status: 'Enabled' },
|
||||
}, next),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObjectByVersionId: next => s3.headObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
VersionId: versionId,
|
||||
}, next),
|
||||
headObjectByNullVersionId: next => s3.headObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
VersionId: 'null',
|
||||
}, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({
|
||||
Bucket: bucketDestination,
|
||||
}, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const firstHeadObjectRes = results.headObjectByVersionId;
|
||||
assert.strictEqual(firstHeadObjectRes.VersionId, versionId);
|
||||
|
||||
const secondHeadObjectRes = results.headObjectByNullVersionId;
|
||||
assert.strictEqual(secondHeadObjectRes.VersionId, 'null');
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, versionId);
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put NULL metadata to a destination that has a null version', done => {
|
||||
let objMD;
|
||||
|
||||
async.series({
|
||||
putObjectDestinationInitial: next => s3.putObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
Body: Buffer.from(testData),
|
||||
}, next),
|
||||
enableVersioningDestination: next => s3.putBucketVersioning({
|
||||
Bucket: bucketDestination,
|
||||
VersioningConfiguration: { Status: 'Enabled' },
|
||||
}, next),
|
||||
putObjectSource: next => s3.putObject({
|
||||
Bucket: bucketSource,
|
||||
Key: keyName,
|
||||
Body: Buffer.from(testData),
|
||||
}, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning({
|
||||
Bucket: bucketSource,
|
||||
VersioningConfiguration: { Status: 'Enabled' },
|
||||
}, next),
|
||||
putObjectTaggingSource: next => s3.putObjectTagging({
|
||||
Bucket: bucketSource,
|
||||
Key: keyName,
|
||||
VersionId: 'null',
|
||||
Tagging: { TagSet: [{ Key: 'key1', Value: 'value1' }] },
|
||||
}, next),
|
||||
getMetadata: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMD = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
replicateMetadata: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMD,
|
||||
}, next),
|
||||
headObjectNullVersion: next => s3.headObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
VersionId: 'null',
|
||||
}, next),
|
||||
getObjectTaggingNullVersion: next => s3.getObjectTagging({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
VersionId: 'null',
|
||||
}, next),
|
||||
listObjectVersions: next => s3.listObjectVersions({
|
||||
Bucket: bucketDestination,
|
||||
}, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const headObjectRes = results.headObjectNullVersion;
|
||||
assert.strictEqual(headObjectRes.VersionId, 'null');
|
||||
|
||||
const getObjectTaggingRes = results.getObjectTaggingNullVersion;
|
||||
assert.deepStrictEqual(getObjectTaggingRes.TagSet, [{ Key: 'key1', Value: 'value1' }]);
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersions;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 1);
|
||||
const [currentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should replicate/put a lifecycled NULL metadata to a destination that has a version', done => {
|
||||
let objMDUpdated;
|
||||
let objMDReplicated;
|
||||
let versionId;
|
||||
|
||||
async.series({
|
||||
enableVersioningDestination: next => s3.putBucketVersioning({
|
||||
Bucket: bucketDestination,
|
||||
VersioningConfiguration: { Status: 'Enabled' },
|
||||
}, next),
|
||||
putObjectDestination: next => s3.putObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
Body: Buffer.from(testData),
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
versionId = data.VersionId;
|
||||
return next();
|
||||
}),
|
||||
putObjectSource: next => s3.putObject({
|
||||
Bucket: bucketSource,
|
||||
Key: keyName,
|
||||
Body: Buffer.from(testData),
|
||||
}, next),
|
||||
enableVersioningSource: next => s3.putBucketVersioning({
|
||||
Bucket: bucketSource,
|
||||
VersioningConfiguration: { Status: 'Enabled' },
|
||||
}, next),
|
||||
simulateLifecycleNullVersion: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMDUpdated = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
updateMetadataSource: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMDUpdated,
|
||||
}, next),
|
||||
getReplicatedNullVersion: next => makeBackbeatRequest({
|
||||
method: 'GET',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketSource,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
objMDReplicated = JSON.parse(data.body).Body;
|
||||
return next();
|
||||
}),
|
||||
putReplicatedNullVersion: next => makeBackbeatRequest({
|
||||
method: 'PUT',
|
||||
resourceType: 'metadata',
|
||||
bucket: bucketDestination,
|
||||
objectKey: keyName,
|
||||
queryObj: { versionId: 'null' },
|
||||
authCredentials: backbeatAuthCredentials,
|
||||
requestBody: objMDReplicated,
|
||||
}, next),
|
||||
headObjectByVersionId: next => s3.headObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
VersionId: versionId,
|
||||
}, next),
|
||||
headObjectByNullVersion: next => s3.headObject({
|
||||
Bucket: bucketDestination,
|
||||
Key: keyName,
|
||||
VersionId: 'null',
|
||||
}, next),
|
||||
listObjectVersionsDestination: next => s3.listObjectVersions({
|
||||
Bucket: bucketDestination,
|
||||
}, next),
|
||||
}, (err, results) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const firstHeadObjectRes = results.headObjectByVersionId;
|
||||
assert.strictEqual(firstHeadObjectRes.VersionId, versionId);
|
||||
|
||||
const secondHeadObjectRes = results.headObjectByNullVersion;
|
||||
assert.strictEqual(secondHeadObjectRes.VersionId, 'null');
|
||||
|
||||
const listObjectVersionsRes = results.listObjectVersionsDestination;
|
||||
const { Versions } = listObjectVersionsRes;
|
||||
|
||||
assert.strictEqual(Versions.length, 2);
|
||||
const [currentVersion, nonCurrentVersion] = Versions;
|
||||
|
||||
assert.strictEqual(currentVersion.VersionId, 'null');
|
||||
assert.strictEqual(currentVersion.IsLatest, true);
|
||||
|
||||
assert.strictEqual(nonCurrentVersion.VersionId, versionId);
|
||||
assert.strictEqual(nonCurrentVersion.IsLatest, false);
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -0,0 +1,254 @@
|
|||
const assert = require('assert');
|
||||
const { parseRedisConfig } = require('../../../lib/Config');
|
||||
|
||||
describe('parseRedisConfig', () => {
|
||||
[
|
||||
{
|
||||
desc: 'with host and port',
|
||||
input: {
|
||||
host: 'localhost',
|
||||
port: 6479,
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with host, port and password',
|
||||
input: {
|
||||
host: 'localhost',
|
||||
port: 6479,
|
||||
password: 'mypass',
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with host, port and an empty password',
|
||||
input: {
|
||||
host: 'localhost',
|
||||
port: 6479,
|
||||
password: '',
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with host, port and an empty retry config',
|
||||
input: {
|
||||
host: 'localhost',
|
||||
port: 6479,
|
||||
retry: {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with host, port and a custom retry config',
|
||||
input: {
|
||||
host: 'localhost',
|
||||
port: 6479,
|
||||
retry: {
|
||||
connectBackoff: {
|
||||
min: 10,
|
||||
max: 1000,
|
||||
jitter: 0.1,
|
||||
factor: 1.5,
|
||||
deadline: 10000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a single sentinel and no sentinel password',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: [
|
||||
{
|
||||
host: 'localhost',
|
||||
port: 16479,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with two sentinels and a sentinel password',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: [
|
||||
{
|
||||
host: '10.20.30.40',
|
||||
port: 16479,
|
||||
},
|
||||
{
|
||||
host: '10.20.30.41',
|
||||
port: 16479,
|
||||
},
|
||||
],
|
||||
sentinelPassword: 'mypass',
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a sentinel and an empty sentinel password',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: [
|
||||
{
|
||||
host: '10.20.30.40',
|
||||
port: 16479,
|
||||
},
|
||||
],
|
||||
sentinelPassword: '',
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a basic production-like config with sentinels',
|
||||
input: {
|
||||
name: 'scality-s3',
|
||||
password: '',
|
||||
sentinelPassword: '',
|
||||
sentinels: [
|
||||
{
|
||||
host: 'storage-1',
|
||||
port: 16379,
|
||||
},
|
||||
{
|
||||
host: 'storage-2',
|
||||
port: 16379,
|
||||
},
|
||||
{
|
||||
host: 'storage-3',
|
||||
port: 16379,
|
||||
},
|
||||
{
|
||||
host: 'storage-4',
|
||||
port: 16379,
|
||||
},
|
||||
{
|
||||
host: 'storage-5',
|
||||
port: 16379,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a single sentinel passed as a string',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: '10.20.30.40:16479',
|
||||
},
|
||||
output: {
|
||||
name: 'myname',
|
||||
sentinels: [
|
||||
{
|
||||
host: '10.20.30.40',
|
||||
port: 16479,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a list of sentinels passed as a string',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: '10.20.30.40:16479,another-host:16480,10.20.30.42:16481',
|
||||
sentinelPassword: 'mypass',
|
||||
},
|
||||
output: {
|
||||
name: 'myname',
|
||||
sentinels: [
|
||||
{
|
||||
host: '10.20.30.40',
|
||||
port: 16479,
|
||||
},
|
||||
{
|
||||
host: 'another-host',
|
||||
port: 16480,
|
||||
},
|
||||
{
|
||||
host: '10.20.30.42',
|
||||
port: 16481,
|
||||
},
|
||||
],
|
||||
sentinelPassword: 'mypass',
|
||||
},
|
||||
},
|
||||
].forEach(testCase => {
|
||||
it(`should parse a valid config ${testCase.desc}`, () => {
|
||||
const redisConfig = parseRedisConfig(testCase.input);
|
||||
assert.deepStrictEqual(redisConfig, testCase.output || testCase.input);
|
||||
});
|
||||
});
|
||||
|
||||
[
|
||||
{
|
||||
desc: 'that is empty',
|
||||
input: {},
|
||||
},
|
||||
{
|
||||
desc: 'with only a host',
|
||||
input: {
|
||||
host: 'localhost',
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with only a port',
|
||||
input: {
|
||||
port: 6479,
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a custom retry config with missing values',
|
||||
input: {
|
||||
host: 'localhost',
|
||||
port: 6479,
|
||||
retry: {
|
||||
connectBackoff: {
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a sentinel but no name',
|
||||
input: {
|
||||
sentinels: [
|
||||
{
|
||||
host: 'localhost',
|
||||
port: 16479,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with a sentinel but an empty name',
|
||||
input: {
|
||||
name: '',
|
||||
sentinels: [
|
||||
{
|
||||
host: 'localhost',
|
||||
port: 16479,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with an empty list of sentinels',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with an empty list of sentinels passed as a string',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: '',
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: 'with an invalid list of sentinels passed as a string (missing port)',
|
||||
input: {
|
||||
name: 'myname',
|
||||
sentinels: '10.20.30.40:16479,10.20.30.50',
|
||||
},
|
||||
},
|
||||
].forEach(testCase => {
|
||||
it(`should fail to parse an invalid config ${testCase.desc}`, () => {
|
||||
assert.throws(() => {
|
||||
parseRedisConfig(testCase.input);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
18
yarn.lock
18
yarn.lock
|
@ -499,9 +499,9 @@ arraybuffer.slice@~0.0.7:
|
|||
optionalDependencies:
|
||||
ioctl "^2.0.2"
|
||||
|
||||
"arsenal@git+https://github.com/scality/arsenal#7.70.25":
|
||||
version "7.70.25"
|
||||
resolved "git+https://github.com/scality/arsenal#7423fac674529efb69fe81bab4404bfa9737382e"
|
||||
"arsenal@git+https://github.com/scality/arsenal#7.70.25-1":
|
||||
version "7.70.25-1"
|
||||
resolved "git+https://github.com/scality/arsenal#9ba0900ff857d50c14bf3c5adf3ae3bb643d7fe6"
|
||||
dependencies:
|
||||
"@js-sdsl/ordered-set" "^4.4.2"
|
||||
"@types/async" "^3.2.12"
|
||||
|
@ -5476,9 +5476,9 @@ user-home@^2.0.0:
|
|||
dependencies:
|
||||
os-homedir "^1.0.0"
|
||||
|
||||
"utapi@git+https://github.com/scality/utapi#7.70.3":
|
||||
version "7.70.3"
|
||||
resolved "git+https://github.com/scality/utapi#e8882a28cc888b2a96479f0301a16f45ce2b0603"
|
||||
"utapi@git+https://github.com/scality/utapi#7.70.5":
|
||||
version "7.70.5"
|
||||
resolved "git+https://github.com/scality/utapi#20667ff74176879da22d745fc6db0896270dd610"
|
||||
dependencies:
|
||||
"@hapi/joi" "^17.1.1"
|
||||
"@senx/warp10" "^1.0.14"
|
||||
|
@ -5606,6 +5606,12 @@ werelogs@scality/werelogs#8.1.0:
|
|||
dependencies:
|
||||
safe-json-stringify "1.0.3"
|
||||
|
||||
werelogs@scality/werelogs#8.1.0-1:
|
||||
version "8.1.0-1"
|
||||
resolved "https://codeload.github.com/scality/werelogs/tar.gz/1a3a7b12aa15e9b72f7fa3231b7c3adbd74f75b3"
|
||||
dependencies:
|
||||
safe-json-stringify "1.0.3"
|
||||
|
||||
werelogs@scality/werelogs#GA7.2.0.5:
|
||||
version "7.2.0"
|
||||
resolved "https://codeload.github.com/scality/werelogs/tar.gz/bc034589ebf7810d6e6d61932f94327976de6eef"
|
||||
|
|
Loading…
Reference in New Issue