Compare commits
37 Commits
developmen
...
master
Author | SHA1 | Date |
---|---|---|
|
740cb42df1 | |
|
5008866629 | |
|
d009b9f0a7 | |
|
29021ae22b | |
|
b564dc1b02 | |
|
100a991c16 | |
|
c8fd6eb20f | |
|
d74b26787b | |
|
3f0da690ee | |
|
6d88042b86 | |
|
feade986ab | |
|
f8cd54e80b | |
|
877189fc0e | |
|
86626079e4 | |
|
f11f179147 | |
|
212098e28f | |
|
b89574ea53 | |
|
fe902ad08c | |
|
7bf1e3d975 | |
|
c8d42e0cee | |
|
ec422b4567 | |
|
bf124b6ddf | |
|
b372386103 | |
|
9b15988e5a | |
|
0c07ff2a07 | |
|
5b8da0afa4 | |
|
168f3f779c | |
|
a853ccbaac | |
|
215cca02a5 | |
|
704f28aa53 | |
|
25abf4b28b | |
|
c0bf437113 | |
|
7f9425221f | |
|
23d3c85f8c | |
|
8b3886d6b5 | |
|
00f6b68f40 | |
|
a7331d2d4c |
lib
auth
data
kms
metadata
utapi
utilities
tests/unit
testConfigs/allOptsConfig
|
@ -1,4 +1,9 @@
|
|||
node_modules
|
||||
**/node_modules
|
||||
build
|
||||
*/build
|
||||
package-lock.json
|
||||
Dockerfile
|
||||
localData/*
|
||||
localMetadata/*
|
||||
# Keep the .git/HEAD file in order to properly report version
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
[submodule "arsenal"]
|
||||
path = arsenal
|
||||
url = ../zenko-arsenal
|
||||
branch = master
|
||||
[submodule "scubaclient"]
|
||||
path = scubaclient
|
||||
url = ../zenko-scubaclient
|
||||
branch = development/1.0
|
||||
[submodule "utapi"]
|
||||
path = utapi
|
||||
url = ../zenko-utapi
|
||||
branch = development/8.1
|
||||
[submodule "werelogs"]
|
||||
path = werelogs
|
||||
url = ../zenko-werelogs
|
||||
branch = development/8.1
|
||||
[submodule "fcntl"]
|
||||
path = fcntl
|
||||
url = ../zenko-fcntl
|
||||
[submodule "httpagent"]
|
||||
path = httpagent
|
||||
url = ../zenko-httpagent
|
||||
branch = development/1.0
|
||||
[submodule "eslint-config-scality"]
|
||||
path = eslint-config-scality
|
||||
url = ../zenko-eslint-config-scality
|
153
Dockerfile
153
Dockerfile
|
@ -1,60 +1,115 @@
|
|||
ARG NODE_VERSION=16.20.2-bookworm-slim
|
||||
# NOTE: Put Vitastor Debian packages in ./vitastor-bookworm/ if you want them
|
||||
# to be used instead of the public package repository (e.g., a development version)
|
||||
|
||||
FROM node:${NODE_VERSION} AS builder
|
||||
FROM debian:bookworm AS builder
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
ADD https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key /etc/apt/trusted.gpg.d/
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
gnupg2 \
|
||||
jq \
|
||||
python3 \
|
||||
ssh \
|
||||
wget \
|
||||
libffi-dev \
|
||||
zlib1g-dev \
|
||||
&& apt-get clean \
|
||||
&& mkdir -p /root/ssh \
|
||||
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
|
||||
RUN set -e ; \
|
||||
apt-get update ; \
|
||||
apt-get install -y --no-install-recommends ca-certificates ; \
|
||||
echo deb https://deb.nodesource.com/node_22.x nodistro main > /etc/apt/sources.list.d/nodesource.list ; \
|
||||
(grep -vP '^----|^=' /etc/apt/trusted.gpg.d/nodesource-repo.gpg.key | base64 -d) > /etc/apt/trusted.gpg.d/nodesource-repo.gpg ; \
|
||||
apt-get update ; \
|
||||
apt-get install -y --no-install-recommends build-essential ca-certificates curl git nodejs jq ; \
|
||||
apt-get clean ; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PYTHON=python3
|
||||
COPY package.json yarn.lock /usr/src/app/
|
||||
RUN npm install typescript@4.9.5 -g
|
||||
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
|
||||
ADD package.json /app/package.json
|
||||
ADD arsenal/package.json /app/arsenal/package.json
|
||||
ADD eslint-config-scality/package.json /app/eslint-config-scality/package.json
|
||||
ADD fcntl/package.json /app/fcntl/package.json
|
||||
ADD httpagent/package.json /app/httpagent/package.json
|
||||
ADD scubaclient/package.json /app/scubaclient/package.json
|
||||
ADD utapi/package.json /app/utapi/package.json
|
||||
ADD werelogs/package.json /app/werelogs/package.json
|
||||
|
||||
################################################################################
|
||||
FROM node:${NODE_VERSION}
|
||||
RUN cd /app/ && \
|
||||
for i in arsenal eslint-config-scality fcntl httpagent scubaclient utapi werelogs; do \
|
||||
jq 'del(.scripts)' < $i/package.json > t; \
|
||||
mv t $i/package.json; \
|
||||
done && \
|
||||
npm install --verbose
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
jq \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ADD https://git.yourcmc.ru/vitalif/vitastor/raw/branch/master/debian/changelog /app/vitastor-changelog
|
||||
|
||||
ENV NO_PROXY localhost,127.0.0.1
|
||||
ENV no_proxy localhost,127.0.0.1
|
||||
ADD . /app/
|
||||
|
||||
RUN cd /app/ && \
|
||||
npm install --verbose
|
||||
|
||||
RUN set -e ; \
|
||||
mkdir -p /deb/ /app/vitastor-bookworm/ ; \
|
||||
(mv /app/vitastor-bookworm/vitastor-client*.deb /deb/ || true) ; \
|
||||
echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list ; \
|
||||
echo deb-src http://vitastor.io/debian bookworm main >> /etc/apt/sources.list.d/vitastor.list ; \
|
||||
curl -o /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg ; \
|
||||
apt-get update ; \
|
||||
cd /app/vitastor-bookworm/ ; \
|
||||
(if ls /deb/*.deb; then \
|
||||
apt-get install -y --no-install-recommends pkg-config /deb/*.deb ; \
|
||||
dpkg-source -x *.dsc ; \
|
||||
else \
|
||||
apt-get install -y --no-install-recommends pkg-config vitastor-client ; \
|
||||
apt-get source vitastor ; \
|
||||
fi) ; \
|
||||
mv vitastor-*/ vitastor ; \
|
||||
cd vitastor/node-binding ; \
|
||||
npm install
|
||||
|
||||
RUN cd /app/ && \
|
||||
cp -r /app/vitastor-bookworm/vitastor/node-binding node_modules/vitastor && \
|
||||
perl -i -pe "s!require\('bindings'\)\('!require('./build/Release/!" \
|
||||
node_modules/vitastor/index.js node_modules/ioctl/index.js node_modules/fcntl/index.js && \
|
||||
perl -i -pe "s!require\('node-gyp-build'\)\(__dirname\)!require('./prebuilds/linux-x64/node.napi.node')!" \
|
||||
node_modules/utf-8-validate/index.js node_modules/bufferutil/index.js && \
|
||||
perl -i -pe "s!require\('node-gyp-build'\)\(__dirname\)!require('./prebuilds/linux-x64/classic-level.node')!" \
|
||||
node_modules/classic-level/binding.js && \
|
||||
npm exec webpack -- --mode=development
|
||||
|
||||
####################
|
||||
|
||||
FROM debian:bookworm
|
||||
|
||||
ADD https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key /etc/apt/trusted.gpg.d/
|
||||
|
||||
RUN set -e ; \
|
||||
apt-get update ; \
|
||||
apt-get install -y --no-install-recommends ca-certificates ; \
|
||||
echo deb https://deb.nodesource.com/node_22.x nodistro main > /etc/apt/sources.list.d/nodesource.list ; \
|
||||
(grep -vP '^----|^=' /etc/apt/trusted.gpg.d/nodesource-repo.gpg.key | base64 -d) > /etc/apt/trusted.gpg.d/nodesource-repo.gpg ; \
|
||||
apt-get update ; \
|
||||
apt-get install -y --no-install-recommends curl ca-certificates nodejs jq node-bindings libibverbs1 librdmacm1 libtcmalloc-minimal4 liburing2 ; \
|
||||
apt-get clean ; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ADD https://vitastor.io/debian/pubkey.gpg /etc/apt/trusted.gpg.d/vitastor.gpg
|
||||
|
||||
ADD https://vitastor.io/debian/dists/bookworm/Release /app/debian-release
|
||||
|
||||
COPY --from=builder /deb/ /deb/
|
||||
|
||||
RUN set -e ; \
|
||||
echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list ; \
|
||||
chmod 644 /etc/apt/trusted.gpg.d/vitastor.gpg ; \
|
||||
apt-get update ; \
|
||||
(if ls /deb/*.deb; then \
|
||||
apt-get install -y --no-install-recommends /deb/*.deb ; \
|
||||
else \
|
||||
apt-get install -y --no-install-recommends vitastor-client ; \
|
||||
fi) ; \
|
||||
apt-get clean ; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /app/node_modules/vitastor /app/node_modules/vitastor
|
||||
COPY --from=builder /app/dist /app/
|
||||
COPY --from=builder /app/authdata.json.example /app/
|
||||
COPY --from=builder /app/config.json.vitastor /app/config.json.example
|
||||
COPY --from=builder /app/locationConfig.json.vitastor /app/locationConfig.json.example
|
||||
COPY bin/configure.sh /usr/bin/
|
||||
|
||||
WORKDIR /app
|
||||
EXPOSE 8000
|
||||
EXPOSE 8002
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
jq \
|
||||
tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Keep the .git directory in order to properly report version
|
||||
COPY . /usr/src/app
|
||||
COPY --from=builder /usr/src/app/node_modules ./node_modules/
|
||||
|
||||
|
||||
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
|
||||
|
||||
ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
|
||||
|
||||
CMD [ "yarn", "start" ]
|
||||
CMD [ "nodejs", "/app/zenko-vitastor.js" ]
|
||||
|
|
|
@ -1,22 +0,0 @@
|
|||
FROM node:6-slim
|
||||
MAINTAINER Giorgio Regni <gr@scality.com>
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
COPY . /usr/src/app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y jq python git build-essential --no-install-recommends \
|
||||
&& yarn install --production \
|
||||
&& apt-get autoremove --purge -y python git build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& yarn cache clean \
|
||||
&& rm -rf ~/.node-gyp \
|
||||
&& rm -rf /tmp/yarn-*
|
||||
|
||||
ENV S3BACKEND mem
|
||||
|
||||
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
|
||||
CMD [ "yarn", "start" ]
|
||||
|
||||
EXPOSE 8000
|
175
README.md
175
README.md
|
@ -1,10 +1,7 @@
|
|||
# Zenko CloudServer
|
||||
# Zenko CloudServer with Vitastor Backend
|
||||
|
||||

|
||||
|
||||
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/zenko/cloudserver)
|
||||
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
|
||||
|
||||
## Overview
|
||||
|
||||
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
|
||||
|
@ -14,137 +11,85 @@ Scality’s Open Source Multi-Cloud Data Controller.
|
|||
CloudServer provides a single AWS S3 API interface to access multiple
|
||||
backend data storage both on-premise or public in the cloud.
|
||||
|
||||
CloudServer is useful for Developers, either to run as part of a
|
||||
continous integration test environment to emulate the AWS S3 service locally
|
||||
or as an abstraction layer to develop object storage enabled
|
||||
application on the go.
|
||||
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
|
||||
backend support.
|
||||
|
||||
## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
|
||||
## Quick Start with Vitastor
|
||||
|
||||
## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
|
||||
Vitastor S3 Backend is now released. Installation instructions:
|
||||
|
||||
## Docker
|
||||
### Install Vitastor
|
||||
|
||||
[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/zenko/cloudserver/)
|
||||
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
|
||||
|
||||
## Contributing
|
||||
Then create a separate pool for S3 object data in your Vitastor cluster, for example:
|
||||
|
||||
In order to contribute, please follow the
|
||||
[Contributing Guidelines](
|
||||
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
|
||||
`vitastor-cli create-pool --ec 2+1 -n 512 s3-data`
|
||||
|
||||
## Installation
|
||||
Retrieve the ID of your pool with `vitastor-cli ls-pools s3-data --detail`.
|
||||
|
||||
### Dependencies
|
||||
### Install and Configure MongoDB
|
||||
|
||||
Building and running the Zenko CloudServer requires node.js 10.x and yarn v1.17.x
|
||||
. Up-to-date versions can be found at
|
||||
[Nodesource](https://github.com/nodesource/distributions).
|
||||
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
|
||||
|
||||
### Clone source code
|
||||
### Install Zenko with Vitastor Backend
|
||||
|
||||
```shell
|
||||
git clone https://github.com/scality/S3.git
|
||||
1. Download Docker image: `docker pull vitalif/vitastor-zenko`
|
||||
|
||||
2. Extract configuration file examples from the Docker image:
|
||||
```
|
||||
docker run --rm -it -v /etc/vitastor:/etc/vitastor -v /etc/vitastor/s3:/conf vitalif/vitastor-zenko configure.sh
|
||||
```
|
||||
|
||||
3. Edit configuration files in `/etc/vitastor/s3/`:
|
||||
- `config.json` - common settings.
|
||||
- `authdata.json` - user accounts and access keys.
|
||||
- `locationConfig.json` - S3 storage class list with placement settings.
|
||||
Note: it actually contains storage classes (like STANDARD, COLD, etc)
|
||||
instead of "locations" (zones like us-east-1) as in the original Zenko CloudServer.
|
||||
- Put your MongoDB connection data into `config.json` and `locationConfig.json`.
|
||||
- Put your Vitastor pool ID into `locationConfig.json`.
|
||||
- For now, the complete list of Vitastor backend settings is only available [in the code](https://git.yourcmc.ru/vitalif/zenko-arsenal/src/branch/master/lib/storage/data/vitastor/VitastorBackend.ts#L94).
|
||||
|
||||
### Start Zenko
|
||||
|
||||
Start the S3 server with:
|
||||
|
||||
```
|
||||
docker run --restart always --security-opt seccomp:unconfined --ulimit memlock=-1 --network=host \
|
||||
-v /etc/vitastor:/etc/vitastor -v /etc/vitastor/s3:/conf --name zenko vitalif/vitastor-zenko
|
||||
```
|
||||
|
||||
### Install js dependencies
|
||||
If you use default settings, Zenko CloudServer starts on port 8000.
|
||||
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
|
||||
|
||||
Go to the ./S3 folder,
|
||||
Now you can access your S3 with, for example, [s3cmd](https://s3tools.org/s3cmd):
|
||||
|
||||
```shell
|
||||
yarn install --frozen-lockfile
|
||||
```
|
||||
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
|
||||
```
|
||||
|
||||
If you get an error regarding installation of the diskUsage module,
|
||||
please install g++.
|
||||
Or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs):
|
||||
|
||||
If you get an error regarding level-down bindings, try clearing your yarn cache:
|
||||
|
||||
```shell
|
||||
yarn cache clean
|
||||
```
|
||||
AWS_ACCESS_KEY_ID=accessKey1 \
|
||||
AWS_SECRET_ACCESS_KEY=verySecretKey1 \
|
||||
geesefs --endpoint http://localhost:8000 testbucket mountdir
|
||||
```
|
||||
|
||||
## Run it with a file backend
|
||||
## Building from source
|
||||
|
||||
```shell
|
||||
yarn start
|
||||
```
|
||||
- Clone this repository: `git clone --recurse-submodules https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
|
||||
- Install dependencies: `npm install --omit dev` or just `npm install`
|
||||
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
|
||||
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
|
||||
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
|
||||
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
|
||||
|
||||
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
|
||||
9991 are also open locally for internal transfer of metadata and data,
|
||||
respectively.
|
||||
# Author & License
|
||||
|
||||
The default access key is accessKey1 with
|
||||
a secret key of verySecretKey1.
|
||||
|
||||
By default the metadata files will be saved in the
|
||||
localMetadata directory and the data files will be saved
|
||||
in the localData directory within the ./S3 directory on your
|
||||
machine. These directories have been pre-created within the
|
||||
repository. If you would like to save the data or metadata in
|
||||
different locations of your choice, you must specify them with absolute paths.
|
||||
So, when starting the server:
|
||||
|
||||
```shell
|
||||
mkdir -m 700 $(pwd)/myFavoriteDataPath
|
||||
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
|
||||
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
|
||||
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
|
||||
yarn start
|
||||
```
|
||||
|
||||
## Run it with multiple data backends
|
||||
|
||||
```shell
|
||||
export S3DATA='multiple'
|
||||
yarn start
|
||||
```
|
||||
|
||||
This starts a Zenko CloudServer on port 8000.
|
||||
The default access key is accessKey1 with
|
||||
a secret key of verySecretKey1.
|
||||
|
||||
With multiple backends, you have the ability to
|
||||
choose where each object will be saved by setting
|
||||
the following header with a locationConstraint on
|
||||
a PUT request:
|
||||
|
||||
```shell
|
||||
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
|
||||
```
|
||||
|
||||
If no header is sent with a PUT object request, the
|
||||
location constraint of the bucket will determine
|
||||
where the data is saved. If the bucket has no location
|
||||
constraint, the endpoint of the PUT request will be
|
||||
used to determine location.
|
||||
|
||||
See the Configuration section in our documentation
|
||||
[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
|
||||
to learn how to set location constraints.
|
||||
|
||||
## Run it with an in-memory backend
|
||||
|
||||
```shell
|
||||
yarn run mem_backend
|
||||
```
|
||||
|
||||
This starts a Zenko CloudServer on port 8000.
|
||||
The default access key is accessKey1 with
|
||||
a secret key of verySecretKey1.
|
||||
|
||||
## Run it with Vault user management
|
||||
|
||||
Note: Vault is proprietary and must be accessed separately.
|
||||
|
||||
```shell
|
||||
export S3VAULT=vault
|
||||
yarn start
|
||||
```
|
||||
|
||||
This starts a Zenko CloudServer using Vault for user management.
|
||||
|
||||
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
|
||||
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
|
||||
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
|
||||
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
|
||||
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality,
|
||||
licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
||||
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is
|
||||
Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
|
||||
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way)
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
Subproject commit ab7861b3b001132b64ec7027fcbe1300149d017c
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
[[ -f /conf/config.json ]] || cp /app/config.json.example /conf/config.json
|
||||
[[ -f /conf/locationConfig.json ]] || cp /app/locationConfig.json.example /conf/locationConfig.json
|
||||
[[ -f /conf/authdata.json ]] || cp /app/authdata.json.example /conf/authdata.json
|
|
@ -1,46 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const {
|
||||
startWSManagementClient,
|
||||
startPushConnectionHealthCheckServer,
|
||||
} = require('../lib/management/push');
|
||||
|
||||
const logger = require('../lib/utilities/logger');
|
||||
|
||||
const {
|
||||
PUSH_ENDPOINT: pushEndpoint,
|
||||
INSTANCE_ID: instanceId,
|
||||
MANAGEMENT_TOKEN: managementToken,
|
||||
} = process.env;
|
||||
|
||||
if (!pushEndpoint) {
|
||||
logger.error('missing push endpoint env var');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!instanceId) {
|
||||
logger.error('missing instance id env var');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!managementToken) {
|
||||
logger.error('missing management token env var');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
startPushConnectionHealthCheckServer(err => {
|
||||
if (err) {
|
||||
logger.error('could not start healthcheck server', { error: err });
|
||||
process.exit(1);
|
||||
}
|
||||
const url = `${pushEndpoint}/${instanceId}/ws?metrics=1`;
|
||||
startWSManagementClient(url, managementToken, err => {
|
||||
if (err) {
|
||||
logger.error('connection failed, exiting', { error: err });
|
||||
process.exit(1);
|
||||
}
|
||||
logger.info('no more connection, exiting');
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
|
@ -1,46 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const {
|
||||
startWSManagementClient,
|
||||
startPushConnectionHealthCheckServer,
|
||||
} = require('../lib/management/push');
|
||||
|
||||
const logger = require('../lib/utilities/logger');
|
||||
|
||||
const {
|
||||
PUSH_ENDPOINT: pushEndpoint,
|
||||
INSTANCE_ID: instanceId,
|
||||
MANAGEMENT_TOKEN: managementToken,
|
||||
} = process.env;
|
||||
|
||||
if (!pushEndpoint) {
|
||||
logger.error('missing push endpoint env var');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!instanceId) {
|
||||
logger.error('missing instance id env var');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!managementToken) {
|
||||
logger.error('missing management token env var');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
startPushConnectionHealthCheckServer(err => {
|
||||
if (err) {
|
||||
logger.error('could not start healthcheck server', { error: err });
|
||||
process.exit(1);
|
||||
}
|
||||
const url = `${pushEndpoint}/${instanceId}/ws?proxy=1`;
|
||||
startWSManagementClient(url, managementToken, err => {
|
||||
if (err) {
|
||||
logger.error('connection failed, exiting', { error: err });
|
||||
process.exit(1);
|
||||
}
|
||||
logger.info('no more connection, exiting');
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
|
@ -4,6 +4,7 @@
|
|||
"metricsPort": 8002,
|
||||
"metricsListenOn": [],
|
||||
"replicationGroupId": "RG001",
|
||||
"workers": 4,
|
||||
"restEndpoints": {
|
||||
"localhost": "us-east-1",
|
||||
"127.0.0.1": "us-east-1",
|
||||
|
@ -58,7 +59,6 @@
|
|||
"host": "localhost",
|
||||
"port": 8500
|
||||
},
|
||||
"clusters": 1,
|
||||
"log": {
|
||||
"logLevel": "info",
|
||||
"dumpLevel": "error"
|
||||
|
@ -101,6 +101,14 @@
|
|||
"readPreference": "primary",
|
||||
"database": "metadata"
|
||||
},
|
||||
"authdata": "authdata.json",
|
||||
"backends": {
|
||||
"auth": "file",
|
||||
"data": "file",
|
||||
"metadata": "mongodb",
|
||||
"kms": "file",
|
||||
"quota": "none"
|
||||
},
|
||||
"externalBackends": {
|
||||
"aws_s3": {
|
||||
"httpAgent": {
|
|
@ -0,0 +1,72 @@
|
|||
{
|
||||
"port": 8000,
|
||||
"workers": 4,
|
||||
"listenOn": [],
|
||||
"metricsPort": 8002,
|
||||
"metricsListenOn": [],
|
||||
"replicationGroupId": "RG001",
|
||||
"restEndpoints": {
|
||||
"localhost": "STANDARD",
|
||||
"127.0.0.1": "STANDARD",
|
||||
"yourhostname.ru": "STANDARD"
|
||||
},
|
||||
"websiteEndpoints": [
|
||||
"static.yourhostname.ru"
|
||||
],
|
||||
"replicationEndpoints": [ {
|
||||
"site": "zenko",
|
||||
"servers": ["127.0.0.1:8000"],
|
||||
"default": true
|
||||
} ],
|
||||
"log": {
|
||||
"logLevel": "info",
|
||||
"dumpLevel": "error"
|
||||
},
|
||||
"healthChecks": {
|
||||
"allowFrom": ["127.0.0.1/8", "::1"]
|
||||
},
|
||||
"backends": {
|
||||
"metadata": "mongodb"
|
||||
},
|
||||
"mongodb": {
|
||||
"replicaSetHosts": "127.0.0.1:27017",
|
||||
"writeConcern": "majority",
|
||||
"replicaSet": "rs0",
|
||||
"readPreference": "primary",
|
||||
"database": "s3",
|
||||
"authCredentials": {
|
||||
"username": "s3",
|
||||
"password": ""
|
||||
}
|
||||
},
|
||||
"externalBackends": {
|
||||
"aws_s3": {
|
||||
"httpAgent": {
|
||||
"keepAlive": false,
|
||||
"keepAliveMsecs": 1000,
|
||||
"maxFreeSockets": 256,
|
||||
"maxSockets": null
|
||||
}
|
||||
},
|
||||
"gcp": {
|
||||
"httpAgent": {
|
||||
"keepAlive": true,
|
||||
"keepAliveMsecs": 1000,
|
||||
"maxFreeSockets": 256,
|
||||
"maxSockets": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"requests": {
|
||||
"viaProxy": false,
|
||||
"trustedProxyCIDRs": [],
|
||||
"extractClientIPFromHeader": ""
|
||||
},
|
||||
"bucketNotificationDestinations": [
|
||||
{
|
||||
"resource": "target1",
|
||||
"type": "dummy",
|
||||
"host": "localhost:6000"
|
||||
}
|
||||
]
|
||||
}
|
|
@ -116,7 +116,7 @@ const constants = {
|
|||
],
|
||||
|
||||
// user metadata header to set object locationConstraint
|
||||
objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
|
||||
objectLocationConstraintHeader: 'x-amz-storage-class',
|
||||
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
|
||||
legacyLocations: ['sproxyd', 'legacy'],
|
||||
// declare here all existing service accounts and their properties
|
||||
|
@ -205,9 +205,6 @@ const constants = {
|
|||
],
|
||||
allowedUtapiEventFilterStates: ['allow', 'deny'],
|
||||
allowedRestoreObjectRequestTierValues: ['Standard'],
|
||||
validStorageClasses: [
|
||||
'STANDARD',
|
||||
],
|
||||
lifecycleListing: {
|
||||
CURRENT_TYPE: 'current',
|
||||
NON_CURRENT_TYPE: 'noncurrent',
|
||||
|
|
|
@ -1,220 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# set -e stops the execution of a script if a command or pipeline has an error
|
||||
set -e
|
||||
|
||||
# modifying config.json
|
||||
JQ_FILTERS_CONFIG="."
|
||||
|
||||
# ENDPOINT var can accept comma separated values
|
||||
# for multiple endpoint locations
|
||||
if [[ "$ENDPOINT" ]]; then
|
||||
IFS="," read -ra HOST_NAMES <<< "$ENDPOINT"
|
||||
for host in "${HOST_NAMES[@]}"; do
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .restEndpoints[\"$host\"]=\"us-east-1\""
|
||||
done
|
||||
echo "Host name has been modified to ${HOST_NAMES[@]}"
|
||||
echo "Note: In your /etc/hosts file on Linux, OS X, or Unix with root permissions, make sure to associate 127.0.0.1 with ${HOST_NAMES[@]}"
|
||||
fi
|
||||
|
||||
if [[ "$LOG_LEVEL" ]]; then
|
||||
if [[ "$LOG_LEVEL" == "info" || "$LOG_LEVEL" == "debug" || "$LOG_LEVEL" == "trace" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .log.logLevel=\"$LOG_LEVEL\""
|
||||
echo "Log level has been modified to $LOG_LEVEL"
|
||||
else
|
||||
echo "The log level you provided is incorrect (info/debug/trace)"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$SSL" && "$HOST_NAMES" ]]; then
|
||||
# This condition makes sure that the certificates are not generated twice. (for docker restart)
|
||||
if [ ! -f ./ca.key ] || [ ! -f ./ca.crt ] || [ ! -f ./server.key ] || [ ! -f ./server.crt ] ; then
|
||||
# Compute config for utapi tests
|
||||
cat >>req.cfg <<EOF
|
||||
[req]
|
||||
distinguished_name = req_distinguished_name
|
||||
prompt = no
|
||||
req_extensions = s3_req
|
||||
|
||||
[req_distinguished_name]
|
||||
CN = ${HOST_NAMES[0]}
|
||||
|
||||
[s3_req]
|
||||
subjectAltName = @alt_names
|
||||
extendedKeyUsage = serverAuth, clientAuth
|
||||
|
||||
[alt_names]
|
||||
DNS.1 = *.${HOST_NAMES[0]}
|
||||
DNS.2 = ${HOST_NAMES[0]}
|
||||
|
||||
EOF
|
||||
|
||||
## Generate SSL key and certificates
|
||||
# Generate a private key for your CSR
|
||||
openssl genrsa -out ca.key 2048
|
||||
# Generate a self signed certificate for your local Certificate Authority
|
||||
openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=US/ST=Country/L=City/O=Organization/CN=S3 CA Server"
|
||||
# Generate a key for S3 Server
|
||||
openssl genrsa -out server.key 2048
|
||||
# Generate a Certificate Signing Request for S3 Server
|
||||
openssl req -new -key server.key -out server.csr -config req.cfg
|
||||
# Generate a local-CA-signed certificate for S3 Server
|
||||
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 -sha256 -extfile req.cfg -extensions s3_req
|
||||
fi
|
||||
## Update S3Server config.json
|
||||
# This condition makes sure that certFilePaths section is not added twice. (for docker restart)
|
||||
if ! grep -q "certFilePaths" ./config.json; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .certFilePaths= { \"key\": \".\/server.key\", \"cert\": \".\/server.crt\", \"ca\": \".\/ca.crt\" }"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$LISTEN_ADDR" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
|
||||
fi
|
||||
|
||||
if [[ "$REPLICATION_GROUP_ID" ]] ; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
|
||||
fi
|
||||
|
||||
if [[ "$DATA_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$METADATA_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$PFSD_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$MONGODB_HOSTS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
|
||||
fi
|
||||
|
||||
if [[ "$MONGODB_RS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSet=\"$MONGODB_RS\""
|
||||
fi
|
||||
|
||||
if [[ "$MONGODB_DATABASE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.database=\"$MONGODB_DATABASE\""
|
||||
fi
|
||||
|
||||
if [ -z "$REDIS_HA_NAME" ]; then
|
||||
REDIS_HA_NAME='mymaster'
|
||||
fi
|
||||
|
||||
if [[ "$REDIS_SENTINELS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.name=\"$REDIS_HA_NAME\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.sentinels=\"$REDIS_SENTINELS\""
|
||||
elif [[ "$REDIS_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.host=\"$REDIS_HOST\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=6379"
|
||||
fi
|
||||
|
||||
if [[ "$REDIS_PORT" ]] && [[ ! "$REDIS_SENTINELS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=$REDIS_PORT"
|
||||
fi
|
||||
|
||||
if [[ "$REDIS_SENTINELS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.name=\"$REDIS_HA_NAME\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.sentinels=\"$REDIS_SENTINELS\""
|
||||
elif [[ "$REDIS_HA_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.host=\"$REDIS_HA_HOST\""
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=6379"
|
||||
fi
|
||||
|
||||
if [[ "$REDIS_HA_PORT" ]] && [[ ! "$REDIS_SENTINELS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=$REDIS_HA_PORT"
|
||||
fi
|
||||
|
||||
if [[ "$RECORDLOG_ENABLED" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .recordLog.enabled=true"
|
||||
fi
|
||||
|
||||
if [[ "$STORAGE_LIMIT_ENABLED" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.metrics[.utapi.metrics | length]=\"location\""
|
||||
fi
|
||||
|
||||
if [[ "$CRR_METRICS_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.host=\"$CRR_METRICS_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$CRR_METRICS_PORT" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
|
||||
fi
|
||||
|
||||
if [[ "$WE_OPERATOR_HOST" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
|
||||
fi
|
||||
|
||||
if [[ "$WE_OPERATOR_PORT" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
|
||||
fi
|
||||
|
||||
if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
|
||||
fi
|
||||
|
||||
# external backends http(s) agent config
|
||||
|
||||
# AWS
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
|
||||
fi
|
||||
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
|
||||
fi
|
||||
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
|
||||
fi
|
||||
|
||||
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
|
||||
fi
|
||||
|
||||
#GCP
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
|
||||
fi
|
||||
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
|
||||
fi
|
||||
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
|
||||
fi
|
||||
|
||||
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
|
||||
fi
|
||||
|
||||
if [[ -n "$BUCKET_DENY_FILTER" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
|
||||
fi
|
||||
|
||||
if [[ "$TESTING_MODE" ]]; then
|
||||
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
|
||||
fi
|
||||
|
||||
if [[ $JQ_FILTERS_CONFIG != "." ]]; then
|
||||
jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
|
||||
mv config.json.tmp config.json
|
||||
fi
|
||||
|
||||
if test -v INITIAL_INSTANCE_ID && test -v S3METADATAPATH && ! test -f ${S3METADATAPATH}/uuid ; then
|
||||
echo -n ${INITIAL_INSTANCE_ID} > ${S3METADATAPATH}/uuid
|
||||
fi
|
||||
|
||||
# s3 secret credentials for Zenko
|
||||
if [ -r /run/secrets/s3-credentials ] ; then
|
||||
. /run/secrets/s3-credentials
|
||||
fi
|
||||
|
||||
exec "$@"
|
|
@ -0,0 +1 @@
|
|||
Subproject commit d383e47d8ec650b547de3d0b5fcb48e97bd48ba1
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 89449bbe583f8ad7531b68cff6cb67ce7682cf18
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 045e9c8282390e92d372f0577d5814fe0ccbeba3
|
|
@ -377,7 +377,7 @@ function dmfLocationConstraintAssert(locationObj) {
|
|||
function locationConstraintAssert(locationConstraints) {
|
||||
const supportedBackends =
|
||||
['mem', 'file', 'scality',
|
||||
'mongodb', 'dmf', 'azure_archive'].concat(Object.keys(validExternalBackends));
|
||||
'mongodb', 'dmf', 'azure_archive', 'vitastor'].concat(Object.keys(validExternalBackends));
|
||||
assert(typeof locationConstraints === 'object',
|
||||
'bad config: locationConstraints must be an object');
|
||||
Object.keys(locationConstraints).forEach(l => {
|
||||
|
@ -502,9 +502,6 @@ function locationConstraintAssert(locationConstraints) {
|
|||
locationConstraints[l].details.connector.hdclient);
|
||||
}
|
||||
});
|
||||
assert(Object.keys(locationConstraints)
|
||||
.includes('us-east-1'), 'bad locationConfig: must ' +
|
||||
'include us-east-1 as a locationConstraint');
|
||||
}
|
||||
|
||||
function parseUtapiReindex(config) {
|
||||
|
@ -633,7 +630,6 @@ class Config extends EventEmitter {
|
|||
// Read config automatically
|
||||
this._getLocationConfig();
|
||||
this._getConfig();
|
||||
this._configureBackends();
|
||||
}
|
||||
|
||||
_parseKmsAWS(config) {
|
||||
|
@ -912,11 +908,11 @@ class Config extends EventEmitter {
|
|||
this.websiteEndpoints = config.websiteEndpoints;
|
||||
}
|
||||
|
||||
this.clusters = false;
|
||||
if (config.clusters !== undefined) {
|
||||
assert(Number.isInteger(config.clusters) && config.clusters > 0,
|
||||
'bad config: clusters must be a positive integer');
|
||||
this.clusters = config.clusters;
|
||||
this.workers = false;
|
||||
if (config.workers !== undefined) {
|
||||
assert(Number.isInteger(config.workers) && config.workers > 0,
|
||||
'bad config: workers must be a positive integer');
|
||||
this.workers = config.workers;
|
||||
}
|
||||
|
||||
if (config.usEastBehavior !== undefined) {
|
||||
|
@ -1374,6 +1370,8 @@ class Config extends EventEmitter {
|
|||
}
|
||||
}
|
||||
|
||||
this.authdata = config.authdata || 'authdata.json';
|
||||
|
||||
this.kms = {};
|
||||
if (config.kms) {
|
||||
assert(typeof config.kms.userName === 'string');
|
||||
|
@ -1605,25 +1603,6 @@ class Config extends EventEmitter {
|
|||
this.outboundProxy.certs = certObj.certs;
|
||||
}
|
||||
|
||||
this.managementAgent = {};
|
||||
this.managementAgent.port = 8010;
|
||||
this.managementAgent.host = 'localhost';
|
||||
if (config.managementAgent !== undefined) {
|
||||
if (config.managementAgent.port !== undefined) {
|
||||
assert(Number.isInteger(config.managementAgent.port)
|
||||
&& config.managementAgent.port > 0,
|
||||
'bad config: managementAgent port must be a positive ' +
|
||||
'integer');
|
||||
this.managementAgent.port = config.managementAgent.port;
|
||||
}
|
||||
if (config.managementAgent.host !== undefined) {
|
||||
assert.strictEqual(typeof config.managementAgent.host, 'string',
|
||||
'bad config: management agent host must ' +
|
||||
'be a string');
|
||||
this.managementAgent.host = config.managementAgent.host;
|
||||
}
|
||||
}
|
||||
|
||||
// Ephemeral token to protect the reporting endpoint:
|
||||
// try inherited from parent first, then hardcoded in conf file,
|
||||
// then create a fresh one as last resort.
|
||||
|
@ -1713,6 +1692,8 @@ class Config extends EventEmitter {
|
|||
'bad config: maxScannedLifecycleListingEntries must be greater than 2');
|
||||
this.maxScannedLifecycleListingEntries = config.maxScannedLifecycleListingEntries;
|
||||
}
|
||||
|
||||
this._configureBackends(config);
|
||||
}
|
||||
|
||||
_setTimeOptions() {
|
||||
|
@ -1751,41 +1732,43 @@ class Config extends EventEmitter {
|
|||
}
|
||||
|
||||
_getAuthData() {
|
||||
return require(findConfigFile(process.env.S3AUTH_CONFIG || 'authdata.json'));
|
||||
return JSON.parse(fs.readFileSync(findConfigFile(process.env.S3AUTH_CONFIG || this.authdata), { encoding: 'utf-8' }));
|
||||
}
|
||||
|
||||
_configureBackends() {
|
||||
_configureBackends(config) {
|
||||
const backends = config.backends || {};
|
||||
/**
|
||||
* Configure the backends for Authentication, Data and Metadata.
|
||||
*/
|
||||
let auth = 'mem';
|
||||
let data = 'multiple';
|
||||
let metadata = 'file';
|
||||
let kms = 'file';
|
||||
let quota = 'none';
|
||||
let auth = backends.auth || 'mem';
|
||||
let data = backends.data || 'multiple';
|
||||
let metadata = backends.metadata || 'file';
|
||||
let kms = backends.kms || 'file';
|
||||
let quota = backends.quota || 'none';
|
||||
if (process.env.S3BACKEND) {
|
||||
const validBackends = ['mem', 'file', 'scality', 'cdmi'];
|
||||
assert(validBackends.indexOf(process.env.S3BACKEND) > -1,
|
||||
'bad environment variable: S3BACKEND environment variable ' +
|
||||
'should be one of mem/file/scality/cdmi'
|
||||
);
|
||||
auth = process.env.S3BACKEND;
|
||||
auth = process.env.S3BACKEND == 'scality' ? 'scality' : 'mem';
|
||||
data = process.env.S3BACKEND;
|
||||
metadata = process.env.S3BACKEND;
|
||||
kms = process.env.S3BACKEND;
|
||||
}
|
||||
if (process.env.S3VAULT) {
|
||||
auth = process.env.S3VAULT;
|
||||
auth = (auth === 'file' || auth === 'mem' || auth === 'cdmi' ? 'mem' : auth);
|
||||
}
|
||||
|
||||
if (auth === 'file' || auth === 'mem' || auth === 'cdmi') {
|
||||
// Auth only checks for 'mem' since mem === file
|
||||
auth = 'mem';
|
||||
let authData;
|
||||
if (process.env.SCALITY_ACCESS_KEY_ID &&
|
||||
process.env.SCALITY_SECRET_ACCESS_KEY) {
|
||||
process.env.SCALITY_SECRET_ACCESS_KEY) {
|
||||
authData = buildAuthDataAccount(
|
||||
process.env.SCALITY_ACCESS_KEY_ID,
|
||||
process.env.SCALITY_SECRET_ACCESS_KEY);
|
||||
process.env.SCALITY_ACCESS_KEY_ID,
|
||||
process.env.SCALITY_SECRET_ACCESS_KEY);
|
||||
} else {
|
||||
authData = this._getAuthData();
|
||||
}
|
||||
|
@ -1793,7 +1776,7 @@ class Config extends EventEmitter {
|
|||
throw new Error('bad config: invalid auth config file.');
|
||||
}
|
||||
this.authData = authData;
|
||||
} else if (auth === 'multiple') {
|
||||
} else if (auth === 'multiple') {
|
||||
const authData = this._getAuthData();
|
||||
if (validateAuthConfig(authData)) {
|
||||
throw new Error('bad config: invalid auth config file.');
|
||||
|
@ -1808,9 +1791,9 @@ class Config extends EventEmitter {
|
|||
'should be one of mem/file/scality/multiple'
|
||||
);
|
||||
data = process.env.S3DATA;
|
||||
}
|
||||
if (data === 'scality' || data === 'multiple') {
|
||||
data = 'multiple';
|
||||
if (data === 'scality' || data === 'multiple') {
|
||||
data = 'multiple';
|
||||
}
|
||||
}
|
||||
assert(this.locationConstraints !== undefined &&
|
||||
this.restEndpoints !== undefined,
|
||||
|
|
|
@ -121,7 +121,7 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
|
|||
}
|
||||
// The locations were sent to metadata as an array
|
||||
// under partLocations. Pull the partLocations.
|
||||
const locations = storedParts.flatMap(item => item.value.partLocations);
|
||||
const locations = storedParts.flatMap(item => item.value.location||item.value.partLocations);
|
||||
if (locations.length === 0) {
|
||||
return next(null, mpuBucket, storedParts, destBucket);
|
||||
}
|
||||
|
|
|
@ -45,9 +45,8 @@ function checkLocationConstraint(request, locationConstraint, log) {
|
|||
} else if (parsedHost && restEndpoints[parsedHost]) {
|
||||
locationConstraintChecked = restEndpoints[parsedHost];
|
||||
} else {
|
||||
log.trace('no location constraint provided on bucket put;' +
|
||||
'setting us-east-1');
|
||||
locationConstraintChecked = 'us-east-1';
|
||||
locationConstraintChecked = Object.keys(locationConstrains)[0];
|
||||
log.trace('no location constraint provided on bucket put; setting '+locationConstraintChecked);
|
||||
}
|
||||
|
||||
if (!locationConstraints[locationConstraintChecked]) {
|
||||
|
|
|
@ -314,7 +314,7 @@ function completeMultipartUpload(authInfo, request, log, callback) {
|
|||
extraPartLocations, completeObjData, totalMPUSize, next) {
|
||||
const metaHeaders = {};
|
||||
const keysNotNeeded =
|
||||
['initiator', 'partLocations', 'key',
|
||||
['initiator', 'location', 'partLocations', 'key',
|
||||
'initiated', 'uploadId', 'content-type', 'expires',
|
||||
'eventualStorageBucket', 'dataStoreName'];
|
||||
const metadataKeysToPull =
|
||||
|
|
|
@ -6,6 +6,7 @@ const convertToXml = s3middleware.convertToXml;
|
|||
const { pushMetric } = require('../utapi/utilities');
|
||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||
const { hasNonPrintables } = require('../utilities/stringChecks');
|
||||
const { config } = require('../Config');
|
||||
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
|
||||
const constants = require('../../constants');
|
||||
const services = require('../services');
|
||||
|
@ -65,7 +66,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
|
|||
const websiteRedirectHeader =
|
||||
request.headers['x-amz-website-redirect-location'];
|
||||
if (request.headers['x-amz-storage-class'] &&
|
||||
!constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) {
|
||||
!config.locationConstraints[request.headers['x-amz-storage-class']]) {
|
||||
log.trace('invalid storage-class header');
|
||||
monitoring.promMetrics('PUT', bucketName,
|
||||
errors.InvalidStorageClass.code, 'initiateMultipartUpload');
|
||||
|
|
|
@ -249,7 +249,7 @@ function objectCopy(authInfo, request, sourceBucket,
|
|||
const responseHeaders = {};
|
||||
|
||||
if (request.headers['x-amz-storage-class'] &&
|
||||
!constants.validStorageClasses.includes(request.headers['x-amz-storage-class'])) {
|
||||
!config.locationConstraints[request.headers['x-amz-storage-class']]) {
|
||||
log.trace('invalid storage-class header');
|
||||
monitoring.promMetrics('PUT', destBucketName,
|
||||
errors.InvalidStorageClass.code, 'copyObject');
|
||||
|
|
|
@ -3,6 +3,7 @@ const { errors, versioning } = require('arsenal');
|
|||
|
||||
const constants = require('../../constants');
|
||||
const aclUtils = require('../utilities/aclUtils');
|
||||
const { config } = require('../Config');
|
||||
const { cleanUpBucket } = require('./apiUtils/bucket/bucketCreation');
|
||||
const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption');
|
||||
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
|
||||
|
@ -71,7 +72,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
|
|||
query,
|
||||
} = request;
|
||||
if (headers['x-amz-storage-class'] &&
|
||||
!constants.validStorageClasses.includes(headers['x-amz-storage-class'])) {
|
||||
!config.locationConstraints[headers['x-amz-storage-class']]) {
|
||||
log.trace('invalid storage-class header');
|
||||
monitoring.promMetrics('PUT', request.bucketName,
|
||||
errors.InvalidStorageClass.code, 'putObject');
|
||||
|
|
|
@ -318,7 +318,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
|
|||
let oldLocations;
|
||||
let prevObjectSize = null;
|
||||
if (result) {
|
||||
oldLocations = result.partLocations;
|
||||
oldLocations = result.location||result.partLocations;
|
||||
prevObjectSize = result['content-length'];
|
||||
// Pull locations to clean up any potential orphans
|
||||
// in data if object put is an overwrite of
|
||||
|
|
|
@ -264,8 +264,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
|
|||
// Pull locations to clean up any potential orphans in
|
||||
// data if object put is an overwrite of a pre-existing
|
||||
// object with the same key and part number.
|
||||
oldLocations = Array.isArray(res.partLocations) ?
|
||||
res.partLocations : [res.partLocations];
|
||||
oldLocations = Array.isArray(res.location||res.partLocations) ?
|
||||
(res.location||res.partLocations) : [(res.location||res.partLocations)];
|
||||
}
|
||||
return next(null, destinationBucket,
|
||||
objectLocationConstraint, cipherBundle,
|
||||
|
@ -321,9 +321,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
|
|||
partLocations[0].sseCipheredDataKey = cipheredDataKey;
|
||||
}
|
||||
const omVal = {
|
||||
// back to Version 3 since number-subparts is not needed
|
||||
'md-model-version': 3,
|
||||
partLocations,
|
||||
// Version 6 changes 'partLocations' to 'location'
|
||||
'md-model-version': 6,
|
||||
'location': partLocations,
|
||||
'key': partKey,
|
||||
'last-modified': new Date().toJSON(),
|
||||
'content-md5': hexDigest,
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
const vaultclient = require('vaultclient');
|
||||
const { auth } = require('arsenal');
|
||||
|
||||
const { config } = require('../Config');
|
||||
|
@ -21,6 +20,7 @@ function getVaultClient(config) {
|
|||
port,
|
||||
https: true,
|
||||
});
|
||||
const vaultclient = require('vaultclient');
|
||||
vaultClient = new vaultclient.Client(host, port, true, key, cert, ca);
|
||||
} else {
|
||||
logger.info('vaultclient configuration', {
|
||||
|
@ -28,6 +28,7 @@ function getVaultClient(config) {
|
|||
port,
|
||||
https: false,
|
||||
});
|
||||
const vaultclient = require('vaultclient');
|
||||
vaultClient = new vaultclient.Client(host, port);
|
||||
}
|
||||
|
||||
|
@ -49,10 +50,6 @@ function getMemBackend(config) {
|
|||
}
|
||||
|
||||
switch (config.backends.auth) {
|
||||
case 'mem':
|
||||
implName = 'vaultMem';
|
||||
client = getMemBackend(config);
|
||||
break;
|
||||
case 'multiple':
|
||||
implName = 'vaultChain';
|
||||
client = new ChainBackend('s3', [
|
||||
|
@ -60,9 +57,14 @@ case 'multiple':
|
|||
getVaultClient(config),
|
||||
]);
|
||||
break;
|
||||
default: // vault
|
||||
case 'vault':
|
||||
implName = 'vault';
|
||||
client = getVaultClient(config);
|
||||
break;
|
||||
default: // mem
|
||||
implName = 'vaultMem';
|
||||
client = getMemBackend(config);
|
||||
break;
|
||||
}
|
||||
|
||||
module.exports = new Vault(client, implName);
|
||||
|
|
|
@ -28,7 +28,7 @@ if (config.backends.data === 'mem') {
|
|||
client = new DataFileInterface(config);
|
||||
implName = 'file';
|
||||
} else if (config.backends.data === 'multiple') {
|
||||
const clients = parseLC(config, vault);
|
||||
const clients = parseLC(config, vault, metadata);
|
||||
client = new MultipleBackendGateway(
|
||||
clients, metadata, locationStorageCheck);
|
||||
implName = 'multipleBackends';
|
||||
|
|
|
@ -12,20 +12,6 @@ const Common = require('./common');
|
|||
const vault = require('../auth/vault');
|
||||
const Cache = require('./Cache');
|
||||
const cache = new Cache();
|
||||
let scalityKMS;
|
||||
let scalityKMSImpl;
|
||||
try {
|
||||
// eslint-disable-next-line import/no-unresolved
|
||||
const ScalityKMS = require('scality-kms');
|
||||
scalityKMS = new ScalityKMS(config.kms);
|
||||
scalityKMSImpl = 'scalityKms';
|
||||
} catch (error) {
|
||||
logger.warn('scality kms unavailable. ' +
|
||||
'Using file kms backend unless mem specified.',
|
||||
{ error });
|
||||
scalityKMS = file;
|
||||
scalityKMSImpl = 'fileKms';
|
||||
}
|
||||
|
||||
let client;
|
||||
let implName;
|
||||
|
@ -37,8 +23,9 @@ if (config.backends.kms === 'mem') {
|
|||
client = file;
|
||||
implName = 'fileKms';
|
||||
} else if (config.backends.kms === 'scality') {
|
||||
client = scalityKMS;
|
||||
implName = scalityKMSImpl;
|
||||
const ScalityKMS = require('scality-kms');
|
||||
client = new ScalityKMS(config.kms);
|
||||
implName = 'scalityKms';
|
||||
} else if (config.backends.kms === 'kmip') {
|
||||
const kmipConfig = { kmip: config.kmip };
|
||||
if (!kmipConfig.kmip) {
|
||||
|
|
|
@ -1,131 +0,0 @@
|
|||
/**
|
||||
* Target service that should handle a message
|
||||
* @readonly
|
||||
* @enum {number}
|
||||
*/
|
||||
const MessageType = {
|
||||
/** Message that contains a configuration overlay */
|
||||
CONFIG_OVERLAY_MESSAGE: 1,
|
||||
/** Message that requests a metrics report */
|
||||
METRICS_REQUEST_MESSAGE: 2,
|
||||
/** Message that contains a metrics report */
|
||||
METRICS_REPORT_MESSAGE: 3,
|
||||
/** Close the virtual TCP socket associated to the channel */
|
||||
CHANNEL_CLOSE_MESSAGE: 4,
|
||||
/** Write data to the virtual TCP socket associated to the channel */
|
||||
CHANNEL_PAYLOAD_MESSAGE: 5,
|
||||
};
|
||||
|
||||
/**
|
||||
* Target service that should handle a message
|
||||
* @readonly
|
||||
* @enum {number}
|
||||
*/
|
||||
const TargetType = {
|
||||
/** Let the dispatcher choose the most appropriate message */
|
||||
TARGET_ANY: 0,
|
||||
};
|
||||
|
||||
const headerSize = 3;
|
||||
|
||||
class ChannelMessageV0 {
|
||||
/**
|
||||
* @param {Buffer} buffer Message bytes
|
||||
*/
|
||||
constructor(buffer) {
|
||||
this.messageType = buffer.readUInt8(0);
|
||||
this.channelNumber = buffer.readUInt8(1);
|
||||
this.target = buffer.readUInt8(2);
|
||||
this.payload = buffer.slice(headerSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} Message type
|
||||
*/
|
||||
getType() {
|
||||
return this.messageType;
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} Channel number if applicable
|
||||
*/
|
||||
getChannelNumber() {
|
||||
return this.channelNumber;
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} Target service, or 0 to choose automatically
|
||||
*/
|
||||
getTarget() {
|
||||
return this.target;
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Buffer} Message payload if applicable
|
||||
*/
|
||||
getPayload() {
|
||||
return this.payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a wire representation of a channel close message
|
||||
*
|
||||
* @param {number} channelId Channel number
|
||||
*
|
||||
* @returns {Buffer} wire representation
|
||||
*/
|
||||
static encodeChannelCloseMessage(channelId) {
|
||||
const buf = Buffer.alloc(headerSize);
|
||||
buf.writeUInt8(MessageType.CHANNEL_CLOSE_MESSAGE, 0);
|
||||
buf.writeUInt8(channelId, 1);
|
||||
buf.writeUInt8(TargetType.TARGET_ANY, 2);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a wire representation of a channel data message
|
||||
*
|
||||
* @param {number} channelId Channel number
|
||||
* @param {Buffer} data Payload
|
||||
*
|
||||
* @returns {Buffer} wire representation
|
||||
*/
|
||||
static encodeChannelDataMessage(channelId, data) {
|
||||
const buf = Buffer.alloc(data.length + headerSize);
|
||||
buf.writeUInt8(MessageType.CHANNEL_PAYLOAD_MESSAGE, 0);
|
||||
buf.writeUInt8(channelId, 1);
|
||||
buf.writeUInt8(TargetType.TARGET_ANY, 2);
|
||||
data.copy(buf, headerSize);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a wire representation of a metrics message
|
||||
*
|
||||
* @param {object} body Metrics report
|
||||
*
|
||||
* @returns {Buffer} wire representation
|
||||
*/
|
||||
static encodeMetricsReportMessage(body) {
|
||||
const report = JSON.stringify(body);
|
||||
const buf = Buffer.alloc(report.length + headerSize);
|
||||
buf.writeUInt8(MessageType.METRICS_REPORT_MESSAGE, 0);
|
||||
buf.writeUInt8(0, 1);
|
||||
buf.writeUInt8(TargetType.TARGET_ANY, 2);
|
||||
buf.write(report, headerSize);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Protocol name used for subprotocol negociation
|
||||
*/
|
||||
static get protocolName() {
|
||||
return 'zenko-secure-channel-v0';
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ChannelMessageV0,
|
||||
MessageType,
|
||||
TargetType,
|
||||
};
|
|
@ -1,94 +0,0 @@
|
|||
const WebSocket = require('ws');
|
||||
const arsenal = require('arsenal');
|
||||
|
||||
const logger = require('../utilities/logger');
|
||||
const _config = require('../Config').config;
|
||||
const { patchConfiguration } = require('./configuration');
|
||||
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||
|
||||
|
||||
const managementAgentMessageType = {
|
||||
/** Message that contains the loaded overlay */
|
||||
NEW_OVERLAY: 1,
|
||||
};
|
||||
|
||||
const CONNECTION_RETRY_TIMEOUT_MS = 5000;
|
||||
|
||||
|
||||
function initManagementClient() {
|
||||
const { host, port } = _config.managementAgent;
|
||||
|
||||
const ws = new WebSocket(`ws://${host}:${port}/watch`);
|
||||
|
||||
ws.on('open', () => {
|
||||
logger.info('connected with management agent');
|
||||
});
|
||||
|
||||
ws.on('close', (code, reason) => {
|
||||
logger.info('disconnected from management agent', { reason });
|
||||
setTimeout(initManagementClient, CONNECTION_RETRY_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
ws.on('error', error => {
|
||||
logger.error('error on connection with management agent', { error });
|
||||
});
|
||||
|
||||
ws.on('message', data => {
|
||||
const method = 'initManagementclient::onMessage';
|
||||
const log = logger.newRequestLogger();
|
||||
let msg;
|
||||
|
||||
if (!data) {
|
||||
log.error('message without data', { method });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
msg = JSON.parse(data);
|
||||
} catch (err) {
|
||||
log.error('data is an invalid json', { method, err, data });
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.payload === undefined) {
|
||||
log.error('message without payload', { method });
|
||||
return;
|
||||
}
|
||||
if (typeof msg.messageType !== 'number') {
|
||||
log.error('messageType is not an integer', {
|
||||
type: typeof msg.messageType,
|
||||
method,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
switch (msg.messageType) {
|
||||
case managementAgentMessageType.NEW_OVERLAY:
|
||||
patchConfiguration(msg.payload, log, err => {
|
||||
if (err) {
|
||||
log.error('failed to patch overlay', {
|
||||
error: reshapeExceptionError(err),
|
||||
method,
|
||||
});
|
||||
}
|
||||
});
|
||||
return;
|
||||
default:
|
||||
log.error('new overlay message with unmanaged message type', {
|
||||
method,
|
||||
type: msg.messageType,
|
||||
});
|
||||
return;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function isManagementAgentUsed() {
|
||||
return process.env.MANAGEMENT_USE_AGENT === '1';
|
||||
}
|
||||
|
||||
|
||||
module.exports = {
|
||||
managementAgentMessageType,
|
||||
initManagementClient,
|
||||
isManagementAgentUsed,
|
||||
};
|
|
@ -1,240 +0,0 @@
|
|||
const arsenal = require('arsenal');
|
||||
|
||||
const { buildAuthDataAccount } = require('../auth/in_memory/builder');
|
||||
const _config = require('../Config').config;
|
||||
const metadata = require('../metadata/wrapper');
|
||||
|
||||
const { getStoredCredentials } = require('./credentials');
|
||||
|
||||
const latestOverlayVersionKey = 'configuration/overlay-version';
|
||||
const managementDatabaseName = 'PENSIEVE';
|
||||
const replicatorEndpoint = 'zenko-cloudserver-replicator';
|
||||
const { decryptSecret } = arsenal.pensieve.credentialUtils;
|
||||
const { patchLocations } = arsenal.patches.locationConstraints;
|
||||
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||
const { replicationBackends } = require('arsenal').constants;
|
||||
|
||||
function overlayHasVersion(overlay) {
|
||||
return overlay && overlay.version !== undefined;
|
||||
}
|
||||
|
||||
function remoteOverlayIsNewer(cachedOverlay, remoteOverlay) {
|
||||
return (overlayHasVersion(remoteOverlay) &&
|
||||
(!overlayHasVersion(cachedOverlay) ||
|
||||
remoteOverlay.version > cachedOverlay.version));
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the live {Config} object with the new overlay configuration.
|
||||
*
|
||||
* No-op if this version was already applied to the live {Config}.
|
||||
*
|
||||
* @param {object} newConf Overlay configuration to apply
|
||||
* @param {werelogs~Logger} log Request-scoped logger
|
||||
* @param {function} cb Function to call with (error, newConf)
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function patchConfiguration(newConf, log, cb) {
|
||||
if (newConf.version === undefined) {
|
||||
log.debug('no remote configuration created yet');
|
||||
return process.nextTick(cb, null, newConf);
|
||||
}
|
||||
|
||||
if (_config.overlayVersion !== undefined &&
|
||||
newConf.version <= _config.overlayVersion) {
|
||||
log.debug('configuration version already applied',
|
||||
{ configurationVersion: newConf.version });
|
||||
return process.nextTick(cb, null, newConf);
|
||||
}
|
||||
return getStoredCredentials(log, (err, creds) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
const accounts = [];
|
||||
if (newConf.users) {
|
||||
newConf.users.forEach(u => {
|
||||
if (u.secretKey && u.secretKey.length > 0) {
|
||||
const secretKey = decryptSecret(creds, u.secretKey);
|
||||
// accountType will be service-replication or service-clueso
|
||||
let serviceName;
|
||||
if (u.accountType && u.accountType.startsWith('service-')) {
|
||||
serviceName = u.accountType.split('-')[1];
|
||||
}
|
||||
const newAccount = buildAuthDataAccount(
|
||||
u.accessKey, secretKey, u.canonicalId, serviceName,
|
||||
u.userName);
|
||||
accounts.push(newAccount.accounts[0]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const restEndpoints = Object.assign({}, _config.restEndpoints);
|
||||
if (newConf.endpoints) {
|
||||
newConf.endpoints.forEach(e => {
|
||||
restEndpoints[e.hostname] = e.locationName;
|
||||
});
|
||||
}
|
||||
|
||||
if (!restEndpoints[replicatorEndpoint]) {
|
||||
restEndpoints[replicatorEndpoint] = 'us-east-1';
|
||||
}
|
||||
|
||||
const locations = patchLocations(newConf.locations, creds, log);
|
||||
if (Object.keys(locations).length !== 0) {
|
||||
try {
|
||||
_config.setLocationConstraints(locations);
|
||||
} catch (error) {
|
||||
const exceptionError = reshapeExceptionError(error);
|
||||
log.error('could not apply configuration version location ' +
|
||||
'constraints', { error: exceptionError,
|
||||
method: 'getStoredCredentials' });
|
||||
return cb(exceptionError);
|
||||
}
|
||||
try {
|
||||
const locationsWithReplicationBackend = Object.keys(locations)
|
||||
// NOTE: In Orbit, we don't need to have Scality location in our
|
||||
// replication endpoind config, since we do not replicate to
|
||||
// any Scality Instance yet.
|
||||
.filter(key => replicationBackends
|
||||
[locations[key].type])
|
||||
.reduce((obj, key) => {
|
||||
/* eslint no-param-reassign:0 */
|
||||
obj[key] = locations[key];
|
||||
return obj;
|
||||
}, {});
|
||||
_config.setReplicationEndpoints(
|
||||
locationsWithReplicationBackend);
|
||||
} catch (error) {
|
||||
const exceptionError = reshapeExceptionError(error);
|
||||
log.error('could not apply replication endpoints',
|
||||
{ error: exceptionError, method: 'getStoredCredentials' });
|
||||
return cb(exceptionError);
|
||||
}
|
||||
}
|
||||
|
||||
_config.setAuthDataAccounts(accounts);
|
||||
_config.setRestEndpoints(restEndpoints);
|
||||
_config.setPublicInstanceId(newConf.instanceId);
|
||||
|
||||
if (newConf.browserAccess) {
|
||||
if (Boolean(_config.browserAccessEnabled) !==
|
||||
Boolean(newConf.browserAccess.enabled)) {
|
||||
_config.browserAccessEnabled =
|
||||
Boolean(newConf.browserAccess.enabled);
|
||||
_config.emit('browser-access-enabled-change');
|
||||
}
|
||||
}
|
||||
|
||||
_config.overlayVersion = newConf.version;
|
||||
|
||||
log.info('applied configuration version',
|
||||
{ configurationVersion: _config.overlayVersion });
|
||||
|
||||
return cb(null, newConf);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes configuration version to the management database
|
||||
*
|
||||
* @param {object} cachedOverlay Latest stored configuration version
|
||||
* for freshness comparison purposes
|
||||
* @param {object} remoteOverlay New configuration version
|
||||
* @param {werelogs~Logger} log Request-scoped logger
|
||||
* @param {function} cb Function to call with (error, remoteOverlay)
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function saveConfigurationVersion(cachedOverlay, remoteOverlay, log, cb) {
|
||||
if (remoteOverlayIsNewer(cachedOverlay, remoteOverlay)) {
|
||||
const objName = `configuration/overlay/${remoteOverlay.version}`;
|
||||
metadata.putObjectMD(managementDatabaseName, objName, remoteOverlay,
|
||||
{}, log, error => {
|
||||
if (error) {
|
||||
const exceptionError = reshapeExceptionError(error);
|
||||
log.error('could not save configuration',
|
||||
{ error: exceptionError,
|
||||
method: 'saveConfigurationVersion',
|
||||
configurationVersion: remoteOverlay.version });
|
||||
cb(exceptionError);
|
||||
return;
|
||||
}
|
||||
metadata.putObjectMD(managementDatabaseName,
|
||||
latestOverlayVersionKey, remoteOverlay.version, {}, log,
|
||||
error => {
|
||||
if (error) {
|
||||
log.error('could not save configuration version', {
|
||||
configurationVersion: remoteOverlay.version,
|
||||
});
|
||||
}
|
||||
cb(error, remoteOverlay);
|
||||
});
|
||||
});
|
||||
} else {
|
||||
log.debug('no remote configuration to cache yet');
|
||||
process.nextTick(cb, null, remoteOverlay);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the latest cached configuration overlay from the management
|
||||
* database, without contacting the Orbit API.
|
||||
*
|
||||
* @param {werelogs~Logger} log Request-scoped logger
|
||||
* @param {function} callback Function called with (error, cachedOverlay)
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function loadCachedOverlay(log, callback) {
|
||||
return metadata.getObjectMD(managementDatabaseName,
|
||||
latestOverlayVersionKey, {}, log, (err, version) => {
|
||||
if (err) {
|
||||
if (err.is.NoSuchKey) {
|
||||
return process.nextTick(callback, null, {});
|
||||
}
|
||||
return callback(err);
|
||||
}
|
||||
return metadata.getObjectMD(managementDatabaseName,
|
||||
`configuration/overlay/${version}`, {}, log, (err, conf) => {
|
||||
if (err) {
|
||||
if (err.is.NoSuchKey) {
|
||||
return process.nextTick(callback, null, {});
|
||||
}
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, conf);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function applyAndSaveOverlay(overlay, log) {
|
||||
patchConfiguration(overlay, log, err => {
|
||||
if (err) {
|
||||
log.error('could not apply pushed overlay', {
|
||||
error: reshapeExceptionError(err),
|
||||
method: 'applyAndSaveOverlay',
|
||||
});
|
||||
return;
|
||||
}
|
||||
saveConfigurationVersion(null, overlay, log, err => {
|
||||
if (err) {
|
||||
log.error('could not cache overlay version', {
|
||||
error: reshapeExceptionError(err),
|
||||
method: 'applyAndSaveOverlay',
|
||||
});
|
||||
return;
|
||||
}
|
||||
log.info('overlay push processed');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
loadCachedOverlay,
|
||||
managementDatabaseName,
|
||||
patchConfiguration,
|
||||
saveConfigurationVersion,
|
||||
remoteOverlayIsNewer,
|
||||
applyAndSaveOverlay,
|
||||
};
|
|
@ -1,145 +0,0 @@
|
|||
const arsenal = require('arsenal');
|
||||
const forge = require('node-forge');
|
||||
const request = require('../utilities/request');
|
||||
|
||||
const metadata = require('../metadata/wrapper');
|
||||
|
||||
const managementDatabaseName = 'PENSIEVE';
|
||||
const tokenConfigurationKey = 'auth/zenko/remote-management-token';
|
||||
const tokenRotationDelay = 3600 * 24 * 7 * 1000; // 7 days
|
||||
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||
|
||||
/**
|
||||
* Retrieves Orbit API token from the management database.
|
||||
*
|
||||
* The token is used to authenticate stat posting and
|
||||
*
|
||||
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
|
||||
* initialization process
|
||||
* @param {function} callback Function called with (error, result)
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function getStoredCredentials(log, callback) {
|
||||
metadata.getObjectMD(managementDatabaseName, tokenConfigurationKey, {},
|
||||
log, callback);
|
||||
}
|
||||
|
||||
function issueCredentials(managementEndpoint, instanceId, log, callback) {
|
||||
log.info('registering with API to get token');
|
||||
|
||||
const keyPair = forge.pki.rsa.generateKeyPair({ bits: 2048, e: 0x10001 });
|
||||
const privateKey = forge.pki.privateKeyToPem(keyPair.privateKey);
|
||||
const publicKey = forge.pki.publicKeyToPem(keyPair.publicKey);
|
||||
|
||||
const postData = {
|
||||
publicKey,
|
||||
};
|
||||
|
||||
request.post(`${managementEndpoint}/${instanceId}/register`,
|
||||
{ body: postData, json: true }, (error, response, body) => {
|
||||
if (error) {
|
||||
return callback(error);
|
||||
}
|
||||
if (response.statusCode !== 201) {
|
||||
log.error('could not register instance', {
|
||||
statusCode: response.statusCode,
|
||||
});
|
||||
return callback(arsenal.errors.InternalError);
|
||||
}
|
||||
/* eslint-disable no-param-reassign */
|
||||
body.privateKey = privateKey;
|
||||
/* eslint-enable no-param-reassign */
|
||||
return callback(null, body);
|
||||
});
|
||||
}
|
||||
|
||||
function confirmInstanceCredentials(
|
||||
managementEndpoint, instanceId, creds, log, callback) {
|
||||
const postData = {
|
||||
serial: creds.serial || 0,
|
||||
publicKey: creds.publicKey,
|
||||
};
|
||||
|
||||
const opts = {
|
||||
headers: {
|
||||
'x-instance-authentication-token': creds.token,
|
||||
},
|
||||
body: postData,
|
||||
};
|
||||
|
||||
request.post(`${managementEndpoint}/${instanceId}/confirm`,
|
||||
opts, (error, response) => {
|
||||
if (error) {
|
||||
return callback(error);
|
||||
}
|
||||
if (response.statusCode === 200) {
|
||||
return callback(null, instanceId, creds.token);
|
||||
}
|
||||
return callback(arsenal.errors.InternalError);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes credentials and PKI in the management database.
|
||||
*
|
||||
* In case the management database is new and empty, the instance
|
||||
* is registered as new against the Orbit API with newly-generated
|
||||
* RSA key pair.
|
||||
*
|
||||
* @param {string} managementEndpoint API endpoint
|
||||
* @param {string} instanceId UUID of this deployment
|
||||
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
|
||||
* initialization process
|
||||
* @param {function} callback Function called with (error, result)
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function initManagementCredentials(
|
||||
managementEndpoint, instanceId, log, callback) {
|
||||
getStoredCredentials(log, (error, value) => {
|
||||
if (error) {
|
||||
if (error.is.NoSuchKey) {
|
||||
return issueCredentials(managementEndpoint, instanceId, log,
|
||||
(error, value) => {
|
||||
if (error) {
|
||||
log.error('could not issue token',
|
||||
{ error: reshapeExceptionError(error),
|
||||
method: 'initManagementCredentials' });
|
||||
return callback(error);
|
||||
}
|
||||
log.debug('saving token');
|
||||
return metadata.putObjectMD(managementDatabaseName,
|
||||
tokenConfigurationKey, value, {}, log, error => {
|
||||
if (error) {
|
||||
log.error('could not save token',
|
||||
{ error: reshapeExceptionError(error),
|
||||
method: 'initManagementCredentials',
|
||||
});
|
||||
return callback(error);
|
||||
}
|
||||
log.info('saved token locally, ' +
|
||||
'confirming instance');
|
||||
return confirmInstanceCredentials(
|
||||
managementEndpoint, instanceId, value, log,
|
||||
callback);
|
||||
});
|
||||
});
|
||||
}
|
||||
log.debug('could not get token', { error });
|
||||
return callback(error);
|
||||
}
|
||||
|
||||
log.info('returning existing token');
|
||||
if (Date.now() - value.issueDate > tokenRotationDelay) {
|
||||
log.warn('management API token is too old, should re-issue');
|
||||
}
|
||||
|
||||
return callback(null, instanceId, value.token);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getStoredCredentials,
|
||||
initManagementCredentials,
|
||||
};
|
|
@ -1,138 +0,0 @@
|
|||
const arsenal = require('arsenal');
|
||||
const async = require('async');
|
||||
|
||||
const metadata = require('../metadata/wrapper');
|
||||
const logger = require('../utilities/logger');
|
||||
|
||||
const {
|
||||
loadCachedOverlay,
|
||||
managementDatabaseName,
|
||||
patchConfiguration,
|
||||
} = require('./configuration');
|
||||
const { initManagementCredentials } = require('./credentials');
|
||||
const { startWSManagementClient } = require('./push');
|
||||
const { startPollingManagementClient } = require('./poll');
|
||||
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||
const { isManagementAgentUsed } = require('./agentClient');
|
||||
|
||||
const initRemoteManagementRetryDelay = 10000;
|
||||
|
||||
const managementEndpointRoot =
|
||||
process.env.MANAGEMENT_ENDPOINT ||
|
||||
'https://api.zenko.io';
|
||||
const managementEndpoint = `${managementEndpointRoot}/api/v1/instance`;
|
||||
|
||||
const pushEndpointRoot =
|
||||
process.env.PUSH_ENDPOINT ||
|
||||
'https://push.api.zenko.io';
|
||||
const pushEndpoint = `${pushEndpointRoot}/api/v1/instance`;
|
||||
|
||||
function initManagementDatabase(log, callback) {
|
||||
// XXX choose proper owner names
|
||||
const md = new arsenal.models.BucketInfo(managementDatabaseName, 'owner',
|
||||
'owner display name', new Date().toJSON());
|
||||
|
||||
metadata.createBucket(managementDatabaseName, md, log, error => {
|
||||
if (error) {
|
||||
if (error.is.BucketAlreadyExists) {
|
||||
log.info('created management database');
|
||||
return callback();
|
||||
}
|
||||
log.error('could not initialize management database',
|
||||
{ error: reshapeExceptionError(error),
|
||||
method: 'initManagementDatabase' });
|
||||
return callback(error);
|
||||
}
|
||||
log.info('initialized management database');
|
||||
return callback();
|
||||
});
|
||||
}
|
||||
|
||||
function startManagementListeners(instanceId, token) {
|
||||
const mode = process.env.MANAGEMENT_MODE || 'push';
|
||||
if (mode === 'push') {
|
||||
const url = `${pushEndpoint}/${instanceId}/ws`;
|
||||
startWSManagementClient(url, token);
|
||||
} else {
|
||||
startPollingManagementClient(managementEndpoint, instanceId, token);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes Orbit-based management by:
|
||||
* - creating the management database in metadata
|
||||
* - generating a key pair for credentials encryption
|
||||
* - generating an instance-unique ID
|
||||
* - getting an authentication token for the API
|
||||
* - loading and applying the latest cached overlay configuration
|
||||
* - starting a configuration update and metrics push background task
|
||||
*
|
||||
* @param {werelogs~Logger} log Request-scoped logger to be able to trace
|
||||
* initialization process
|
||||
* @param {function} callback Function to call once the overlay is loaded
|
||||
* (overlay)
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function initManagement(log, callback) {
|
||||
if ((process.env.REMOTE_MANAGEMENT_DISABLE &&
|
||||
process.env.REMOTE_MANAGEMENT_DISABLE !== '0')
|
||||
|| process.env.S3BACKEND === 'mem') {
|
||||
log.info('remote management disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
/* Temporary check before to fully move to the process management agent. */
|
||||
if (isManagementAgentUsed() ^ typeof callback === 'function') {
|
||||
let msg = 'misuse of initManagement function: ';
|
||||
msg += `MANAGEMENT_USE_AGENT: ${process.env.MANAGEMENT_USE_AGENT}`;
|
||||
msg += `, callback type: ${typeof callback}`;
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
async.waterfall([
|
||||
// eslint-disable-next-line arrow-body-style
|
||||
cb => { return isManagementAgentUsed() ? metadata.setup(cb) : cb(); },
|
||||
cb => initManagementDatabase(log, cb),
|
||||
cb => metadata.getUUID(log, cb),
|
||||
(instanceId, cb) => initManagementCredentials(
|
||||
managementEndpoint, instanceId, log, cb),
|
||||
(instanceId, token, cb) => {
|
||||
if (!isManagementAgentUsed()) {
|
||||
cb(null, instanceId, token, {});
|
||||
return;
|
||||
}
|
||||
loadCachedOverlay(log, (err, overlay) => cb(err, instanceId,
|
||||
token, overlay));
|
||||
},
|
||||
(instanceId, token, overlay, cb) => {
|
||||
if (!isManagementAgentUsed()) {
|
||||
cb(null, instanceId, token, overlay);
|
||||
return;
|
||||
}
|
||||
patchConfiguration(overlay, log,
|
||||
err => cb(err, instanceId, token, overlay));
|
||||
},
|
||||
], (error, instanceId, token, overlay) => {
|
||||
if (error) {
|
||||
log.error('could not initialize remote management, retrying later',
|
||||
{ error: reshapeExceptionError(error),
|
||||
method: 'initManagement' });
|
||||
setTimeout(initManagement,
|
||||
initRemoteManagementRetryDelay,
|
||||
logger.newRequestLogger());
|
||||
} else {
|
||||
log.info(`this deployment's Instance ID is ${instanceId}`);
|
||||
log.end('management init done');
|
||||
startManagementListeners(instanceId, token);
|
||||
if (callback) {
|
||||
callback(overlay);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
initManagement,
|
||||
initManagementDatabase,
|
||||
};
|
|
@ -1,157 +0,0 @@
|
|||
const arsenal = require('arsenal');
|
||||
const async = require('async');
|
||||
const request = require('../utilities/request');
|
||||
|
||||
const _config = require('../Config').config;
|
||||
const logger = require('../utilities/logger');
|
||||
const metadata = require('../metadata/wrapper');
|
||||
const {
|
||||
loadCachedOverlay,
|
||||
patchConfiguration,
|
||||
saveConfigurationVersion,
|
||||
} = require('./configuration');
|
||||
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||
|
||||
const pushReportDelay = 30000;
|
||||
const pullConfigurationOverlayDelay = 60000;
|
||||
|
||||
function loadRemoteOverlay(
|
||||
managementEndpoint, instanceId, remoteToken, cachedOverlay, log, cb) {
|
||||
log.debug('loading remote overlay');
|
||||
const opts = {
|
||||
headers: {
|
||||
'x-instance-authentication-token': remoteToken,
|
||||
'x-scal-request-id': log.getSerializedUids(),
|
||||
},
|
||||
json: true,
|
||||
};
|
||||
request.get(`${managementEndpoint}/${instanceId}/config/overlay`, opts,
|
||||
(error, response, body) => {
|
||||
if (error) {
|
||||
return cb(error);
|
||||
}
|
||||
if (response.statusCode === 200) {
|
||||
return cb(null, cachedOverlay, body);
|
||||
}
|
||||
if (response.statusCode === 404) {
|
||||
return cb(null, cachedOverlay, {});
|
||||
}
|
||||
return cb(arsenal.errors.AccessForbidden, cachedOverlay, {});
|
||||
});
|
||||
}
|
||||
|
||||
// TODO save only after successful patch
|
||||
function applyConfigurationOverlay(
|
||||
managementEndpoint, instanceId, remoteToken, log) {
|
||||
async.waterfall([
|
||||
wcb => loadCachedOverlay(log, wcb),
|
||||
(cachedOverlay, wcb) => patchConfiguration(cachedOverlay,
|
||||
log, wcb),
|
||||
(cachedOverlay, wcb) =>
|
||||
loadRemoteOverlay(managementEndpoint, instanceId, remoteToken,
|
||||
cachedOverlay, log, wcb),
|
||||
(cachedOverlay, remoteOverlay, wcb) =>
|
||||
saveConfigurationVersion(cachedOverlay, remoteOverlay, log, wcb),
|
||||
(remoteOverlay, wcb) => patchConfiguration(remoteOverlay,
|
||||
log, wcb),
|
||||
], error => {
|
||||
if (error) {
|
||||
log.error('could not apply managed configuration',
|
||||
{ error: reshapeExceptionError(error),
|
||||
method: 'applyConfigurationOverlay' });
|
||||
}
|
||||
setTimeout(applyConfigurationOverlay, pullConfigurationOverlayDelay,
|
||||
managementEndpoint, instanceId, remoteToken,
|
||||
logger.newRequestLogger());
|
||||
});
|
||||
}
|
||||
|
||||
function postStats(managementEndpoint, instanceId, remoteToken, report, next) {
|
||||
const toURL = `${managementEndpoint}/${instanceId}/stats`;
|
||||
const toOptions = {
|
||||
json: true,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-instance-authentication-token': remoteToken,
|
||||
},
|
||||
body: report,
|
||||
};
|
||||
const toCallback = (err, response, body) => {
|
||||
if (err) {
|
||||
logger.info('could not post stats', { error: err });
|
||||
}
|
||||
if (response && response.statusCode !== 201) {
|
||||
logger.info('could not post stats', {
|
||||
body,
|
||||
statusCode: response.statusCode,
|
||||
});
|
||||
}
|
||||
if (next) {
|
||||
next(null, instanceId, remoteToken);
|
||||
}
|
||||
};
|
||||
return request.post(toURL, toOptions, toCallback);
|
||||
}
|
||||
|
||||
function getStats(next) {
|
||||
const fromURL = `http://localhost:${_config.port}/_/report`;
|
||||
const fromOptions = {
|
||||
headers: {
|
||||
'x-scal-report-token': process.env.REPORT_TOKEN,
|
||||
},
|
||||
};
|
||||
return request.get(fromURL, fromOptions, next);
|
||||
}
|
||||
|
||||
function pushStats(managementEndpoint, instanceId, remoteToken, next) {
|
||||
if (process.env.PUSH_STATS === 'false') {
|
||||
return;
|
||||
}
|
||||
|
||||
getStats((err, res, report) => {
|
||||
if (err) {
|
||||
logger.info('could not retrieve stats', { error: err });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug('report', { report });
|
||||
postStats(
|
||||
managementEndpoint,
|
||||
instanceId,
|
||||
remoteToken,
|
||||
report,
|
||||
next
|
||||
);
|
||||
return;
|
||||
});
|
||||
|
||||
setTimeout(pushStats, pushReportDelay,
|
||||
managementEndpoint, instanceId, remoteToken);
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts background task that updates configuration and pushes stats.
|
||||
*
|
||||
* Periodically polls for configuration updates, and pushes stats at
|
||||
* a fixed interval.
|
||||
*
|
||||
* @param {string} managementEndpoint API endpoint
|
||||
* @param {string} instanceId UUID of this deployment
|
||||
* @param {string} remoteToken API authentication token
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function startPollingManagementClient(
|
||||
managementEndpoint, instanceId, remoteToken) {
|
||||
metadata.notifyBucketChange(() => {
|
||||
pushStats(managementEndpoint, instanceId, remoteToken);
|
||||
});
|
||||
|
||||
pushStats(managementEndpoint, instanceId, remoteToken);
|
||||
applyConfigurationOverlay(managementEndpoint, instanceId, remoteToken,
|
||||
logger.newRequestLogger());
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startPollingManagementClient,
|
||||
};
|
|
@ -1,301 +0,0 @@
|
|||
const arsenal = require('arsenal');
|
||||
const HttpsProxyAgent = require('https-proxy-agent');
|
||||
const net = require('net');
|
||||
const request = require('../utilities/request');
|
||||
const { URL } = require('url');
|
||||
const WebSocket = require('ws');
|
||||
const assert = require('assert');
|
||||
const http = require('http');
|
||||
|
||||
const _config = require('../Config').config;
|
||||
const logger = require('../utilities/logger');
|
||||
const metadata = require('../metadata/wrapper');
|
||||
|
||||
const { reshapeExceptionError } = arsenal.errorUtils;
|
||||
const { isManagementAgentUsed } = require('./agentClient');
|
||||
const { applyAndSaveOverlay } = require('./configuration');
|
||||
const {
|
||||
ChannelMessageV0,
|
||||
MessageType,
|
||||
} = require('./ChannelMessageV0');
|
||||
|
||||
const {
|
||||
CONFIG_OVERLAY_MESSAGE,
|
||||
METRICS_REQUEST_MESSAGE,
|
||||
CHANNEL_CLOSE_MESSAGE,
|
||||
CHANNEL_PAYLOAD_MESSAGE,
|
||||
} = MessageType;
|
||||
|
||||
const PING_INTERVAL_MS = 10000;
|
||||
const subprotocols = [ChannelMessageV0.protocolName];
|
||||
|
||||
const cloudServerHost = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_HOST
|
||||
|| 'localhost';
|
||||
const cloudServerPort = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_PORT
|
||||
|| _config.port;
|
||||
|
||||
let overlayMessageListener = null;
|
||||
let connected = false;
|
||||
|
||||
// No wildcard nor cidr/mask match for now
|
||||
function createWSAgent(pushEndpoint, env, log) {
|
||||
const url = new URL(pushEndpoint);
|
||||
const noProxy = (env.NO_PROXY || env.no_proxy
|
||||
|| '').split(',');
|
||||
|
||||
if (noProxy.includes(url.hostname)) {
|
||||
log.info('push server ws has proxy exclusion', { noProxy });
|
||||
return null;
|
||||
}
|
||||
|
||||
if (url.protocol === 'https:' || url.protocol === 'wss:') {
|
||||
const httpsProxy = (env.HTTPS_PROXY || env.https_proxy);
|
||||
if (httpsProxy) {
|
||||
log.info('push server ws using https proxy', { httpsProxy });
|
||||
return new HttpsProxyAgent(httpsProxy);
|
||||
}
|
||||
} else if (url.protocol === 'http:' || url.protocol === 'ws:') {
|
||||
const httpProxy = (env.HTTP_PROXY || env.http_proxy);
|
||||
if (httpProxy) {
|
||||
log.info('push server ws using http proxy', { httpProxy });
|
||||
return new HttpsProxyAgent(httpProxy);
|
||||
}
|
||||
}
|
||||
|
||||
const allProxy = (env.ALL_PROXY || env.all_proxy);
|
||||
if (allProxy) {
|
||||
log.info('push server ws using wildcard proxy', { allProxy });
|
||||
return new HttpsProxyAgent(allProxy);
|
||||
}
|
||||
|
||||
log.info('push server ws not using proxy');
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts background task that updates configuration and pushes stats.
|
||||
*
|
||||
* Receives pushed Websocket messages on configuration updates, and
|
||||
* sends stat messages in response to API sollicitations.
|
||||
*
|
||||
* @param {string} url API endpoint
|
||||
* @param {string} token API authentication token
|
||||
* @param {function} cb end-of-connection callback
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
function startWSManagementClient(url, token, cb) {
|
||||
logger.info('connecting to push server', { url });
|
||||
function _logError(error, errorMessage, method) {
|
||||
if (error) {
|
||||
logger.error(`management client error: ${errorMessage}`,
|
||||
{ error: reshapeExceptionError(error), method });
|
||||
}
|
||||
}
|
||||
|
||||
const socketsByChannelId = [];
|
||||
const headers = {
|
||||
'x-instance-authentication-token': token,
|
||||
};
|
||||
const agent = createWSAgent(url, process.env, logger);
|
||||
|
||||
const ws = new WebSocket(url, subprotocols, { headers, agent });
|
||||
let pingTimeout = null;
|
||||
|
||||
function sendPing() {
|
||||
if (ws.readyState === ws.OPEN) {
|
||||
ws.ping(err => _logError(err, 'failed to send a ping', 'sendPing'));
|
||||
}
|
||||
pingTimeout = setTimeout(() => ws.terminate(), PING_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function initiatePing() {
|
||||
clearTimeout(pingTimeout);
|
||||
setTimeout(sendPing, PING_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function pushStats(options) {
|
||||
if (process.env.PUSH_STATS === 'false') {
|
||||
return;
|
||||
}
|
||||
const fromURL = `http://${cloudServerHost}:${cloudServerPort}/_/report`;
|
||||
const fromOptions = {
|
||||
json: true,
|
||||
headers: {
|
||||
'x-scal-report-token': process.env.REPORT_TOKEN,
|
||||
'x-scal-report-skip-cache': Boolean(options && options.noCache),
|
||||
},
|
||||
};
|
||||
request.get(fromURL, fromOptions, (err, response, body) => {
|
||||
if (err) {
|
||||
_logError(err, 'failed to get metrics report', 'pushStats');
|
||||
return;
|
||||
}
|
||||
ws.send(ChannelMessageV0.encodeMetricsReportMessage(body),
|
||||
err => _logError(err, 'failed to send metrics report message',
|
||||
'pushStats'));
|
||||
});
|
||||
}
|
||||
|
||||
function closeChannel(channelId) {
|
||||
const socket = socketsByChannelId[channelId];
|
||||
if (socket) {
|
||||
socket.destroy();
|
||||
delete socketsByChannelId[channelId];
|
||||
}
|
||||
}
|
||||
|
||||
function receiveChannelData(channelId, payload) {
|
||||
let socket = socketsByChannelId[channelId];
|
||||
if (!socket) {
|
||||
socket = net.createConnection(cloudServerPort, cloudServerHost);
|
||||
|
||||
socket.on('data', data => {
|
||||
ws.send(ChannelMessageV0.
|
||||
encodeChannelDataMessage(channelId, data), err =>
|
||||
_logError(err, 'failed to send channel data message',
|
||||
'receiveChannelData'));
|
||||
});
|
||||
|
||||
socket.on('connect', () => {
|
||||
});
|
||||
|
||||
socket.on('drain', () => {
|
||||
});
|
||||
|
||||
socket.on('error', error => {
|
||||
logger.error('failed to connect to S3', {
|
||||
code: error.code,
|
||||
host: error.address,
|
||||
port: error.port,
|
||||
});
|
||||
});
|
||||
|
||||
socket.on('end', () => {
|
||||
socket.destroy();
|
||||
socketsByChannelId[channelId] = null;
|
||||
ws.send(ChannelMessageV0.encodeChannelCloseMessage(channelId),
|
||||
err => _logError(err,
|
||||
'failed to send channel close message',
|
||||
'receiveChannelData'));
|
||||
});
|
||||
|
||||
socketsByChannelId[channelId] = socket;
|
||||
}
|
||||
socket.write(payload);
|
||||
}
|
||||
|
||||
function browserAccessChangeHandler() {
|
||||
if (!_config.browserAccessEnabled) {
|
||||
socketsByChannelId.forEach(s => s.close());
|
||||
}
|
||||
}
|
||||
|
||||
ws.on('open', () => {
|
||||
connected = true;
|
||||
logger.info('connected to push server');
|
||||
|
||||
metadata.notifyBucketChange(() => {
|
||||
pushStats({ noCache: true });
|
||||
});
|
||||
_config.on('browser-access-enabled-change', browserAccessChangeHandler);
|
||||
|
||||
initiatePing();
|
||||
});
|
||||
|
||||
const cbOnce = cb ? arsenal.jsutil.once(cb) : null;
|
||||
|
||||
ws.on('close', () => {
|
||||
logger.info('disconnected from push server, reconnecting in 10s');
|
||||
metadata.notifyBucketChange(null);
|
||||
_config.removeListener('browser-access-enabled-change',
|
||||
browserAccessChangeHandler);
|
||||
setTimeout(startWSManagementClient, 10000, url, token);
|
||||
connected = false;
|
||||
|
||||
if (cbOnce) {
|
||||
process.nextTick(cbOnce);
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('error', err => {
|
||||
connected = false;
|
||||
logger.error('error from push server connection', {
|
||||
error: err,
|
||||
errorMessage: err.message,
|
||||
});
|
||||
if (cbOnce) {
|
||||
process.nextTick(cbOnce, err);
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('ping', () => {
|
||||
ws.pong(err => _logError(err, 'failed to send a pong'));
|
||||
});
|
||||
|
||||
ws.on('pong', () => {
|
||||
initiatePing();
|
||||
});
|
||||
|
||||
ws.on('message', data => {
|
||||
const log = logger.newRequestLogger();
|
||||
const message = new ChannelMessageV0(data);
|
||||
switch (message.getType()) {
|
||||
case CONFIG_OVERLAY_MESSAGE:
|
||||
if (!isManagementAgentUsed()) {
|
||||
applyAndSaveOverlay(JSON.parse(message.getPayload()), log);
|
||||
} else {
|
||||
if (overlayMessageListener) {
|
||||
overlayMessageListener(message.getPayload().toString());
|
||||
}
|
||||
}
|
||||
break;
|
||||
case METRICS_REQUEST_MESSAGE:
|
||||
pushStats();
|
||||
break;
|
||||
case CHANNEL_CLOSE_MESSAGE:
|
||||
closeChannel(message.getChannelNumber());
|
||||
break;
|
||||
case CHANNEL_PAYLOAD_MESSAGE:
|
||||
// browserAccessEnabled defaults to true unless explicitly false
|
||||
if (_config.browserAccessEnabled !== false) {
|
||||
receiveChannelData(
|
||||
message.getChannelNumber(), message.getPayload());
|
||||
}
|
||||
break;
|
||||
default:
|
||||
logger.error('unknown message type from push server',
|
||||
{ messageType: message.getType() });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function addOverlayMessageListener(callback) {
|
||||
assert(typeof callback === 'function');
|
||||
overlayMessageListener = callback;
|
||||
}
|
||||
|
||||
function startPushConnectionHealthCheckServer(cb) {
|
||||
const server = http.createServer((req, res) => {
|
||||
if (req.url !== '/_/healthcheck') {
|
||||
res.writeHead(404);
|
||||
res.write('Not Found');
|
||||
} else if (connected) {
|
||||
res.writeHead(200);
|
||||
res.write('Connected');
|
||||
} else {
|
||||
res.writeHead(503);
|
||||
res.write('Not Connected');
|
||||
}
|
||||
res.end();
|
||||
});
|
||||
|
||||
server.listen(_config.port, cb);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createWSAgent,
|
||||
startWSManagementClient,
|
||||
startPushConnectionHealthCheckServer,
|
||||
addOverlayMessageListener,
|
||||
};
|
|
@ -2,9 +2,9 @@ const MetadataWrapper = require('arsenal').storage.metadata.MetadataWrapper;
|
|||
const { config } = require('../Config');
|
||||
const logger = require('../utilities/logger');
|
||||
const constants = require('../../constants');
|
||||
const bucketclient = require('bucketclient');
|
||||
|
||||
const clientName = config.backends.metadata;
|
||||
let bucketclient;
|
||||
let params;
|
||||
if (clientName === 'mem') {
|
||||
params = {};
|
||||
|
@ -21,6 +21,7 @@ if (clientName === 'mem') {
|
|||
noDbOpen: null,
|
||||
};
|
||||
} else if (clientName === 'scality') {
|
||||
bucketclient = require('bucketclient');
|
||||
params = {
|
||||
bucketdBootstrap: config.bucketd.bootstrap,
|
||||
bucketdLog: config.bucketd.log,
|
||||
|
|
|
@ -18,11 +18,6 @@ const locationStorageCheck =
|
|||
require('./api/apiUtils/object/locationStorageCheck');
|
||||
const vault = require('./auth/vault');
|
||||
const metadata = require('./metadata/wrapper');
|
||||
const { initManagement } = require('./management');
|
||||
const {
|
||||
initManagementClient,
|
||||
isManagementAgentUsed,
|
||||
} = require('./management/agentClient');
|
||||
|
||||
const HttpAgent = require('agentkeepalive');
|
||||
const QuotaService = require('./quotas/quotas');
|
||||
|
@ -56,7 +51,6 @@ const STATS_INTERVAL = 5; // 5 seconds
|
|||
const STATS_EXPIRY = 30; // 30 seconds
|
||||
const statsClient = new StatsClient(localCacheClient, STATS_INTERVAL,
|
||||
STATS_EXPIRY);
|
||||
const enableRemoteManagement = true;
|
||||
|
||||
class S3Server {
|
||||
/**
|
||||
|
@ -327,26 +321,13 @@ class S3Server {
|
|||
QuotaService?.setup(log);
|
||||
}
|
||||
|
||||
// TODO this should wait for metadata healthcheck to be ok
|
||||
// TODO only do this in cluster master
|
||||
if (enableRemoteManagement) {
|
||||
if (!isManagementAgentUsed()) {
|
||||
setTimeout(() => {
|
||||
initManagement(logger.newRequestLogger());
|
||||
}, 5000);
|
||||
} else {
|
||||
initManagementClient();
|
||||
}
|
||||
}
|
||||
|
||||
this.started = true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
// TODO: change config to use workers prop. name for clarity
|
||||
let workers = _config.clusters || 1;
|
||||
let workers = _config.workers || 1;
|
||||
if (process.env.S3BACKEND === 'mem') {
|
||||
workers = 1;
|
||||
}
|
||||
|
|
|
@ -772,10 +772,9 @@ const services = {
|
|||
assert.strictEqual(typeof splitter, 'string');
|
||||
const partKey = `${uploadId}${splitter}${partNumber}`;
|
||||
const omVal = {
|
||||
// Version 3 changes the format of partLocations
|
||||
// from an object to an array
|
||||
'md-model-version': 3,
|
||||
partLocations,
|
||||
// Version 6 changes 'partLocations' to 'location'
|
||||
'md-model-version': 6,
|
||||
'location': partLocations,
|
||||
'key': partKey,
|
||||
'last-modified': dateModified,
|
||||
'content-md5': contentMD5,
|
||||
|
@ -891,6 +890,7 @@ const services = {
|
|||
marker: undefined,
|
||||
delimiter: undefined,
|
||||
maxKeys: 10000,
|
||||
withLocation: true,
|
||||
};
|
||||
metadata.listObject(mpuBucketName, searchArgs, log, cb);
|
||||
},
|
||||
|
@ -906,6 +906,7 @@ const services = {
|
|||
marker: `${uploadId}${params.splitter}${paddedPartNumber}`,
|
||||
delimiter: undefined,
|
||||
maxKeys: maxParts,
|
||||
withLocation: true,
|
||||
};
|
||||
metadata.listObject(mpuBucketName, searchArgs, log, cb);
|
||||
},
|
||||
|
|
|
@ -1,17 +1,12 @@
|
|||
require('werelogs').stderrUtils.catchAndTimestampStderr();
|
||||
const _config = require('../Config').config;
|
||||
const { utapiVersion, UtapiServer: utapiServer } = require('utapi');
|
||||
const vault = require('../auth/vault');
|
||||
|
||||
// start utapi server
|
||||
if (utapiVersion === 1 && _config.utapi) {
|
||||
const fullConfig = Object.assign({}, _config.utapi,
|
||||
{ redis: _config.redis });
|
||||
if (_config.vaultd) {
|
||||
Object.assign(fullConfig, { vaultd: _config.vaultd });
|
||||
}
|
||||
if (_config.https) {
|
||||
Object.assign(fullConfig, { https: _config.https });
|
||||
}
|
||||
{ redis: _config.redis, vaultclient: vault });
|
||||
// copy healthcheck IPs
|
||||
if (_config.healthChecks) {
|
||||
Object.assign(fullConfig, { healthChecks: _config.healthChecks });
|
||||
|
|
|
@ -1,7 +1,11 @@
|
|||
const { Werelogs } = require('werelogs');
|
||||
const { configure, Werelogs } = require('werelogs');
|
||||
|
||||
const _config = require('../Config.js').config;
|
||||
|
||||
configure({
|
||||
level: _config.log.logLevel,
|
||||
dump: _config.log.dumpLevel,
|
||||
});
|
||||
const werelogs = new Werelogs({
|
||||
level: _config.log.logLevel,
|
||||
dump: _config.log.dumpLevel,
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"STANDARD": {
|
||||
"type": "vitastor",
|
||||
"objectId": "std",
|
||||
"legacyAwsBehavior": true,
|
||||
"details": {
|
||||
"config_path": "/etc/vitastor/vitastor.conf",
|
||||
"pool_id": 10,
|
||||
"metadata_mongodb": {
|
||||
"url": "mongodb://USERNAME:PASSWORD@10.10.10.1:27017,10.10.10.2:27017,10.10.10.3:27017/?w=majority&readPreference=primary&replicaSet=rs0",
|
||||
"dbname": "vitastor",
|
||||
"collection": "volume_metadata"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,179 +0,0 @@
|
|||
const Uuid = require('uuid');
|
||||
const WebSocket = require('ws');
|
||||
|
||||
const logger = require('./lib/utilities/logger');
|
||||
const { initManagement } = require('./lib/management');
|
||||
const _config = require('./lib/Config').config;
|
||||
const { managementAgentMessageType } = require('./lib/management/agentClient');
|
||||
const { addOverlayMessageListener } = require('./lib/management/push');
|
||||
const { saveConfigurationVersion } = require('./lib/management/configuration');
|
||||
|
||||
|
||||
// TODO: auth?
|
||||
// TODO: werelogs with a specific name.
|
||||
|
||||
const CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS = 15000;
|
||||
|
||||
|
||||
class ManagementAgentServer {
|
||||
constructor() {
|
||||
this.port = _config.managementAgent.port || 8010;
|
||||
this.wss = null;
|
||||
this.loadedOverlay = null;
|
||||
|
||||
this.stop = this.stop.bind(this);
|
||||
process.on('SIGINT', this.stop);
|
||||
process.on('SIGHUP', this.stop);
|
||||
process.on('SIGQUIT', this.stop);
|
||||
process.on('SIGTERM', this.stop);
|
||||
process.on('SIGPIPE', () => {});
|
||||
}
|
||||
|
||||
start(_cb) {
|
||||
const cb = _cb || function noop() {};
|
||||
|
||||
/* Define REPORT_TOKEN env variable needed by the management
|
||||
* module. */
|
||||
process.env.REPORT_TOKEN = process.env.REPORT_TOKEN
|
||||
|| _config.reportToken
|
||||
|| Uuid.v4();
|
||||
|
||||
initManagement(logger.newRequestLogger(), overlay => {
|
||||
let error = null;
|
||||
|
||||
if (overlay) {
|
||||
this.loadedOverlay = overlay;
|
||||
this.startServer();
|
||||
} else {
|
||||
error = new Error('failed to init management');
|
||||
}
|
||||
return cb(error);
|
||||
});
|
||||
}
|
||||
|
||||
stop() {
|
||||
if (!this.wss) {
|
||||
process.exit(0);
|
||||
return;
|
||||
}
|
||||
this.wss.close(() => {
|
||||
logger.info('server shutdown');
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
|
||||
startServer() {
|
||||
this.wss = new WebSocket.Server({
|
||||
port: this.port,
|
||||
clientTracking: true,
|
||||
path: '/watch',
|
||||
});
|
||||
|
||||
this.wss.on('connection', this.onConnection.bind(this));
|
||||
this.wss.on('listening', this.onListening.bind(this));
|
||||
this.wss.on('error', this.onError.bind(this));
|
||||
|
||||
setInterval(this.checkBrokenConnections.bind(this),
|
||||
CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS);
|
||||
|
||||
addOverlayMessageListener(this.onNewOverlay.bind(this));
|
||||
}
|
||||
|
||||
onConnection(socket, request) {
|
||||
function hearthbeat() {
|
||||
this.isAlive = true;
|
||||
}
|
||||
logger.info('client connected to watch route', {
|
||||
ip: request.connection.remoteAddress,
|
||||
});
|
||||
|
||||
/* eslint-disable no-param-reassign */
|
||||
socket.isAlive = true;
|
||||
socket.on('pong', hearthbeat.bind(socket));
|
||||
|
||||
if (socket.readyState !== socket.OPEN) {
|
||||
logger.error('client socket not in ready state', {
|
||||
state: socket.readyState,
|
||||
client: socket._socket._peername,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const msg = {
|
||||
messageType: managementAgentMessageType.NEW_OVERLAY,
|
||||
payload: this.loadedOverlay,
|
||||
};
|
||||
socket.send(JSON.stringify(msg), error => {
|
||||
if (error) {
|
||||
logger.error('failed to send remoteOverlay to client', {
|
||||
error,
|
||||
client: socket._socket._peername,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
onListening() {
|
||||
logger.info('websocket server listening',
|
||||
{ port: this.port });
|
||||
}
|
||||
|
||||
onError(error) {
|
||||
logger.error('websocket server error', { error });
|
||||
}
|
||||
|
||||
_sendNewOverlayToClient(client) {
|
||||
if (client.readyState !== client.OPEN) {
|
||||
logger.error('client socket not in ready state', {
|
||||
state: client.readyState,
|
||||
client: client._socket._peername,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const msg = {
|
||||
messageType: managementAgentMessageType.NEW_OVERLAY,
|
||||
payload: this.loadedOverlay,
|
||||
};
|
||||
client.send(JSON.stringify(msg), error => {
|
||||
if (error) {
|
||||
logger.error(
|
||||
'failed to send remoteOverlay to management agent client', {
|
||||
error, client: client._socket._peername,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
onNewOverlay(remoteOverlay) {
|
||||
const remoteOverlayObj = JSON.parse(remoteOverlay);
|
||||
saveConfigurationVersion(
|
||||
this.loadedOverlay, remoteOverlayObj, logger, err => {
|
||||
if (err) {
|
||||
logger.error('failed to save remote overlay', { err });
|
||||
return;
|
||||
}
|
||||
this.loadedOverlay = remoteOverlayObj;
|
||||
this.wss.clients.forEach(
|
||||
this._sendNewOverlayToClient.bind(this)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
checkBrokenConnections() {
|
||||
this.wss.clients.forEach(client => {
|
||||
if (!client.isAlive) {
|
||||
logger.info('close broken connection', {
|
||||
client: client._socket._peername,
|
||||
});
|
||||
client.terminate();
|
||||
return;
|
||||
}
|
||||
client.isAlive = false;
|
||||
client.ping();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const server = new ManagementAgentServer();
|
||||
server.start();
|
62
package.json
62
package.json
|
@ -6,7 +6,10 @@
|
|||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"repository": "scality/S3",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor"
|
||||
},
|
||||
"keywords": [
|
||||
"s3",
|
||||
"cloud",
|
||||
|
@ -15,60 +18,69 @@
|
|||
"author": "Scality Inc.",
|
||||
"license": "Apache-2.0",
|
||||
"bugs": {
|
||||
"url": "https://github.com/scality/S3/issues"
|
||||
"url": "https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor/issues"
|
||||
},
|
||||
"homepage": "https://github.com/scality/S3#readme",
|
||||
"homepage": "https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor#zenko-cloudserver-with-vitastor-backend",
|
||||
"dependencies": {
|
||||
"@azure/storage-blob": "^12.12.0",
|
||||
"@hapi/joi": "^17.1.0",
|
||||
"arsenal": "git+https://github.com/scality/arsenal#8.1.146",
|
||||
"async": "~2.5.0",
|
||||
"aws-sdk": "2.905.0",
|
||||
"bucketclient": "scality/bucketclient#8.1.9",
|
||||
"arsenal": "./arsenal",
|
||||
"async": "^2.5.0",
|
||||
"aws-sdk": "^2.905.0",
|
||||
"bufferutil": "^4.0.6",
|
||||
"commander": "^2.9.0",
|
||||
"cron-parser": "^2.11.0",
|
||||
"diskusage": "1.1.3",
|
||||
"diskusage": "^1.1.3",
|
||||
"google-auto-auth": "^0.9.1",
|
||||
"http-proxy": "^1.17.0",
|
||||
"http-proxy-agent": "^4.0.1",
|
||||
"https-proxy-agent": "^2.2.0",
|
||||
"level-mem": "^5.0.1",
|
||||
"moment": "^2.26.0",
|
||||
"mongodb": "^5.2.0",
|
||||
"node-fetch": "^2.6.0",
|
||||
"node-forge": "^0.7.1",
|
||||
"npm-run-all": "~4.1.5",
|
||||
"prom-client": "14.2.0",
|
||||
"npm-run-all": "^4.1.5",
|
||||
"prom-client": "^14.2.0",
|
||||
"request": "^2.81.0",
|
||||
"scubaclient": "git+https://github.com/scality/scubaclient.git#fb7375a9298bda7df0e9f9ed81d7fc5b363590a9",
|
||||
"sql-where-parser": "~2.2.1",
|
||||
"utapi": "github:scality/utapi#8.1.16",
|
||||
"scubaclient": "./scubaclient",
|
||||
"sql-where-parser": "^2.2.1",
|
||||
"utapi": "./utapi",
|
||||
"utf-8-validate": "^5.0.8",
|
||||
"utf8": "~2.1.1",
|
||||
"utf8": "^2.1.1",
|
||||
"uuid": "^8.3.2",
|
||||
"vaultclient": "scality/vaultclient#8.4.1",
|
||||
"werelogs": "scality/werelogs#8.1.5",
|
||||
"werelogs": "./werelogs",
|
||||
"ws": "^5.1.0",
|
||||
"xml2js": "~0.4.16"
|
||||
"xml2js": "^0.4.16",
|
||||
"fcntl": "./fcntl",
|
||||
"httpagent": "./httpagent"
|
||||
},
|
||||
"overrides": {
|
||||
"ltgt": "^2.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.25.2",
|
||||
"@babel/preset-env": "^7.25.3",
|
||||
"babel-loader": "^9.1.3",
|
||||
"bluebird": "^3.3.1",
|
||||
"eslint": "^8.14.0",
|
||||
"eslint-config-airbnb-base": "^13.1.0",
|
||||
"eslint-config-scality": "scality/Guidelines#8.2.0",
|
||||
"eslint-config-airbnb-base": "^15.0.0",
|
||||
"eslint-config-scality": "./eslint-config-scality",
|
||||
"eslint-plugin-import": "^2.14.0",
|
||||
"eslint-plugin-mocha": "^10.1.0",
|
||||
"express": "^4.17.1",
|
||||
"ioredis": "4.9.5",
|
||||
"ioredis": "^4.9.5",
|
||||
"istanbul": "^1.0.0-alpha.2",
|
||||
"istanbul-api": "^1.0.0-alpha.13",
|
||||
"lolex": "^1.4.0",
|
||||
"mocha": "^2.3.4",
|
||||
"mocha": ">=3.1.2",
|
||||
"mocha-junit-reporter": "^1.23.1",
|
||||
"mocha-multi-reporters": "^1.1.7",
|
||||
"node-mocks-http": "1.5.2",
|
||||
"nyc": "^15.1.0",
|
||||
"node-loader": "^2.1.0",
|
||||
"node-mocks-http": "^1.5.2",
|
||||
"sinon": "^13.0.1",
|
||||
"tv4": "^1.2.7"
|
||||
"tv4": "^1.2.7",
|
||||
"webpack": "^5.93.0",
|
||||
"webpack-cli": "^5.1.4"
|
||||
},
|
||||
"scripts": {
|
||||
"cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server",
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 619aca73fc95af9594f8113f700111c479d96b9f
|
|
@ -71,35 +71,35 @@ describe('List Parts API', () => {
|
|||
'last-modified': '2015-11-30T22:41:18.658Z',
|
||||
'content-md5': 'f3a9fb2071d3503b703938a74eb99846',
|
||||
'content-length': '6000000',
|
||||
'partLocations': ['068db6a6745a79d54c1b29ff99f9f131'],
|
||||
'location': ['068db6a6745a79d54c1b29ff99f9f131'],
|
||||
});
|
||||
inMemMetadata.keyMaps.get(mpuBucket).set(partTwoKey, {
|
||||
'key': partTwoKey,
|
||||
'last-modified': '2015-11-30T22:41:40.207Z',
|
||||
'content-md5': 'f3a9fb2071d3503b703938a74eb99846',
|
||||
'content-length': '6000000',
|
||||
'partLocations': ['ff22f316b16956ff5118c93abce7d62d'],
|
||||
'location': ['ff22f316b16956ff5118c93abce7d62d'],
|
||||
});
|
||||
inMemMetadata.keyMaps.get(mpuBucket).set(partThreeKey, {
|
||||
'key': partThreeKey,
|
||||
'last-modified': '2015-11-30T22:41:52.102Z',
|
||||
'content-md5': 'f3a9fb2071d3503b703938a74eb99846',
|
||||
'content-length': '6000000',
|
||||
'partLocations': ['dea282f70edb6fc5f9433cd6f525d4a6'],
|
||||
'location': ['dea282f70edb6fc5f9433cd6f525d4a6'],
|
||||
});
|
||||
inMemMetadata.keyMaps.get(mpuBucket).set(partFourKey, {
|
||||
'key': partFourKey,
|
||||
'last-modified': '2015-11-30T22:42:03.493Z',
|
||||
'content-md5': 'f3a9fb2071d3503b703938a74eb99846',
|
||||
'content-length': '6000000',
|
||||
'partLocations': ['afe24bc40153982e1f7f28066f7af6a4'],
|
||||
'location': ['afe24bc40153982e1f7f28066f7af6a4'],
|
||||
});
|
||||
inMemMetadata.keyMaps.get(mpuBucket).set(partFiveKey, {
|
||||
'key': partFiveKey,
|
||||
'last-modified': '2015-11-30T22:42:22.876Z',
|
||||
'content-md5': '555e4cd2f9eff38109d7a3ab13995a32',
|
||||
'content-length': '18',
|
||||
'partLocations': ['85bc16f5769687070fb13cfe66b5e41f'],
|
||||
'location': ['85bc16f5769687070fb13cfe66b5e41f'],
|
||||
});
|
||||
done();
|
||||
});
|
||||
|
|
|
@ -159,7 +159,7 @@ function putMPU(key, body, cb) {
|
|||
const calculatedHash = md5Hash.digest('hex');
|
||||
const partKey = `${uploadId}${constants.splitter}00001`;
|
||||
const obj = {
|
||||
partLocations: [{
|
||||
location: [{
|
||||
key: 1,
|
||||
dataStoreName: 'scality-internal-mem',
|
||||
dataStoreETag: `1:${calculatedHash}`,
|
||||
|
|
|
@ -1,82 +0,0 @@
|
|||
const assert = require('assert');
|
||||
|
||||
const {
|
||||
createWSAgent,
|
||||
} = require('../../../lib/management/push');
|
||||
|
||||
const proxy = 'http://proxy:3128/';
|
||||
const logger = { info: () => {} };
|
||||
|
||||
function testVariableSet(httpProxy, httpsProxy, allProxy, noProxy) {
|
||||
return () => {
|
||||
it(`should use ${httpProxy} environment variable`, () => {
|
||||
let agent = createWSAgent('https://pushserver', {
|
||||
[httpProxy]: 'http://proxy:3128',
|
||||
}, logger);
|
||||
assert.equal(agent, null);
|
||||
|
||||
agent = createWSAgent('http://pushserver', {
|
||||
[httpProxy]: proxy,
|
||||
}, logger);
|
||||
assert.equal(agent.proxy.href, proxy);
|
||||
});
|
||||
|
||||
it(`should use ${httpsProxy} environment variable`, () => {
|
||||
let agent = createWSAgent('http://pushserver', {
|
||||
[httpsProxy]: proxy,
|
||||
}, logger);
|
||||
assert.equal(agent, null);
|
||||
|
||||
agent = createWSAgent('https://pushserver', {
|
||||
[httpsProxy]: proxy,
|
||||
}, logger);
|
||||
assert.equal(agent.proxy.href, proxy);
|
||||
});
|
||||
|
||||
it(`should use ${allProxy} environment variable`, () => {
|
||||
let agent = createWSAgent('http://pushserver', {
|
||||
[allProxy]: proxy,
|
||||
}, logger);
|
||||
assert.equal(agent.proxy.href, proxy);
|
||||
|
||||
agent = createWSAgent('https://pushserver', {
|
||||
[allProxy]: proxy,
|
||||
}, logger);
|
||||
assert.equal(agent.proxy.href, proxy);
|
||||
});
|
||||
|
||||
it(`should use ${noProxy} environment variable`, () => {
|
||||
let agent = createWSAgent('http://pushserver', {
|
||||
[noProxy]: 'pushserver',
|
||||
}, logger);
|
||||
assert.equal(agent, null);
|
||||
|
||||
agent = createWSAgent('http://pushserver', {
|
||||
[noProxy]: 'pushserver',
|
||||
[httpProxy]: proxy,
|
||||
}, logger);
|
||||
assert.equal(agent, null);
|
||||
|
||||
agent = createWSAgent('http://pushserver', {
|
||||
[noProxy]: 'pushserver2',
|
||||
[httpProxy]: proxy,
|
||||
}, logger);
|
||||
assert.equal(agent.proxy.href, proxy);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
describe('Websocket connection agent', () => {
|
||||
describe('with no proxy env', () => {
|
||||
it('should handle empty proxy environment', () => {
|
||||
const agent = createWSAgent('https://pushserver', {}, logger);
|
||||
assert.equal(agent, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with lowercase proxy env',
|
||||
testVariableSet('http_proxy', 'https_proxy', 'all_proxy', 'no_proxy'));
|
||||
|
||||
describe('with uppercase proxy env',
|
||||
testVariableSet('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY'));
|
||||
});
|
|
@ -1,239 +0,0 @@
|
|||
const assert = require('assert');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const { DummyRequestLogger } = require('../helpers');
|
||||
const log = new DummyRequestLogger();
|
||||
|
||||
const metadata = require('../../../lib/metadata/wrapper');
|
||||
const managementDatabaseName = 'PENSIEVE';
|
||||
const tokenConfigurationKey = 'auth/zenko/remote-management-token';
|
||||
|
||||
const { privateKey, accessKey, decryptedSecretKey, secretKey, canonicalId,
|
||||
userName } = require('./resources.json');
|
||||
const shortid = '123456789012';
|
||||
const email = 'customaccount1@setbyenv.com';
|
||||
const arn = 'arn:aws:iam::123456789012:root';
|
||||
const { config } = require('../../../lib/Config');
|
||||
|
||||
const {
|
||||
remoteOverlayIsNewer,
|
||||
patchConfiguration,
|
||||
} = require('../../../lib/management/configuration');
|
||||
|
||||
const {
|
||||
initManagementDatabase,
|
||||
} = require('../../../lib/management/index');
|
||||
|
||||
function initManagementCredentialsMock(cb) {
|
||||
return metadata.putObjectMD(managementDatabaseName,
|
||||
tokenConfigurationKey, { privateKey }, {},
|
||||
log, error => cb(error));
|
||||
}
|
||||
|
||||
function getConfig() {
|
||||
return config;
|
||||
}
|
||||
|
||||
// Original Config
|
||||
const overlayVersionOriginal = Object.assign({}, config.overlayVersion);
|
||||
const authDataOriginal = Object.assign({}, config.authData);
|
||||
const locationConstraintsOriginal = Object.assign({},
|
||||
config.locationConstraints);
|
||||
const restEndpointsOriginal = Object.assign({}, config.restEndpoints);
|
||||
const browserAccessEnabledOriginal = config.browserAccessEnabled;
|
||||
const instanceId = '19683e55-56f7-4a4c-98a7-706c07e4ec30';
|
||||
const publicInstanceId = crypto.createHash('sha256')
|
||||
.update(instanceId)
|
||||
.digest('hex');
|
||||
|
||||
function resetConfig() {
|
||||
config.overlayVersion = overlayVersionOriginal;
|
||||
config.authData = authDataOriginal;
|
||||
config.locationConstraints = locationConstraintsOriginal;
|
||||
config.restEndpoints = restEndpointsOriginal;
|
||||
config.browserAccessEnabled = browserAccessEnabledOriginal;
|
||||
}
|
||||
|
||||
function assertConfig(actualConf, expectedConf) {
|
||||
Object.keys(expectedConf).forEach(key => {
|
||||
assert.deepStrictEqual(actualConf[key], expectedConf[key]);
|
||||
});
|
||||
}
|
||||
|
||||
describe('patchConfiguration', () => {
|
||||
before(done => initManagementDatabase(log, err => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
return initManagementCredentialsMock(done);
|
||||
}));
|
||||
beforeEach(() => {
|
||||
resetConfig();
|
||||
});
|
||||
|
||||
it('should modify config using the new config', done => {
|
||||
const newConf = {
|
||||
version: 1,
|
||||
instanceId,
|
||||
users: [
|
||||
{
|
||||
secretKey,
|
||||
accessKey,
|
||||
canonicalId,
|
||||
userName,
|
||||
},
|
||||
],
|
||||
endpoints: [
|
||||
{
|
||||
hostname: '1.1.1.1',
|
||||
locationName: 'us-east-1',
|
||||
},
|
||||
],
|
||||
locations: {
|
||||
'us-east-1': {
|
||||
name: 'us-east-1',
|
||||
objectId: 'us-east-1',
|
||||
locationType: 'location-file-v1',
|
||||
legacyAwsBehavior: true,
|
||||
details: {},
|
||||
},
|
||||
},
|
||||
browserAccess: {
|
||||
enabled: true,
|
||||
},
|
||||
};
|
||||
return patchConfiguration(newConf, log, err => {
|
||||
assert.ifError(err);
|
||||
const actualConf = getConfig();
|
||||
const expectedConf = {
|
||||
overlayVersion: 1,
|
||||
publicInstanceId,
|
||||
browserAccessEnabled: true,
|
||||
authData: {
|
||||
accounts: [{
|
||||
name: userName,
|
||||
email,
|
||||
arn,
|
||||
canonicalID: canonicalId,
|
||||
shortid,
|
||||
keys: [{
|
||||
access: accessKey,
|
||||
secret: decryptedSecretKey,
|
||||
}],
|
||||
}],
|
||||
},
|
||||
locationConstraints: {
|
||||
'us-east-1': {
|
||||
type: 'file',
|
||||
objectId: 'us-east-1',
|
||||
legacyAwsBehavior: true,
|
||||
isTransient: false,
|
||||
sizeLimitGB: null,
|
||||
details: { supportsVersioning: true },
|
||||
name: 'us-east-1',
|
||||
locationType: 'location-file-v1',
|
||||
},
|
||||
},
|
||||
};
|
||||
assertConfig(actualConf, expectedConf);
|
||||
assert.deepStrictEqual(actualConf.restEndpoints['1.1.1.1'],
|
||||
'us-east-1');
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should apply second configuration if version (2) is greater than ' +
|
||||
'overlayVersion (1)', done => {
|
||||
const newConf1 = {
|
||||
version: 1,
|
||||
instanceId,
|
||||
};
|
||||
const newConf2 = {
|
||||
version: 2,
|
||||
instanceId,
|
||||
browserAccess: {
|
||||
enabled: true,
|
||||
},
|
||||
};
|
||||
patchConfiguration(newConf1, log, err => {
|
||||
assert.ifError(err);
|
||||
return patchConfiguration(newConf2, log, err => {
|
||||
assert.ifError(err);
|
||||
const actualConf = getConfig();
|
||||
const expectedConf = {
|
||||
overlayVersion: 2,
|
||||
browserAccessEnabled: true,
|
||||
};
|
||||
assertConfig(actualConf, expectedConf);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should not apply the second configuration if version equals ' +
|
||||
'overlayVersion', done => {
|
||||
const newConf1 = {
|
||||
version: 1,
|
||||
instanceId,
|
||||
};
|
||||
const newConf2 = {
|
||||
version: 1,
|
||||
instanceId,
|
||||
browserAccess: {
|
||||
enabled: true,
|
||||
},
|
||||
};
|
||||
patchConfiguration(newConf1, log, err => {
|
||||
assert.ifError(err);
|
||||
return patchConfiguration(newConf2, log, err => {
|
||||
assert.ifError(err);
|
||||
const actualConf = getConfig();
|
||||
const expectedConf = {
|
||||
overlayVersion: 1,
|
||||
browserAccessEnabled: undefined,
|
||||
};
|
||||
assertConfig(actualConf, expectedConf);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('remoteOverlayIsNewer', () => {
|
||||
it('should return remoteOverlayIsNewer equals false if remote overlay ' +
|
||||
'is less than the cached', () => {
|
||||
const cachedOverlay = {
|
||||
version: 2,
|
||||
};
|
||||
const remoteOverlay = {
|
||||
version: 1,
|
||||
};
|
||||
const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay,
|
||||
remoteOverlay);
|
||||
assert.equal(isRemoteOverlayNewer, false);
|
||||
});
|
||||
it('should return remoteOverlayIsNewer equals false if remote overlay ' +
|
||||
'and the cached one are equal', () => {
|
||||
const cachedOverlay = {
|
||||
version: 1,
|
||||
};
|
||||
const remoteOverlay = {
|
||||
version: 1,
|
||||
};
|
||||
const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay,
|
||||
remoteOverlay);
|
||||
assert.equal(isRemoteOverlayNewer, false);
|
||||
});
|
||||
it('should return remoteOverlayIsNewer equals true if remote overlay ' +
|
||||
'version is greater than the cached one ', () => {
|
||||
const cachedOverlay = {
|
||||
version: 0,
|
||||
};
|
||||
const remoteOverlay = {
|
||||
version: 1,
|
||||
};
|
||||
const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay,
|
||||
remoteOverlay);
|
||||
assert.equal(isRemoteOverlayNewer, true);
|
||||
});
|
||||
});
|
|
@ -1,9 +0,0 @@
|
|||
{
|
||||
"privateKey": "-----BEGIN RSA PRIVATE KEY-----\r\nMIIEowIBAAKCAQEAj13sSYE40lAX2qpBvfdGfcSVNtBf8i5FH+E8FAhORwwPu+2S\r\n3yBQbgwHq30WWxunGb1NmZL1wkVZ+vf12DtxqFRnMA08LfO4oO6oC4V8XfKeuHyJ\r\n1qlaKRINz6r9yDkTHtwWoBnlAINurlcNKgGD5p7D+G26Chbr/Oo0ZwHula9DxXy6\r\neH8/bJ5/BynyNyyWRPoAO+UkUdY5utkFCUq2dbBIhovMgjjikf5p2oWqnRKXc+JK\r\nBegr6lSHkkhyqNhTmd8+wA+8Cace4sy1ajY1t5V4wfRZea5vwl/HlyyKodvHdxng\r\nJgg6H61JMYPkplY6Gr9OryBKEAgq02zYoYTDfwIDAQABAoIBAAuDYGlavkRteCzw\r\nRU1LIVcSRWVcgIgDXTu9K8T0Ec0008Kkxomyn6LmxmroJbZ1VwsDH8s4eRH73ckA\r\nxrZxt6Pr+0lplq6eBvKtl8MtGhq1VDe+kJczjHEF6SQHOFAu/TEaPZrn2XMcGvRX\r\nO1BnRL9tepFlxm3u/06VRFYNWqqchM+tFyzLu2AuiuKd5+slSX7KZvVgdkY1ErKH\r\ngB75lPyhPb77C/6ptqUisVMSO4JhLhsD0+ekDVY982Sb7KkI+szdWSbtMx9Ek2Wo\r\ntXwJz7I8T7IbODy9aW9G+ydyhMDFmaEYIaDVFKJj5+fluNza3oQ5PtFNVE50GQJA\r\nsisGqfECgYEAwpkwt0KpSamSEH6qknNYPOwxgEuXWoFVzibko7is2tFPvY+YJowb\r\n68MqHIYhf7gHLq2dc5Jg1TTbGqLECjVxp4xLU4c95KBy1J9CPAcuH4xQLDXmeLzP\r\nJ2YgznRocbzAMCDAwafCr3uY9FM7oGDHAi5bE5W11xWx+9MlFExL3JkCgYEAvJp5\r\nf+JGN1W037bQe2QLYUWGszewZsvplnNOeytGQa57w4YdF42lPhMz6Kc/zdzKZpN9\r\njrshiIDhAD5NCno6dwqafBAW9WZl0sn7EnlLhD4Lwm8E9bRHnC9H82yFuqmNrzww\r\nzxBCQogJISwHiVz4EkU48B283ecBn0wT/fAa19cCgYEApKWsnEHgrhy1IxOpCoRh\r\nUhqdv2k1xDPN/8DUjtnAFtwmVcLa/zJopU/Zn4y1ZzSzjwECSTi+iWZRQ/YXXHPf\r\nl92SFjhFW92Niuy8w8FnevXjF6T7PYiy1SkJ9OR1QlZrXc04iiGBDazLu115A7ce\r\nanACS03OLw+CKgl6Q/RR83ECgYBCUngDVoimkMcIHHt3yJiP3ikeAKlRnMdJlsa0\r\nXWVZV4hCG3lDfRXsnEgWuimftNKf+6GdfYSvQdLdiQsCcjT5A4uLsQTByv5nf4uA\r\n1ZKOsFrmRrARzxGXhLDikvj7yP//7USkq+0BBGFhfuAvl7fMhPceyPZPehqB7/jf\r\nxX1LBQKBgAn5GgSXzzS0e06ZlP/VrKxreOHa5Z8wOmqqYQ0QTeczAbNNmuITdwwB\r\nNkbRqpVXRIfuj0BQBegAiix8om1W4it0cwz54IXBwQULxJR1StWxj3jo4QtpMQ+z\r\npVPdB1Ilb9zPV1YvDwRfdS1xsobzznAx56ecsXduZjs9mF61db8Q\r\n-----END RSA PRIVATE KEY-----\r\n",
|
||||
"publicKey": "-----BEGIN PUBLIC KEY-----\r\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAj13sSYE40lAX2qpBvfdG\r\nfcSVNtBf8i5FH+E8FAhORwwPu+2S3yBQbgwHq30WWxunGb1NmZL1wkVZ+vf12Dtx\r\nqFRnMA08LfO4oO6oC4V8XfKeuHyJ1qlaKRINz6r9yDkTHtwWoBnlAINurlcNKgGD\r\n5p7D+G26Chbr/Oo0ZwHula9DxXy6eH8/bJ5/BynyNyyWRPoAO+UkUdY5utkFCUq2\r\ndbBIhovMgjjikf5p2oWqnRKXc+JKBegr6lSHkkhyqNhTmd8+wA+8Cace4sy1ajY1\r\nt5V4wfRZea5vwl/HlyyKodvHdxngJgg6H61JMYPkplY6Gr9OryBKEAgq02zYoYTD\r\nfwIDAQAB\r\n-----END PUBLIC KEY-----\r\n",
|
||||
"accessKey": "QXP3VDG3SALNBX2QBJ1C",
|
||||
"secretKey": "K5FyqZo5uFKfw9QBtn95o6vuPuD0zH/1seIrqPKqGnz8AxALNSx6EeRq7G1I6JJpS1XN13EhnwGn2ipsml3Uf2fQ00YgEmImG8wzGVZm8fWotpVO4ilN4JGyQCah81rNX4wZ9xHqDD7qYR5MyIERxR/osoXfctOwY7GGUjRKJfLOguNUlpaovejg6mZfTvYAiDF+PTO1sKUYqHt1IfKQtsK3dov1EFMBB5pWM7sVfncq/CthKN5M+VHx9Y87qdoP3+7AW+RCBbSDOfQgxvqtS7PIAf10mDl8k2kEURLz+RqChu4O4S0UzbEmtja7wa7WYhYKv/tM/QeW7kyNJMmnPg==",
|
||||
"decryptedSecretKey": "n7PSZ3U6SgerF9PCNhXYsq3S3fRKVGdZTicGV8Ur",
|
||||
"canonicalId": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be",
|
||||
"userName": "orbituser"
|
||||
}
|
|
@ -1,43 +0,0 @@
|
|||
const assert = require('assert');
|
||||
|
||||
const { getCapabilities } = require('../../../lib/utilities/reportHandler');
|
||||
|
||||
// Ensures that expected features are enabled even if they
|
||||
// rely on optional dependencies (such as secureChannelOptimizedPath)
|
||||
describe('report handler', () => {
|
||||
it('should report current capabilities', () => {
|
||||
const c = getCapabilities();
|
||||
assert.strictEqual(c.locationTypeDigitalOcean, true);
|
||||
assert.strictEqual(c.locationTypeS3Custom, true);
|
||||
assert.strictEqual(c.locationTypeSproxyd, true);
|
||||
assert.strictEqual(c.locationTypeHyperdriveV2, true);
|
||||
assert.strictEqual(c.locationTypeLocal, true);
|
||||
assert.strictEqual(c.preferredReadLocation, true);
|
||||
assert.strictEqual(c.managedLifecycle, true);
|
||||
assert.strictEqual(c.secureChannelOptimizedPath, true);
|
||||
assert.strictEqual(c.s3cIngestLocation, true);
|
||||
});
|
||||
|
||||
[
|
||||
{ value: 'true', result: true },
|
||||
{ value: 'TRUE', result: true },
|
||||
{ value: 'tRuE', result: true },
|
||||
{ value: '1', result: true },
|
||||
{ value: 'false', result: false },
|
||||
{ value: 'FALSE', result: false },
|
||||
{ value: 'FaLsE', result: false },
|
||||
{ value: '0', result: false },
|
||||
{ value: 'foo', result: false },
|
||||
{ value: '', result: true },
|
||||
{ value: undefined, result: true },
|
||||
].forEach(param =>
|
||||
it(`should allow set local file system capability ${param.value}`, () => {
|
||||
const OLD_ENV = process.env;
|
||||
|
||||
if (param.value !== undefined) process.env.LOCAL_VOLUME_CAPABILITY = param.value;
|
||||
assert.strictEqual(getCapabilities().locationTypeLocal, param.result);
|
||||
|
||||
process.env = OLD_ENV;
|
||||
})
|
||||
);
|
||||
});
|
|
@ -1,72 +0,0 @@
|
|||
const assert = require('assert');
|
||||
|
||||
const {
|
||||
ChannelMessageV0,
|
||||
MessageType,
|
||||
TargetType,
|
||||
} = require('../../../lib/management/ChannelMessageV0');
|
||||
|
||||
const {
|
||||
CONFIG_OVERLAY_MESSAGE,
|
||||
METRICS_REQUEST_MESSAGE,
|
||||
METRICS_REPORT_MESSAGE,
|
||||
CHANNEL_CLOSE_MESSAGE,
|
||||
CHANNEL_PAYLOAD_MESSAGE,
|
||||
} = MessageType;
|
||||
|
||||
const { TARGET_ANY } = TargetType;
|
||||
|
||||
describe('ChannelMessageV0', () => {
|
||||
describe('codec', () => {
|
||||
it('should roundtrip metrics report', () => {
|
||||
const b = ChannelMessageV0.encodeMetricsReportMessage({ a: 1 });
|
||||
const m = new ChannelMessageV0(b);
|
||||
|
||||
assert.strictEqual(METRICS_REPORT_MESSAGE, m.getType());
|
||||
assert.strictEqual(0, m.getChannelNumber());
|
||||
assert.strictEqual(m.getTarget(), TARGET_ANY);
|
||||
assert.strictEqual(m.getPayload().toString(), '{"a":1}');
|
||||
});
|
||||
|
||||
it('should roundtrip channel data', () => {
|
||||
const data = new Buffer('dummydata');
|
||||
const b = ChannelMessageV0.encodeChannelDataMessage(50, data);
|
||||
const m = new ChannelMessageV0(b);
|
||||
|
||||
assert.strictEqual(CHANNEL_PAYLOAD_MESSAGE, m.getType());
|
||||
assert.strictEqual(50, m.getChannelNumber());
|
||||
assert.strictEqual(m.getTarget(), TARGET_ANY);
|
||||
assert.strictEqual(m.getPayload().toString(), 'dummydata');
|
||||
});
|
||||
|
||||
it('should roundtrip channel close', () => {
|
||||
const b = ChannelMessageV0.encodeChannelCloseMessage(3);
|
||||
const m = new ChannelMessageV0(b);
|
||||
|
||||
assert.strictEqual(CHANNEL_CLOSE_MESSAGE, m.getType());
|
||||
assert.strictEqual(3, m.getChannelNumber());
|
||||
assert.strictEqual(m.getTarget(), TARGET_ANY);
|
||||
});
|
||||
});
|
||||
|
||||
describe('decoder', () => {
|
||||
it('should parse metrics request', () => {
|
||||
const b = new Buffer([METRICS_REQUEST_MESSAGE, 0, 0]);
|
||||
const m = new ChannelMessageV0(b);
|
||||
|
||||
assert.strictEqual(METRICS_REQUEST_MESSAGE, m.getType());
|
||||
assert.strictEqual(0, m.getChannelNumber());
|
||||
assert.strictEqual(m.getTarget(), TARGET_ANY);
|
||||
});
|
||||
|
||||
it('should parse overlay push', () => {
|
||||
const b = new Buffer([CONFIG_OVERLAY_MESSAGE, 0, 0, 34, 65, 34]);
|
||||
const m = new ChannelMessageV0(b);
|
||||
|
||||
assert.strictEqual(CONFIG_OVERLAY_MESSAGE, m.getType());
|
||||
assert.strictEqual(0, m.getChannelNumber());
|
||||
assert.strictEqual(m.getTarget(), TARGET_ANY);
|
||||
assert.strictEqual(m.getPayload().toString(), '"A"');
|
||||
});
|
||||
});
|
||||
});
|
|
@ -46,7 +46,6 @@
|
|||
"host": "localhost",
|
||||
"port": 8500
|
||||
},
|
||||
"clusters": 10,
|
||||
"log": {
|
||||
"logLevel": "info",
|
||||
"dumpLevel": "error"
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 84953ff86820bfa2bb6a24bdc79b1ae94ce6ff68
|
|
@ -0,0 +1,49 @@
|
|||
module.exports = {
|
||||
entry: './index.js',
|
||||
resolve: {
|
||||
extensions: ["...", ".node"],
|
||||
},
|
||||
target: 'node',
|
||||
devtool: 'source-map',
|
||||
output: {
|
||||
filename: 'zenko-vitastor.js'
|
||||
},
|
||||
module: {
|
||||
rules: [
|
||||
{
|
||||
test: /.jsx?$/,
|
||||
use: {
|
||||
loader: 'babel-loader',
|
||||
options: {
|
||||
presets: [ [ "@babel/preset-env", { "targets": { "node": "18.0" } } ] ]
|
||||
//presets: [ [ "@babel/preset-env", { "targets": { "node": "4.0" } } ] ],
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
test: /\.node$/,
|
||||
loader: "node-loader",
|
||||
},
|
||||
{
|
||||
test: /.json$/,
|
||||
type: 'json'
|
||||
}
|
||||
]
|
||||
},
|
||||
externals: {
|
||||
'vitastor': 'commonjs vitastor',
|
||||
'vaultclient': 'commonjs vaultclient',
|
||||
'bucketclient': 'commonjs bucketclient',
|
||||
'scality-kms': 'commonjs scality-kms',
|
||||
'sproxydclient': 'commonjs sproxydclient',
|
||||
'hdclient': 'commonjs hdclient',
|
||||
'cdmiclient': 'commonjs cdmiclient',
|
||||
'kerberos': 'commonjs kerberos',
|
||||
'@mongodb-js/zstd': 'commonjs @mongodb-js/zstd',
|
||||
'@aws-sdk/credential-providers': 'commonjs @aws-sdk/credential-providers',
|
||||
'mongodb-client-encryption': 'commonjs mongodb-client-encryption'
|
||||
},
|
||||
node: {
|
||||
__dirname: false
|
||||
}
|
||||
};
|
|
@ -0,0 +1 @@
|
|||
Subproject commit b57d072e51f39a3112b0719bbd1974c0e292e233
|
Loading…
Reference in New Issue