Compare commits

...

14 Commits

Author SHA1 Message Date
Thomas Carmet b51357d22a Adding gcp credentials
Signed-off-by: Thomas Carmet <thomas.carmet@scality.com>
2018-02-07 14:27:22 +01:00
Thomas Carmet 0e3d4c8fee Setting up node0 tests 2018-02-07 14:10:22 +01:00
Salim 9bdea54239 env utf8 2018-02-07 13:25:07 +01:00
Thomas Carmet 4d8fba27c2 Update worker package.json 2018-02-07 13:25:07 +01:00
Thomas Carmet bdc9302338 TMP - trying to run mocha via npm 2018-02-07 13:25:07 +01:00
Akim Sadaoui f14fc6476a debugging multiple-backend tests 2018-02-07 13:25:07 +01:00
Akim Sadaoui 09f580f471 switch to trusty for more up to date pkg 2018-02-07 13:25:07 +01:00
Akim Sadaoui 58a4a94ccb Launch server and test on same step 2018-02-07 13:25:07 +01:00
Akim Sadaoui b5ee85a6dc Extract multi-backend unit tests 2018-02-07 13:25:07 +01:00
Akim Sadaoui cddad288d8 Extract unit tests 2018-02-07 13:25:07 +01:00
Akim Sadaoui f28e4a53d2 Extract static-analysis 2018-02-07 13:25:07 +01:00
Akim Sadaoui eba106b0d6 comment out old way testing 2018-02-07 13:25:07 +01:00
Akim Sadaoui 5697993af1 fix aws creds 2018-02-07 13:25:07 +01:00
Josh Toklu 41df0d5859 Adding eve support to S3 2018-02-07 13:25:07 +01:00
11 changed files with 403 additions and 51 deletions

View File

@ -47,7 +47,7 @@
}, },
"clusters": 10, "clusters": 10,
"log": { "log": {
"logLevel": "info", "logLevel": "debug",
"dumpLevel": "error" "dumpLevel": "error"
}, },
"healthChecks": { "healthChecks": {

190
eve/main.yml Normal file
View File

@ -0,0 +1,190 @@
---
version: 0.2
branches:
default:
stage: pre-merge
models:
- Git: &clone
name: fetch source
repourl: '%(prop:git_reference)s'
shallow: True
retryFetch: True
haltOnFailure: True
- env: &circle_ci_env
CIRCLE_ARTIFACTS: '/tmp'
CIRCLE_NODE_TOTAL: '4'
CIRCLE_TEST_REPORTS: '/tmp'
- env: &s3_ci_env
CI: 'true'
CI_REPORTS: '/tmp'
ENABLE_LOCAL_CACHE: '1'
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- env: &gcp_credentials_env
GOOGLE_SERVICE_KEY: $GOOGLE_S3_SERVICE_KEY
GOOGLE_SERVICE_EMAIL: $GOOGLE_S3_SERVICE_EMAIL
- ShellCommand: &node_modules
name: make node_modules available
command: ln -s /usr/lib/node_modules .
- ShellCommandWithSecrets: &aws_credentials
name: add AWS credentials
command: |-
mkdir -p ~/.aws;
cat >>~/.aws/credentials <<EOF
[default]
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
[default_2]
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
EOF
- ShellCommandWithSecrets: &gcp_credentials
name: Add GCP credentials
command: |-
mkdir -p ~/.gcp;
cat >>~/.gcp/servicekey <<EOF
{
"type": "service_account",
"private_key": "$GOOGLE_S3_SERVICE_KEY",
"client_email": "$GOOGLE_S3_SERVICE_EMAIL"
}
EOF
stages:
pre-merge:
worker: &master-worker
type: docker
path: eve/workers/master
volumes:
- '/home/eve/workspace'
steps:
- TriggerStages:
name: trigger all tests in parallel
stage_names:
- static-analysis
- unit-tests
- multi-backends-unit-tests
- node0
# - node1
# - node2
# - node3
waitForFinish: True
haltOnFailure: True
static-analysis:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand:
name: run lint
command: npm run --silent lint -- --max-warnings 0
- ShellCommand:
name: run lint_md
command: npm run --silent lint_md
- ShellCommand:
name: run flake8
command: flake8 $(git ls-files "*.py")
- ShellCommand:
name: run yamllint
command: yamllint $(git ls-files "*.yml")
unit-tests:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *node_modules
- ShellCommand:
name: create report directory
command: mkdir -p $CIRCLE_TEST_REPORTS/unit
env: *circle_ci_env
- ShellCommand:
name: run legacy unit test
command: npm run unit_coverage_legacy_location
env: *circle_ci_env
# - ShellCommand:
# name: run unit tests
# command: npm run unit_coverage
# env: *circle_ci_env
multi-backends-unit-tests:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *node_modules
- ShellCommandWithSecrets: *aws_credentials
- ShellCommandWithSecrets: *gcp_credentials
- ShellCommandWithSecrets:
name: run multi-backends unit tests
command: mocha -t 20000 --recursive tests/multipleBackend
env:
CI: 'true'
S3BACKEND: mem
S3DATA: multiple
GOOGLE_SERVICE_KEY: '$GOOGLE_S3_SERVICE_KEY'
GOOGLE_SERVICE_EMAIL: '$GOOGLE_S3_SERVICE_EMAIL'
- ShellCommand:
name: Process
command: ps -aux
# TODO: deduplicate using a setvar for node number
node0:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *node_modules
- ShellCommandWithSecrets: *aws_credentials
- ShellCommandWithSecrets: *gcp_credentials
- ShellCommandWithSecrets:
name: run tests
command: sudo -E bash tests.bash
env:
<<:
- *circle_ci_env
- *s3_ci_env
CIRCLE_NODE_INDEX: '0'
node1:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *node_modules
- ShellCommand:
name: run tests
command: sudo -E bash tests.bash
env:
<<:
- *circle_ci_env
- *s3_ci_env
CIRCLE_NODE_INDEX: '1'
node2:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *node_modules
- ShellCommand:
name: run tests
command: sudo -E bash tests.bash
env:
<<:
- *circle_ci_env
- *s3_ci_env
CIRCLE_NODE_INDEX: '2'
node3:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *node_modules
- ShellCommand:
name: run tests
command: sudo -E bash tests.bash
env:
<<:
- *circle_ci_env
- *s3_ci_env
CIRCLE_NODE_INDEX: '3'

View File

@ -0,0 +1,66 @@
FROM ubuntu:trusty
#
# Install apt packages needed by the buildchain
#
ENV LANG C.UTF-8
COPY s3_packages.list buildbot_worker_packages.list /tmp/
RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
&& apt-get install -qy software-properties-common python-software-properties \
&& curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
&& echo "deb https://deb.nodesource.com/node_6.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
&& add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-add-repository ppa:brightbox/ruby-ng \
&& apt-get update -q \
&& cat /tmp/*_packages.list | xargs apt-get install -qy \
&& pip install pip==9.0.1 \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /tmp/*_packages.list
#
# Install dependencies
#
# Install python packages
COPY requirements.txt /tmp/
RUN pip install -r /tmp/requirements.txt \
&& rm -f /tmp/requirements.txt
# Install gems
COPY gems.list /tmp/
RUN cat /tmp/gems.list | xargs gem install \
&& rm /tmp/gems.list
# Install nodejs packages
WORKDIR /usr/lib/
COPY package.json .
RUN npm install
#
# Add user eve
#
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
&& adduser eve sudo \
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
#
# Misc
#
RUN echo '127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com' >> /etc/hosts
#
# Run buildbot-worker on startup
#
ARG BUILDBOT_VERSION=0.9.7
RUN pip install buildbot-worker==$BUILDBOT_VERSION
ENV CXX=g++-4.9
USER eve
ENV LANG C.UTF-8
WORKDIR /home/eve/workspace
CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
&& sudo service redis-server start \
&& buildbot-worker start --nodaemon

View File

@ -0,0 +1,9 @@
ca-certificates
git
libffi-dev
libssl-dev
python2.7
python2.7-dev
python-pip
software-properties-common
sudo

View File

@ -0,0 +1,5 @@
digest
fog-aws:1.3.0
json
mime-types:3.1
rspec:3.5

View File

@ -0,0 +1,91 @@
{
"name": "s3",
"version": "7.2.0",
"description": "S3 connector",
"main": "index.js",
"engines": {
"node": ">=6.9.5"
},
"repository": "scality/S3",
"keywords": [
"s3",
"cloud",
"server"
],
"author": "Giorgio Regni",
"license": "Apache-2.0",
"bugs": {
"url": "https://github.com/scality/S3/issues"
},
"homepage": "https://github.com/scality/S3#readme",
"dependencies": {
"aws-sdk": "2.28.0",
"arsenal": "scality/Arsenal",
"async": "~2.5.0",
"google-auto-auth": "^0.9.1",
"azure-storage": "^2.1.0",
"bucketclient": "scality/bucketclient",
"commander": "^2.9.0",
"mongodb": "^2.2.31",
"node-uuid": "^1.4.3",
"npm-run-all": "~4.0.2",
"sproxydclient": "scality/sproxydclient",
"utapi": "scality/utapi",
"utf8": "~2.1.1",
"uuid": "^3.0.1",
"vaultclient": "scality/vaultclient",
"werelogs": "scality/werelogs",
"xml2js": "~0.4.16"
},
"devDependencies": {
"bluebird": "^3.3.1",
"eslint": "^2.4.0",
"eslint-config-airbnb": "^6.0.0",
"eslint-config-scality": "scality/Guidelines",
"ioredis": "2.4.0",
"istanbul": "1.0.0-alpha.2",
"istanbul-api": "1.0.0-alpha.13",
"lolex": "^1.4.0",
"mocha": "^5.0.0",
"mocha-junit-reporter": "1.11.1",
"node-mocks-http": "^1.5.2",
"s3blaster": "scality/s3blaster",
"tv4": "^1.2.7"
},
"optionalDependencies": {
"cdmiclient": "scality/cdmiclient"
},
"scripts": {
"ft_awssdk": "cd tests/functional/aws-node-sdk && mocha test/",
"ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha test/",
"ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha test/bucket",
"ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha test/legacy test/object test/service test/support",
"ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha test/versioning/",
"ft_awssdk_external_backends": "cd tests/functional/aws-node-sdk && mocha test/multipleBackend",
"ft_management": "cd tests/functional/report && npm test",
"ft_node": "cd tests/functional/raw-node && npm test",
"ft_gcp": "cd tests/functional/raw-node && npm run test-gcp",
"ft_healthchecks": "cd tests/functional/healthchecks && npm test",
"ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
"ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
"ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management",
"install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
"lint": "eslint $(git ls-files '*.js')",
"lint_md": "mdlint $(git ls-files '*.md')",
"mem_backend": "S3BACKEND=mem node index.js",
"perf": "mocha tests/performance/s3standard.js",
"start": "npm-run-all --parallel start_dmd start_s3server",
"start_mongo": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server",
"start_mdserver": "node mdserver.js",
"start_dataserver": "node dataserver.js",
"start_s3server": "node index.js",
"start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
"start_utapi": "node lib/utapi/utapi.js",
"utapi_replay": "node lib/utapi/utapiReplay.js",
"test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
"test_legacy_location": "CI=true S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --recursive tests/unit",
"multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend",
"unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --reporter mocha-junit-reporter --recursive tests/unit",
"unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unitlegacylocation.xml istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-junit-reporter --recursive tests/unit"
}
}

View File

@ -0,0 +1,3 @@
flake8
s3cmd==1.6.1
yamllint

View File

@ -0,0 +1,8 @@
default-jdk
lsof
maven
netcat
nodejs
redis-server
ruby2.4
ruby2.4-dev

View File

@ -3,7 +3,6 @@ const { EventEmitter } = require('events');
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const url = require('url'); const url = require('url');
const uuid = require('node-uuid'); const uuid = require('node-uuid');
const { isValidBucketName } = require('arsenal').s3routes.routesUtils; const { isValidBucketName } = require('arsenal').s3routes.routesUtils;
@ -279,6 +278,7 @@ class Config extends EventEmitter {
if (process.env.S3_LOCATION_FILE !== undefined) { if (process.env.S3_LOCATION_FILE !== undefined) {
this.locationConfigPath = process.env.S3_LOCATION_FILE; this.locationConfigPath = process.env.S3_LOCATION_FILE;
} }
process.stdout.write(this.locationConfigPath);
// Read config automatically // Read config automatically
this._getLocationConfig(); this._getLocationConfig();

View File

@ -83,7 +83,7 @@
"utapi_replay": "node lib/utapi/utapiReplay.js", "utapi_replay": "node lib/utapi/utapiReplay.js",
"test": "CI=true S3BACKEND=mem mocha --recursive tests/unit", "test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
"test_legacy_location": "CI=true S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --recursive tests/unit", "test_legacy_location": "CI=true S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --recursive tests/unit",
"multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend", "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple S3_LOCATION_FILE=tests/locationConfig/locationConfigTests.json mocha -t 20000 --recursive tests/multipleBackend",
"unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --reporter mocha-junit-reporter --recursive tests/unit", "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --reporter mocha-junit-reporter --recursive tests/unit",
"unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unitlegacylocation.xml istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-junit-reporter --recursive tests/unit" "unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unitlegacylocation.xml istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-junit-reporter --recursive tests/unit"
} }

View File

@ -2,29 +2,30 @@
set -x #echo on set -x #echo on
set -e #exit at the first error set -e #exit at the first error
cat >>~/.aws/credentials <<EOF # mkdir -p ~/.aws
[default] # cat >>~/.aws/credentials <<EOF
aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT # [default]
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT # aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT
[default_2] # aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT
aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT_2 # [default_2]
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT_2 # aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT_2
[google] # aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT_2
aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE # [google]
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE # aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE
[google_2] # aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE
aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE_2 # [google_2]
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE_2 # aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE_2
EOF # aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE_2
# EOF
mkdir ${HOME}/.gcp #create directory for GCP service credential # mkdir ${HOME}/.gcp #create directory for GCP service credential
cat >>${HOME}/.gcp/servicekey <<EOF # cat >>${HOME}/.gcp/servicekey <<EOF
{ # {
"type": "service_account", # "type": "service_account",
"private_key": "$GOOGLE_SERVICE_KEY", # "private_key": "$GOOGLE_SERVICE_KEY",
"client_email": "$GOOGLE_SERVICE_EMAIL" # "client_email": "$GOOGLE_SERVICE_EMAIL"
} # }
EOF # EOF
MYPWD=$(pwd) MYPWD=$(pwd)
@ -35,47 +36,26 @@ killandsleep () {
if [ $CIRCLE_NODE_INDEX -eq 0 ] if [ $CIRCLE_NODE_INDEX -eq 0 ]
then then
npm run --silent lint -- --max-warnings 0
npm run --silent lint_md
flake8 $(git ls-files "*.py")
yamllint $(git ls-files "*.yml")
mkdir -p $CIRCLE_TEST_REPORTS/unit
npm run unit_coverage
npm run unit_coverage_legacy_location
npm run start_dmd &
bash wait_for_local_port.bash 9990 40 &&
npm run multiple_backend_test
killandsleep 9990
# Run S3 with multiple data backends ; run ft_tests # Run S3 with multiple data backends ; run ft_tests
S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_java.txt & bash wait_for_local_port.bash 8000 40 && cd ./tests/functional/jaws && mvn test S3BACKEND=mem S3DATA=multiple npm start # > $CIRCLE_ARTIFACTS/server_multiple_java.txt & bash wait_for_local_port.bash 8000 40 && cd ./tests/functional/jaws && mvn test
killandsleep 8000 killandsleep 8000
cd $MYPWD cd $MYPWD
S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_fog.txt & bash wait_for_local_port.bash 8000 40 && cd tests/functional/fog && rspec tests.rb S3BACKEND=mem S3DATA=multiple npm start #> $CIRCLE_ARTIFACTS/server_multiple_fog.txt & bash wait_for_local_port.bash 8000 40 && cd tests/functional/fog && rspec tests.rb
cd $MYPWD cd $MYPWD
killandsleep 8000 killandsleep 8000
S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple npm run ft_awssdk S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start # > $CIRCLE_ARTIFACTS/server_multiple_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple npm run ft_awssdk
cd $MYPWD cd $MYPWD
killandsleep 8000 killandsleep 8000
# Run external backend tests with proxy ; run ft_awssdk_external_backends # Run external backend tests with proxy ; run ft_awssdk_external_backends
S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple CI_PROXY=true npm start > $CIRCLE_ARTIFACTS/server_external_backends_proxy_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple CI_PROXY=true npm run ft_awssdk_external_backends S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple CI_PROXY=true npm start #> $CIRCLE_ARTIFACTS/server_external_backends_proxy_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple CI_PROXY=true npm run ft_awssdk_external_backends
killandsleep 8000 killandsleep 8000