Compare commits

...

1 Commits

Author SHA1 Message Date
Josh Toklu c5bfcc2057 Adding eve support to S3 2017-10-06 17:35:09 -07:00
7 changed files with 194 additions and 84 deletions

39
eve/main.yml Normal file
View File

@ -0,0 +1,39 @@
---
version: 0.2
branches:
default:
stage: pre-merge
stages:
pre-merge:
worker:
type: docker
path: eve/workers/unit_and_feature_tests
volumes:
- '/home/eve/workspace'
steps:
- Git:
name: fetch source
repourl: '%(prop:git_reference)s'
shallow: True
retryFetch: True
haltOnFailure: True
- ShellCommand:
name: get api node modules from cache
command: mv /home/eve/node_reqs/node_modules .
- ShellCommand:
name: create aws creds dir (required for tests.bash to run)
command: mkdir ~/.aws
# - ShellCommand:
# command: npm install
- ShellCommand:
name: run tests
command: sudo -E bash tests.bash
timeout: 30000
env:
CIRCLE_NODE_INDEX: '0'
CIRCLE_TEST_REPORTS: /tmp
ENABLE_LOCAL_CACHE: '1'
REPORT_TOKEN: report-token-1
CIRCLE_ARTIFACTS: '/tmp'

View File

@ -0,0 +1,55 @@
FROM scality/nodesvc-base
#
# Install apt packages needed by s3 and buildbot_worker
#
COPY s3_packages.list buildbot_worker_packages.list /tmp/
RUN apt-get update -q \
&& apt-get install -qy software-properties-common python-software-properties \
&& add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-add-repository ppa:brightbox/ruby-ng \
&& apt-get update -q \
&& cat /tmp/s3_packages.list | xargs apt-get install -qy \
&& cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
&& pip install pip==9.0.1 \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /tmp/packages.list
RUN pip install flake8 yamllint \
&& pip install s3cmd==1.6.1 \
&& gem install fog-aws -v 1.3.0 \
&& gem install mime-types -v 3.1 \
&& gem install rspec -v 3.5 \
&& gem install json \
&& gem install digest
ENV CXX=g++-4.9
# eve configuration
#
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
&& adduser eve sudo \
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
ARG BUILDBOT_VERSION=0.9.7
RUN pip install buildbot-worker==$BUILDBOT_VERSION
USER eve
#
# Install nodejs packages needed by s3
#
RUN mkdir /home/eve/node_reqs
COPY package.json /home/eve/node_reqs/package.json
RUN cd /home/eve/node_reqs && npm install
WORKDIR /home/eve/workspace
CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
&& sudo service redis-server start \
&& twistd -ny buildbot.tac

View File

@ -0,0 +1,9 @@
ca-certificates
git
libffi-dev
libssl-dev
python2.7
python2.7-dev
python-pip
software-properties-common
sudo

View File

@ -0,0 +1,82 @@
{
"name": "s3",
"version": "7.0.0",
"description": "S3 connector",
"main": "index.js",
"engines": {
"node": ">=6.9.5"
},
"repository": "scality/S3",
"keywords": [
"s3",
"cloud",
"server"
],
"author": "Giorgio Regni",
"license": "Apache-2.0",
"bugs": {
"url": "https://github.com/scality/S3/issues"
},
"homepage": "https://github.com/scality/S3#readme",
"dependencies": {
"aws-sdk": "2.28.0",
"arsenal": "scality/Arsenal",
"async": "~2.5.0",
"azure-storage": "^2.1.0",
"bucketclient": "scality/bucketclient",
"commander": "^2.9.0",
"ioredis": "2.4.0",
"node-uuid": "^1.4.3",
"npm-run-all": "~4.0.2",
"sproxydclient": "scality/sproxydclient",
"utapi": "scality/utapi",
"utf8": "~2.1.1",
"vaultclient": "scality/vaultclient",
"werelogs": "scality/werelogs",
"xml2js": "~0.4.16"
},
"devDependencies": {
"bluebird": "^3.3.1",
"eslint": "^2.4.0",
"eslint-config-airbnb": "^6.0.0",
"eslint-config-scality": "scality/Guidelines",
"istanbul": "1.0.0-alpha.2",
"istanbul-api": "1.0.0-alpha.13",
"lolex": "^1.4.0",
"mocha": "^2.3.4",
"mocha-junit-reporter": "1.11.1",
"node-mocks-http": "^1.5.2",
"s3blaster": "scality/s3blaster",
"tv4": "^1.2.7"
},
"optionalDependencies": {
"cdmiclient": "scality/cdmiclient"
},
"scripts": {
"ft_awssdk": "cd tests/functional/aws-node-sdk && mocha test/",
"ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha test/",
"ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha test/bucket",
"ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha test/legacy test/object test/service test/support test/multipleBackend",
"ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha test/versioning/",
"ft_management": "cd tests/functional/report && npm test",
"ft_node": "cd tests/functional/raw-node && npm test",
"ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
"ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
"ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_management",
"install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
"lint": "eslint $(git ls-files '*.js')",
"lint_md": "mdlint $(git ls-files '*.md')",
"mem_backend": "S3BACKEND=mem node index.js",
"perf": "mocha tests/performance/s3standard.js",
"start": "npm-run-all --parallel start_dmd start_s3server",
"start_mdserver": "node mdserver.js",
"start_dataserver": "node dataserver.js",
"start_s3server": "node index.js",
"start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
"start_utapi": "node lib/utapi/utapi.js",
"utapi_replay": "node lib/utapi/utapiReplay.js",
"test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
"multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend",
"unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml mocha --recursive tests/unit"
}
}

View File

@ -0,0 +1,6 @@
build-essential
g++-4.9
redis-server
ruby2.4
ruby2.4-dev
default-jdk

View File

@ -1,82 +0,0 @@
{
"name": "s3",
"version": "7.0.0",
"description": "S3 connector",
"main": "index.js",
"engines": {
"node": ">=6.9.5"
},
"repository": "scality/S3",
"keywords": [
"s3",
"cloud",
"server"
],
"author": "Giorgio Regni",
"license": "Apache-2.0",
"bugs": {
"url": "https://github.com/scality/S3/issues"
},
"homepage": "https://github.com/scality/S3#readme",
"dependencies": {
"aws-sdk": "2.28.0",
"arsenal": "scality/Arsenal",
"async": "~2.5.0",
"azure-storage": "^2.1.0",
"bucketclient": "scality/bucketclient",
"commander": "^2.9.0",
"ioredis": "2.4.0",
"node-uuid": "^1.4.3",
"npm-run-all": "~4.0.2",
"sproxydclient": "scality/sproxydclient",
"utapi": "scality/utapi",
"utf8": "~2.1.1",
"vaultclient": "scality/vaultclient",
"werelogs": "scality/werelogs",
"xml2js": "~0.4.16"
},
"devDependencies": {
"bluebird": "^3.3.1",
"eslint": "^2.4.0",
"eslint-config-airbnb": "^6.0.0",
"eslint-config-scality": "scality/Guidelines",
"istanbul": "1.0.0-alpha.2",
"istanbul-api": "1.0.0-alpha.13",
"lolex": "^1.4.0",
"mocha": "^2.3.4",
"mocha-junit-reporter": "1.11.1",
"node-mocks-http": "^1.5.2",
"s3blaster": "scality/s3blaster",
"tv4": "^1.2.7"
},
"optionalDependencies": {
"cdmiclient": "scality/cdmiclient"
},
"scripts": {
"ft_awssdk": "cd tests/functional/aws-node-sdk && mocha test/",
"ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha test/",
"ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha test/bucket",
"ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha test/legacy test/object test/service test/support test/multipleBackend",
"ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha test/versioning/",
"ft_management": "cd tests/functional/report && npm test",
"ft_node": "cd tests/functional/raw-node && npm test",
"ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
"ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
"ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_management",
"install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
"lint": "eslint $(git ls-files '*.js')",
"lint_md": "mdlint $(git ls-files '*.md')",
"mem_backend": "S3BACKEND=mem node index.js",
"perf": "mocha tests/performance/s3standard.js",
"start": "npm-run-all --parallel start_dmd start_s3server",
"start_mdserver": "node mdserver.js",
"start_dataserver": "node dataserver.js",
"start_s3server": "node index.js",
"start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
"start_utapi": "node lib/utapi/utapi.js",
"utapi_replay": "node lib/utapi/utapiReplay.js",
"test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
"multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend",
"unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --reporter mocha-junit-reporter --recursive tests/unit"
}
}

1
package.json Symbolic link
View File

@ -0,0 +1 @@
eve/workers/unit_and_feature_tests/package.json

View File

@ -31,13 +31,13 @@ then
mkdir -p $CIRCLE_TEST_REPORTS/unit mkdir -p $CIRCLE_TEST_REPORTS/unit
npm run unit_coverage #npm run unit_coverage
npm run start_dmd & npm run start_dmd &
bash wait_for_local_port.bash 9990 40 && bash wait_for_local_port.bash 9990 40 &&
npm run multiple_backend_test npm run multiple_backend_test
killandsleep 9990 #killandsleep 9990
# Run S3 with multiple data backends ; run ft_tests # Run S3 with multiple data backends ; run ft_tests