Adding eve support to S3

eve/main.yml

@@ -0,0 +1,39 @@
+---
+version: 0.2
+
+branches:
+  default:
+    stage: pre-merge
+
+stages:
+  pre-merge:
+    worker:
+      type: docker
+      path: eve/workers/unit_and_feature_tests
+      volumes:
+        - '/home/eve/workspace'
+    steps:
+      - Git:
+          name: fetch source
+          repourl: '%(prop:git_reference)s'
+          shallow: True
+          retryFetch: True
+          haltOnFailure: True
+      - ShellCommand:
+          name: get api node modules from cache
+          command: mv /home/eve/node_reqs/node_modules .
+      - ShellCommand:
+          name: create aws creds dir (required for tests.bash to run)
+          command: mkdir ~/.aws
+#      - ShellCommand:
+#          command: npm install
+      - ShellCommand:
+          name: run tests
+          command: sudo -E bash tests.bash
+          timeout: 30000
+          env:
+            CIRCLE_NODE_INDEX: '0'
+            CIRCLE_TEST_REPORTS: /tmp
+            ENABLE_LOCAL_CACHE: '1'
+            REPORT_TOKEN: report-token-1
+            CIRCLE_ARTIFACTS: '/tmp'

eve/workers/unit_and_feature_tests/Dockerfile

@@ -0,0 +1,55 @@
+FROM scality/nodesvc-base
+
+#
+# Install apt packages needed by s3 and buildbot_worker
+#
+COPY s3_packages.list buildbot_worker_packages.list /tmp/
+
+RUN apt-get update -q \
+    && apt-get install -qy software-properties-common python-software-properties \
+    && add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-add-repository ppa:brightbox/ruby-ng \
+    && apt-get update -q \
+    && cat /tmp/s3_packages.list | xargs apt-get install -qy \
+    && cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
+    && pip install pip==9.0.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/packages.list
+
+RUN pip install flake8 yamllint \
+    && pip install s3cmd==1.6.1 \
+    && gem install fog-aws -v 1.3.0 \
+    && gem install mime-types -v 3.1 \
+    && gem install rspec -v 3.5 \
+    && gem install json \
+    && gem install digest 
+
+ENV CXX=g++-4.9
+
+# eve configuration
+#
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+
+ARG BUILDBOT_VERSION=0.9.7
+
+
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+USER eve
+
+#
+# Install nodejs packages needed by s3
+#
+
+RUN mkdir /home/eve/node_reqs
+
+COPY package.json /home/eve/node_reqs/package.json
+RUN cd /home/eve/node_reqs && npm install
+
+WORKDIR /home/eve/workspace
+
+CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
+    && sudo service redis-server start \
+    && twistd -ny buildbot.tac

eve/workers/unit_and_feature_tests/buildbot_worker_packages.list

@@ -0,0 +1,9 @@
+ca-certificates
+git
+libffi-dev
+libssl-dev
+python2.7
+python2.7-dev
+python-pip
+software-properties-common
+sudo

eve/workers/unit_and_feature_tests/package.json

@@ -0,0 +1,82 @@
+{
+  "name": "s3",
+  "version": "7.0.0",
+  "description": "S3 connector",
+  "main": "index.js",
+  "engines": {
+    "node": ">=6.9.5"
+  },
+  "repository": "scality/S3",
+  "keywords": [
+    "s3",
+    "cloud",
+    "server"
+  ],
+  "author": "Giorgio Regni",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/scality/S3/issues"
+  },
+  "homepage": "https://github.com/scality/S3#readme",
+  "dependencies": {
+    "aws-sdk": "2.28.0",
+    "arsenal": "scality/Arsenal",
+    "async": "~2.5.0",
+    "azure-storage": "^2.1.0",
+    "bucketclient": "scality/bucketclient",
+    "commander": "^2.9.0",
+    "ioredis": "2.4.0",
+    "node-uuid": "^1.4.3",
+    "npm-run-all": "~4.0.2",
+    "sproxydclient": "scality/sproxydclient",
+    "utapi": "scality/utapi",
+    "utf8": "~2.1.1",
+    "vaultclient": "scality/vaultclient",
+    "werelogs": "scality/werelogs",
+    "xml2js": "~0.4.16"
+  },
+  "devDependencies": {
+    "bluebird": "^3.3.1",
+    "eslint": "^2.4.0",
+    "eslint-config-airbnb": "^6.0.0",
+    "eslint-config-scality": "scality/Guidelines",
+    "istanbul": "1.0.0-alpha.2",
+    "istanbul-api": "1.0.0-alpha.13",
+    "lolex": "^1.4.0",
+    "mocha": "^2.3.4",
+    "mocha-junit-reporter": "1.11.1",
+    "node-mocks-http": "^1.5.2",
+    "s3blaster": "scality/s3blaster",
+    "tv4": "^1.2.7"
+  },
+  "optionalDependencies": {
+    "cdmiclient": "scality/cdmiclient"
+  },
+  "scripts": {
+    "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha test/",
+    "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha test/",
+    "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha test/bucket",
+    "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha test/legacy test/object test/service test/support test/multipleBackend",
+    "ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha test/versioning/",
+    "ft_management": "cd tests/functional/report && npm test",
+    "ft_node": "cd tests/functional/raw-node && npm test",
+    "ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
+    "ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
+    "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_management",
+    "install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
+    "lint": "eslint $(git ls-files '*.js')",
+    "lint_md": "mdlint $(git ls-files '*.md')",
+    "mem_backend": "S3BACKEND=mem node index.js",
+    "perf": "mocha tests/performance/s3standard.js",
+    "start": "npm-run-all --parallel start_dmd start_s3server",
+    "start_mdserver": "node mdserver.js",
+    "start_dataserver": "node dataserver.js",
+    "start_s3server": "node index.js",
+    "start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
+    "start_utapi": "node lib/utapi/utapi.js",
+    "utapi_replay": "node lib/utapi/utapiReplay.js",
+    "test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
+    "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend",
+    "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml mocha --recursive tests/unit"
+  }
+}

eve/workers/unit_and_feature_tests/s3_packages.list

@@ -0,0 +1,6 @@
+build-essential
+g++-4.9
+redis-server
+ruby2.4
+ruby2.4-dev
+default-jdk

package.json

@@ -1,82 +0,0 @@
-{
-  "name": "s3",
-  "version": "7.0.0",
-  "description": "S3 connector",
-  "main": "index.js",
-  "engines": {
-    "node": ">=6.9.5"
-  },
-  "repository": "scality/S3",
-  "keywords": [
-    "s3",
-    "cloud",
-    "server"
-  ],
-  "author": "Giorgio Regni",
-  "license": "Apache-2.0",
-  "bugs": {
-    "url": "https://github.com/scality/S3/issues"
-  },
-  "homepage": "https://github.com/scality/S3#readme",
-  "dependencies": {
-    "aws-sdk": "2.28.0",
-    "arsenal": "scality/Arsenal",
-    "async": "~2.5.0",
-    "azure-storage": "^2.1.0",
-    "bucketclient": "scality/bucketclient",
-    "commander": "^2.9.0",
-    "ioredis": "2.4.0",
-    "node-uuid": "^1.4.3",
-    "npm-run-all": "~4.0.2",
-    "sproxydclient": "scality/sproxydclient",
-    "utapi": "scality/utapi",
-    "utf8": "~2.1.1",
-    "vaultclient": "scality/vaultclient",
-    "werelogs": "scality/werelogs",
-    "xml2js": "~0.4.16"
-  },
-  "devDependencies": {
-    "bluebird": "^3.3.1",
-    "eslint": "^2.4.0",
-    "eslint-config-airbnb": "^6.0.0",
-    "eslint-config-scality": "scality/Guidelines",
-    "istanbul": "1.0.0-alpha.2",
-    "istanbul-api": "1.0.0-alpha.13",
-    "lolex": "^1.4.0",
-    "mocha": "^2.3.4",
-    "mocha-junit-reporter": "1.11.1",
-    "node-mocks-http": "^1.5.2",
-    "s3blaster": "scality/s3blaster",
-    "tv4": "^1.2.7"
-  },
-  "optionalDependencies": {
-    "cdmiclient": "scality/cdmiclient"
-  },
-  "scripts": {
-    "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha test/",
-    "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha test/",
-    "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha test/bucket",
-    "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha test/legacy test/object test/service test/support test/multipleBackend",
-    "ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha test/versioning/",
-    "ft_management": "cd tests/functional/report && npm test",
-    "ft_node": "cd tests/functional/raw-node && npm test",
-    "ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
-    "ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
-    "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_management",
-    "install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
-    "lint": "eslint $(git ls-files '*.js')",
-    "lint_md": "mdlint $(git ls-files '*.md')",
-    "mem_backend": "S3BACKEND=mem node index.js",
-    "perf": "mocha tests/performance/s3standard.js",
-    "start": "npm-run-all --parallel start_dmd start_s3server",
-    "start_mdserver": "node mdserver.js",
-    "start_dataserver": "node dataserver.js",
-    "start_s3server": "node index.js",
-    "start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
-    "start_utapi": "node lib/utapi/utapi.js",
-    "utapi_replay": "node lib/utapi/utapiReplay.js",
-    "test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
-    "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend",
-    "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --reporter mocha-junit-reporter --recursive tests/unit"
-  }
-}

package.json

@@ -0,0 +1 @@
+eve/workers/unit_and_feature_tests/package.json

tests.bash

@@ -31,13 +31,13 @@ then
 
   mkdir -p $CIRCLE_TEST_REPORTS/unit
 
-  npm run unit_coverage
+  #npm run unit_coverage
 
   npm run start_dmd &
   bash wait_for_local_port.bash 9990 40 &&
   npm run multiple_backend_test
 
-  killandsleep 9990
+  #killandsleep 9990
 
   # Run S3 with multiple data backends ; run ft_tests