Adding gcp credentials

Signed-off-by: Thomas Carmet <thomas.carmet@scality.com>

config.json

@@ -47,7 +47,7 @@
     },
     "clusters": 10,
     "log": {
-        "logLevel": "info",
+        "logLevel": "debug",
         "dumpLevel": "error"
     },
     "healthChecks": {

eve/main.yml

@@ -0,0 +1,190 @@
+---
+version: 0.2
+
+branches:
+  default:
+    stage: pre-merge
+
+models:
+  - Git: &clone
+      name: fetch source
+      repourl: '%(prop:git_reference)s'
+      shallow: True
+      retryFetch: True
+      haltOnFailure: True
+  - env: &circle_ci_env
+      CIRCLE_ARTIFACTS: '/tmp'
+      CIRCLE_NODE_TOTAL: '4'
+      CIRCLE_TEST_REPORTS: '/tmp'
+  - env: &s3_ci_env
+      CI: 'true'
+      CI_REPORTS: '/tmp'
+      ENABLE_LOCAL_CACHE: '1'
+      S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
+  - env: &gcp_credentials_env
+      GOOGLE_SERVICE_KEY: $GOOGLE_S3_SERVICE_KEY
+      GOOGLE_SERVICE_EMAIL: $GOOGLE_S3_SERVICE_EMAIL
+  - ShellCommand: &node_modules
+      name: make node_modules available
+      command: ln -s /usr/lib/node_modules .
+  - ShellCommandWithSecrets: &aws_credentials
+      name: add AWS credentials
+      command: |-
+          mkdir -p ~/.aws;
+          cat >>~/.aws/credentials <<EOF
+          [default]
+          aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
+          aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
+          [default_2]
+          aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
+          aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
+          EOF
+
+  - ShellCommandWithSecrets: &gcp_credentials
+      name: Add GCP credentials
+      command: |-
+        mkdir -p ~/.gcp;
+        cat >>~/.gcp/servicekey <<EOF
+        {
+          "type": "service_account",
+          "private_key": "$GOOGLE_S3_SERVICE_KEY",
+          "client_email": "$GOOGLE_S3_SERVICE_EMAIL"
+        }
+        EOF
+
+stages:
+  pre-merge:
+    worker: &master-worker
+      type: docker
+      path: eve/workers/master
+      volumes:
+        - '/home/eve/workspace'
+    steps:
+      - TriggerStages:
+          name: trigger all tests in parallel
+          stage_names:
+            - static-analysis
+            - unit-tests
+            - multi-backends-unit-tests
+            - node0
+            # - node1
+            # - node2
+            # - node3
+          waitForFinish: True
+          haltOnFailure: True
+
+  static-analysis:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand:
+          name: run lint
+          command: npm run --silent lint -- --max-warnings 0
+      - ShellCommand:
+          name: run lint_md
+          command: npm run --silent lint_md
+      - ShellCommand:
+          name: run flake8
+          command: flake8 $(git ls-files "*.py")
+      - ShellCommand:
+          name: run yamllint
+          command: yamllint $(git ls-files "*.yml")
+
+  unit-tests:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *node_modules
+      - ShellCommand:
+          name: create report directory
+          command: mkdir -p $CIRCLE_TEST_REPORTS/unit
+          env: *circle_ci_env
+      - ShellCommand:
+          name: run legacy unit test
+          command: npm run unit_coverage_legacy_location
+          env: *circle_ci_env
+      # - ShellCommand:
+      #     name: run unit tests
+      #     command: npm run unit_coverage
+      #     env: *circle_ci_env
+
+  multi-backends-unit-tests:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *node_modules
+      - ShellCommandWithSecrets: *aws_credentials
+      - ShellCommandWithSecrets: *gcp_credentials
+      - ShellCommandWithSecrets:
+          name: run multi-backends unit tests
+          command: mocha -t 20000 --recursive tests/multipleBackend
+          env:
+            CI: 'true'
+            S3BACKEND: mem
+            S3DATA: multiple
+            GOOGLE_SERVICE_KEY: '$GOOGLE_S3_SERVICE_KEY'
+            GOOGLE_SERVICE_EMAIL: '$GOOGLE_S3_SERVICE_EMAIL'
+
+
+      - ShellCommand:
+          name: Process
+          command: ps -aux
+
+  # TODO: deduplicate using a setvar for node number
+  node0:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *node_modules
+      - ShellCommandWithSecrets: *aws_credentials
+      - ShellCommandWithSecrets: *gcp_credentials
+      - ShellCommandWithSecrets:
+          name: run tests
+          command: sudo -E bash tests.bash
+          env:
+            <<:
+              - *circle_ci_env
+              - *s3_ci_env
+            CIRCLE_NODE_INDEX: '0'
+
+  node1:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *node_modules
+      - ShellCommand:
+          name: run tests
+          command: sudo -E bash tests.bash
+          env:
+            <<:
+              - *circle_ci_env
+              - *s3_ci_env
+            CIRCLE_NODE_INDEX: '1'
+
+  node2:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *node_modules
+      - ShellCommand:
+          name: run tests
+          command: sudo -E bash tests.bash
+          env:
+            <<:
+              - *circle_ci_env
+              - *s3_ci_env
+            CIRCLE_NODE_INDEX: '2'
+
+  node3:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *node_modules
+      - ShellCommand:
+          name: run tests
+          command: sudo -E bash tests.bash
+          env:
+            <<:
+              - *circle_ci_env
+              - *s3_ci_env
+            CIRCLE_NODE_INDEX: '3'

eve/workers/master/Dockerfile

@@ -0,0 +1,66 @@
+FROM ubuntu:trusty
+
+#
+# Install apt packages needed by the buildchain
+#
+ENV LANG C.UTF-8
+COPY s3_packages.list buildbot_worker_packages.list /tmp/
+RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
+    && apt-get install -qy software-properties-common python-software-properties \
+    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && echo "deb https://deb.nodesource.com/node_6.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
+    && add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-add-repository ppa:brightbox/ruby-ng \
+    && apt-get update -q \
+    && cat /tmp/*_packages.list | xargs apt-get install -qy \
+    && pip install pip==9.0.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/*_packages.list
+
+#
+# Install dependencies
+#
+
+# Install python packages
+COPY requirements.txt /tmp/
+RUN pip install -r /tmp/requirements.txt \
+    && rm -f /tmp/requirements.txt
+
+# Install gems
+COPY gems.list /tmp/
+RUN cat /tmp/gems.list | xargs gem install \
+    && rm /tmp/gems.list
+
+# Install nodejs packages
+WORKDIR /usr/lib/
+COPY package.json .
+RUN npm install
+
+#
+# Add user eve
+#
+
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+
+#
+# Misc
+#
+
+RUN echo '127.0.0.1    bucketwebsitetester.s3-website-us-east-1.amazonaws.com' >> /etc/hosts
+
+#
+# Run buildbot-worker on startup
+#
+
+ARG BUILDBOT_VERSION=0.9.7
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+ENV CXX=g++-4.9
+USER eve
+ENV LANG C.UTF-8
+WORKDIR /home/eve/workspace
+CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
+    && sudo service redis-server start \
+    && buildbot-worker start --nodaemon

eve/workers/master/buildbot_worker_packages.list

@@ -0,0 +1,9 @@
+ca-certificates
+git
+libffi-dev
+libssl-dev
+python2.7
+python2.7-dev
+python-pip
+software-properties-common
+sudo

eve/workers/master/gems.list

@@ -0,0 +1,5 @@
+digest
+fog-aws:1.3.0
+json
+mime-types:3.1
+rspec:3.5

eve/workers/master/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "s3",
+  "version": "7.2.0",
+  "description": "S3 connector",
+  "main": "index.js",
+  "engines": {
+    "node": ">=6.9.5"
+  },
+  "repository": "scality/S3",
+  "keywords": [
+    "s3",
+    "cloud",
+    "server"
+  ],
+  "author": "Giorgio Regni",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/scality/S3/issues"
+  },
+  "homepage": "https://github.com/scality/S3#readme",
+  "dependencies": {
+    "aws-sdk": "2.28.0",
+    "arsenal": "scality/Arsenal",
+    "async": "~2.5.0",
+    "google-auto-auth": "^0.9.1",
+    "azure-storage": "^2.1.0",
+    "bucketclient": "scality/bucketclient",
+    "commander": "^2.9.0",
+    "mongodb": "^2.2.31",
+    "node-uuid": "^1.4.3",
+    "npm-run-all": "~4.0.2",
+    "sproxydclient": "scality/sproxydclient",
+    "utapi": "scality/utapi",
+    "utf8": "~2.1.1",
+    "uuid": "^3.0.1",
+    "vaultclient": "scality/vaultclient",
+    "werelogs": "scality/werelogs",
+    "xml2js": "~0.4.16"
+  },
+  "devDependencies": {
+    "bluebird": "^3.3.1",
+    "eslint": "^2.4.0",
+    "eslint-config-airbnb": "^6.0.0",
+    "eslint-config-scality": "scality/Guidelines",
+    "ioredis": "2.4.0",
+    "istanbul": "1.0.0-alpha.2",
+    "istanbul-api": "1.0.0-alpha.13",
+    "lolex": "^1.4.0",
+    "mocha": "^5.0.0",
+    "mocha-junit-reporter": "1.11.1",
+    "node-mocks-http": "^1.5.2",
+    "s3blaster": "scality/s3blaster",
+    "tv4": "^1.2.7"
+  },
+  "optionalDependencies": {
+    "cdmiclient": "scality/cdmiclient"
+  },
+  "scripts": {
+    "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha test/",
+    "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha test/",
+    "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha test/bucket",
+    "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha test/legacy test/object test/service test/support",
+    "ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha test/versioning/",
+    "ft_awssdk_external_backends": "cd tests/functional/aws-node-sdk && mocha test/multipleBackend",
+    "ft_management": "cd tests/functional/report && npm test",
+    "ft_node": "cd tests/functional/raw-node && npm test",
+    "ft_gcp": "cd tests/functional/raw-node && npm run test-gcp",
+    "ft_healthchecks": "cd tests/functional/healthchecks && npm test",
+    "ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
+    "ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
+    "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management",
+    "install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
+    "lint": "eslint $(git ls-files '*.js')",
+    "lint_md": "mdlint $(git ls-files '*.md')",
+    "mem_backend": "S3BACKEND=mem node index.js",
+    "perf": "mocha tests/performance/s3standard.js",
+    "start": "npm-run-all --parallel start_dmd start_s3server",
+    "start_mongo": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server",
+    "start_mdserver": "node mdserver.js",
+    "start_dataserver": "node dataserver.js",
+    "start_s3server": "node index.js",
+    "start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
+    "start_utapi": "node lib/utapi/utapi.js",
+    "utapi_replay": "node lib/utapi/utapiReplay.js",
+    "test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
+    "test_legacy_location": "CI=true S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --recursive tests/unit",
+    "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend",
+    "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --reporter mocha-junit-reporter --recursive tests/unit",
+    "unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unitlegacylocation.xml istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-junit-reporter --recursive tests/unit"
+  }
+}

eve/workers/master/requirements.txt

@@ -0,0 +1,3 @@
+flake8
+s3cmd==1.6.1
+yamllint

eve/workers/master/s3_packages.list

@@ -0,0 +1,8 @@
+default-jdk
+lsof
+maven
+netcat
+nodejs
+redis-server
+ruby2.4
+ruby2.4-dev

lib/Config.js

@@ -3,7 +3,6 @@ const { EventEmitter } = require('events');
 const fs = require('fs');
 const path = require('path');
 const url = require('url');
-
 const uuid = require('node-uuid');
 
 const { isValidBucketName } = require('arsenal').s3routes.routesUtils;
@@ -279,6 +278,7 @@ class Config extends EventEmitter {
         if (process.env.S3_LOCATION_FILE !== undefined) {
             this.locationConfigPath = process.env.S3_LOCATION_FILE;
         }
+        process.stdout.write(this.locationConfigPath);
 
         // Read config automatically
         this._getLocationConfig();

package.json

@@ -83,7 +83,7 @@
     "utapi_replay": "node lib/utapi/utapiReplay.js",
     "test": "CI=true S3BACKEND=mem mocha --recursive tests/unit",
     "test_legacy_location": "CI=true S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --recursive tests/unit",
-    "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple mocha -t 20000 --recursive tests/multipleBackend",
+    "multiple_backend_test": "CI=true S3BACKEND=mem S3DATA=multiple S3_LOCATION_FILE=tests/locationConfig/locationConfigTests.json mocha -t 20000 --recursive tests/multipleBackend",
     "unit_coverage": "CI=true mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --reporter mocha-junit-reporter --recursive tests/unit",
     "unit_coverage_legacy_location": "CI=true mkdir -p coverage/unitlegacylocation/ && S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unitlegacylocation.xml istanbul cover --dir coverage/unitlegacylocation _mocha -- --reporter mocha-junit-reporter --recursive tests/unit"
   }

tests.bash

@@ -2,29 +2,30 @@
 set -x #echo on
 set -e #exit at the first error
 
-cat >>~/.aws/credentials <<EOF
+# mkdir -p ~/.aws
-[default]
+# cat >>~/.aws/credentials <<EOF
-aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT
+# [default]
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT
+# aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT
-[default_2]
+# aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT
-aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT_2
+# [default_2]
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT_2
+# aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT_2
-[google]
+# aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT_2
-aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE
+# [google]
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE
+# aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE
-[google_2]
+# aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE
-aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE_2
+# [google_2]
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE_2
+# aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE_2
-EOF
+# aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE_2
+# EOF
 
-mkdir ${HOME}/.gcp #create directory for GCP service credential
+# mkdir ${HOME}/.gcp #create directory for GCP service credential
-cat >>${HOME}/.gcp/servicekey <<EOF
+# cat >>${HOME}/.gcp/servicekey <<EOF
-{
+# {
-  "type": "service_account",
+#   "type": "service_account",
-  "private_key": "$GOOGLE_SERVICE_KEY",
+#   "private_key": "$GOOGLE_SERVICE_KEY",
-  "client_email": "$GOOGLE_SERVICE_EMAIL"
+#   "client_email": "$GOOGLE_SERVICE_EMAIL"
-}
+# }
-EOF
+# EOF
 
 MYPWD=$(pwd)
 
@@ -35,47 +36,26 @@ killandsleep () {
 
 if [ $CIRCLE_NODE_INDEX -eq 0 ]
 then
-
-  npm run --silent lint -- --max-warnings 0
-
-  npm run --silent lint_md
-
-  flake8 $(git ls-files "*.py")
-
-  yamllint $(git ls-files "*.yml")
-
-  mkdir -p $CIRCLE_TEST_REPORTS/unit
-
-  npm run unit_coverage
-
-  npm run unit_coverage_legacy_location
-
-  npm run start_dmd &
-  bash wait_for_local_port.bash 9990 40 &&
-  npm run multiple_backend_test
-
-  killandsleep 9990
-
   # Run S3 with multiple data backends ; run ft_tests
 
-  S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_java.txt & bash wait_for_local_port.bash 8000 40 && cd ./tests/functional/jaws && mvn test
+  S3BACKEND=mem S3DATA=multiple npm start # > $CIRCLE_ARTIFACTS/server_multiple_java.txt & bash wait_for_local_port.bash 8000 40 && cd ./tests/functional/jaws && mvn test
 
   killandsleep 8000
   cd $MYPWD
 
-  S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_fog.txt & bash wait_for_local_port.bash 8000 40 && cd tests/functional/fog && rspec tests.rb
+  S3BACKEND=mem S3DATA=multiple npm start #> $CIRCLE_ARTIFACTS/server_multiple_fog.txt & bash wait_for_local_port.bash 8000 40 && cd tests/functional/fog && rspec tests.rb
 
   cd $MYPWD
   killandsleep 8000
 
-  S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple npm run ft_awssdk
+  S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start # > $CIRCLE_ARTIFACTS/server_multiple_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple npm run ft_awssdk
 
   cd $MYPWD
   killandsleep 8000
 
   # Run external backend tests with proxy ; run ft_awssdk_external_backends
 
-  S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple CI_PROXY=true npm start > $CIRCLE_ARTIFACTS/server_external_backends_proxy_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple CI_PROXY=true npm run ft_awssdk_external_backends
+  S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple CI_PROXY=true npm start #> $CIRCLE_ARTIFACTS/server_external_backends_proxy_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple CI_PROXY=true npm run ft_awssdk_external_backends
 
   killandsleep 8000
 
@@ -148,7 +128,7 @@ then
   S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_awssdk.txt & bash wait_for_local_port.bash 8000 40 && npm run ft_test
 
   killandsleep 8000
-  
+
 fi
 
 if [ $CIRCLE_NODE_INDEX -eq 3 ]