fixup aws creds bash

Dockerfile

@@ -1,20 +1,10 @@
-FROM node:6-slim
+FROM zenko/cloudserver:pensieve-0
-MAINTAINER Giorgio Regni <gr@scality.com>
 
-WORKDIR /usr/src/app
+RUN apt-get update && apt-get install -qqy git
 
-# Keep the .git directory in order to properly report version
 COPY . /usr/src/app
-
+RUN npm cache clear
-RUN apt-get update \
+RUN mkdir -p /root/.ssh/ && touch ~/.ssh/known_hosts && ssh-keygen -R github.com && npm install uuid && npm install --production
-    && apt-get install -y jq python git build-essential --no-install-recommends \
-    && npm install --production \
-    && apt-get autoremove --purge -y python git build-essential \
-    && rm -rf /var/lib/apt/lists/* \
-    && npm cache clear \
-    && rm -rf ~/.node-gyp \
-    && rm -rf /tmp/npm-*
-
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
 ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]

dataserver.js

@@ -4,11 +4,14 @@ const arsenal = require('arsenal');
 const { config } = require('./lib/Config.js');
 const logger = require('./lib/utilities/logger');
 
+console.log(config.backends.data);
+console.log(config.backends.metadata);
 if (config.backends.data === 'file' ||
     (config.backends.data === 'multiple' &&
      config.backends.metadata !== 'scality') &&
      (config.backends.auth !== 'scality' &&
       config.backends.metadata !== 'mongodb')) {
+    console.log('!!Data Server Setup');
     const dataServer = new arsenal.network.rest.RESTServer(
         { bindAddress: config.dataDaemon.bindAddress,
             port: config.dataDaemon.port,

eve/main.yml

@@ -0,0 +1,175 @@
+---
+version: 0.2
+
+branches:
+  default:
+    stage: pre-merge
+models:
+  - env: &global_env
+      CI: 'true'
+      CIRCLE_ARTIFACTS: '/tmp'
+      CIRCLE_NODE_TOTAL: "4"
+      CIRCLE_TEST_REPORTS: '/tmp'
+      CI_REPORTS: '/tmp'
+      ENABLE_LOCAL_CACHE: "1"
+      REPORT_TOKEN: 'report-token-1'
+  - Git: &clone
+      name: Pull repo
+      repourl: '%(prop:git_reference)s'
+      shallow: True
+      retryFetch: True
+      haltOnFailure: True
+      # Machine Setup
+  - ShellCommand: &update_hosts
+      name: Update hosts
+      command: sudo bash -c "echo '
+               127.0.0.1     \
+               bucketwebsitetester.s3-website-us-east-1.amazonaws.com
+               ' >> /etc/hosts"
+      haltOnFailure: True
+  - ShellCommandWthSecrets: &aws-credentials
+      name: Setup AWS Credentials
+      command: bash %(prop:worker_path)s/aws.bash
+      haltOnFailure: True
+  - ShellCommandWithSecrets: &npm-install
+      name: install modules
+      command: npm install
+      haltOnFailure: True
+  - ShellCommand: &kill-s3
+      name: Kill S3
+      command: bash %(prop:worker_path)s/killandsleep.bash 8000
+      haltOnFailure: True
+  - ShellCommand: &kill-dmd
+      name: Kill DMD
+      command: bash %(prop:worker_path)s/killandsleep.bash 9990
+      haltOnFailure: True
+stages:
+  pre-merge:
+    worker: &master-worker
+      type: docker
+      path: eve/workers/build
+      volumes: &default_volumes
+        - '/home/eve/workspace'
+    steps:
+      - TriggerStages:
+          name: Launch all nodes simultaneously
+          stage_names:
+            - node0
+            - node1
+            - node2
+            - node3
+          waitForFinish: True
+          haltOnFailure: True
+  node0:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *update_hosts
+      - ShellCommand:
+          command: rm -rf node_modules
+#      - ShellCommandWithSecrets: *aws-credentials
+      - ShellCommandWithSecrets: *npm-install
+      - ShellCommandWithSecrets:
+          command: bash -l tests.bash
+          env:
+            <<: *global_env
+            CIRCLE_NODE_INDEX: '0'
+          timeout: 120
+#      - ShellCommandWithSecrets:
+#          name: Linting
+#          command: |
+#            npm run --silent lint -- --max-warnings 0
+#            npm run --silent lint_md
+#            flake8 $(git ls-files "*.py")
+#            yamllint $(git ls-files "*.yml")
+#      - ShellCommandWithSecrets:
+#          name: Unit Coverage
+#          command: |
+#            mkdir -p $CIRCLE_TEST_REPORTS/unit
+#            npm run unit_coverage
+#            npm run unit_coverage_legacy_location
+#          env: *global_env
+#          haltOnFailure: True
+# testing with supervisor
+#      - ShellCommand:
+#          command: ln -s $PWD /home/eve/S3 && ls ~/S3
+#      - ShellCommandWithSecrets:
+#          name: Run all tests
+#          command: sudo -E supervisord -c /etc/supervisor/supervisord.conf -n
+#          env: 
+#            <<: *global_env
+#            CIRCLE_NODE_INDEX: '0'
+#            WORK_DIR: /home/eve/S3
+#      - ShellCommandWithSecrets:
+#          name: Node 1 Tests
+#          command: |
+#            . ~/.bashrc &> /dev/null \
+#            && bash wait_for_local_port.bash 9991 40 \
+#            && npm run multiple_backend_test || exit 1
+#          env: 
+#            <<: *global_env
+#            CIRCLE_NODE_INDEX: '0'
+#          timeout: 120
+#      - ShellCommand: *kill-dmd
+#      - ShellCommandWithSecrets:
+#          command: ls -la && cat /tmp/*.txt
+#          env:
+#            <<: *global_env
+#          haltOnFailure: True
+  node1:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *update_hosts
+      - ShellCommand:
+          command: rm -rf node_modules
+      - ShellCommandWithSecrets: *aws-credentials
+      - ShellCommandWithSecrets: *npm-install
+      - ShellCommandWithSecrets:
+          command: bash -l tests.bash
+          env:
+            <<: *global_env
+            CIRCLE_NODE_INDEX: '1'
+      - ShellCommandWithSecrets:
+          command: ls -la && cat /tmp/*.txt
+          env:
+            <<: *global_env
+          haltOnFailure: True
+  node2:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *update_hosts
+      - ShellCommand:
+          command: rm -rf node_modules
+      - ShellCommandWithSecrets: *aws-credentials
+      - ShellCommandWithSecrets: *npm-install
+      - ShellCommandWithSecrets:
+          command: bash -l tests.bash
+          env:
+            <<: *global_env
+            CIRCLE_NODE_INDEX: '2'
+      - ShellCommandWithSecrets:
+          command: ls -la && cat /tmp/*.txt
+          env:
+            <<: *global_env
+          haltOnFailure: True
+  node3:
+    worker: *master-worker
+    steps:
+      - Git: *clone
+      - ShellCommand: *update_hosts
+      - ShellCommand:
+          command: rm -rf node_modules
+      - ShellCommandWithSecrets: *aws-credentials
+      - ShellCommandWithSecrets: *npm-install
+      - ShellCommandWithSecrets:
+          command: bash -l tests.bash
+          env:
+            <<: *global_env
+            CIRCLE_NODE_INDEX: '3'
+      - ShellCommandWithSecrets:
+          command: ls -la && cat /tmp/*.txt
+          env:
+            <<: *global_env
+          haltOnFailure: True

eve/workers/build/Dockerfile

@@ -0,0 +1,64 @@
+FROM buildpack-deps:jessie-curl
+
+#
+# Install packages needed by the buildchain
+#
+ENV LANG C.UTF-8
+COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
+RUN apt-get update \
+    && cat /tmp/*packages.list | xargs apt-get install -y \
+    && wget https://nodejs.org/dist/v6.9.5/node-v6.9.5-linux-x64.tar.xz \
+    && tar -C /usr/local --strip-components 1 -xvf node-v6.9.5-linux-x64.tar.xz \
+    && pip install pip==9.0.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/packages.list
+
+#
+# Add user eve
+#
+
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+#
+# Install Dependencies
+#
+
+# Install RVM and gems
+ENV RUBY_VERSION="2.4.1"
+COPY ./gems.list /tmp/
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
+    && curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
+    && usermod -a -G rvm eve
+RUN /bin/bash -l -c "\
+    source /usr/local/rvm/scripts/rvm \
+    && cat /tmp/gems.list | xargs gem install \
+    && rm /tmp/gems.list"
+
+# Install Pip packages
+COPY ./pip_packages.list /tmp/
+RUN cat /tmp/pip_packages.list | xargs pip install \
+    && rm -f /tmp/pip_packages.list \
+    && mkdir /home/eve/.aws \
+    && chown eve /home/eve/.aws 
+
+
+# environment vars into the bash profile
+COPY ./bash_profile /home/eve/.bash_profile
+RUN chown eve /home/eve/.bash_profile
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+#
+# Run buildbot-worker on startup
+#
+
+ARG BUILDBOT_VERSION
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+#USER eve
+COPY ./bash_profile /home/eve/.bashrc
+COPY ./bash_profile /home/eve/.bash_profile
+#RUN exec bash -l
+#WORKDIR /home/eve/workspace
+#CMD ["sudo", "supervisord", "-n"]
+CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS   && buildbot-worker start --nodaemon"]

eve/workers/build/aws.bash

@@ -0,0 +1,12 @@
+#!/bin/bash -x
+set -x #echo on
+set -e #exit at the first error
+mkdir -p ~/.aws
+cat >>~/.aws/credentials <<EOF
+[default]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
+[default_2]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
+EOF

eve/workers/build/bash_profile

@@ -0,0 +1,16 @@
+export GOOGLE_SERVICE_EMAIL="$GCP_BACKEND_SERVICE_EMAIL"
+export GOOGLE_SERVICE_KEY="$GCP_BACKEND_SERVICE_KEY"
+export azurebackend2_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY_2"
+export azurebackend2_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME_2"
+export azurebackend2_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT_2"
+export azurebackend_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurebackend_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurebackend_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurebackendmismatch_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurenonexistcontainer_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azuretest_AZURE_BLOB_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+

eve/workers/build/buildbot_worker_packages.list

@@ -0,0 +1,11 @@
+tcl
+ca-certificates
+git
+gnupg
+libffi-dev
+libssl-dev
+python2.7
+python2.7-dev
+python-pip
+software-properties-common
+sudo

eve/workers/build/gems.list

@@ -0,0 +1,4 @@
+fog-aws:1.3.0
+json
+mime-types:3.1
+rspec:3.5

eve/workers/build/killandsleep.bash

@@ -0,0 +1,8 @@
+#!/bin/bash -x
+set -x #echo on
+set -e #exit at the first error
+
+killandsleep () {
+  kill -9 $(lsof -t -i:$1) || true
+  sleep 10
+}

eve/workers/build/pip_packages.list

@@ -0,0 +1,3 @@
+flake8
+s3cmd==1.6.1
+yamllint

eve/workers/build/s3_packages.list

@@ -0,0 +1,7 @@
+default-jdk
+lsof
+maven
+netcat
+nodejs
+redis-server
+supervisor

eve/workers/build/supervisord.conf

@@ -0,0 +1,22 @@
+#[program:start_dmd]
+#command=/bin/bash -l /home/eve/S3/tests.bash
+#directory=/home/eve/S3
+#autostart=true
+#autorestart=true
+#process_name=dmd
+#numprocs=1
+#stdout_logfile=/dev/fd/1
+##stdout_logfile=/tmp/dmd_server.txt
+#stdout_logfile_maxbytes=0
+#stderr_logfile=/dev/fd/2
+##stderr_logfile=/tmp/dmd_server.txt
+#stderr_logfile_maxbytes=0
+#stopsignal=QUIT
+##environment=
+##    FOO=bar
+##    FOO2=bar
+
+[program:redis]
+command=/usr/bin/redis-server
+autostart=true
+autorestart=true

tests.bash

@@ -2,21 +2,26 @@
 set -x #echo on
 set -e #exit at the first error
 
-cat >>~/.aws/credentials <<EOF
+source /home/eve/.bashrc
+source ./eve/workers/build/bash_profile
+mkdir -p /root/.aws/
+cat >>$(pwd)/credentials <<EOF
 [default]
-aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
 [default_2]
-aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT_2
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT_2
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
 [google]
-aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY
 [google_2]
-aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE_2
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY_2
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE_2
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY_2
 EOF
-
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+cp $(pwd)/credentials /root/.aws/
 MYPWD=$(pwd)
 
 killandsleep () {
@@ -29,13 +34,13 @@ export REMOTE_MANAGEMENT_DISABLE=1
 if [ $CIRCLE_NODE_INDEX -eq 0 ]
 then
 
-  npm run --silent lint -- --max-warnings 0
+#  npm run --silent lint -- --max-warnings 0
-
+#
-  npm run --silent lint_md
+#  npm run --silent lint_md
-
+#
-  flake8 $(git ls-files "*.py")
+#  flake8 $(git ls-files "*.py")
-
+#
-  yamllint $(git ls-files "*.yml")
+#  yamllint $(git ls-files "*.yml")
 
   mkdir -p $CIRCLE_TEST_REPORTS/unit
 
@@ -46,11 +51,9 @@ then
   npm run start_dmd &
   bash wait_for_local_port.bash 9990 40 &&
   npm run multiple_backend_test
-
   killandsleep 9990
 
   # Run S3 with multiple data backends ; run ft_tests
-
   S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_java.txt & bash wait_for_local_port.bash 8000 40 && cd ./tests/functional/jaws && mvn test
 
   killandsleep 8000

tests/multipleBackend/backendHealthcheckResponse.js

@@ -14,7 +14,6 @@ const {
 const log = new DummyRequestLogger();
 const locConstraints = Object.keys(config.locationConstraints);
 const azureClient = getAzureClient();
-
 describe('Healthcheck response', () => {
     it('should return result for every location constraint in ' +
     'locationConfig and every external locations with flightCheckOnStartUp ' +