feature: Speed up Docker rebuild using cache

Allows faster CI and dev builds of the Docker image through the use
of cached layers.

Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Giorgio Regni <gr@scality.com>
 WORKDIR /usr/src/app
 
 # Keep the .git directory in order to properly report version
-COPY . /usr/src/app
+COPY ./package.json .
 
 RUN apt-get update \
     && apt-get install -y jq python git build-essential --no-install-recommends \
@@ -15,6 +15,8 @@ RUN apt-get update \
     && rm -rf ~/.node-gyp \
     && rm -rf /tmp/npm-*
 
+COPY ./ ./
+
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
 ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]

eve/main.yml

@@ -0,0 +1,102 @@
+---
+version: 0.2
+
+branches:
+  feature/*, improvement/*, bugfix/*, w/*, q/*, hotfix/*:
+    stage: pre-merge
+  development/*:
+    stage: post-merge
+
+models:
+  - Git: &clone
+      name: Pull repo
+      repourl: '%(prop:git_reference)s'
+      shallow: True
+      retryFetch: True
+      haltOnFailure: True
+      # Machine Setup
+  - ShellCommandWthSecrets: &credentials
+      name: Setup Credentials
+      command: bash eve/workers/build/credentials.bash
+      haltOnFailure: True
+  - ShellCommandWithSecrets: &npm-install
+      name: install modules
+      command: npm install
+      haltOnFailure: True
+stages:
+  pre-merge:
+    worker:
+      type: local
+    steps:
+      - SetProperty:
+          property: artifacts_name
+          value: "scality-s3-%(prop:buildnumber)s"
+          haltOnFailure: True
+
+      - TriggerStages:
+          name: Launch all workers
+          stage_names:
+            - linting-coverage
+            - mongo-ft-tests
+          waitForFinish: True
+          haltOnFailure: True
+
+  linting-coverage:
+    worker:
+      type: docker
+      path: eve/workers/build
+      volumes: &default_volumes
+        - '/home/eve/workspace'
+    steps:
+      - Git: *clone
+      - ShellCommand: *npm-install
+      - ShellCommand:
+          name: Linting
+          command: |
+            npm run --silent lint -- --max-warnings 0
+            npm run --silent lint_md
+            flake8 $(git ls-files "*.py")
+            yamllint $(git ls-files "*.yml")
+      - ShellCommand:
+          name: Unit Coverage
+          command: |
+            mkdir -p $CIRCLE_TEST_REPORTS/unit
+            npm run unit_coverage
+            npm run unit_coverage_legacy_location
+          haltOnFailure: True
+          env:
+            CIRCLE_TEST_REPORTS: /tmp
+
+  mongo-ft-tests:
+    worker:
+      type: kube_pod
+      path: eve/workers/pod.yaml
+      images:
+        aggressor: eve/workers/build
+        s3: "."
+      vars: &mongo-vars
+        S3BACKEND: "mem"
+        S3DATA: ""
+        MPU_TESTING: "yes"
+        S3METADATA: mongodb
+        CI_PROXY: "false"
+    steps:
+      - Git: *clone
+      - ShellCommandWithSecrets: *credentials
+      - ShellCommand: *npm-install
+      - ShellCommandWithSecrets:
+          command: |
+            . /root/.aws/exports &> /dev/null
+            bash wait_for_local_port.bash 8000 40
+            npm run ft_test
+          env:
+            <<: *mongo-vars
+
+  post-merge:
+    worker:
+      type: local
+    steps:
+      - Git: *clone
+      - ShellCommand:
+          name: Nightly Build
+          command: docker build -t zenko/cloudserver:nightly .

eve/workers/build/Dockerfile

@@ -0,0 +1,52 @@
+FROM ubuntu:xenial
+
+#
+# Install packages needed by the buildchain
+#
+ENV LANG C.UTF-8
+COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
+RUN apt-get update \
+    && cat /tmp/*packages.list | xargs apt-get install -y \
+    && wget https://nodejs.org/dist/v6.9.5/node-v6.9.5-linux-x64.tar.xz \
+    && tar -C /usr/local --strip-components 1 -xvf node-v6.9.5-linux-x64.tar.xz \
+    && pip install pip==9.0.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/packages.list
+
+#
+# Add user eve
+#
+
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+#
+# Install Dependencies
+#
+
+# Install RVM and gems
+ENV RUBY_VERSION="2.4.1"
+COPY ./gems.list /tmp/
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
+    && curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
+    && usermod -a -G rvm eve
+RUN /bin/bash -l -c "\
+    source /usr/local/rvm/scripts/rvm \
+    && cat /tmp/gems.list | xargs gem install \
+    && rm /tmp/gems.list"
+
+# Install Pip packages
+COPY ./pip_packages.list /tmp/
+RUN cat /tmp/pip_packages.list | xargs pip install \
+    && rm -f /tmp/pip_packages.list \
+    && mkdir /home/eve/.aws \
+    && chown eve /home/eve/.aws 
+
+#
+# Run buildbot-worker on startup
+#
+
+ARG BUILDBOT_VERSION
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS   && buildbot-worker start --nodaemon"]

eve/workers/build/buildbot_worker_packages.list

@@ -0,0 +1,12 @@
+ca-certificates
+git
+gnupg
+libffi-dev
+libssl-dev
+python-pip
+python2.7
+python2.7-dev
+software-properties-common
+sudo
+tcl
+wget

eve/workers/build/credentials.bash

@@ -0,0 +1,41 @@
+#!/bin/bash -x
+set -x #echo on
+set -e #exit at the first error
+
+mkdir -p ~/.aws
+cat >>/root/.aws/exports <<EOF
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+export GOOGLE_SERVICE_EMAIL="$GCP_BACKEND_SERVICE_EMAIL"
+export GOOGLE_SERVICE_KEY="$GCP_BACKEND_SERVICE_KEY"
+export azurebackend2_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY_2"
+export azurebackend2_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME_2"
+export azurebackend2_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT_2"
+export azurebackend_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurebackend_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurebackend_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurebackendmismatch_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurenonexistcontainer_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azuretest_AZURE_BLOB_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+EOF
+
+source /root/.aws/exports &> /dev/null
+cat >>/root/.aws/credentials <<EOF
+[default]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
+[default_2]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
+[google]
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY
+[google_2]
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY_2
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY_2
+EOF
+

eve/workers/build/gems.list

@@ -0,0 +1,4 @@
+fog-aws:1.3.0
+json
+mime-types:3.1
+rspec:3.5

eve/workers/build/pip_packages.list

@@ -0,0 +1,3 @@
+flake8
+s3cmd==1.6.1
+yamllint

eve/workers/build/s3_packages.list

@@ -0,0 +1,7 @@
+curl
+default-jdk
+libdigest-hmac-perl
+lsof
+maven
+netcat
+redis-server

eve/workers/pod.yaml

@@ -0,0 +1,163 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "proxy-ci-test-pod"
+spec:
+  activeDeadlineSeconds: 3600
+  restartPolicy: Never
+  terminationGracePeriodSeconds: 10
+  hostAliases:
+  - ip: "127.0.0.1"
+    hostnames:
+    - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com"
+  containers:
+    - name: aggressor
+      image: {{ images.aggressor }}
+      imagePullPolicy: IfNotPresent
+      resources:
+        requests:
+          cpu: 100m
+          memory: 512Mi
+        limits:
+          cpu: "1"
+          memory: 2Gi
+      volumeMounts:
+        - name: creds
+          readOnly: false
+          mountPath: /root/.aws
+      command:
+        - bash
+        - -lc
+        - |
+          buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS
+          buildbot-worker start --nodaemon
+      env:
+        - name: CI
+          value: "true"
+        - name: ENABLE_LOCAL_CACHE
+          value: "true"
+        - name: S3BACKEND
+          value: "{{ vars.S3BACKEND }}"
+        - name: MPU_TESTING
+          value: "{{ vars.MPU_TESTING }}"
+        - name: S3DATA
+          value: "{{ vars.S3DATA }}"
+        - name: CI_PROXY
+          value: "{{ vars.CI_PROXY }}"
+        - name: REPORT_TOKEN
+          value: "report-token-1"
+        - name: REMOTE_MANAGEMENT_DISABLE
+          value: "1"
+    - name: s3
+      image: {{ images.s3 }}
+      imagePullPolicy: IfNotPresent
+      resources:
+        requests:
+          cpu: 200m
+          memory: 512Mi
+        limits:
+          cpu: 500m
+          memory: 1Gi
+      volumeMounts:
+        - name: creds
+          readOnly: false
+          mountPath: /root/.aws
+        - name: certs
+          readOnly: true
+          mountPath: /tmp
+      command:
+        - bash
+        - -ec
+        - |
+          # wait for exports to be created
+          sleep 10
+          source /root/.aws/exports || exit 1
+          /usr/src/app/docker-entrypoint.sh npm start
+      env:
+        {% if vars.S3DATA == "multiple" -%}
+        - name: S3_LOCATION_FILE
+          value: "/usr/src/app/tests/locationConfig/locationConfigTests.json"
+        {%- endif %}
+        - name: CI
+          value: "true"
+        - name: CI_PROXY
+          value: "{{ vars.CI_PROXY }}"
+        - name: ENABLE_LOCAL_CACHE
+          value: "true"
+        - name: S3BACKEND
+          value: "{{ vars.S3BACKEND }}"
+        - name: S3DATA
+          value: "{{ vars.S3DATA }}"
+        - name: S3METADATA
+          value: "{{ vars.S3METADATA }}"
+        - name: MONGODB_HOSTS
+          value: "localhost:27018"
+        - name: MONGODB_RS
+          value: "rs0"
+        - name: MPU_TESTING
+          value: "{{ vars.MPU_TESTING }}"
+        - name: REDIS_HOST
+          value: "localhost"
+        - name: REDIS_PORT
+          value: "6379"
+        - name: REPORT_TOKEN
+          value: "report-token-1"
+        - name: REMOTE_MANAGEMENT_DISABLE
+          value: "1"
+        - name: HEALTHCHECKS_ALLOWFROM
+          value: "0.0.0.0/0"
+    {% if vars.S3METADATA == "mongodb" -%}
+    - name: mongo
+      image: scality/ci-mongo:3.4
+      imagePullPolicy: IfNotPresent
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 500m
+          memory: 512Mi
+    {%- endif %}
+    - name: redis
+      image: redis:alpine
+      imagePullPolicy: IfNotPresent
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 200m
+          memory: 512Mi
+    {% if vars.CI_PROXY == "true" -%}
+    - name: squid
+      image: scality/ci-squid
+      imagePullPolicy: IfNotPresent
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 250m
+          memory: 256Mi
+      volumeMounts:
+        - name: certs
+          readOnly: false
+          mountPath: /ssl
+      command:
+        - sh
+        - -exc
+        - |
+          mkdir -p /ssl
+          openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
+           -subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
+           -keyout /ssl/myca.pem  -out /ssl/myca.pem
+          cp /ssl/myca.pem /ssl/CA.pem
+          squid -f /etc/squid/squid.conf -N -z
+          squid -f /etc/squid/squid.conf -NYCd 1
+    {%- endif %}
+  volumes:
+    - name: creds
+      emptyDir: {}
+    - name: certs
+      emptyDir: {}

wait_for_local_port.bash

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-
+set -e
 wait_for_local_port() {
     local port=$1
     local timeout=$2