Compare commits

...

54 Commits

Author SHA1 Message Date
Salim b589a1e36d unify dockerfile 2018-06-18 10:06:51 -07:00
Salim 63b672622e unify Dockerfiles WIP 2018-06-12 16:08:03 -07:00
Salim 3c3f9d9ede unify Dockerfiles 2018-06-11 18:14:47 -07:00
Salim aedba84e0b debug 2018-06-11 16:01:03 -07:00
Salim c633fe9325 passing node0 tests 2018-06-11 15:43:54 -07:00
Salim 128c5aaa09 revert 2018-06-11 12:50:22 -07:00
Salim 5a601d88ca ci true 2018-06-11 12:44:08 -07:00
Salim 79fd3392a2 cleanup disable tests 2018-06-07 15:42:52 -07:00
Salim f37e26495e enable all nodes 2018-06-07 14:58:08 -07:00
Salim cf8a21f38c cleanup 2018-06-07 14:58:08 -07:00
Salim 2e0be383aa location config 2018-06-07 14:58:08 -07:00
Salim 48f23206d7 increase cpu 2018-06-07 10:32:20 -07:00
Salim 4877ef8782 locations 2018-06-07 10:19:55 -07:00
Salim dfc4254885 exports 2018-06-07 10:10:23 -07:00
Salim 8c679895a9 exports 2018-06-07 10:02:40 -07:00
Salim 7efe002cc0 share secrets with s3 2018-06-07 09:39:14 -07:00
Salim 7c6c71dc34 add redis and certs shared mount 2018-06-07 09:21:20 -07:00
Salim 56218a35f8 add mongo tests 2018-06-06 21:55:17 -07:00
Salim 3bbb4ffd4a add mongo tests 2018-06-06 21:47:36 -07:00
Salim 0f89695c8c add mongo tests 2018-06-06 21:40:53 -07:00
Salim 0d3227b6fe debug 2018-06-06 18:07:18 -07:00
Salim 13ccdc222b disable s3 container 2018-06-06 18:01:05 -07:00
Salim 97861c7d75 debug creds 2018-06-06 18:01:05 -07:00
Salim f378d9657a test dir 2018-06-06 17:03:46 -07:00
Salim 05876a0d09 fix hidden char 2018-06-06 16:56:02 -07:00
Salim 65984e2636 wait for credentials 2018-06-06 16:53:27 -07:00
Salim 87255d06e8 volumes 2018-06-06 16:43:41 -07:00
Salim 13164fbc32 cred volmount 2018-06-06 16:20:18 -07:00
Salim 666b693674 locationconfig ci vars 2018-06-06 15:56:27 -07:00
Salim a0cd0c0582 uncomment 2018-06-06 15:40:43 -07:00
Salim df200fef94 squid ssl 2018-06-06 15:21:55 -07:00
Salim f38de61775 aggressor fixes 2018-06-06 14:33:54 -07:00
Salim 650f8e58e2 aggressor fixes 2018-06-06 14:26:41 -07:00
Salim c959af45cd aggressor script 2018-06-06 14:03:54 -07:00
Salim 2795d1d454 fix aws for kube 2018-06-06 13:36:53 -07:00
Salim b531959353 resource limits 2018-06-06 13:28:48 -07:00
Salim eb8b1c0bcd typo 2018-06-06 12:59:12 -07:00
Salim 93c26c8e1e typo 2018-06-06 12:32:27 -07:00
Salim 20a784dbac dropme disable linting 2018-06-06 12:28:06 -07:00
Salim 613557153f kube pod typo 2018-06-06 12:28:06 -07:00
Salim 5442a5ca2d refactor dockerfile 2018-06-06 12:28:03 -07:00
Salim ee4ab36fe2 kube proxy pod 2018-06-06 11:43:58 -07:00
Salim 8d081cf9a4 tests.bash refactor 2018-06-06 10:17:00 -07:00
Salim e2864d6c98 cleanup 2018-06-06 09:55:53 -07:00
Salim cfa124f17e missing packages 2018-06-05 21:17:50 -07:00
Salim 5fafb20d48 missing packages 2018-06-05 21:04:58 -07:00
Salim f841f25b6c missing packages 2018-06-05 20:53:28 -07:00
Salim 93187f3a20 rm caching for now 2018-06-05 20:45:47 -07:00
Salim 08b35d3bc7 missing packages 2018-06-05 20:39:18 -07:00
Salim be9854baaf chages tests script 2018-06-05 20:29:03 -07:00
Salim 9860e5a97e cache node modules 2018-06-05 20:17:41 -07:00
Salim f040d08ec2 add missing packages 2018-06-05 17:48:42 -07:00
Salim 3cd0d682f9 cleanup 2018-06-05 17:46:24 -07:00
Salim 30d844f405 Eve CI migration 2018-06-05 17:41:57 -07:00
17 changed files with 709 additions and 85 deletions

View File

@ -1,19 +1,24 @@
FROM node:6-slim FROM node:6-slim
MAINTAINER Giorgio Regni <gr@scality.com> MAINTAINER Giorgio Regni <gr@scality.com>
WORKDIR /usr/src/app ENV LANG C.UTF-8
ARG BUILDBOT_VERSION
# Keep the .git directory in order to properly report version # Keep the .git directory in order to properly report version
COPY . /usr/src/app WORKDIR /usr/src/app
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y jq python git build-essential --no-install-recommends \ && apt-get install -y jq python git build-essential --no-install-recommends
&& npm install --production \
&& apt-get autoremove --purge -y python git build-essential \ COPY ./package.json ./
&& rm -rf /var/lib/apt/lists/* \
RUN npm install --production \
&& npm cache clear \ && npm cache clear \
&& rm -rf ~/.node-gyp \ && rm -rf ~/.node-gyp \
&& rm -rf /tmp/npm-* && rm -rf /tmp/npm-*
# For CI builds
COPY ./eve/workers/build ./eve/workers/build
RUN bash -l ./eve/workers/build/build.sh
COPY . ./
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"] VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]

263
eve/main.yml Normal file
View File

@ -0,0 +1,263 @@
---
version: 0.2
branches:
default:
stage: pre-merge
models:
- env: &global_env
CI: 'true'
CIRCLE_ARTIFACTS: '/tmp'
CIRCLE_NODE_TOTAL: "4"
CIRCLE_TEST_REPORTS: '/tmp'
CI_REPORTS: '/tmp'
ENABLE_LOCAL_CACHE: "1"
REPORT_TOKEN: 'report-token-1'
- Git: &clone
name: Pull repo
repourl: '%(prop:git_reference)s'
shallow: True
retryFetch: True
haltOnFailure: True
# Machine Setup
- ShellCommand: &update_hosts
name: Update hosts
command: sudo bash -c "echo '
127.0.0.1 \
bucketwebsitetester.s3-website-us-east-1.amazonaws.com
' >> /etc/hosts"
haltOnFailure: True
- ShellCommandWthSecrets: &aws-credentials
name: Setup AWS Credentials
command: bash eve/workers/build/aws.bash
haltOnFailure: True
- ShellCommandWithSecrets: &npm-install
name: install modules
command: npm install
haltOnFailure: True
stages:
pre-merge:
worker: &master-worker
type: docker
path: eve/workers/build
volumes: &default_volumes
- '/home/eve/workspace'
steps:
- SetProperty:
property: artifacts_name
value: "scality-s3-%(prop:buildnumber)s"
haltOnFailure: True
- Git: *clone
- ShellCommandWithSecrets: *npm-install
- ShellCommandWithSecrets:
name: Linting
command: |
npm run --silent lint -- --max-warnings 0
npm run --silent lint_md
flake8 $(git ls-files "*.py")
yamllint $(git ls-files "*.yml" | grep -v eve/main.yml)
- ShellCommandWithSecrets:
name: Unit Coverage
command: |
mkdir -p $CIRCLE_TEST_REPORTS/unit
npm run unit_coverage
npm run unit_coverage_legacy_location
env: *global_env
haltOnFailure: True
- TriggerStages:
name: Launch all nodes simultaneously
stage_names:
# - external-backend-with-proxy
# - mongo-ft-tests
- node0
# - node1
# - node2
# - node3
waitForFinish: True
haltOnFailure: True
external-backend-with-proxy:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars: &proxy-vars
S3BACKEND: "mem"
MPU_TESTING: "yes"
S3DATA: "multiple"
S3METADATA: ""
CI_PROXY: "true"
steps:
- Git: *clone
- ShellCommandWithSecrets: *aws-credentials
- ShellCommand: *npm-install
- ShellCommandWithSecrets:
command: |
set -ex
. /root/.aws/exports &> /dev/null
# npm start &
bash wait_for_local_port.bash 8000 40
npm run ft_awssdk_external_backends
# kill -9 $(lsof -t -i:8000) || exit 1
env:
<<: *global_env
<<: *proxy-vars
mongo-ft-tests:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars: &mongo-vars
S3BACKEND: "mem"
S3DATA: ""
MPU_TESTING: "yes"
S3METADATA: mongodb
CI_PROXY: "false"
steps:
- Git: *clone
- ShellCommandWithSecrets: *aws-credentials
- ShellCommand: *npm-install
- ShellCommandWithSecrets:
command: |
bash wait_for_local_port.bash 8000 40
npm run ft_test
env:
<<: *global_env
<<: *mongo-vars
node0:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *update_hosts
- ShellCommand:
command: rm -rf node_modules
- ShellCommandWithSecrets: *aws-credentials
- ShellCommandWithSecrets: *npm-install
- ShellCommandWithSecrets:
command: |
source /home/eve/.bashrc &> /dev/null
source /root/.aws/exports &> /dev/null
npm run start_dmd &
. wait_for_local_port.bash 9990 40
npm run multiple_backend_test
kill -9 $(lsof -t -i:9990) || true
timeout: 300
env: &node0
<<: *global_env
CIRCLE_NODE_INDEX: '0'
haltOnFailure: True
want_stderr: False
# Run S3 with multiple data backends ; run ft_tests
- ShellCommandWithSecrets:
command: |
source /home/eve/.bashrc &> /dev/null
source /root/.aws/exports &> /dev/null
S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_java.txt &
S3PID=$!
. wait_for_local_port.bash 8000 40
cd ./tests/functional/jaws && mvn test
kill -9 $S3PID && exit 0
timeout: 300
env: *node0
haltOnFailure: True
want_stderr: False
- ShellCommandWithSecrets:
command: |
source /home/eve/.bashrc &> /dev/null
source /root/.aws/exports &> /dev/null
S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_fog.txt &
S3PID=$!
. wait_for_local_port.bash 8000 40
cd tests/functional/fog && rspec tests.rb
kill -9 $S3PID && exit 0
timeout: 300
env: *node0
haltOnFailure: True
want_stderr: False
- ShellCommandWithSecrets:
command: |
source /home/eve/.bashrc &> /dev/null
source /root/.aws/exports &> /dev/null
S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_awssdk.txt &
S3PID=$!
. wait_for_local_port.bash 8000 40
S3DATA=multiple npm run ft_awssdk
kill -9 $S3PID && exit 0
timeout: 300
env: *node0
haltOnFailure: True
want_stderr: False
- ShellCommand:
name: prepare artifacts
command: |
mkdir -p artifacts/node0
cp /tmp/*.txt artifacts/node0
- Upload: &upload_artifacts
source: artifacts
urls:
- "*"
node1:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *update_hosts
- ShellCommand:
command: rm -rf node_modules
- ShellCommandWithSecrets: *aws-credentials
- ShellCommandWithSecrets: *npm-install
- ShellCommandWithSecrets:
command: bash -l tests.bash
timeout: 300
env:
<<: *global_env
CIRCLE_NODE_INDEX: '1'
- ShellCommand:
name: prepare artifacts
command: |
mkdir -p artifacts/node1
cp /tmp/*.txt artifacts/node1
- Upload: *upload_artifacts
node2:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *update_hosts
- ShellCommand:
command: rm -rf node_modules
- ShellCommandWithSecrets: *aws-credentials
- ShellCommandWithSecrets: *npm-install
- ShellCommandWithSecrets:
command: bash -l tests.bash
env:
<<: *global_env
CIRCLE_NODE_INDEX: '2'
- ShellCommand:
name: prepare artifacts
command: |
mkdir -p artifacts/node2
cp /tmp/*.txt artifacts/node2
- Upload: *upload_artifacts
node3:
worker: *master-worker
steps:
- Git: *clone
- ShellCommand: *update_hosts
- ShellCommand:
command: rm -rf node_modules
- ShellCommandWithSecrets: *aws-credentials
- ShellCommandWithSecrets: *npm-install
- ShellCommandWithSecrets:
command: bash -l tests.bash
timeout: 300
env:
<<: *global_env
CIRCLE_NODE_INDEX: '3'
- ShellCommand:
name: prepare artifacts
command: |
mkdir -p artifacts/node3
cp /tmp/*.txt artifacts/node3
- Upload: *upload_artifacts

View File

@ -0,0 +1,57 @@
FROM ubuntu:xenial
#
# Install packages needed by the buildchain
#
ENV LANG C.UTF-8
COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
RUN apt-get update \
&& cat /tmp/*packages.list | xargs apt-get install -y \
&& wget https://nodejs.org/dist/v6.9.5/node-v6.9.5-linux-x64.tar.xz \
&& tar -C /usr/local --strip-components 1 -xvf node-v6.9.5-linux-x64.tar.xz \
&& pip install pip==9.0.1 \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /tmp/packages.list
#
# Add user eve
#
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
&& adduser eve sudo \
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
#
# Install Dependencies
#
# Install RVM and gems
ENV RUBY_VERSION="2.4.1"
COPY ./gems.list /tmp/
RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
&& curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
&& usermod -a -G rvm eve
RUN /bin/bash -l -c "\
source /usr/local/rvm/scripts/rvm \
&& cat /tmp/gems.list | xargs gem install \
&& rm /tmp/gems.list"
# Install Pip packages
COPY ./pip_packages.list /tmp/
RUN cat /tmp/pip_packages.list | xargs pip install \
&& rm -f /tmp/pip_packages.list \
&& mkdir /home/eve/.aws \
&& chown eve /home/eve/.aws
#
# Run buildbot-worker on startup
#
ARG BUILDBOT_VERSION
RUN pip install buildbot-worker==$BUILDBOT_VERSION
#USER eve
#WORKDIR /home/eve/workspace
#CMD ["sudo", "supervisord", "-n"]
COPY ./bash_profile /home/eve/.bashrc
COPY ./bash_profile /home/eve/.bash_profile
CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS && buildbot-worker start --nodaemon"]

View File

@ -0,0 +1,41 @@
#!/bin/bash -x
set -x #echo on
set -e #exit at the first error
mkdir -p ~/.aws
cat >>/root/.aws/exports <<EOF
export AWS_ACCESS_KEY_ID=
export AWS_SECRET_ACCESS_KEY=
export GOOGLE_SERVICE_EMAIL="$GCP_BACKEND_SERVICE_EMAIL"
export GOOGLE_SERVICE_KEY="$GCP_BACKEND_SERVICE_KEY"
export azurebackend2_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY_2"
export azurebackend2_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME_2"
export azurebackend2_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT_2"
export azurebackend_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
export azurebackend_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
export azurebackend_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
export azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
export azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
export azurebackendmismatch_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
export azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
export azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
export azurenonexistcontainer_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
export azuretest_AZURE_BLOB_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
EOF
source /root/.aws/exports &> /dev/null
cat >>/root/.aws/credentials <<EOF
[default]
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
[default_2]
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
[google]
aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY
aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY
[google_2]
aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY_2
aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY_2
EOF

View File

@ -0,0 +1,17 @@
export AWS_ACCESS_KEY_ID=
export AWS_SECRET_ACCESS_KEY=
export GOOGLE_SERVICE_EMAIL="$GCP_BACKEND_SERVICE_EMAIL"
export GOOGLE_SERVICE_KEY="$GCP_BACKEND_SERVICE_KEY"
export azurebackend2_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY_2"
export azurebackend2_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME_2"
export azurebackend2_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT_2"
export azurebackend_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
export azurebackend_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
export azurebackend_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
export azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
export azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
export azurebackendmismatch_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
export azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
export azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
export azurenonexistcontainer_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
export azuretest_AZURE_BLOB_ENDPOINT="$AZURE_BACKEND_ENDPOINT"

View File

@ -0,0 +1,12 @@
ca-certificates
git
gnupg
libffi-dev
libssl-dev
python-pip
python2.7
python2.7-dev
software-properties-common
sudo
tcl
wget

View File

@ -0,0 +1,4 @@
fog-aws:1.3.0
json
mime-types:3.1
rspec:3.5

View File

@ -0,0 +1,8 @@
#!/bin/bash -x
set -x #echo on
set -e #exit at the first error
killandsleep () {
kill -9 $(lsof -t -i:$1) || true
sleep 10
}

View File

@ -0,0 +1,7 @@
curl
default-jdk
libdigest-hmac-perl
lsof
maven
netcat
redis-server

View File

@ -0,0 +1,8 @@
#!/bin/bash
set -exo pipefail
buildbot-worker create-worker ${HOME} \
"${BUILDMASTER}:${BUILDMASTER_PORT}" \
"${WORKERNAME}" "${WORKERPASS}"
exec buildbot-worker start --nodaemon

View File

@ -0,0 +1,22 @@
#[program:start_dmd]
#command=/bin/bash -l /home/eve/S3/tests.bash
#directory=/home/eve/S3
#autostart=true
#autorestart=true
#process_name=dmd
#numprocs=1
#stdout_logfile=/dev/fd/1
##stdout_logfile=/tmp/dmd_server.txt
#stdout_logfile_maxbytes=0
#stderr_logfile=/dev/fd/2
##stderr_logfile=/tmp/dmd_server.txt
#stderr_logfile_maxbytes=0
#stopsignal=QUIT
##environment=
## FOO=bar
## FOO2=bar
[program:redis]
command=/usr/bin/redis-server
autostart=true
autorestart=true

144
eve/workers/pod.yaml Normal file
View File

@ -0,0 +1,144 @@
---
apiVersion: v1
kind: Pod
metadata:
name: "proxy-ci-test-pod"
spec:
activeDeadlineSeconds: 3600
restartPolicy: Never
terminationGracePeriodSeconds: 10
containers:
- name: aggressor
image: {{ images.aggressor }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: "1"
memory: 2Gi
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
command:
- bash
- -lc
- |
buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS
buildbot-worker start --nodaemon
env:
- name: CI
value: "true"
- name: S3BACKEND
value: "{{ vars.S3BACKEND }}"
- name: MPU_TESTING
value: "{{ vars.MPU_TESTING }}"
- name: S3DATA
value: "{{ vars.S3DATA }}"
- name: CI_PROXY
value: "{{ vars.CI_PROXY }}"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
- name: s3
image: {{ images.s3 }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
- name: certs
readOnly: true
mountPath: /tmp
command:
- bash
- -ec
- |
sleep 10
source /root/.aws/exports || exit 1
/usr/src/app/docker-entrypoint.sh npm start
env:
{% if vars.S3DATA == "multiple" -%}
- name: S3_LOCATION_FILE
value: "/usr/src/app/tests/locationConfig/locationConfigTests.json"
{%- endif %}
- name: CI
value: "true"
- name: S3BACKEND
value: "{{ vars.S3BACKEND }}"
- name: MPU_TESTING
value: "{{ vars.MPU_TESTING }}"
- name: S3DATA
value: "{{ vars.S3DATA }}"
- name: CI_PROXY
value: "{{ vars.CI_PROXY }}"
- name: S3METADATA
value: "{{ vars.S3METADATA }}"
- name: MONGODB_HOSTS
value: "localhost:27018"
- name: MONGODB_RS
value: "rs0"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
{% if vars.S3METADATA == "mongodb" -%}
- name: mongo
image: scality/ci-mongo:3.4
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
{%- endif %}
- name: redis
image: redis:4-alpine
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 200m
memory: 512Mi
{% if vars.CI_PROXY == "true" -%}
- name: squid
image: scality/ci-squid
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi
volumeMounts:
- name: certs
readOnly: false
mountPath: /ssl
command:
- sh
- -exc
- |
mkdir -p /ssl
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-keyout /ssl/myca.pem -out /ssl/myca.pem
cp /ssl/myca.pem /ssl/CA.pem
squid -f /etc/squid/squid.conf -N -z
squid -f /etc/squid/squid.conf -NYCd 1
{%- endif %}
volumes:
- name: creds
emptyDir: {}
- name: certs
emptyDir: {}

View File

@ -1,21 +1,9 @@
#!/bin/bash -x #!/bin/bash -x
set -x #echo on set -x #echo on
set -e #exit at the first error set -e #exit at the first error
set -o pipefail
cat >>~/.aws/credentials <<EOF source /home/eve/.bashrc &> /dev/null
[default] source /root/.aws/exports &> /dev/null
aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT
[default_2]
aws_access_key_id = $AWS_ACCESS_KEY_ID_DEFAULT_2
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_DEFAULT_2
[google]
aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE
[google_2]
aws_access_key_id = $AWS_ACCESS_KEY_ID_GOOGLE_2
aws_secret_access_key = $AWS_SECRET_ACCESS_KEY_GOOGLE_2
EOF
MYPWD=$(pwd) MYPWD=$(pwd)
@ -29,48 +17,42 @@ export REMOTE_MANAGEMENT_DISABLE=1
if [ $CIRCLE_NODE_INDEX -eq 0 ] if [ $CIRCLE_NODE_INDEX -eq 0 ]
then then
npm run --silent lint -- --max-warnings 0
npm run --silent lint_md
flake8 $(git ls-files "*.py")
yamllint $(git ls-files "*.yml")
mkdir -p $CIRCLE_TEST_REPORTS/unit
npm run unit_coverage
npm run unit_coverage_legacy_location
npm run start_dmd & npm run start_dmd &
bash wait_for_local_port.bash 9990 40 && bash wait_for_local_port.bash 9990 40
npm run multiple_backend_test npm run multiple_backend_test
killandsleep 9990 killandsleep 9990
# Run S3 with multiple data backends ; run ft_tests # Run S3 with multiple data backends ; run ft_tests
S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_java.txt &
S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_java.txt & bash wait_for_local_port.bash 8000 40 && cd ./tests/functional/jaws && mvn test bash wait_for_local_port.bash 8000 40
pushd ./tests/functional/jaws
mvn test
killandsleep 8000 killandsleep 8000
cd $MYPWD cd $MYPWD
S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_fog.txt & bash wait_for_local_port.bash 8000 40 && cd tests/functional/fog && rspec tests.rb S3BACKEND=mem S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_fog.txt &
bash wait_for_local_port.bash 8000 40
pushd tests/functional/fog
rspec tests.rb
cd $MYPWD cd $MYPWD
killandsleep 8000 killandsleep 8000
S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple npm run ft_awssdk S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_awssdk.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=multiple npm run ft_awssdk
cd $MYPWD cd $MYPWD
killandsleep 8000 killandsleep 8000
# Run external backend tests with proxy ; run ft_awssdk_external_backends # # Run external backend tests with proxy ; run ft_awssdk_external_backends
#
S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple CI_PROXY=true npm start > $CIRCLE_ARTIFACTS/server_external_backends_proxy_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple CI_PROXY=true npm run ft_awssdk_external_backends # S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple CI_PROXY=true npm start > $CIRCLE_ARTIFACTS/server_external_backends_proxy_awssdk.txt &
# bash wait_for_local_port.bash 8000 40
killandsleep 8000 # S3DATA=multiple CI_PROXY=true npm run ft_awssdk_external_backends
#
# killandsleep 8000
fi fi
@ -79,18 +61,26 @@ then
# Run S3 with multiple data backends + KMS Encryption; run ft_awssdk # Run S3 with multiple data backends + KMS Encryption; run ft_awssdk
S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_kms_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=multiple ENABLE_KMS_ENCRYPTION=true npm run ft_awssdk S3BACKEND=mem MPU_TESTING=yes S3DATA=multiple npm start > $CIRCLE_ARTIFACTS/server_multiple_kms_awssdk.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=multiple ENABLE_KMS_ENCRYPTION=true npm run ft_awssdk
killandsleep 8000 killandsleep 8000
# Run S3 with mem Backend ; run ft_tests # Run S3 with mem Backend ; run ft_tests
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_java.txt & bash wait_for_local_port.bash 8000 40 && cd ./tests/functional/jaws && mvn test S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_java.txt &
bash wait_for_local_port.bash 8000 40
pushd ./tests/functional/jaws
mvn test
cd $MYPWD cd $MYPWD
killandsleep 8000 killandsleep 8000
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_fog.txt & bash wait_for_local_port.bash 8000 40 && cd tests/functional/fog && rspec tests.rb S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_fog.txt &
bash wait_for_local_port.bash 8000 40
cd tests/functional/fog
rspec tests.rb
cd $MYPWD cd $MYPWD
killandsleep 8000 killandsleep 8000
@ -100,61 +90,85 @@ fi
if [ $CIRCLE_NODE_INDEX -eq 2 ] if [ $CIRCLE_NODE_INDEX -eq 2 ]
then then
S3BACKEND=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_mem_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_awssdk S3BACKEND=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_mem_awssdk.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_awssdk
killandsleep 8000 killandsleep 8000
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_s3cmd.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_s3cmd S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_s3cmd.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_s3cmd
killandsleep 8000 killandsleep 8000
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_s3curl.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_s3curl S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_s3curl.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_s3curl
killandsleep 8000 killandsleep 8000
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_rawnode.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_node S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_rawnode.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_node
killandsleep 8000 killandsleep 8000
# Run S3 with mem Backend + KMS Encryption ; run ft_tests # Run S3 with mem Backend + KMS Encryption ; run ft_tests
S3BACKEND=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_mem_kms_awssdk.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_awssdk S3BACKEND=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_mem_kms_awssdk.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_awssdk
killandsleep 8000 killandsleep 8000
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_kms_s3cmd.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3cmd S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_kms_s3cmd.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3cmd
killandsleep 8000 killandsleep 8000
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_kms_s3curl.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3curl S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_kms_s3curl.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3curl
killandsleep 8000 killandsleep 8000
S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_kms_rawnode.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_node S3BACKEND=mem npm start > $CIRCLE_ARTIFACTS/server_mem_kms_rawnode.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_node
killandsleep 8000 killandsleep 8000
# Run with mongdb backend ; run ft_tests # # Run with mongdb backend ; run ft_tests
S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file S3METADATA=mongodb npm run ft_awssdk # S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_awssdk.txt &
# bash wait_for_local_port.bash 8000 40 && S3DATA=file S3METADATA=mongodb npm run ft_awssdk
killandsleep 8000 # killandsleep 8000
S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_s3cmd.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file S3METADATA=mongodb npm run ft_s3cmd # S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_s3cmd.txt &
# bash wait_for_local_port.bash 8000 40
# S3DATA=file S3METADATA=mongodb npm run ft_s3cmd
killandsleep 8000 # killandsleep 8000
S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_s3curl.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file S3METADATA=mongodb npm run ft_s3curl # S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_s3curl.txt &
# bash wait_for_local_port.bash 8000 40
# S3DATA=file S3METADATA=mongodb npm run ft_s3curl
killandsleep 8000 # killandsleep 8000
S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_healthchecks.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file S3METADATA=mongodb npm run ft_healthchecks # S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_healthchecks.txt &
# bash wait_for_local_port.bash 8000 40
# S3DATA=file S3METADATA=mongodb npm run ft_healthchecks
killandsleep 8000 # killandsleep 8000
S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_management.txt & bash wait_for_local_port.bash 8000 40 && S3METADATA=mongodb npm run ft_management # S3BACKEND=mem MPU_TESTING=yes S3METADATA=mongodb npm start > $CIRCLE_ARTIFACTS/server_mongodb_management.txt &
# bash wait_for_local_port.bash 8000 40 && S3METADATA=mongodb npm run ft_management
killandsleep 8000 # killandsleep 8000
fi fi
@ -163,49 +177,71 @@ then
# Run S3 with file Backend ; run ft_tests # Run S3 with file Backend ; run ft_tests
S3BACKEND=file S3VAULT=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_file_awssdk.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_awssdk S3BACKEND=file S3VAULT=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_file_awssdk.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_awssdk
killandsleep 8000 killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_management.txt & bash wait_for_local_port.bash 8000 40 && npm run ft_management S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_management.txt &
bash wait_for_local_port.bash 8000 40
npm run ft_management
killandsleep 8000 killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_s3cmd.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_s3cmd S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_s3cmd.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_s3cmd
killandsleep 8000 killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_s3curl.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_s3curl S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_s3curl.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_s3curl
killandsleep 8000 killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_rawnode.txt & bash wait_for_local_port.bash 8000 40 && S3DATA=file npm run ft_node S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_rawnode.txt &
bash wait_for_local_port.bash 8000 40
S3DATA=file npm run ft_node
killandsleep 8000 killandsleep 8000
# Run S3 with file Backend + KMS Encryption ; run ft_tests # Run S3 with file Backend + KMS Encryption ; run ft_tests
S3BACKEND=file S3VAULT=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_file_kms_awssdk.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_awssdk S3BACKEND=file S3VAULT=mem MPU_TESTING=yes npm start > $CIRCLE_ARTIFACTS/server_file_kms_awssdk.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_awssdk
killandsleep 8000 killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_s3cmd.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3cmd S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_s3cmd.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3cmd
killandsleep 8000 killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_s3curl.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3curl S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_s3curl.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_s3curl
killandsleep 8000 killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_rawnode.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_node S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_rawnode.txt &
bash wait_for_local_port.bash 8000 40
ENABLE_KMS_ENCRYPTION=true S3DATA=file npm run ft_node
killandsleep 8000 killandsleep 8000
# Disabled see jira ZENKO-478
# S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_management.txt &
# bash wait_for_local_port.bash 8000 40
# ENABLE_KMS_ENCRYPTION=true npm run ft_management
#
# killandsleep 8000
S3BACKEND=file S3VAULT=mem npm start > $CIRCLE_ARTIFACTS/server_file_kms_management.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_KMS_ENCRYPTION=true npm run ft_management S3BACKEND=mem ENABLE_LOCAL_CACHE=true npm start > $CIRCLE_ARTIFACTS/server_mem_healthchecks.txt &
bash wait_for_local_port.bash 8000 40
killandsleep 8000 ENABLE_LOCAL_CACHE=true S3DATA=file npm run ft_healthchecks
S3BACKEND=mem ENABLE_LOCAL_CACHE=true npm start > $CIRCLE_ARTIFACTS/server_mem_healthchecks.txt & bash wait_for_local_port.bash 8000 40 && ENABLE_LOCAL_CACHE=true S3DATA=file npm run ft_healthchecks
killandsleep 8000 killandsleep 8000

View File

@ -14,7 +14,6 @@ const {
const log = new DummyRequestLogger(); const log = new DummyRequestLogger();
const locConstraints = Object.keys(config.locationConstraints); const locConstraints = Object.keys(config.locationConstraints);
const azureClient = getAzureClient(); const azureClient = getAzureClient();
describe('Healthcheck response', () => { describe('Healthcheck response', () => {
it('should return result for every location constraint in ' + it('should return result for every location constraint in ' +
'locationConfig and every external locations with flightCheckOnStartUp ' + 'locationConfig and every external locations with flightCheckOnStartUp ' +

View File

@ -1,4 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -e
wait_for_local_port() { wait_for_local_port() {
local port=$1 local port=$1