Compare commits
2 Commits
developmen
...
feature/CL
Author | SHA1 | Date |
---|---|---|
Nicolas Humbert | 6211be12b2 | |
Alexander Chan | fae374455b |
|
@ -0,0 +1,25 @@
|
||||||
|
FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
|
||||||
|
|
||||||
|
ENV CEPH_DAEMON demo
|
||||||
|
ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
|
||||||
|
|
||||||
|
ENV CEPH_DEMO_UID zenko
|
||||||
|
ENV CEPH_DEMO_ACCESS_KEY accessKey1
|
||||||
|
ENV CEPH_DEMO_SECRET_KEY verySecretKey1
|
||||||
|
ENV CEPH_DEMO_BUCKET zenkobucket
|
||||||
|
|
||||||
|
ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
|
||||||
|
ENV MON_IP 0.0.0.0
|
||||||
|
ENV NETWORK_AUTO_DETECT 4
|
||||||
|
ENV RGW_CIVETWEB_PORT 8001
|
||||||
|
|
||||||
|
RUN rm /etc/yum.repos.d/tcmu-runner.repo
|
||||||
|
|
||||||
|
ADD ./entrypoint-wrapper.sh /
|
||||||
|
RUN chmod +x /entrypoint-wrapper.sh && \
|
||||||
|
yum install -y python-pip && \
|
||||||
|
yum clean all && \
|
||||||
|
pip install awscli && \
|
||||||
|
rm -rf /root/.cache/pip
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/entrypoint-wrapper.sh" ]
|
|
@ -0,0 +1,37 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
touch /artifacts/ceph.log
|
||||||
|
mkfifo /tmp/entrypoint_output
|
||||||
|
# We run this in the background so that we can tail the RGW log after init,
|
||||||
|
# because entrypoint.sh never returns
|
||||||
|
|
||||||
|
# The next line will be needed when ceph builds 3.2.2 so I'll leave it here
|
||||||
|
# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
|
||||||
|
|
||||||
|
bash /entrypoint.sh > /tmp/entrypoint_output &
|
||||||
|
entrypoint_pid="$!"
|
||||||
|
while read -r line; do
|
||||||
|
echo $line
|
||||||
|
# When we find this line server has started
|
||||||
|
if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done < /tmp/entrypoint_output
|
||||||
|
|
||||||
|
# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
|
||||||
|
s3cmd mb s3://cephbucket s3://cephbucket2
|
||||||
|
|
||||||
|
mkdir /root/.aws
|
||||||
|
cat > /root/.aws/credentials <<EOF
|
||||||
|
[default]
|
||||||
|
aws_access_key_id = accessKey1
|
||||||
|
aws_secret_access_key = verySecretKey1
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Enable versioning on them
|
||||||
|
for bucket in cephbucket cephbucket2; do
|
||||||
|
echo "Enabling versiong for $bucket"
|
||||||
|
aws --endpoint http://127.0.0.1:8001 s3api put-bucket-versioning --bucket $bucket --versioning Status=Enabled
|
||||||
|
done
|
||||||
|
tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
|
||||||
|
wait $entrypoint_pid
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# This script is needed because RADOS Gateway
|
||||||
|
# will open the port before beginning to serve traffic
|
||||||
|
# causing wait_for_local_port.bash to exit immediately
|
||||||
|
|
||||||
|
echo 'Waiting for ceph'
|
||||||
|
while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
|
||||||
|
sleep 1
|
||||||
|
echo -n "."
|
||||||
|
done
|
|
@ -34,4 +34,3 @@ gcpbackendmismatch_GCP_SERVICE_KEY
|
||||||
gcpbackend_GCP_SERVICE_KEYFILE
|
gcpbackend_GCP_SERVICE_KEYFILE
|
||||||
gcpbackendmismatch_GCP_SERVICE_KEYFILE
|
gcpbackendmismatch_GCP_SERVICE_KEYFILE
|
||||||
gcpbackendnoproxy_GCP_SERVICE_KEYFILE
|
gcpbackendnoproxy_GCP_SERVICE_KEYFILE
|
||||||
gcpbackendproxy_GCP_SERVICE_KEYFILE
|
|
|
@ -20,6 +20,7 @@ services:
|
||||||
- METADATA_HOST=0.0.0.0
|
- METADATA_HOST=0.0.0.0
|
||||||
- S3BACKEND
|
- S3BACKEND
|
||||||
- S3DATA
|
- S3DATA
|
||||||
|
- S3METADATA
|
||||||
- MPU_TESTING
|
- MPU_TESTING
|
||||||
- S3VAULT
|
- S3VAULT
|
||||||
- S3_LOCATION_FILE
|
- S3_LOCATION_FILE
|
||||||
|
@ -34,6 +35,10 @@ services:
|
||||||
- S3KMIP_KEY
|
- S3KMIP_KEY
|
||||||
- S3KMIP_CERT
|
- S3KMIP_CERT
|
||||||
- S3KMIP_CA
|
- S3KMIP_CA
|
||||||
|
- MONGODB_HOSTS=0.0.0.0:27018
|
||||||
|
- MONGODB_RS=rs0
|
||||||
|
- DEFAULT_BUCKET_KEY_FORMAT
|
||||||
|
- METADATA_MAX_CACHED_BUCKETS
|
||||||
env_file:
|
env_file:
|
||||||
- creds.env
|
- creds.env
|
||||||
depends_on:
|
depends_on:
|
||||||
|
@ -64,3 +69,11 @@ services:
|
||||||
image: registry.scality.com/cloudserver-dev/pykmip
|
image: registry.scality.com/cloudserver-dev/pykmip
|
||||||
volumes:
|
volumes:
|
||||||
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
- /tmp/artifacts/${JOB_NAME}:/artifacts
|
||||||
|
mongo:
|
||||||
|
network_mode: "host"
|
||||||
|
profiles: ['mongo']
|
||||||
|
image: scality/ci-mongo:3.6.8
|
||||||
|
# ceph:
|
||||||
|
# network_mode: "host"
|
||||||
|
# profiles: ['mongo']
|
||||||
|
# image: scality/ci-mongo:3.6.8
|
||||||
|
|
|
@ -153,15 +153,63 @@ jobs:
|
||||||
cache-from: type=gha,scope=cloudserver
|
cache-from: type=gha,scope=cloudserver
|
||||||
cache-to: type=gha,mode=max,scope=cloudserver
|
cache-to: type=gha,mode=max,scope=cloudserver
|
||||||
|
|
||||||
multiple-backend:
|
# multiple-backend:
|
||||||
|
# runs-on: ubuntu-latest
|
||||||
|
# needs: build
|
||||||
|
# env:
|
||||||
|
# CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
|
# S3BACKEND: mem
|
||||||
|
# S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||||
|
# S3DATA: multiple
|
||||||
|
# JOB_NAME: ${{ github.job }}
|
||||||
|
# steps:
|
||||||
|
# - name: Checkout
|
||||||
|
# uses: actions/checkout@v3
|
||||||
|
# - uses: actions/setup-python@v4
|
||||||
|
# with:
|
||||||
|
# python-version: 3.9
|
||||||
|
# - name: Setup CI environment
|
||||||
|
# uses: ./.github/actions/setup-ci
|
||||||
|
# - name: Setup CI services
|
||||||
|
# run: docker-compose up -d
|
||||||
|
# working-directory: .github/docker
|
||||||
|
# - name: Run multiple backend test
|
||||||
|
# run: |-
|
||||||
|
# set -o pipefail;
|
||||||
|
# bash wait_for_local_port.bash 8000 40
|
||||||
|
# yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
|
# env:
|
||||||
|
# S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
|
# - name: Upload logs to artifacts
|
||||||
|
# uses: scality/action-artifacts@v3
|
||||||
|
# with:
|
||||||
|
# method: upload
|
||||||
|
# url: https://artifacts.scality.net
|
||||||
|
# user: ${{ secrets.ARTIFACTS_USER }}
|
||||||
|
# password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||||
|
# source: /tmp/artifacts
|
||||||
|
# if: always()
|
||||||
|
|
||||||
|
mongo-ft-tests:
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- version: v0
|
||||||
|
md-max-cache-buckets: ''
|
||||||
|
- version: v1
|
||||||
|
md-max-cache-buckets: '1'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
env:
|
env:
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
|
||||||
S3BACKEND: mem
|
S3BACKEND: mem
|
||||||
|
MPU_TESTING: true
|
||||||
|
S3METADATA: mongodb
|
||||||
|
S3KMS: file
|
||||||
|
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
|
||||||
S3DATA: multiple
|
DEFAULT_BUCKET_KEY_FORMAT: ${{ matrix.version }}
|
||||||
JOB_NAME: ${{ github.job }}
|
METADATA_MAX_CACHED_BUCKETS: ${{ matrix.md-max-cache-buckets }}
|
||||||
|
JOB_NAME: ${{ github.job }}-${{ matrix.version }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v3
|
||||||
|
@ -171,13 +219,13 @@ jobs:
|
||||||
- name: Setup CI environment
|
- name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
uses: ./.github/actions/setup-ci
|
||||||
- name: Setup CI services
|
- name: Setup CI services
|
||||||
run: docker-compose up -d
|
run: docker-compose --profile mongo up -d
|
||||||
working-directory: .github/docker
|
working-directory: .github/docker
|
||||||
- name: Run multiple backend test
|
- name: Run multiple backend test
|
||||||
run: |-
|
run: |-
|
||||||
set -o pipefail;
|
set -o pipefail;
|
||||||
bash wait_for_local_port.bash 8000 40
|
bash wait_for_local_port.bash 8000 40
|
||||||
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
env:
|
env:
|
||||||
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
|
||||||
- name: Upload logs to artifacts
|
- name: Upload logs to artifacts
|
||||||
|
@ -190,121 +238,121 @@ jobs:
|
||||||
source: /tmp/artifacts
|
source: /tmp/artifacts
|
||||||
if: always()
|
if: always()
|
||||||
|
|
||||||
file-ft-tests:
|
# file-ft-tests:
|
||||||
runs-on: ubuntu-latest
|
# runs-on: ubuntu-latest
|
||||||
needs: build
|
# needs: build
|
||||||
env:
|
# env:
|
||||||
S3BACKEND: file
|
# S3BACKEND: file
|
||||||
S3VAULT: mem
|
# S3VAULT: mem
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
# CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
MPU_TESTING: "yes"
|
# MPU_TESTING: "yes"
|
||||||
JOB_NAME: ${{ github.job }}
|
# JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
# steps:
|
||||||
- name: Checkout
|
# - name: Checkout
|
||||||
uses: actions/checkout@v3
|
# uses: actions/checkout@v3
|
||||||
- uses: actions/setup-python@v4
|
# - uses: actions/setup-python@v4
|
||||||
with:
|
# with:
|
||||||
python-version: |
|
# python-version: |
|
||||||
2.7
|
# 2.7
|
||||||
3.9
|
# 3.9
|
||||||
- name: Setup CI environment
|
# - name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
# uses: ./.github/actions/setup-ci
|
||||||
- name: Setup python2 test environment
|
# - name: Setup python2 test environment
|
||||||
run: |
|
# run: |
|
||||||
sudo apt-get install -y libdigest-hmac-perl
|
# sudo apt-get install -y libdigest-hmac-perl
|
||||||
pip install virtualenv
|
# pip install virtualenv
|
||||||
virtualenv -p $(which python2) ~/.virtualenv/py2
|
# virtualenv -p $(which python2) ~/.virtualenv/py2
|
||||||
source ~/.virtualenv/py2/bin/activate
|
# source ~/.virtualenv/py2/bin/activate
|
||||||
pip install 's3cmd==1.6.1'
|
# pip install 's3cmd==1.6.1'
|
||||||
- name: Setup CI services
|
# - name: Setup CI services
|
||||||
run: docker-compose up -d
|
# run: docker-compose up -d
|
||||||
working-directory: .github/docker
|
# working-directory: .github/docker
|
||||||
- name: Run file ft tests
|
# - name: Run file ft tests
|
||||||
run: |-
|
# run: |-
|
||||||
set -o pipefail;
|
# set -o pipefail;
|
||||||
bash wait_for_local_port.bash 8000 40
|
# bash wait_for_local_port.bash 8000 40
|
||||||
source ~/.virtualenv/py2/bin/activate
|
# source ~/.virtualenv/py2/bin/activate
|
||||||
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
# yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
# - name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v3
|
# uses: scality/action-artifacts@v3
|
||||||
with:
|
# with:
|
||||||
method: upload
|
# method: upload
|
||||||
url: https://artifacts.scality.net
|
# url: https://artifacts.scality.net
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
# user: ${{ secrets.ARTIFACTS_USER }}
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
# password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||||
source: /tmp/artifacts
|
# source: /tmp/artifacts
|
||||||
if: always()
|
# if: always()
|
||||||
|
|
||||||
utapi-v2-tests:
|
# utapi-v2-tests:
|
||||||
runs-on: ubuntu-latest
|
# runs-on: ubuntu-latest
|
||||||
needs: build
|
# needs: build
|
||||||
env:
|
# env:
|
||||||
ENABLE_UTAPI_V2: t
|
# ENABLE_UTAPI_V2: t
|
||||||
S3BACKEND: mem
|
# S3BACKEND: mem
|
||||||
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
# BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
# CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
# JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
# steps:
|
||||||
- name: Checkout
|
# - name: Checkout
|
||||||
uses: actions/checkout@v3
|
# uses: actions/checkout@v3
|
||||||
- uses: actions/setup-python@v4
|
# - uses: actions/setup-python@v4
|
||||||
with:
|
# with:
|
||||||
python-version: 3.9
|
# python-version: 3.9
|
||||||
- name: Setup CI environment
|
# - name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
# uses: ./.github/actions/setup-ci
|
||||||
- name: Setup CI services
|
# - name: Setup CI services
|
||||||
run: docker-compose up -d
|
# run: docker-compose up -d
|
||||||
working-directory: .github/docker
|
# working-directory: .github/docker
|
||||||
- name: Run file utapi v2 tests
|
# - name: Run file utapi v2 tests
|
||||||
run: |-
|
# run: |-
|
||||||
set -ex -o pipefail;
|
# set -ex -o pipefail;
|
||||||
bash wait_for_local_port.bash 8000 40
|
# bash wait_for_local_port.bash 8000 40
|
||||||
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
# yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
# - name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v3
|
# uses: scality/action-artifacts@v3
|
||||||
with:
|
# with:
|
||||||
method: upload
|
# method: upload
|
||||||
url: https://artifacts.scality.net
|
# url: https://artifacts.scality.net
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
# user: ${{ secrets.ARTIFACTS_USER }}
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
# password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||||
source: /tmp/artifacts
|
# source: /tmp/artifacts
|
||||||
if: always()
|
# if: always()
|
||||||
|
|
||||||
kmip-ft-tests:
|
# kmip-ft-tests:
|
||||||
runs-on: ubuntu-latest
|
# runs-on: ubuntu-latest
|
||||||
needs: build
|
# needs: build
|
||||||
env:
|
# env:
|
||||||
S3BACKEND: file
|
# S3BACKEND: file
|
||||||
S3VAULT: mem
|
# S3VAULT: mem
|
||||||
MPU_TESTING: true
|
# MPU_TESTING: true
|
||||||
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
# CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}/cloudserver:${{ github.sha }}
|
||||||
JOB_NAME: ${{ github.job }}
|
# JOB_NAME: ${{ github.job }}
|
||||||
steps:
|
# steps:
|
||||||
- name: Checkout
|
# - name: Checkout
|
||||||
uses: actions/checkout@v3
|
# uses: actions/checkout@v3
|
||||||
- uses: actions/setup-python@v4
|
# - uses: actions/setup-python@v4
|
||||||
with:
|
# with:
|
||||||
python-version: 3.9
|
# python-version: 3.9
|
||||||
- name: Setup CI environment
|
# - name: Setup CI environment
|
||||||
uses: ./.github/actions/setup-ci
|
# uses: ./.github/actions/setup-ci
|
||||||
- name: Copy KMIP certs
|
# - name: Copy KMIP certs
|
||||||
run: cp -r ./certs /tmp/ssl-kmip
|
# run: cp -r ./certs /tmp/ssl-kmip
|
||||||
working-directory: .github/pykmip
|
# working-directory: .github/pykmip
|
||||||
- name: Setup CI services
|
# - name: Setup CI services
|
||||||
run: docker-compose --profile pykmip up -d
|
# run: docker-compose --profile pykmip up -d
|
||||||
working-directory: .github/docker
|
# working-directory: .github/docker
|
||||||
- name: Run file KMIP tests
|
# - name: Run file KMIP tests
|
||||||
run: |-
|
# run: |-
|
||||||
set -ex -o pipefail;
|
# set -ex -o pipefail;
|
||||||
bash wait_for_local_port.bash 8000 40
|
# bash wait_for_local_port.bash 8000 40
|
||||||
bash wait_for_local_port.bash 5696 40
|
# bash wait_for_local_port.bash 5696 40
|
||||||
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
# yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||||
- name: Upload logs to artifacts
|
# - name: Upload logs to artifacts
|
||||||
uses: scality/action-artifacts@v3
|
# uses: scality/action-artifacts@v3
|
||||||
with:
|
# with:
|
||||||
method: upload
|
# method: upload
|
||||||
url: https://artifacts.scality.net
|
# url: https://artifacts.scality.net
|
||||||
user: ${{ secrets.ARTIFACTS_USER }}
|
# user: ${{ secrets.ARTIFACTS_USER }}
|
||||||
password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
# password: ${{ secrets.ARTIFACTS_PASSWORD }}
|
||||||
source: /tmp/artifacts
|
# source: /tmp/artifacts
|
||||||
if: always()
|
# if: always()
|
||||||
|
|
Loading…
Reference in New Issue