Compare commits

..

10 Commits

Author SHA1 Message Date
Taylor McKinnon 1e1520bb5b linting 2021-09-30 08:43:06 -07:00
Taylor McKinnon 7c502c2bee override python interpreter in CI 2021-09-30 08:32:48 -07:00
Taylor McKinnon 6f5066e5b5 add override for used python interpreter 2021-09-30 08:31:22 -07:00
Taylor McKinnon 1eba9e4793 bump vaultclient 2021-09-29 17:03:38 -07:00
Taylor McKinnon df0522aeb2 fix runner script 2021-09-29 16:50:48 -07:00
Taylor McKinnon 0bb8704689 set vault backend 2021-09-29 16:44:22 -07:00
Taylor McKinnon 7142ea735c add logging 2021-09-29 16:36:07 -07:00
Taylor McKinnon ac758a8bff bump vault limits 2021-09-29 16:27:06 -07:00
Taylor McKinnon b0bb7c9408 fix sentinel 2021-09-29 16:20:18 -07:00
Taylor McKinnon aeb22c3eec f 2021-09-29 16:13:07 -07:00
7 changed files with 133 additions and 73 deletions

View File

@ -42,12 +42,17 @@ models:
set -ex
bash tests/utils/wait_for_local_port.bash 8500 60"
haltOnFailure: true
logfiles:
vault:
filename: "/artifacts/vault.log"
follow: true
- ShellCommand: &wait_for_warp10
name: Wait for Warp 10
command: |
bash -c "
set -ex
bash tests/utils/wait_for_local_port.bash 4802 60"
haltOnFailure: true
logfiles:
warp10:
filename: "/artifacts/warp10.log"
@ -124,6 +129,8 @@ stages:
- ShellCommand:
name: run cron tests
command: bash ./eve/workers/unit_and_feature_tests/run_ft_tests.bash false ft_test:cron
env:
REINDEX_PYTHON_INTERPRETER: python3
logfiles:
utapi:
filename: "/artifacts/setup_ft_test:cron.log"
@ -164,6 +171,9 @@ stages:
utapi:
filename: "/artifacts/setup_ft_test:v2.log"
follow: true
vault:
filename: "/artifacts/vault.log"
follow: true
run-v2-limit-tests:
worker:
<< : *workspace
@ -187,6 +197,10 @@ stages:
follow: true
utapi:
filename: "/artifacts/setup_ft_test:softLimit.log"
follow: true
vault:
filename: "/artifacts/vault.log"
follow: true
- ShellCommand:
name: run v2 hard limit tests
command: bash ./eve/workers/unit_and_feature_tests/run_ft_tests.bash true ft_test:hardLimit
@ -201,4 +215,7 @@ stages:
utapi:
filename: "/artifacts/setup_ft_test:hardLimit.log"
follow: true
vault:
filename: "/artifacts/vault.log"
follow: true
- Upload: *upload_artifacts

View File

@ -53,11 +53,11 @@ spec:
- /init | tee -a /artifacts/redis.log
resources:
requests:
cpu: 200m
memory: 1Gi
cpu: 150m
memory: 500Mi
limits:
cpu: 200m
memory: 1Gi
memory: 500Mi
volumeMounts:
- name: artifacts
readOnly: false
@ -70,11 +70,11 @@ spec:
- redis-server --port 6380 --slaveof localhost 6379 --slave-announce-ip localhost | tee -a /artifacts/redis.log
resources:
requests:
cpu: 200m
memory: 1Gi
cpu: 150m
memory: 500Mi
limits:
cpu: 200m
memory: 1Gi
cpu: 150m
memory: 500Mi
volumeMounts:
- name: artifacts
readOnly: false
@ -82,7 +82,7 @@ spec:
- name: redis-sentinel
image: "{{ images.redis }}"
command:
- bash
- sh
- -c
- |-
cat > /tmp/sentinel.conf <<EOF
@ -96,11 +96,11 @@ spec:
redis-sentinel /tmp/sentinel.conf
resources:
requests:
cpu: 200m
memory: 1Gi
cpu: 150m
memory: 500Mi
limits:
cpu: 200m
memory: 1Gi
cpu: 150m
memory: 500Mi
volumeMounts:
- name: artifacts
readOnly: false
@ -113,15 +113,23 @@ spec:
command:
- bash
- -c
- 'chmod 400 tests/utils/keyfile && yarn start'
- 'chmod 400 tests/utils/keyfile && yarn start | tee -a /artifacts/vault.log'
env:
- name: VAULT_DB_BACKEND
value: LEVELDB
resources:
requests:
cpu: 10m
memory: 64Mi
cpu: 150m
memory: 1Gi
limits:
cpu: 50m
memory: 128Mi
cpu: 150m
memory: 1Gi
volumeMounts:
- name: artifacts
readOnly: false
mountPath: /artifacts
{% endif %}
volumes:
- name: artifacts
emptyDir: {}

View File

@ -18,5 +18,5 @@ if [ -z "$SETUP_CMD" ]; then
fi
UTAPI_INTERVAL_TEST_MODE=$1 npm $SETUP_CMD 2>&1 | tee -a "/artifacts/setup_$2.log" &
bash tests/utils/wait_for_port.bash localhost $PORT 40
bash tests/utils/wait_for_local_port.bash $PORT 40
UTAPI_INTERVAL_TEST_MODE=$1 npm run $2 | tee -a "/artifacts/test_$2.log"

View File

@ -12,6 +12,9 @@ const RedisClient = require('../libV2/redis');
const REINDEX_SCHEDULE = '0 0 * * Sun';
const REINDEX_LOCK_KEY = 's3:utapireindex:lock';
const REINDEX_LOCK_TTL = (60 * 60) * 24;
const REINDEX_PYTHON_INTERPRETER = process.env.REINDEX_PYTHON_INTERPRETER !== undefined
? process.env.REINDEX_PYTHON_INTERPRETER
: 'python3.4';
class UtapiReindex {
constructor(config) {
@ -109,7 +112,7 @@ class UtapiReindex {
const flags = this._buildFlags();
this._requestLogger.debug(`launching subprocess ${path} `
+ `with flags: ${flags}`);
const process = childProcess.spawn('python3.4', [
const process = childProcess.spawn(REINDEX_PYTHON_INTERPRETER, [
path,
...flags,
]);

View File

@ -38,10 +38,11 @@
"oas-tools": "^2.1.8",
"prom-client": "^13.1.0",
"uuid": "^3.3.2",
"vaultclient": "scality/vaultclient#ff9e92f",
"vaultclient": "scality/vaultclient#9013faab804a7ee61bd8bc4c71de89c3498bb6c0",
"werelogs": "scality/werelogs#8.1.0"
},
"devDependencies": {
"aws-sdk": "^2.997.0",
"eslint": "^5.12.0",
"eslint-config-airbnb-base": "13.1.0",
"eslint-config-scality": "scality/Guidelines#7cc5ff1",
@ -53,10 +54,10 @@
},
"resolutions": {
"**/@yarnpkg/fslib": "2.4.0",
"**/@yarnpkg/libzip" : "2.2.1",
"**/@yarnpkg/libzip": "2.2.1",
"**/@yarnpkg/json-proxy": "2.1.0",
"**/@yarnpkg/parsers": "2.3.0"
},
},
"scripts": {
"ft_test": "mocha --recursive tests/functional",
"ft_test:client": "mocha --recursive tests/functional/client",

View File

@ -1,8 +1,7 @@
/* eslint-disable no-undef-init */
const { promisify } = require('util');
const { IAM } = require('aws-sdk');
const vaultclient = require('vaultclient');
const fs = require('fs');
const IAM = require('aws-sdk').IAM;
const adminCredentials = {
accessKey: 'D4IT2AWSB588GO5J9T00',

130
yarn.lock
View File

@ -710,31 +710,6 @@ arraybuffer.slice@~0.0.7:
resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675"
integrity sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==
arsenal@scality/Arsenal#580e25a:
version "7.5.0"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/580e25a9e84acf2a2a8dec6c83f69f862804d85f"
dependencies:
"@hapi/joi" "^15.1.0"
JSONStream "^1.0.0"
ajv "6.12.2"
async "~2.1.5"
debug "~2.3.3"
diskusage "^1.1.1"
ioredis "4.9.5"
ipaddr.js "1.9.1"
level "~5.0.1"
level-sublevel "~6.6.5"
node-forge "^0.7.1"
simple-glob "^0.2"
socket.io "~2.3.0"
socket.io-client "~2.3.0"
utf8 "2.1.2"
uuid "^3.0.1"
werelogs scality/werelogs#0ff7ec82
xml2js "~0.4.23"
optionalDependencies:
ioctl "2.0.0"
arsenal@scality/Arsenal#65966f5:
version "7.7.0"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/65966f5ddf93b048906d14a8c26056abfd4c22ba"
@ -761,6 +736,35 @@ arsenal@scality/Arsenal#65966f5:
optionalDependencies:
ioctl "2.0.0"
arsenal@scality/Arsenal#7.10.1:
version "7.10.1"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/ce7bba1f8d8b4078335f65e8eaa49f6454ddbf46"
dependencies:
"@hapi/joi" "^15.1.0"
JSONStream "^1.0.0"
agentkeepalive "^4.1.3"
ajv "6.12.2"
async "~2.1.5"
base-x "3.0.8"
base62 "2.0.1"
debug "~2.6.9"
diskusage "^1.1.1"
ioredis "4.9.5"
ipaddr.js "1.9.1"
level "~5.0.1"
level-sublevel "~6.6.5"
node-forge "^0.7.1"
prom-client "10.2.3"
simple-glob "^0.2"
socket.io "~2.3.0"
socket.io-client "~2.3.0"
utf8 "2.1.2"
uuid "^3.0.1"
werelogs scality/werelogs#8.1.0
xml2js "~0.4.23"
optionalDependencies:
ioctl "2.0.0"
arsenal@scality/Arsenal#c57cde8:
version "8.1.4"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/c57cde88bb04fe9803ec08c3a883f6eb986e4149"
@ -879,6 +883,21 @@ aws-sdk@2.80.0:
xml2js "0.4.17"
xmlbuilder "4.2.1"
aws-sdk@^2.997.0:
version "2.997.0"
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.997.0.tgz#8598a5dd7bc6b6833a2fc3d737fba89020a79418"
integrity sha512-PiuDmC5hN+FsyLvl7GsZAnS6hQpo1pP+Ax2u8gyL19QlbBLwlhsFQF29vPcYatyv6WUxr51o6uymJdPxQg6uEA==
dependencies:
buffer "4.9.2"
events "1.1.1"
ieee754 "1.1.13"
jmespath "0.15.0"
querystring "0.2.0"
sax "1.2.1"
url "0.10.3"
uuid "3.3.2"
xml2js "0.4.19"
aws-sign2@~0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@ -921,6 +940,18 @@ balanced-match@^1.0.0:
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
base-x@3.0.8:
version "3.0.8"
resolved "https://registry.yarnpkg.com/base-x/-/base-x-3.0.8.tgz#1e1106c2537f0162e8b52474a557ebb09000018d"
integrity sha512-Rl/1AWP4J/zRrk54hhlxH4drNxPJXYUaKffODVI53/dAsV4t9fBxyxYKAVPU1XBHxYwOWP9h9H0hM2MVw4YfJA==
dependencies:
safe-buffer "^5.0.1"
base62@2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/base62/-/base62-2.0.1.tgz#729cfe179ed34c61e4a489490105b44ce4ea1197"
integrity sha512-4t4WQK7mdbcWzqEBiq6tfo2qDCeIZGXvjifJZyxHIVcjQkZJxpFtu/pa2Va69OouCkg6izZ08hKnPxroeDyzew==
base64-arraybuffer@0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz#9818c79e059b1355f97e0428a017c838e90ba812"
@ -1108,6 +1139,15 @@ buffer@4.9.1:
ieee754 "^1.1.4"
isarray "^1.0.0"
buffer@4.9.2:
version "4.9.2"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==
dependencies:
base64-js "^1.0.2"
ieee754 "^1.1.4"
isarray "^1.0.0"
buffer@^5.1.0, buffer@^5.5.0, buffer@^5.6.0:
version "5.6.0"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
@ -1597,13 +1637,6 @@ debug@^3.1.0, debug@^3.2.5, debug@^3.2.6:
dependencies:
ms "^2.1.1"
debug@~2.3.3:
version "2.3.3"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.3.3.tgz#40c453e67e6e13c901ddec317af8986cda9eff8c"
integrity sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=
dependencies:
ms "0.7.2"
debug@~4.1.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791"
@ -2212,6 +2245,11 @@ event-loop-spinner@^2.0.0:
dependencies:
tslib "^1.10.0"
events@1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
execa@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8"
@ -2801,7 +2839,7 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4:
dependencies:
safer-buffer ">= 2.1.2 < 3"
ieee754@^1.1.4:
ieee754@1.1.13, ieee754@^1.1.4:
version "1.1.13"
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
@ -4080,11 +4118,6 @@ mpath@~0.5.0:
resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.5.2.tgz#b1eac586dffb5175d2f51ca9aacba35d9940dd41"
integrity sha512-NOeCoW6AYc3hLi30npe7uzbD9b4FQZKH40YKABUCCvaKKL5agj6YzvHoNx8jQpDMNPgIa5bvSZQbQpWBAVD0Kw==
ms@0.7.2:
version "0.7.2"
resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765"
integrity sha1-riXPJRKziFodldfwN4aNhDESR2U=
ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@ -6408,6 +6441,11 @@ uuid@3.0.1:
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
integrity sha1-ZUS7ot/ajBzxfmKaOjBeK7H+5sE=
uuid@3.3.2:
version "3.3.2"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
uuid@^3.0.0, uuid@^3.0.1, uuid@^3.3.2:
version "3.4.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
@ -6451,14 +6489,14 @@ vary@~1.1.2:
resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=
vaultclient@scality/vaultclient#ff9e92f:
version "7.5.1"
resolved "https://codeload.github.com/scality/vaultclient/tar.gz/ff9e92fd8e35c16bbc20f5e4ee0531d8bffedbf2"
vaultclient@scality/vaultclient#9013faab804a7ee61bd8bc4c71de89c3498bb6c0:
version "7.10.1"
resolved "https://codeload.github.com/scality/vaultclient/tar.gz/9013faab804a7ee61bd8bc4c71de89c3498bb6c0"
dependencies:
agentkeepalive "^4.1.3"
arsenal scality/Arsenal#580e25a
arsenal scality/Arsenal#7.10.1
commander "2.20.0"
werelogs scality/werelogs#4e0d97c
werelogs scality/werelogs#8.1.0
xml2js "0.4.19"
verror@1.10.0:
@ -6487,12 +6525,6 @@ werelogs@scality/werelogs#351a2a3:
dependencies:
safe-json-stringify "1.0.3"
werelogs@scality/werelogs#4e0d97c:
version "7.4.1"
resolved "https://codeload.github.com/scality/werelogs/tar.gz/4e0d97cf69ea7ed60bea90756278513e7e7ea9b1"
dependencies:
safe-json-stringify "1.0.3"
werelogs@scality/werelogs#8.1.0:
version "8.1.0"
resolved "https://codeload.github.com/scality/werelogs/tar.gz/e8f828725642c54c511cdbe580b18f43d3589313"