linting

eve/main.yml

@@ -42,12 +42,17 @@ models:
         set -ex
         bash tests/utils/wait_for_local_port.bash 8500 60"
       haltOnFailure: true
+      logfiles:
+        vault:
+          filename: "/artifacts/vault.log"
+          follow: true
   - ShellCommand: &wait_for_warp10
       name: Wait for Warp 10
       command: |
         bash -c "
         set -ex
         bash tests/utils/wait_for_local_port.bash 4802 60"
+      haltOnFailure: true
       logfiles:
         warp10:
           filename: "/artifacts/warp10.log"
@@ -124,6 +129,8 @@ stages:
       - ShellCommand:
           name: run cron tests
           command: bash ./eve/workers/unit_and_feature_tests/run_ft_tests.bash false ft_test:cron
+          env:
+            REINDEX_PYTHON_INTERPRETER: python3
           logfiles:
             utapi:
               filename: "/artifacts/setup_ft_test:cron.log"
@@ -164,6 +171,9 @@ stages:
             utapi:
               filename: "/artifacts/setup_ft_test:v2.log"
               follow: true
+            vault:
+              filename: "/artifacts/vault.log"
+              follow: true
   run-v2-limit-tests:
     worker:
       << : *workspace
@@ -187,6 +197,10 @@ stages:
               follow: true
             utapi:
               filename: "/artifacts/setup_ft_test:softLimit.log"
+              follow: true
+            vault:
+              filename: "/artifacts/vault.log"
+              follow: true
       - ShellCommand:
           name: run v2 hard limit tests
           command: bash ./eve/workers/unit_and_feature_tests/run_ft_tests.bash true ft_test:hardLimit
@@ -201,4 +215,7 @@ stages:
             utapi:
               filename: "/artifacts/setup_ft_test:hardLimit.log"
               follow: true
+            vault:
+              filename: "/artifacts/vault.log"
+              follow: true
       - Upload: *upload_artifacts

eve/workers/pod.yml

@@ -53,11 +53,11 @@ spec:
       - /init | tee -a /artifacts/redis.log
     resources:
       requests:
-        cpu: 200m
+        cpu: 150m
-        memory: 1Gi
+        memory: 500Mi
       limits:
         cpu: 200m
-        memory: 1Gi
+        memory: 500Mi
     volumeMounts:
     - name: artifacts
       readOnly: false
@@ -70,11 +70,11 @@ spec:
       - redis-server --port 6380 --slaveof localhost 6379 --slave-announce-ip localhost | tee -a /artifacts/redis.log
     resources:
       requests:
-        cpu: 200m
+        cpu: 150m
-        memory: 1Gi
+        memory: 500Mi
       limits:
-        cpu: 200m
+        cpu: 150m
-        memory: 1Gi
+        memory: 500Mi
     volumeMounts:
     - name: artifacts
       readOnly: false
@@ -82,7 +82,7 @@ spec:
   - name: redis-sentinel
     image: "{{ images.redis }}"
     command:
-      - bash
+      - sh
       - -c
       - |-
         cat > /tmp/sentinel.conf <<EOF
@@ -96,11 +96,11 @@ spec:
         redis-sentinel /tmp/sentinel.conf
     resources:
       requests:
-        cpu: 200m
+        cpu: 150m
-        memory: 1Gi
+        memory: 500Mi
       limits:
-        cpu: 200m
+        cpu: 150m
-        memory: 1Gi
+        memory: 500Mi
     volumeMounts:
     - name: artifacts
       readOnly: false
@@ -113,15 +113,23 @@ spec:
     command:
       - bash
       - -c
-      - 'chmod 400 tests/utils/keyfile && yarn start'
+      - 'chmod 400 tests/utils/keyfile && yarn start | tee -a /artifacts/vault.log'
+    env:
+      - name:  VAULT_DB_BACKEND
+        value: LEVELDB
     resources:
       requests:
-        cpu: 10m
+        cpu: 150m
-        memory: 64Mi
+        memory: 1Gi
       limits:
-        cpu: 50m
+        cpu: 150m
-        memory: 128Mi
+        memory: 1Gi
+    volumeMounts:
+    - name: artifacts
+      readOnly: false
+      mountPath: /artifacts
 {% endif %}
+  volumes:
   - name: artifacts
     emptyDir: {}
 

eve/workers/unit_and_feature_tests/run_ft_tests.bash

@@ -18,5 +18,5 @@ if [ -z "$SETUP_CMD" ]; then
 fi
 
 UTAPI_INTERVAL_TEST_MODE=$1 npm $SETUP_CMD 2>&1 | tee -a "/artifacts/setup_$2.log" &
-bash tests/utils/wait_for_port.bash localhost $PORT 40
+bash tests/utils/wait_for_local_port.bash $PORT 40
 UTAPI_INTERVAL_TEST_MODE=$1 npm run $2 | tee -a "/artifacts/test_$2.log"

lib/UtapiReindex.js

@@ -12,6 +12,9 @@ const RedisClient = require('../libV2/redis');
 const REINDEX_SCHEDULE = '0 0 * * Sun';
 const REINDEX_LOCK_KEY = 's3:utapireindex:lock';
 const REINDEX_LOCK_TTL = (60 * 60) * 24;
+const REINDEX_PYTHON_INTERPRETER = process.env.REINDEX_PYTHON_INTERPRETER !== undefined
+    ? process.env.REINDEX_PYTHON_INTERPRETER
+    : 'python3.4';
 
 class UtapiReindex {
     constructor(config) {
@@ -109,7 +112,7 @@ class UtapiReindex {
         const flags = this._buildFlags();
         this._requestLogger.debug(`launching subprocess ${path} `
             + `with flags: ${flags}`);
-        const process = childProcess.spawn('python3.4', [
+        const process = childProcess.spawn(REINDEX_PYTHON_INTERPRETER, [
             path,
             ...flags,
         ]);

package.json

@@ -38,10 +38,11 @@
     "oas-tools": "^2.1.8",
     "prom-client": "^13.1.0",
     "uuid": "^3.3.2",
-    "vaultclient": "scality/vaultclient#ff9e92f",
+    "vaultclient": "scality/vaultclient#9013faab804a7ee61bd8bc4c71de89c3498bb6c0",
     "werelogs": "scality/werelogs#8.1.0"
   },
   "devDependencies": {
+    "aws-sdk": "^2.997.0",
     "eslint": "^5.12.0",
     "eslint-config-airbnb-base": "13.1.0",
     "eslint-config-scality": "scality/Guidelines#7cc5ff1",
@@ -53,10 +54,10 @@
   },
   "resolutions": {
     "**/@yarnpkg/fslib": "2.4.0",
-    "**/@yarnpkg/libzip" : "2.2.1",
+    "**/@yarnpkg/libzip": "2.2.1",
     "**/@yarnpkg/json-proxy": "2.1.0",
     "**/@yarnpkg/parsers": "2.3.0"
-},
+  },
   "scripts": {
     "ft_test": "mocha --recursive tests/functional",
     "ft_test:client": "mocha --recursive tests/functional/client",

tests/utils/vaultclient.js

@@ -1,8 +1,7 @@
 /* eslint-disable no-undef-init */
-const { promisify } = require('util');
+const { IAM } = require('aws-sdk');
 const vaultclient = require('vaultclient');
 const fs = require('fs');
-const IAM = require('aws-sdk').IAM;
 
 const adminCredentials = {
     accessKey: 'D4IT2AWSB588GO5J9T00',

yarn.lock

@@ -710,31 +710,6 @@ arraybuffer.slice@~0.0.7:
   resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675"
   integrity sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==
 
-arsenal@scality/Arsenal#580e25a:
-  version "7.5.0"
-  resolved "https://codeload.github.com/scality/Arsenal/tar.gz/580e25a9e84acf2a2a8dec6c83f69f862804d85f"
-  dependencies:
-    "@hapi/joi" "^15.1.0"
-    JSONStream "^1.0.0"
-    ajv "6.12.2"
-    async "~2.1.5"
-    debug "~2.3.3"
-    diskusage "^1.1.1"
-    ioredis "4.9.5"
-    ipaddr.js "1.9.1"
-    level "~5.0.1"
-    level-sublevel "~6.6.5"
-    node-forge "^0.7.1"
-    simple-glob "^0.2"
-    socket.io "~2.3.0"
-    socket.io-client "~2.3.0"
-    utf8 "2.1.2"
-    uuid "^3.0.1"
-    werelogs scality/werelogs#0ff7ec82
-    xml2js "~0.4.23"
-  optionalDependencies:
-    ioctl "2.0.0"
-
 arsenal@scality/Arsenal#65966f5:
   version "7.7.0"
   resolved "https://codeload.github.com/scality/Arsenal/tar.gz/65966f5ddf93b048906d14a8c26056abfd4c22ba"
@@ -761,6 +736,35 @@ arsenal@scality/Arsenal#65966f5:
   optionalDependencies:
     ioctl "2.0.0"
 
+arsenal@scality/Arsenal#7.10.1:
+  version "7.10.1"
+  resolved "https://codeload.github.com/scality/Arsenal/tar.gz/ce7bba1f8d8b4078335f65e8eaa49f6454ddbf46"
+  dependencies:
+    "@hapi/joi" "^15.1.0"
+    JSONStream "^1.0.0"
+    agentkeepalive "^4.1.3"
+    ajv "6.12.2"
+    async "~2.1.5"
+    base-x "3.0.8"
+    base62 "2.0.1"
+    debug "~2.6.9"
+    diskusage "^1.1.1"
+    ioredis "4.9.5"
+    ipaddr.js "1.9.1"
+    level "~5.0.1"
+    level-sublevel "~6.6.5"
+    node-forge "^0.7.1"
+    prom-client "10.2.3"
+    simple-glob "^0.2"
+    socket.io "~2.3.0"
+    socket.io-client "~2.3.0"
+    utf8 "2.1.2"
+    uuid "^3.0.1"
+    werelogs scality/werelogs#8.1.0
+    xml2js "~0.4.23"
+  optionalDependencies:
+    ioctl "2.0.0"
+
 arsenal@scality/Arsenal#c57cde8:
   version "8.1.4"
   resolved "https://codeload.github.com/scality/Arsenal/tar.gz/c57cde88bb04fe9803ec08c3a883f6eb986e4149"
@@ -879,6 +883,21 @@ aws-sdk@2.80.0:
     xml2js "0.4.17"
     xmlbuilder "4.2.1"
 
+aws-sdk@^2.997.0:
+  version "2.997.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.997.0.tgz#8598a5dd7bc6b6833a2fc3d737fba89020a79418"
+  integrity sha512-PiuDmC5hN+FsyLvl7GsZAnS6hQpo1pP+Ax2u8gyL19QlbBLwlhsFQF29vPcYatyv6WUxr51o6uymJdPxQg6uEA==
+  dependencies:
+    buffer "4.9.2"
+    events "1.1.1"
+    ieee754 "1.1.13"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.3.2"
+    xml2js "0.4.19"
+
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -921,6 +940,18 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
   integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
 
+base-x@3.0.8:
+  version "3.0.8"
+  resolved "https://registry.yarnpkg.com/base-x/-/base-x-3.0.8.tgz#1e1106c2537f0162e8b52474a557ebb09000018d"
+  integrity sha512-Rl/1AWP4J/zRrk54hhlxH4drNxPJXYUaKffODVI53/dAsV4t9fBxyxYKAVPU1XBHxYwOWP9h9H0hM2MVw4YfJA==
+  dependencies:
+    safe-buffer "^5.0.1"
+
+base62@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/base62/-/base62-2.0.1.tgz#729cfe179ed34c61e4a489490105b44ce4ea1197"
+  integrity sha512-4t4WQK7mdbcWzqEBiq6tfo2qDCeIZGXvjifJZyxHIVcjQkZJxpFtu/pa2Va69OouCkg6izZ08hKnPxroeDyzew==
+
 base64-arraybuffer@0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz#9818c79e059b1355f97e0428a017c838e90ba812"
@@ -1108,6 +1139,15 @@ buffer@4.9.1:
     ieee754 "^1.1.4"
     isarray "^1.0.0"
 
+buffer@4.9.2:
+  version "4.9.2"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
+  integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==
+  dependencies:
+    base64-js "^1.0.2"
+    ieee754 "^1.1.4"
+    isarray "^1.0.0"
+
 buffer@^5.1.0, buffer@^5.5.0, buffer@^5.6.0:
   version "5.6.0"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
@@ -1597,13 +1637,6 @@ debug@^3.1.0, debug@^3.2.5, debug@^3.2.6:
   dependencies:
     ms "^2.1.1"
 
-debug@~2.3.3:
-  version "2.3.3"
-  resolved "https://registry.yarnpkg.com/debug/-/debug-2.3.3.tgz#40c453e67e6e13c901ddec317af8986cda9eff8c"
-  integrity sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=
-  dependencies:
-    ms "0.7.2"
-
 debug@~4.1.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791"
@@ -2212,6 +2245,11 @@ event-loop-spinner@^2.0.0:
   dependencies:
     tslib "^1.10.0"
 
+events@1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
+  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
+
 execa@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8"
@@ -2801,7 +2839,7 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
-ieee754@^1.1.4:
+ieee754@1.1.13, ieee754@^1.1.4:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
   integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
@@ -4080,11 +4118,6 @@ mpath@~0.5.0:
   resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.5.2.tgz#b1eac586dffb5175d2f51ca9aacba35d9940dd41"
   integrity sha512-NOeCoW6AYc3hLi30npe7uzbD9b4FQZKH40YKABUCCvaKKL5agj6YzvHoNx8jQpDMNPgIa5bvSZQbQpWBAVD0Kw==
 
-ms@0.7.2:
-  version "0.7.2"
-  resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765"
-  integrity sha1-riXPJRKziFodldfwN4aNhDESR2U=
-
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -6408,6 +6441,11 @@ uuid@3.0.1:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
   integrity sha1-ZUS7ot/ajBzxfmKaOjBeK7H+5sE=
 
+uuid@3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
+  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
+
 uuid@^3.0.0, uuid@^3.0.1, uuid@^3.3.2:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
@@ -6451,14 +6489,14 @@ vary@~1.1.2:
   resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
   integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=
 
-vaultclient@scality/vaultclient#ff9e92f:
+vaultclient@scality/vaultclient#9013faab804a7ee61bd8bc4c71de89c3498bb6c0:
-  version "7.5.1"
+  version "7.10.1"
-  resolved "https://codeload.github.com/scality/vaultclient/tar.gz/ff9e92fd8e35c16bbc20f5e4ee0531d8bffedbf2"
+  resolved "https://codeload.github.com/scality/vaultclient/tar.gz/9013faab804a7ee61bd8bc4c71de89c3498bb6c0"
   dependencies:
     agentkeepalive "^4.1.3"
-    arsenal scality/Arsenal#580e25a
+    arsenal scality/Arsenal#7.10.1
     commander "2.20.0"
-    werelogs scality/werelogs#4e0d97c
+    werelogs scality/werelogs#8.1.0
     xml2js "0.4.19"
 
 verror@1.10.0:
@@ -6487,12 +6525,6 @@ werelogs@scality/werelogs#351a2a3:
   dependencies:
     safe-json-stringify "1.0.3"
 
-werelogs@scality/werelogs#4e0d97c:
-  version "7.4.1"
-  resolved "https://codeload.github.com/scality/werelogs/tar.gz/4e0d97cf69ea7ed60bea90756278513e7e7ea9b1"
-  dependencies:
-    safe-json-stringify "1.0.3"
-
 werelogs@scality/werelogs#8.1.0:
   version "8.1.0"
   resolved "https://codeload.github.com/scality/werelogs/tar.gz/e8f828725642c54c511cdbe580b18f43d3589313"