S3C-6474 [proto] bump utapi

lib/Config.js

@@ -88,6 +88,47 @@ function parseSproxydConfig(configSproxyd) {
     return joi.attempt(configSproxyd, joiSchema, 'bad config');
 }
 
+function parseRedisConfig(redisConfig) {
+    const joiSchema = joi.object({
+        password: joi.string().allow(''),
+        host: joi.string(),
+        port: joi.number(),
+        retry: joi.object({
+            connectBackoff: joi.object({
+                min: joi.number().required(),
+                max: joi.number().required(),
+                jitter: joi.number().required(),
+                factor: joi.number().required(),
+                deadline: joi.number().required(),
+            }),
+        }),
+        // sentinel config
+        sentinels: joi.alternatives().try(
+            joi.string()
+                .pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
+                .custom(hosts => hosts.split(',').map(item => {
+                    const [host, port] = item.split(':');
+                    return { host, port: Number.parseInt(port, 10) };
+                })),
+            joi.array().items(
+                joi.object({
+                    host: joi.string().required(),
+                    port: joi.number().required(),
+                })
+            ).min(1),
+        ),
+        name: joi.string(),
+        sentinelPassword: joi.string().allow(''),
+    })
+          .and('host', 'port')
+          .and('sentinels', 'name')
+          .xor('host', 'sentinels')
+          .without('sentinels', ['host', 'port'])
+          .without('host', ['sentinels', 'sentinelPassword']);
+
+    return joi.attempt(redisConfig, joiSchema, 'bad config');
+}
+
 function restEndpointsAssert(restEndpoints, locationConstraints) {
     assert(typeof restEndpoints === 'object',
         'bad config: restEndpoints must be an object of endpoints');
@@ -294,18 +335,17 @@ function parseUtapiReindex(config) {
     const {
         enabled,
         schedule,
-        sentinel,
+        redis,
         bucketd,
         onlyCountLatestWhenObjectLocked,
     } = config;
     assert(typeof enabled === 'boolean',
            'bad config: utapi.reindex.enabled must be a boolean');
-    assert(typeof sentinel === 'object',
+
-        'bad config: utapi.reindex.sentinel must be an object');
+    const parsedRedis = parseRedisConfig(redis);
-    assert(typeof sentinel.port === 'number',
+    assert(Array.isArray(parsedRedis.sentinels),
-        'bad config: utapi.reindex.sentinel.port must be a number');
+           'bad config: utapi reindex redis config requires a list of sentinels');
-    assert(typeof sentinel.name === 'string',
+
-        'bad config: utapi.reindex.sentinel.name must be a string');
     assert(typeof bucketd === 'object',
         'bad config: utapi.reindex.bucketd must be an object');
     assert(typeof bucketd.port === 'number',
@@ -323,6 +363,13 @@ function parseUtapiReindex(config) {
             'bad config: utapi.reindex.schedule must be a valid ' +
             `cron schedule. ${e.message}.`);
     }
+    return {
+        enabled,
+        schedule,
+        redis: parsedRedis,
+        bucketd,
+        onlyCountLatestWhenObjectLocked,
+    };
 }
 
 function requestsConfigAssert(requestsConfig) {
@@ -761,8 +808,7 @@ class Config extends EventEmitter {
             assert(typeof config.localCache.port === 'number',
                 'config: invalid port for localCache. port must be a number');
             if (config.localCache.password !== undefined) {
-                assert(
+                assert(typeof config.localCache.password === 'string',
-                    this._verifyRedisPassword(config.localCache.password),
                     'config: invalid password for localCache. password must' +
                     ' be a string');
             }
@@ -774,55 +820,7 @@ class Config extends EventEmitter {
         }
 
         if (config.redis) {
-            if (config.redis.sentinels) {
+            this.redis = parseRedisConfig(config.redis);
-                this.redis = { sentinels: [], name: null };
-
-                assert(typeof config.redis.name === 'string',
-                    'bad config: redis sentinel name must be a string');
-                this.redis.name = config.redis.name;
-                assert(Array.isArray(config.redis.sentinels) ||
-                    typeof config.redis.sentinels === 'string',
-                    'bad config: redis sentinels must be an array or string');
-
-                if (typeof config.redis.sentinels === 'string') {
-                    config.redis.sentinels.split(',').forEach(item => {
-                        const [host, port] = item.split(':');
-                        this.redis.sentinels.push({ host,
-                            port: Number.parseInt(port, 10) });
-                    });
-                } else if (Array.isArray(config.redis.sentinels)) {
-                    config.redis.sentinels.forEach(item => {
-                        const { host, port } = item;
-                        assert(typeof host === 'string',
-                            'bad config: redis sentinel host must be a string');
-                        assert(typeof port === 'number',
-                            'bad config: redis sentinel port must be a number');
-                        this.redis.sentinels.push({ host, port });
-                    });
-                }
-
-                if (config.redis.sentinelPassword !== undefined) {
-                    assert(
-                    this._verifyRedisPassword(config.redis.sentinelPassword));
-                    this.redis.sentinelPassword = config.redis.sentinelPassword;
-                }
-            } else {
-                // check for standalone configuration
-                this.redis = {};
-                assert(typeof config.redis.host === 'string',
-                    'bad config: redis.host must be a string');
-                assert(typeof config.redis.port === 'number',
-                    'bad config: redis.port must be a number');
-                this.redis.host = config.redis.host;
-                this.redis.port = config.redis.port;
-            }
-            if (config.redis.password !== undefined) {
-                assert(
-                    this._verifyRedisPassword(config.redis.password),
-                    'bad config: invalid password for redis. password must ' +
-                    'be a string');
-                this.redis.password = config.redis.password;
-            }
         }
         if (config.utapi) {
             this.utapi = { component: 's3' };
@@ -851,65 +849,8 @@ class Config extends EventEmitter {
                 this.utapi.localCache = config.localCache;
                 assert(config.utapi.redis, 'missing required property of utapi ' +
                     'configuration: redis');
-                if (config.utapi.redis.sentinels) {
+                this.utapi.redis = parseRedisConfig(config.utapi.redis);
-                    this.utapi.redis = { sentinels: [], name: null };
+                if (this.utapi.redis.retry === undefined) {
-
-                    assert(typeof config.utapi.redis.name === 'string',
-                        'bad config: redis sentinel name must be a string');
-                    this.utapi.redis.name = config.utapi.redis.name;
-
-                    assert(Array.isArray(config.utapi.redis.sentinels),
-                        'bad config: redis sentinels must be an array');
-                    config.utapi.redis.sentinels.forEach(item => {
-                        const { host, port } = item;
-                        assert(typeof host === 'string',
-                            'bad config: redis sentinel host must be a string');
-                        assert(typeof port === 'number',
-                            'bad config: redis sentinel port must be a number');
-                        this.utapi.redis.sentinels.push({ host, port });
-                    });
-                } else {
-                    // check for standalone configuration
-                    this.utapi.redis = {};
-                    assert(typeof config.utapi.redis.host === 'string',
-                        'bad config: redis.host must be a string');
-                    assert(typeof config.utapi.redis.port === 'number',
-                        'bad config: redis.port must be a number');
-                    this.utapi.redis.host = config.utapi.redis.host;
-                    this.utapi.redis.port = config.utapi.redis.port;
-                }
-                if (config.utapi.redis.password !== undefined) {
-                    assert(
-                        this._verifyRedisPassword(config.utapi.redis.password),
-                        'config: invalid password for utapi redis. password' +
-                        ' must be a string');
-                    this.utapi.redis.password = config.utapi.redis.password;
-                }
-                if (config.utapi.redis.sentinelPassword !== undefined) {
-                    assert(
-                    this._verifyRedisPassword(config.utapi.redis.sentinelPassword),
-                        'config: invalid password for utapi redis. password' +
-                        ' must be a string');
-                    this.utapi.redis.sentinelPassword =
-                        config.utapi.redis.sentinelPassword;
-                }
-                if (config.utapi.redis.retry !== undefined) {
-                    if (config.utapi.redis.retry.connectBackoff !== undefined) {
-                        const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
-                        assert.strictEqual(typeof min, 'number',
-                        'utapi.redis.retry.connectBackoff: min must be a number');
-                        assert.strictEqual(typeof max, 'number',
-                        'utapi.redis.retry.connectBackoff: max must be a number');
-                        assert.strictEqual(typeof jitter, 'number',
-                        'utapi.redis.retry.connectBackoff: jitter must be a number');
-                        assert.strictEqual(typeof factor, 'number',
-                        'utapi.redis.retry.connectBackoff: factor must be a number');
-                        assert.strictEqual(typeof deadline, 'number',
-                        'utapi.redis.retry.connectBackoff: deadline must be a number');
-                    }
-
-                    this.utapi.redis.retry = config.utapi.redis.retry;
-                } else {
                     this.utapi.redis.retry = {
                         connectBackoff: {
                             min: 10,
@@ -920,6 +861,7 @@ class Config extends EventEmitter {
                         },
                     };
                 }
+
                 if (config.utapi.metrics) {
                     this.utapi.metrics = config.utapi.metrics;
                 }
@@ -988,8 +930,7 @@ class Config extends EventEmitter {
                 }
 
                 if (config.utapi && config.utapi.reindex) {
-                    parseUtapiReindex(config.utapi.reindex);
+                    this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
-                    this.utapi.reindex = config.utapi.reindex;
                 }
             }
 
@@ -1430,10 +1371,6 @@ class Config extends EventEmitter {
         };
     }
 
-    _verifyRedisPassword(password) {
-        return typeof password === 'string';
-    }
-
     setAuthDataAccounts(accounts) {
         this.authData.accounts = accounts;
         this.emit('authdata-update');
@@ -1547,6 +1484,7 @@ class Config extends EventEmitter {
 
 module.exports = {
     parseSproxydConfig,
+    parseRedisConfig,
     locationConstraintAssert,
     ConfigObject: Config,
     config: new Config(),

package.json

@@ -35,7 +35,7 @@
     "moment": "^2.26.0",
     "npm-run-all": "~4.1.5",
     "prom-client": "14.2.0",
-    "utapi": "git+https://github.com/scality/utapi#7.70.4",
+    "utapi": "git+https://github.com/scality/utapi#cbc28e8bca7bdcc650f4f9947fa56ad07a482d53",
     "utf8": "~2.1.1",
     "uuid": "^3.0.1",
     "vaultclient": "scality/vaultclient#7.10.13",

tests/unit/testConfigs/parseRedisConfig.spec.js

@@ -0,0 +1,254 @@
+const assert = require('assert');
+const { parseRedisConfig } = require('../../../lib/Config');
+
+describe('parseRedisConfig', () => {
+    [
+        {
+            desc: 'with host and port',
+            input: {
+                host: 'localhost',
+                port: 6479,
+            },
+        },
+        {
+            desc: 'with host, port and password',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                password: 'mypass',
+            },
+        },
+        {
+            desc: 'with host, port and an empty password',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                password: '',
+            },
+        },
+        {
+            desc: 'with host, port and an empty retry config',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                retry: {
+                },
+            },
+        },
+        {
+            desc: 'with host, port and a custom retry config',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                retry: {
+                    connectBackoff: {
+                        min: 10,
+                        max: 1000,
+                        jitter: 0.1,
+                        factor: 1.5,
+                        deadline: 10000,
+                    },
+                },
+            },
+        },
+        {
+            desc: 'with a single sentinel and no sentinel password',
+            input: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: 'localhost',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with two sentinels and a sentinel password',
+            input: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                    {
+                        host: '10.20.30.41',
+                        port: 16479,
+                    },
+                ],
+                sentinelPassword: 'mypass',
+            },
+        },
+        {
+            desc: 'with a sentinel and an empty sentinel password',
+            input: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                ],
+                sentinelPassword: '',
+            },
+        },
+        {
+            desc: 'with a basic production-like config with sentinels',
+            input: {
+                name: 'scality-s3',
+                password: '',
+                sentinelPassword: '',
+                sentinels: [
+                    {
+                        host: 'storage-1',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-2',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-3',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-4',
+                        port: 16379,
+                    },
+                    {
+                        host: 'storage-5',
+                        port: 16379,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with a single sentinel passed as a string',
+            input: {
+                name: 'myname',
+                sentinels: '10.20.30.40:16479',
+            },
+            output: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with a list of sentinels passed as a string',
+            input: {
+                name: 'myname',
+                sentinels: '10.20.30.40:16479,another-host:16480,10.20.30.42:16481',
+                sentinelPassword: 'mypass',
+            },
+            output: {
+                name: 'myname',
+                sentinels: [
+                    {
+                        host: '10.20.30.40',
+                        port: 16479,
+                    },
+                    {
+                        host: 'another-host',
+                        port: 16480,
+                    },
+                    {
+                        host: '10.20.30.42',
+                        port: 16481,
+                    },
+                ],
+                sentinelPassword: 'mypass',
+            },
+        },
+    ].forEach(testCase => {
+        it(`should parse a valid config ${testCase.desc}`, () => {
+            const redisConfig = parseRedisConfig(testCase.input);
+            assert.deepStrictEqual(redisConfig, testCase.output || testCase.input);
+        });
+    });
+
+    [
+        {
+            desc: 'that is empty',
+            input: {},
+        },
+        {
+            desc: 'with only a host',
+            input: {
+                host: 'localhost',
+            },
+        },
+        {
+            desc: 'with only a port',
+            input: {
+                port: 6479,
+            },
+        },
+        {
+            desc: 'with a custom retry config with missing values',
+            input: {
+                host: 'localhost',
+                port: 6479,
+                retry: {
+                    connectBackoff: {
+                    },
+                },
+            },
+        },
+        {
+            desc: 'with a sentinel but no name',
+            input: {
+                sentinels: [
+                    {
+                        host: 'localhost',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with a sentinel but an empty name',
+            input: {
+                name: '',
+                sentinels: [
+                    {
+                        host: 'localhost',
+                        port: 16479,
+                    },
+                ],
+            },
+        },
+        {
+            desc: 'with an empty list of sentinels',
+            input: {
+                name: 'myname',
+                sentinels: [],
+            },
+        },
+        {
+            desc: 'with an empty list of sentinels passed as a string',
+            input: {
+                name: 'myname',
+                sentinels: '',
+            },
+        },
+        {
+            desc: 'with an invalid list of sentinels passed as a string (missing port)',
+            input: {
+                name: 'myname',
+                sentinels: '10.20.30.40:16479,10.20.30.50',
+            },
+        },
+    ].forEach(testCase => {
+        it(`should fail to parse an invalid config ${testCase.desc}`, () => {
+            assert.throws(() => {
+                parseRedisConfig(testCase.input);
+            });
+        });
+    });
+});

yarn.lock

@@ -5481,9 +5481,9 @@ user-home@^2.0.0:
   dependencies:
     os-homedir "^1.0.0"
 
-"utapi@git+https://github.com/scality/utapi#7.70.4":
+"utapi@git+https://github.com/scality/utapi#cbc28e8bca7bdcc650f4f9947fa56ad07a482d53":
   version "7.70.4"
-  resolved "git+https://github.com/scality/utapi#960d990e899bc6d90f9e835ac2befdb319f6ee0b"
+  resolved "git+https://github.com/scality/utapi#cbc28e8bca7bdcc650f4f9947fa56ad07a482d53"
   dependencies:
     "@hapi/joi" "^17.1.1"
     "@senx/warp10" "^1.0.14"