Compare commits

...

3 Commits

Author SHA1 Message Date
Jonathan Gramain eae3f0b2fa S3C-6474 [proto] bump utapi 2024-06-25 14:51:20 -07:00
Jonathan Gramain f989d5a1d1 S3C-6474 [proto] fix parsing of reindex redis config 2024-06-25 13:26:09 -07:00
Jonathan Gramain 80c62db053 S3C-6474 [proto] bump utapi 2024-06-24 18:23:06 -07:00
4 changed files with 319 additions and 127 deletions

View File

@ -88,6 +88,47 @@ function parseSproxydConfig(configSproxyd) {
return joi.attempt(configSproxyd, joiSchema, 'bad config');
}
function parseRedisConfig(redisConfig) {
const joiSchema = joi.object({
password: joi.string().allow(''),
host: joi.string(),
port: joi.number(),
retry: joi.object({
connectBackoff: joi.object({
min: joi.number().required(),
max: joi.number().required(),
jitter: joi.number().required(),
factor: joi.number().required(),
deadline: joi.number().required(),
}),
}),
// sentinel config
sentinels: joi.alternatives().try(
joi.string()
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
.custom(hosts => hosts.split(',').map(item => {
const [host, port] = item.split(':');
return { host, port: Number.parseInt(port, 10) };
})),
joi.array().items(
joi.object({
host: joi.string().required(),
port: joi.number().required(),
})
).min(1),
),
name: joi.string(),
sentinelPassword: joi.string().allow(''),
})
.and('host', 'port')
.and('sentinels', 'name')
.xor('host', 'sentinels')
.without('sentinels', ['host', 'port'])
.without('host', ['sentinels', 'sentinelPassword']);
return joi.attempt(redisConfig, joiSchema, 'bad config');
}
function restEndpointsAssert(restEndpoints, locationConstraints) {
assert(typeof restEndpoints === 'object',
'bad config: restEndpoints must be an object of endpoints');
@ -294,18 +335,17 @@ function parseUtapiReindex(config) {
const {
enabled,
schedule,
sentinel,
redis,
bucketd,
onlyCountLatestWhenObjectLocked,
} = config;
assert(typeof enabled === 'boolean',
'bad config: utapi.reindex.enabled must be a boolean');
assert(typeof sentinel === 'object',
'bad config: utapi.reindex.sentinel must be an object');
assert(typeof sentinel.port === 'number',
'bad config: utapi.reindex.sentinel.port must be a number');
assert(typeof sentinel.name === 'string',
'bad config: utapi.reindex.sentinel.name must be a string');
const parsedRedis = parseRedisConfig(redis);
assert(Array.isArray(parsedRedis.sentinels),
'bad config: utapi reindex redis config requires a list of sentinels');
assert(typeof bucketd === 'object',
'bad config: utapi.reindex.bucketd must be an object');
assert(typeof bucketd.port === 'number',
@ -323,6 +363,13 @@ function parseUtapiReindex(config) {
'bad config: utapi.reindex.schedule must be a valid ' +
`cron schedule. ${e.message}.`);
}
return {
enabled,
schedule,
redis: parsedRedis,
bucketd,
onlyCountLatestWhenObjectLocked,
};
}
function requestsConfigAssert(requestsConfig) {
@ -761,8 +808,7 @@ class Config extends EventEmitter {
assert(typeof config.localCache.port === 'number',
'config: invalid port for localCache. port must be a number');
if (config.localCache.password !== undefined) {
assert(
this._verifyRedisPassword(config.localCache.password),
assert(typeof config.localCache.password === 'string',
'config: invalid password for localCache. password must' +
' be a string');
}
@ -774,55 +820,7 @@ class Config extends EventEmitter {
}
if (config.redis) {
if (config.redis.sentinels) {
this.redis = { sentinels: [], name: null };
assert(typeof config.redis.name === 'string',
'bad config: redis sentinel name must be a string');
this.redis.name = config.redis.name;
assert(Array.isArray(config.redis.sentinels) ||
typeof config.redis.sentinels === 'string',
'bad config: redis sentinels must be an array or string');
if (typeof config.redis.sentinels === 'string') {
config.redis.sentinels.split(',').forEach(item => {
const [host, port] = item.split(':');
this.redis.sentinels.push({ host,
port: Number.parseInt(port, 10) });
});
} else if (Array.isArray(config.redis.sentinels)) {
config.redis.sentinels.forEach(item => {
const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.redis.sentinels.push({ host, port });
});
}
if (config.redis.sentinelPassword !== undefined) {
assert(
this._verifyRedisPassword(config.redis.sentinelPassword));
this.redis.sentinelPassword = config.redis.sentinelPassword;
}
} else {
// check for standalone configuration
this.redis = {};
assert(typeof config.redis.host === 'string',
'bad config: redis.host must be a string');
assert(typeof config.redis.port === 'number',
'bad config: redis.port must be a number');
this.redis.host = config.redis.host;
this.redis.port = config.redis.port;
}
if (config.redis.password !== undefined) {
assert(
this._verifyRedisPassword(config.redis.password),
'bad config: invalid password for redis. password must ' +
'be a string');
this.redis.password = config.redis.password;
}
this.redis = parseRedisConfig(config.redis);
}
if (config.utapi) {
this.utapi = { component: 's3' };
@ -851,65 +849,8 @@ class Config extends EventEmitter {
this.utapi.localCache = config.localCache;
assert(config.utapi.redis, 'missing required property of utapi ' +
'configuration: redis');
if (config.utapi.redis.sentinels) {
this.utapi.redis = { sentinels: [], name: null };
assert(typeof config.utapi.redis.name === 'string',
'bad config: redis sentinel name must be a string');
this.utapi.redis.name = config.utapi.redis.name;
assert(Array.isArray(config.utapi.redis.sentinels),
'bad config: redis sentinels must be an array');
config.utapi.redis.sentinels.forEach(item => {
const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.utapi.redis.sentinels.push({ host, port });
});
} else {
// check for standalone configuration
this.utapi.redis = {};
assert(typeof config.utapi.redis.host === 'string',
'bad config: redis.host must be a string');
assert(typeof config.utapi.redis.port === 'number',
'bad config: redis.port must be a number');
this.utapi.redis.host = config.utapi.redis.host;
this.utapi.redis.port = config.utapi.redis.port;
}
if (config.utapi.redis.password !== undefined) {
assert(
this._verifyRedisPassword(config.utapi.redis.password),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.password = config.utapi.redis.password;
}
if (config.utapi.redis.sentinelPassword !== undefined) {
assert(
this._verifyRedisPassword(config.utapi.redis.sentinelPassword),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.sentinelPassword =
config.utapi.redis.sentinelPassword;
}
if (config.utapi.redis.retry !== undefined) {
if (config.utapi.redis.retry.connectBackoff !== undefined) {
const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
assert.strictEqual(typeof min, 'number',
'utapi.redis.retry.connectBackoff: min must be a number');
assert.strictEqual(typeof max, 'number',
'utapi.redis.retry.connectBackoff: max must be a number');
assert.strictEqual(typeof jitter, 'number',
'utapi.redis.retry.connectBackoff: jitter must be a number');
assert.strictEqual(typeof factor, 'number',
'utapi.redis.retry.connectBackoff: factor must be a number');
assert.strictEqual(typeof deadline, 'number',
'utapi.redis.retry.connectBackoff: deadline must be a number');
}
this.utapi.redis.retry = config.utapi.redis.retry;
} else {
this.utapi.redis = parseRedisConfig(config.utapi.redis);
if (this.utapi.redis.retry === undefined) {
this.utapi.redis.retry = {
connectBackoff: {
min: 10,
@ -920,6 +861,7 @@ class Config extends EventEmitter {
},
};
}
if (config.utapi.metrics) {
this.utapi.metrics = config.utapi.metrics;
}
@ -988,8 +930,7 @@ class Config extends EventEmitter {
}
if (config.utapi && config.utapi.reindex) {
parseUtapiReindex(config.utapi.reindex);
this.utapi.reindex = config.utapi.reindex;
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
}
}
@ -1430,10 +1371,6 @@ class Config extends EventEmitter {
};
}
_verifyRedisPassword(password) {
return typeof password === 'string';
}
setAuthDataAccounts(accounts) {
this.authData.accounts = accounts;
this.emit('authdata-update');
@ -1547,6 +1484,7 @@ class Config extends EventEmitter {
module.exports = {
parseSproxydConfig,
parseRedisConfig,
locationConstraintAssert,
ConfigObject: Config,
config: new Config(),

View File

@ -35,7 +35,7 @@
"moment": "^2.26.0",
"npm-run-all": "~4.1.5",
"prom-client": "14.2.0",
"utapi": "git+https://github.com/scality/utapi#7.70.4",
"utapi": "git+https://github.com/scality/utapi#cbc28e8bca7bdcc650f4f9947fa56ad07a482d53",
"utf8": "~2.1.1",
"uuid": "^3.0.1",
"vaultclient": "scality/vaultclient#7.10.13",

View File

@ -0,0 +1,254 @@
const assert = require('assert');
const { parseRedisConfig } = require('../../../lib/Config');
describe('parseRedisConfig', () => {
[
{
desc: 'with host and port',
input: {
host: 'localhost',
port: 6479,
},
},
{
desc: 'with host, port and password',
input: {
host: 'localhost',
port: 6479,
password: 'mypass',
},
},
{
desc: 'with host, port and an empty password',
input: {
host: 'localhost',
port: 6479,
password: '',
},
},
{
desc: 'with host, port and an empty retry config',
input: {
host: 'localhost',
port: 6479,
retry: {
},
},
},
{
desc: 'with host, port and a custom retry config',
input: {
host: 'localhost',
port: 6479,
retry: {
connectBackoff: {
min: 10,
max: 1000,
jitter: 0.1,
factor: 1.5,
deadline: 10000,
},
},
},
},
{
desc: 'with a single sentinel and no sentinel password',
input: {
name: 'myname',
sentinels: [
{
host: 'localhost',
port: 16479,
},
],
},
},
{
desc: 'with two sentinels and a sentinel password',
input: {
name: 'myname',
sentinels: [
{
host: '10.20.30.40',
port: 16479,
},
{
host: '10.20.30.41',
port: 16479,
},
],
sentinelPassword: 'mypass',
},
},
{
desc: 'with a sentinel and an empty sentinel password',
input: {
name: 'myname',
sentinels: [
{
host: '10.20.30.40',
port: 16479,
},
],
sentinelPassword: '',
},
},
{
desc: 'with a basic production-like config with sentinels',
input: {
name: 'scality-s3',
password: '',
sentinelPassword: '',
sentinels: [
{
host: 'storage-1',
port: 16379,
},
{
host: 'storage-2',
port: 16379,
},
{
host: 'storage-3',
port: 16379,
},
{
host: 'storage-4',
port: 16379,
},
{
host: 'storage-5',
port: 16379,
},
],
},
},
{
desc: 'with a single sentinel passed as a string',
input: {
name: 'myname',
sentinels: '10.20.30.40:16479',
},
output: {
name: 'myname',
sentinels: [
{
host: '10.20.30.40',
port: 16479,
},
],
},
},
{
desc: 'with a list of sentinels passed as a string',
input: {
name: 'myname',
sentinels: '10.20.30.40:16479,another-host:16480,10.20.30.42:16481',
sentinelPassword: 'mypass',
},
output: {
name: 'myname',
sentinels: [
{
host: '10.20.30.40',
port: 16479,
},
{
host: 'another-host',
port: 16480,
},
{
host: '10.20.30.42',
port: 16481,
},
],
sentinelPassword: 'mypass',
},
},
].forEach(testCase => {
it(`should parse a valid config ${testCase.desc}`, () => {
const redisConfig = parseRedisConfig(testCase.input);
assert.deepStrictEqual(redisConfig, testCase.output || testCase.input);
});
});
[
{
desc: 'that is empty',
input: {},
},
{
desc: 'with only a host',
input: {
host: 'localhost',
},
},
{
desc: 'with only a port',
input: {
port: 6479,
},
},
{
desc: 'with a custom retry config with missing values',
input: {
host: 'localhost',
port: 6479,
retry: {
connectBackoff: {
},
},
},
},
{
desc: 'with a sentinel but no name',
input: {
sentinels: [
{
host: 'localhost',
port: 16479,
},
],
},
},
{
desc: 'with a sentinel but an empty name',
input: {
name: '',
sentinels: [
{
host: 'localhost',
port: 16479,
},
],
},
},
{
desc: 'with an empty list of sentinels',
input: {
name: 'myname',
sentinels: [],
},
},
{
desc: 'with an empty list of sentinels passed as a string',
input: {
name: 'myname',
sentinels: '',
},
},
{
desc: 'with an invalid list of sentinels passed as a string (missing port)',
input: {
name: 'myname',
sentinels: '10.20.30.40:16479,10.20.30.50',
},
},
].forEach(testCase => {
it(`should fail to parse an invalid config ${testCase.desc}`, () => {
assert.throws(() => {
parseRedisConfig(testCase.input);
});
});
});
});

View File

@ -5481,9 +5481,9 @@ user-home@^2.0.0:
dependencies:
os-homedir "^1.0.0"
"utapi@git+https://github.com/scality/utapi#7.70.4":
"utapi@git+https://github.com/scality/utapi#cbc28e8bca7bdcc650f4f9947fa56ad07a482d53":
version "7.70.4"
resolved "git+https://github.com/scality/utapi#960d990e899bc6d90f9e835ac2befdb319f6ee0b"
resolved "git+https://github.com/scality/utapi#cbc28e8bca7bdcc650f4f9947fa56ad07a482d53"
dependencies:
"@hapi/joi" "^17.1.1"
"@senx/warp10" "^1.0.14"