Compare commits

...

6 Commits

Author SHA1 Message Date
Maxime Lubin 806e546c67 RING-28500 Add location-scality-hdclient-v1 config patching
Mapping Orbit config to actual HypderdriveClient
configuration requires a custom translation.
2018-11-20 11:01:01 +01:00
Maxime Lubin c7e4a90f72 RING-28500 Add clientType='scality' to hdclient instances 2018-11-20 11:01:01 +01:00
Maxime Lubin bf441f7384 RING-28500 Check hdclient configuration on startup
Add hook in locationConstraintAssert function to
verify hdclient in-file configuration. Check
is done by using provided hdclient API, not
manually inside S3.

NB: this code path is valid only for CloudServer
not running under Orbit (as far as I can tell...)
2018-11-20 11:01:01 +01:00
Maxime Lubin cfe565a30b RING-28500 Add hyperdrive new locationType 2018-11-20 11:00:17 +01:00
Maxime Lubin ee7ec88966 RING-28500 Plug Hyperdrive client as new 'multiple' data backend
Signed-off-by: Maxime Lubin <maxime.lubin@scality.com>
2018-11-20 11:00:17 +01:00
Maxime Lubin 50d7af6e4a WIP Add hdclient dependency to package.json
Note: version is not fixed yet!
Signed-off-by: Maxime Lubin <maxime.lubin@scality.com>
2018-11-20 10:59:14 +01:00
7 changed files with 92 additions and 15 deletions

View File

@ -12,6 +12,8 @@ const { buildAuthDataAccount } = require('./auth/in_memory/builder');
const validExternalBackends = require('../constants').externalBackends;
const { azureAccountNameRegex, base64Regex } = require('../constants');
const hdclient = require('hdclient');
// whitelist IP, CIDR for health checks
const defaultHealthChecks = { allowFrom: ['127.0.0.1/8', '::1'] };
@ -164,6 +166,14 @@ function azureLocationConstraintAssert(location, locationObj) {
'azureContainerName is an invalid container name');
}
function hyperdriveClientLocationConstraintAssert(location, locationObj) {
const { configIsValid, configError } = hdclient.config.validate(
locationObj.details.connector.hdclient);
if (!configIsValid) {
throw configError;
}
}
function locationConstraintAssert(locationConstraints) {
const supportedBackends =
['mem', 'file', 'scality',
@ -272,6 +282,10 @@ function locationConstraintAssert(locationConstraints) {
assert(typeof details.pfsDaemonEndpoint === 'object',
'bad config: pfsDaemonEndpoint is mandatory and must be an object');
}
if (locationConstraints[l].type === 'scality' &&
locationConstraints[l].details.connector.hdclient !== undefined) {
hyperdriveClientLocationConstraintAssert(l, locationConstraints[l]);
}
});
assert(Object.keys(locationConstraints)
.includes('us-east-1'), 'bad locationConfig: must ' +

View File

@ -3,6 +3,7 @@ const http = require('http');
const url = require('url');
const AWS = require('aws-sdk');
const Sproxy = require('sproxydclient');
const Hyperdrive = require('hdclient');
const HttpsProxyAgent = require('https-proxy-agent');
@ -27,21 +28,26 @@ function parseLC() {
if (locationObj.type === 'file') {
clients[location] = new DataFileBackend();
}
if (locationObj.type === 'scality'
&& locationObj.details.connector.sproxyd) {
clients[location] = new Sproxy({
bootstrap: locationObj.details.connector
.sproxyd.bootstrap,
// Might be undefined which is ok since there is a default
// set in sproxydclient if chordCos is undefined
chordCos: locationObj.details.connector.sproxyd.chordCos,
// Might also be undefined, but there is a default path set
// in sproxydclient as well
path: locationObj.details.connector.sproxyd.path,
// enable immutable optim for all objects
immutable: true,
});
clients[location].clientType = 'scality';
if (locationObj.type === 'scality') {
if (locationObj.details.connector.sproxyd) {
clients[location] = new Sproxy({
bootstrap: locationObj.details.connector
.sproxyd.bootstrap,
// Might be undefined which is ok since there is a default
// set in sproxydclient if chordCos is undefined
chordCos: locationObj.details.connector.sproxyd.chordCos,
// Might also be undefined, but there is a default path set
// in sproxydclient as well
path: locationObj.details.connector.sproxyd.path,
// enable immutable optim for all objects
immutable: true,
});
clients[location].clientType = 'scality';
} else if (locationObj.details.connector.hdclient) {
clients[location] = new Hyperdrive.hdclient.HyperdriveClient(
locationObj.details.connector.hdclient);
clients[location].clientType = 'scality';
}
}
if (locationObj.type === 'aws_s3' || locationObj.type === 'gcp') {
let connectionAgent;

View File

@ -183,6 +183,47 @@ function patchConfiguration(newConf, log, cb) {
};
}
break;
case 'location-scality-hdclient-v1':
location.type = 'scality';
if (l.details) {
const dataParts = l.details.dataParts;
let codingParts = 0;
if (l.details.codingParts !== null ||
l.details.codingParts !== undefined) {
codingParts = l.details.codingParts;
}
const code = codingParts > 0 ? 'RS' : 'CP';
const uuidmapping = {};
const components = [];
l.details.bootstrapList.forEach(
entry => {
const [uuid, endpoint, writeEnabled] =
entry.split(',');
uuidmapping[uuid] = endpoint;
components.push({
name: uuid,
staticWeight: writeEnabled === 'enabled' ?
1 : 0,
});
});
location.details = {
connector: {
hdclient: {
code,
dataParts,
codingParts,
uuidmapping,
requestTimeoutMs: 10000,
errorAgent: { kafkaBrokers: 'unused' },
policy: {
minSplitSize: 512000,
cluster: { components },
},
},
},
};
}
break;
default:
log.info('unknown location type', { locationType:
l.locationType });

View File

@ -34,6 +34,7 @@ function getCapabilities() {
locationTypeDigitalOcean: true,
locationTypeS3Custom: true,
locationTypeSproxyd: true,
locationTypeHyperdrive: true,
preferredReadLocation: true,
managedLifecycle: true,
secureChannelOptimizedPath: hasWSOptionalDependencies(),

View File

@ -29,6 +29,7 @@
"commander": "^2.9.0",
"diskusage": "0.2.4",
"google-auto-auth": "^0.9.1",
"hdclient": "scality/hdclient",
"http-proxy": "^1.17.0",
"https-proxy-agent": "^2.2.0",
"mongodb": "^2.2.31",

View File

@ -168,6 +168,19 @@ describe('patchConfiguration', () => {
sizeLimitGB: 0,
details: {},
},
'hyperdrivebackendtest': {
name: 'hyperdrivebackendtest',
objectId: 'hyperdrivebackendtest',
locationType: 'location-scality-hdclient-v1',
details: {
nDataParts: 3,
nCodingParts: 0,
bootstraplist: ['hd1,fakehyperdrive:8444,enbled',
'hd2,localhost:7777,disabled',
'hd3,localhost:4244,enabled',
'hd4,localhost:4245,enabled'],
},
},
},
browserAccess: {
enabled: true,

View File

@ -10,6 +10,7 @@ describe('report handler', () => {
assert.strictEqual(c.locationTypeDigitalOcean, true);
assert.strictEqual(c.locationTypeS3Custom, true);
assert.strictEqual(c.locationTypeSproxyd, true);
assert.strictEqual(c.locationTypeHyperdrive, true);
assert.strictEqual(c.preferredReadLocation, true);
assert.strictEqual(c.managedLifecycle, true);
assert.strictEqual(c.secureChannelOptimizedPath, true);