Compare commits

...

19 Commits

Author SHA1 Message Date
Salim 60dab39ace Merge remote-tracking branch 'origin/bugfix/ZENKO-2131' into w/8.2/bugfix/ZENKOIO-2131 2019-10-07 16:20:58 -07:00
bert-e dc5efbafdc Merge branch 'w/8.1/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens' into tmp/octopus/w/8.2/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens 2019-10-07 21:06:09 +00:00
naren-scality de3ff7507e Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens' into w/8.1/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens 2019-10-07 13:51:40 -07:00
naren-scality 42fba9e2bb bugfix: S3C-1805 Consecutive hyphen in bucketnames 2019-10-07 11:41:52 -07:00
naren-scality 14d3bc8b2d Merge remote-tracking branch 'origin/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens' into w/7.6/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens 2019-10-04 16:55:05 -07:00
naren-scality f2b5eb3ec7 bugfix: S3C-1805 Consecutive hyphen in bucketnames 2019-10-04 11:36:35 -07:00
bert-e 1f201b8b3a Merge branches 'development/8.2' and 'w/8.1/bugfix/S3C-2410/handle_unsupported_operations' into tmp/octopus/w/8.2/bugfix/S3C-2410/handle_unsupported_operations 2019-10-02 17:28:15 +00:00
bert-e bd8f92ee6f Merge branches 'w/8.2/improvement/ZENKOIO-100' and 'q/2157/8.1/improvement/ZENKOIO-100' into tmp/octopus/q/8.2 2019-10-01 22:03:41 +00:00
Salim 6322e527af Merge branch 'improvement/ZENKOIO-100' into w/8.2/improvement/ZENKOIO-100 2019-10-01 13:26:44 -07:00
bert-e 9c0ef12b10 Merge branch 'w/8.1/bugfix/S3C-2410/handle_unsupported_operations' into tmp/octopus/w/8.2/bugfix/S3C-2410/handle_unsupported_operations 2019-10-01 18:01:37 +00:00
vrancurel 2aab4be5d4 feature: workflow engine operator proxy 2019-09-27 09:59:03 -07:00
bert-e f04608a0ce Merge branches 'w/8.2/bugfix/S3C-2440-get-policy-xml-error' and 'q/2154/8.1/bugfix/S3C-2440-get-policy-xml-error' into tmp/octopus/q/8.2 2019-09-25 20:41:21 +00:00
bert-e 19415e36d5 Merge branch 'w/8.1/bugfix/S3C-2440-get-policy-xml-error' into tmp/octopus/w/8.2/bugfix/S3C-2440-get-policy-xml-error 2019-09-24 20:07:04 +00:00
bert-e 589a465c57 Merge branches 'w/8.2/bugfix/S3C-2439-precise-request-types-mpu' and 'q/2151/8.1/bugfix/S3C-2439-precise-request-types-mpu' into tmp/octopus/q/8.2 2019-09-24 01:10:11 +00:00
bert-e 957631cea0 Merge branch 'w/8.1/bugfix/S3C-2439-precise-request-types-mpu' into tmp/octopus/w/8.2/bugfix/S3C-2439-precise-request-types-mpu 2019-09-23 22:52:39 +00:00
bert-e 0faa470247 Merge branch 'w/8.1/bugfix/S3C-2399/incorrect_part_number_size' into tmp/octopus/w/8.2/bugfix/S3C-2399/incorrect_part_number_size 2019-09-20 07:15:46 +00:00
bert-e aa874cf6fd Merge branch 'w/8.1/bugfix/S3C-2435-fix-obj-action-parse' into tmp/octopus/w/8.2/bugfix/S3C-2435-fix-obj-action-parse 2019-09-17 22:55:41 +00:00
bert-e 3ab5d23dab Merge branch 'w/8.2/bugfix/S3C-2391/list_objects_v2_no_fetch_owner_corrections' into tmp/octopus/q/8.2 2019-09-17 16:40:48 +00:00
Salim 6ea92cf9ce ft: enable AWS and Ceph OOB on reportHandler 2019-09-12 15:23:47 -07:00
8 changed files with 121 additions and 3 deletions

View File

@ -39,6 +39,10 @@
"host": "localhost", "host": "localhost",
"port": 8900 "port": 8900
}, },
"workflowEngineOperator": {
"host": "localhost",
"port": 3001
},
"cdmi": { "cdmi": {
"host": "localhost", "host": "localhost",
"port": 81, "port": 81,

View File

@ -147,6 +147,14 @@ if [[ "$CRR_METRICS_PORT" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
fi fi
if [[ "$WE_OPERATOR_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
fi
if [[ "$WE_OPERATOR_PORT" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
fi
if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]" JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
fi fi

View File

@ -586,6 +586,16 @@ class Config extends EventEmitter {
this.backbeat = backbeat; this.backbeat = backbeat;
} }
if (config.workflowEngineOperator) {
const { workflowEngineOperator } = config;
assert.strictEqual(typeof workflowEngineOperator.host, 'string',
'bad config: workflowEngineOperator host must be a string');
assert(Number.isInteger(workflowEngineOperator.port) &&
workflowEngineOperator.port > 0,
'bad config: workflowEngineOperator port not a positive integer');
this.workflowEngineOperator = workflowEngineOperator;
}
// legacy // legacy
if (config.regions !== undefined) { if (config.regions !== undefined) {
throw new Error('bad config: regions key is deprecated. ' + throw new Error('bad config: regions key is deprecated. ' +

View File

@ -0,0 +1,91 @@
const url = require('url');
const httpProxy = require('http-proxy');
const workflowEngineOperatorProxy = httpProxy.createProxyServer({
ignorePath: true,
});
const { auth, errors, s3routes } =
require('arsenal');
const { responseJSONBody } = s3routes.routesUtils;
const vault = require('../auth/vault');
const prepareRequestContexts = require(
'../api/apiUtils/authorization/prepareRequestContexts');
const { config } = require('../Config');
const constants = require('../../constants');
auth.setHandler(vault);
function _decodeURI(uri) {
// do the same decoding than in S3 server
return decodeURIComponent(uri.replace(/\+/g, ' '));
}
function _normalizeRequest(req) {
/* eslint-disable no-param-reassign */
const parsedUrl = url.parse(req.url, true);
req.path = _decodeURI(parsedUrl.pathname);
const pathArr = req.path.split('/');
req.query = parsedUrl.query;
req.resourceType = pathArr[3];
req.bucketName = pathArr[4];
req.objectKey = pathArr.slice(5).join('/');
/* eslint-enable no-param-reassign */
}
function routeWorkflowEngineOperator(clientIP, request, response, log) {
log.debug('routing request', {
method: 'routeWorkflowEngineOperator',
url: request.url,
});
_normalizeRequest(request);
const requestContexts = prepareRequestContexts('objectReplicate', request);
// proxy api requests to Workflow Engine Operator API server
if (request.resourceType === 'api') {
if (!config.workflowEngineOperator) {
log.debug('unable to proxy workflow engine operator request', {
workflowEngineConfig: config.workflowEngineOperator,
});
return responseJSONBody(errors.MethodNotAllowed, null, response,
log);
}
const path = request.url.replace('/_/workflow-engine-operator/api', '/_/');
const { host, port } = config.workflowEngineOperator;
const target = `http://${host}:${port}${path}`;
return auth.server.doAuth(request, log, (err, userInfo) => {
if (err) {
log.debug('authentication error', {
error: err,
method: request.method,
bucketName: request.bucketName,
objectKey: request.objectKey,
});
return responseJSONBody(err, null, response, log);
}
// FIXME for now, any authenticated user can access API
// routes. We should introduce admin accounts or accounts
// with admin privileges, and restrict access to those
// only.
if (userInfo.getCanonicalID() === constants.publicId) {
log.debug('unauthenticated access to API routes', {
method: request.method,
bucketName: request.bucketName,
objectKey: request.objectKey,
});
return responseJSONBody(
errors.AccessDenied, null, response, log);
}
return workflowEngineOperatorProxy.web(
request, response, { target }, err => {
log.error('error proxying request to api server',
{ error: err.message });
return responseJSONBody(errors.ServiceUnavailable, null,
response, log);
});
}, 's3', requestContexts);
}
return undefined;
}
module.exports = routeWorkflowEngineOperator;

View File

@ -1,6 +1,8 @@
const { healthcheckHandler } = require('./healthcheckHandler'); const { healthcheckHandler } = require('./healthcheckHandler');
const routeBackbeat = require('../routes/routeBackbeat'); const routeBackbeat = require('../routes/routeBackbeat');
const routeMetadata = require('../routes/routeMetadata'); const routeMetadata = require('../routes/routeMetadata');
const routeWorkflowEngineOperator =
require('../routes/routeWorkflowEngineOperator');
const { reportHandler } = require('./reportHandler'); const { reportHandler } = require('./reportHandler');
const { monitoringHandler } = require('./monitoringHandler'); const { monitoringHandler } = require('./monitoringHandler');
@ -10,6 +12,7 @@ const internalHandlers = {
report: reportHandler, report: reportHandler,
monitoring: monitoringHandler, monitoring: monitoringHandler,
metadata: routeMetadata, metadata: routeMetadata,
'workflow-engine-operator': routeWorkflowEngineOperator,
}; };
module.exports = { module.exports = {

View File

@ -46,6 +46,8 @@ function getCapabilities() {
secureChannelOptimizedPath: hasWSOptionalDependencies(), secureChannelOptimizedPath: hasWSOptionalDependencies(),
s3cIngestLocation: true, s3cIngestLocation: true,
nfsIngestLocation: true, nfsIngestLocation: true,
cephIngestLocation: true,
awsIngestLocation: true,
}; };
} }

View File

@ -19,7 +19,7 @@
}, },
"homepage": "https://github.com/scality/S3#readme", "homepage": "https://github.com/scality/S3#readme",
"dependencies": { "dependencies": {
"arsenal": "github:scality/Arsenal#2bb0e17", "arsenal": "github:scality/Arsenal#635d2fe",
"async": "~2.5.0", "async": "~2.5.0",
"aws-sdk": "2.28.0", "aws-sdk": "2.28.0",
"azure-storage": "^2.1.0", "azure-storage": "^2.1.0",

View File

@ -231,9 +231,9 @@ arraybuffer.slice@~0.0.7:
resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675" resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675"
integrity sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog== integrity sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==
"arsenal@github:scality/Arsenal#2bb0e17": "arsenal@github:scality/Arsenal#635d2fe":
version "8.1.4" version "8.1.4"
resolved "https://codeload.github.com/scality/Arsenal/tar.gz/2bb0e171d8bada09e40491680b82ab33a882e7eb" resolved "https://codeload.github.com/scality/Arsenal/tar.gz/635d2fe6d9421e2a59c4fa213594ef2785b17709"
dependencies: dependencies:
"@hapi/joi" "^15.1.0" "@hapi/joi" "^15.1.0"
JSONStream "^1.0.0" JSONStream "^1.0.0"