Compare commits

...

6 Commits

Author SHA1 Message Date
Rahul Padigela 543c737f04 more controllers 2017-06-15 10:23:44 -07:00
Lauren Spiegel d6e607f8a5 LOG: Add endpoint log 2017-06-15 10:23:44 -07:00
Lauren Spiegel 4782f57dad FIX: Remove excess logging info 2017-06-15 10:23:44 -07:00
Alexandre Merle 0ea7145d1f S3C-380: Get security token
Send security token to vault

See https://scality.atlassian.net/browse/S3C-380
2017-06-15 10:23:44 -07:00
Rahul Padigela bbc050335c wip: controller, middleware, models [ci skip] 2017-06-06 17:55:58 -07:00
Rahul Padigela 113d0f699b ft: controller and router 2017-06-06 17:25:42 -07:00
24 changed files with 649 additions and 2 deletions

View File

@ -0,0 +1,25 @@
class BucketController {
static putBucket(req, res, cb) {
}
static putBucketVersioning(req, res, cb) {
}
static putObject(req, res, cb) {
}
static putObjectCors(req, res, cb) {
}
static putObjectVersioning(req, res, cb) {
}
}
module.exports = BucketController;

View File

View File

@ -0,0 +1,14 @@
class ObjectController {
static putObject(req, res, cb) {
}
static getObject(req, res, cb) {
}
}
module.exports = ObjectController;

View File

@ -102,6 +102,7 @@ function authenticateV2Request(params, requestContexts, callback) {
algo: params.data.algo, algo: params.data.algo,
reqUid: params.log.getSerializedUids(), reqUid: params.log.getSerializedUids(),
logger: params.log, logger: params.log,
securityToken: params.data.securityToken,
requestContext: serializedRCsArr, requestContext: serializedRCsArr,
}, },
(err, userInfo) => vaultSignatureCb(err, userInfo, (err, userInfo) => vaultSignatureCb(err, userInfo,
@ -156,6 +157,7 @@ function authenticateV4Request(params, requestContexts, callback) {
{ {
reqUid: params.log.getSerializedUids(), reqUid: params.log.getSerializedUids(),
logger: params.log, logger: params.log,
securityToken: params.data.securityToken,
requestContext: serializedRCs, requestContext: serializedRCs,
}, },
(err, userInfo) => vaultSignatureCb(err, userInfo, (err, userInfo) => vaultSignatureCb(err, userInfo,

View File

@ -54,6 +54,7 @@ function routes(req, res, logger) {
clientPort: req.socket.remotePort, clientPort: req.socket.remotePort,
httpMethod: req.method, httpMethod: req.method,
httpURL: req.url, httpURL: req.url,
endpoint: req.endpoint,
}; };
const log = logger.newRequestLogger(); const log = logger.newRequestLogger();

31
lib/routes/Router.js Normal file
View File

@ -0,0 +1,31 @@
const { normalizeRequest } = require('./utils');
const BucketController = require('../controllers/BucketController');
class Router {
constructor(routes) {
this._routes = routes;
this._controllers = { BucketController };
}
_matchRoute(route, req) {
const query = route.query || [];
return req.method === route.method &&
Boolean(req.bucketName) === Boolean(route.bucket) &&
Boolean(req.objectKey) === Boolean(route.object) &&
query.every(q => q in req.query);
}
exec(request, response, cb) {
const _req = normalizeRequest(request);
const index = this._routes.findIndex(r => this._matchRoute(r, _req));
if (index === -1) {
// TODO: return NotImplemented / Invalid Request
}
const { controller, action } = this._routes[index];
this._controllers[controller][action](request, response, cb);
}
}
module.exports = Router;

88
lib/routes/routes.json Normal file
View File

@ -0,0 +1,88 @@
[
{
bucket: true,
method: 'GET',
controller: 'BucketController',
action: 'getService',
auth: ['bucket']
},
{
bucket: true,
method: 'GET',
controller: 'BucketController',
action: 'getBucket',
auth: ['bucket']
},
{
bucket: true,
method: 'PUT',
controller: 'BucketController',
action: 'putBucket',
auth: ['bucket']
},
{
bucket: true,
method: 'PUT',
controller: 'BucketController',
action: 'putBucketAcl',
query: ['acl'],
auth: ['bucket']
},
{
bucket: true,
method: 'PUT',
controller: 'BucketController',
action: 'putBucketVersioning',
query: ['versioning'],
auth: ['bucket']
},
{
bucket: true,
method: 'PUT',
controller: 'BucketController',
action: 'putBucketWebsite',
query: ['website'],
auth: ['bucket']
},
{
bucket: true,
method: 'PUT',
controller: 'BucketController',
action: 'putBucketCors',
query: ['cors'],
auth: ['bucket']
},
{
bucket: true,
object: true,
method: 'GET',
controller: 'ObjectController',
action: 'getObject',,
auth: ['bucket', 'object']
},
{
bucket: true,
object: true,
method: 'PUT',
controller: 'ObjectController',
action: 'putObject',,
auth: ['bucket', 'object']
},
{
bucket: false,
object: false,
method: 'GET',
controller: 'HealthcheckController',
action: 'basic',
admin: true,
},
{
bucket: false,
object: false,
method: 'GET',
controller: 'HealthcheckController',
query: ['deep'],
action: 'deep',
admin: true,
}
];

View File

@ -0,0 +1,33 @@
const constants = require('../conf/constants');
const ipAddressRegex = new RegExp(/^(\d+\.){3}\d+$/);
const dnsRegex = new RegExp(/^[a-z0-9]+([\.\-]{1}[a-z0-9]+)*$/);
/**
* Validate bucket name per naming rules and restrictions
* @param {string} bucketname - name of the bucket to be created
* @return {boolean} - returns true/false by testing
* bucket name against validation rules
*/
function isValidBucketName(bucketname) {
// Must be at least 3 and no more than 63 characters long.
if (bucketname.length < 3 || bucketname.length > 63) {
return false;
}
// Must not start with the mpuBucketPrefix since this is
// reserved for the shadow bucket used for multipart uploads
if (bucketname.startsWith(constants.mpuBucketPrefix)) {
return false;
}
// Must not contain more than one consecutive period
if (bucketname.indexOf('..') > 1) {
return false;
}
// Must not be an ip address
if (bucketname.match(ipAddressRegex)) {
return false;
}
// Must be dns compatible
return !!bucketname.match(dnsRegex);
}
module.exports = isValidBucketName;

16
middleware/middleware.js Normal file
View File

@ -0,0 +1,16 @@
class Middleware {
constructor() {
this.order = [
'normalizeRequest',
'capturePostData',
'router',
'auth',
];
}
exec(req, res, cb) {
}
}
module.exports = Middleware;

View File

@ -0,0 +1,182 @@
const url = require('url');
const invalidBucketName = require('./invalidBucketName');
/**
* Get bucket name from the request of a virtually hosted bucket
* @param {object} request - HTTP request object
* @return {string|undefined} - returns bucket name if dns-style query
* returns undefined if path-style query
* @throws {Error} in case the type of query could not be infered
*/
function getBucketNameFromHost(request) {
const headers = request.headers;
if (headers === undefined || headers.host === undefined) {
throw new Error('bad request: no host in headers');
}
const reqHost = headers.host;
const bracketIndex = reqHost.indexOf(']');
const colonIndex = reqHost.lastIndexOf(':');
const hostLength = colonIndex > bracketIndex ? colonIndex : reqHost.length;
// If request is made using IPv6 (indicated by presence of brackets),
// surrounding brackets should not be included in host var
const host = bracketIndex > -1 ?
reqHost.slice(1, hostLength - 1) : reqHost.slice(0, hostLength);
// parseIp returns empty object if host is not valid IP
// If host is an IP address, it's path-style
if (Object.keys(ipCheck.parseIp(host)).length !== 0) {
return undefined;
}
// All endpoints from all regions + `websiteEndpoints
const validHosts = getAllEndpoints().concat(config.websiteEndpoints);
let bucketName;
for (let i = 0; i < validHosts.length; ++i) {
if (host === validHosts[i]) {
// It's path-style
return undefined;
} else if (host.endsWith(`.${validHosts[i]}`)) {
const potentialBucketName = host.split(`.${validHosts[i]}`)[0];
if (!bucketName) {
bucketName = potentialBucketName;
} else {
// bucketName should be shortest so that takes into account
// most specific potential hostname
bucketName = potentialBucketName.length < bucketName.length ?
potentialBucketName : bucketName;
}
}
}
if (bucketName) {
return bucketName;
}
throw new Error(`bad request: hostname ${host} is not in valid endpoints`);
};
/**
* Get all valid endpoints, according to our configuration
*
* @returns {string[]} - list of valid endpoints
*/
function getAllEndpoints() {
return Object.keys(config.restEndpoints);
};
/**
* Get bucket name and object name from the request
* @param {object} request - http request object
* @param {string} pathname - http request path parsed from request url
* @returns {object} result - returns object containing bucket
* name and objectKey as key
*/
function getResourceNames(request, pathname) {
return this.getNamesFromReq(request, pathname,
utils.getBucketNameFromHost(request));
};
/**
* Get bucket name and/or object name from the path of a request
* @param {object} request - http request object
* @param {string} pathname - http request path parsed from request url
* @param {string} bucketNameFromHost - name of bucket obtained from host name
* @returns {object} resources - returns object with bucket and object as keys
*/
function getNamesFromReq(request, pathname,
bucketNameFromHost) {
const resources = {
bucket: undefined,
object: undefined,
host: undefined,
gotBucketNameFromHost: undefined,
path: undefined,
};
// If there are spaces in a key name, s3cmd sends them as "+"s.
// Actual "+"s are uri encoded as "%2B" so by switching "+"s to
// spaces here, you still retain any "+"s in the final decoded path
const pathWithSpacesInsteadOfPluses = pathname.replace(/\+/g, ' ');
const path = decodeURIComponent(pathWithSpacesInsteadOfPluses);
resources.path = path;
let fullHost;
if (request.headers && request.headers.host) {
const reqHost = request.headers.host;
const bracketIndex = reqHost.indexOf(']');
const colonIndex = reqHost.lastIndexOf(':');
const hostLength = colonIndex > bracketIndex ?
colonIndex : reqHost.length;
fullHost = reqHost.slice(0, hostLength);
} else {
fullHost = undefined;
}
if (bucketNameFromHost) {
resources.bucket = bucketNameFromHost;
const bucketNameLength = bucketNameFromHost.length;
resources.host = fullHost.slice(bucketNameLength + 1);
// Slice off leading '/'
resources.object = path.slice(1);
resources.gotBucketNameFromHost = true;
} else {
resources.host = fullHost;
const urlArr = path.split('/');
if (urlArr.length > 1) {
resources.bucket = urlArr[1];
resources.object = urlArr.slice(2).join('/');
} else if (urlArr.length === 1) {
resources.bucket = urlArr[0];
}
}
// remove any empty strings or nulls
if (resources.bucket === '' || resources.bucket === null) {
resources.bucket = undefined;
}
if (resources.object === '' || resources.object === null) {
resources.object = undefined;
}
return resources;
};
function normalizeRequest(request) {
/* eslint-disable no-param-reassign */
const parsedUrl = url.parse(request.url, true);
request.query = parsedUrl.query;
// TODO: make the namespace come from a config variable.
request.namespace = 'default';
// Parse bucket and/or object names from request
const resources = this.getResourceNames(request, parsedUrl.pathname);
request.gotBucketNameFromHost = resources.gotBucketNameFromHost;
request.bucketName = resources.bucket;
request.objectKey = resources.object;
request.parsedHost = resources.host;
request.path = resources.path;
// For streaming v4 auth, the total body content length
// without the chunk metadata is sent as
// the x-amz-decoded-content-length
const contentLength = request.headers['x-amz-decoded-content-length'] ?
request.headers['x-amz-decoded-content-length'] :
request.headers['content-length'];
request.parsedContentLength =
Number.parseInt(contentLength, 10);
if (process.env.ALLOW_INVALID_META_HEADERS) {
const headersArr = Object.keys(request.headers);
const length = headersArr.length;
if (headersArr.indexOf('x-invalid-metadata') > 1) {
for (let i = 0; i < length; i++) {
const headerName = headersArr[i];
if (headerName.startsWith('x-amz-')) {
const translatedHeaderName =
headerName.replace(/\|\+2f/g, '/');
request.headers[translatedHeaderName] =
request.headers[headerName];
if (translatedHeaderName !== headerName) {
delete request.headers[headerName];
}
}
}
}
}
return request;
}
module.exports = normalizeRequest;

View File

@ -0,0 +1,136 @@
const ipCheck = require('ipCheck');
/**
* Get all valid endpoints, according to our configuration
*
* @returns {string[]} - list of valid endpoints
*/
function _getAllEndpoints() {
return Object.keys(config.restEndpoints);
};
/**
* Get bucket name from the request of a virtually hosted bucket
* @param {object} request - HTTP request object
* @return {string|undefined} - returns bucket name if dns-style query
* returns undefined if path-style query
* @throws {Error} in case the type of query could not be infered
*/
function getBucketNameFromHost(request) {
const headers = request.headers;
if (headers === undefined || headers.host === undefined) {
throw new Error('bad request: no host in headers');
}
const reqHost = headers.host;
const bracketIndex = reqHost.indexOf(']');
const colonIndex = reqHost.lastIndexOf(':');
const hostLength = colonIndex > bracketIndex ? colonIndex : reqHost.length;
// If request is made using IPv6 (indicated by presence of brackets),
// surrounding brackets should not be included in host var
const host = bracketIndex > -1 ?
reqHost.slice(1, hostLength - 1) : reqHost.slice(0, hostLength);
// parseIp returns empty object if host is not valid IP
// If host is an IP address, it's path-style
if (Object.keys(ipCheck.parseIp(host)).length !== 0) {
return undefined;
}
// All endpoints from all regions + `websiteEndpoints
const validHosts = _getAllEndpoints().concat(config.websiteEndpoints);
let bucketName;
for (let i = 0; i < validHosts.length; ++i) {
if (host === validHosts[i]) {
// It's path-style
return undefined;
} else if (host.endsWith(`.${validHosts[i]}`)) {
const potentialBucketName = host.split(`.${validHosts[i]}`)[0];
if (!bucketName) {
bucketName = potentialBucketName;
} else {
// bucketName should be shortest so that takes into account
// most specific potential hostname
bucketName = potentialBucketName.length < bucketName.length ?
potentialBucketName : bucketName;
}
}
}
if (bucketName) {
return bucketName;
}
throw new Error(`bad request: hostname ${host} is not in valid endpoints`);
}
/**
* Get bucket name and object name from the request
* @param {object} request - http request object
* @param {string} pathname - http request path parsed from request url
* @returns {object} result - returns object containing bucket
* name and objectKey as key
*/
function getResourceNames(request, pathname) {
return this.getNamesFromReq(request, pathname,
utils.getBucketNameFromHost(request));
};
/**
* Get bucket name and/or object name from the path of a request
* @param {object} request - http request object
* @param {string} pathname - http request path parsed from request url
* @param {string} bucketNameFromHost - name of bucket obtained from host name
* @returns {object} resources - returns object with bucket and object as keys
*/
function getNamesFromReq(request, pathname,
bucketNameFromHost) {
const resources = {
bucket: undefined,
object: undefined,
host: undefined,
gotBucketNameFromHost: undefined,
path: undefined,
};
// If there are spaces in a key name, s3cmd sends them as "+"s.
// Actual "+"s are uri encoded as "%2B" so by switching "+"s to
// spaces here, you still retain any "+"s in the final decoded path
const pathWithSpacesInsteadOfPluses = pathname.replace(/\+/g, ' ');
const path = decodeURIComponent(pathWithSpacesInsteadOfPluses);
resources.path = path;
let fullHost;
if (request.headers && request.headers.host) {
const reqHost = request.headers.host;
const bracketIndex = reqHost.indexOf(']');
const colonIndex = reqHost.lastIndexOf(':');
const hostLength = colonIndex > bracketIndex ?
colonIndex : reqHost.length;
fullHost = reqHost.slice(0, hostLength);
} else {
fullHost = undefined;
}
if (bucketNameFromHost) {
resources.bucket = bucketNameFromHost;
const bucketNameLength = bucketNameFromHost.length;
resources.host = fullHost.slice(bucketNameLength + 1);
// Slice off leading '/'
resources.object = path.slice(1);
resources.gotBucketNameFromHost = true;
} else {
resources.host = fullHost;
const urlArr = path.split('/');
if (urlArr.length > 1) {
resources.bucket = urlArr[1];
resources.object = urlArr.slice(2).join('/');
} else if (urlArr.length === 1) {
resources.bucket = urlArr[0];
}
}
// remove any empty strings or nulls
if (resources.bucket === '' || resources.bucket === null) {
resources.bucket = undefined;
}
if (resources.object === '' || resources.object === null) {
resources.object = undefined;
}
return resources;
};

0
models/ObjectModel.js Normal file
View File

0
models/RequestModel.js Normal file
View File

31
routes/Router.js Normal file
View File

@ -0,0 +1,31 @@
const { normalizeRequest } = require('./utils');
const BucketController = require('../controllers/BucketController');
class Router {
constructor(routes) {
this._routes = routes;
this._controllers = { BucketController };
}
_matchRoute(route, req) {
const query = route.query || [];
return req.method === route.method &&
Boolean(req.bucketName) === Boolean(route.bucket) &&
Boolean(req.objectKey) === Boolean(route.object) &&
query.every(q => q in req.query);
}
exec(request, response, cb) {
const _req = normalizeRequest(request);
const index = this._routes.findIndex(r => this._matchRoute(r, _req));
if (index === -1) {
// TODO: return NotImplemented / Invalid Request
}
const { controller, action } = this._routes[index];
this._controllers[controller][action](request, response, cb);
}
}
module.exports = Router;

90
routes/routes.json Normal file
View File

@ -0,0 +1,90 @@
[
{
"bucket": true,
"method": "GET",
"controller": "BucketController",
"action": "getService",
"auth": true
},
{
"bucket": true,
"method": "GET",
"controller": "BucketController",
"action": "getBucket",
"auth": true
},
{
"bucket": true,
"method": "PUT",
"controller": "BucketController",
"action": "putBucket",
"auth": false
},
{
"bucket": true,
"method": "PUT",
"controller": "BucketController",
"action": "putBucketAcl",
"query": ["acl"],
"auth": true
},
{
"bucket": true,
"method": "PUT",
"controller": "BucketController",
"action": "putBucketVersioning",
"query": ["versioning"],
"auth": true
},
{
"bucket": true,
"method": "PUT",
"controller": "BucketController",
"action": "putBucketWebsite",
"query": ["website"],
"auth": true
},
{
"bucket": true,
"method": "PUT",
"controller": "BucketController",
"action": "putBucketCors",
"query": ["cors"],
"auth": true
},
{
"bucket": true,
"object": true,
"method": "GET",
"controller": "ObjectController",
"action": "getObject",
"auth": true
},
{
"bucket": true,
"object": true,
"method": "PUT",
"controller": "ObjectController",
"action": "putObject",
"auth": true
},
{
"bucket": false,
"object": false,
"method": "GET",
"controller": "HealthcheckController",
"action": "basic",
"admin": true,
"auth": false
},
{
"bucket": false,
"object": false,
"method": "GET",
"controller": "HealthcheckController",
"query": ["deep"],
"action": "deep",
"admin": true,
"auth": false
}
];

View File

@ -111,8 +111,6 @@ function errorXMLResponse(errCode, response, log, corsHeaders) {
return response.end(xmlStr, 'utf8', () => { return response.end(xmlStr, 'utf8', () => {
log.end().info('responded with error XML', { log.end().info('responded with error XML', {
httpCode: response.statusCode, httpCode: response.statusCode,
xmlStr,
corsHeaders,
}); });
}); });
} }