Merge branch 'feature/S3C-3231_add_api_version_header' into tmp/octopus/w/8.1/feature/S3C-3231_add_api_version_header

libV2/constants.js

@@ -90,6 +90,8 @@ const constants = {
     checkpointLagSecs: 300,
     snapshotLagSecs: 900,
     repairLagSecs: 5,
+    legacyApiVersion: '2016-08-15',
+    currentApiVersion: '2020-09-01',
 };
 
 constants.operationToResponse = constants.operations

libV2/errors/errors.json

@@ -7,6 +7,10 @@
         "code": 500,
         "description": "The server encountered an internal error. Please retry the request."
     },
+    "InvalidQueryParameter" : {
+        "code": 400,
+        "description": "The query string is malformed."
+    },
     "InvalidUri": {
         "code": 400,
         "description": "The requested URI does not represent any resource on the server."

libV2/server/API/metrics.js

@@ -1,8 +1,5 @@
 const ApiController = require('../controller');
-const { authenticateV4 } = require('../../vault');
 
-const controller = new ApiController('metrics', [
-    authenticateV4,
-]);
+const controller = new ApiController('metrics');
 
 module.exports = controller.buildMap();

libV2/server/index.js

@@ -8,6 +8,7 @@ const { initializeOasTools, middleware } = require('./middleware');
 const { spec: apiSpec } = require('./spec');
 const { client: cacheClient } = require('../cache');
 const { LoggerContext } = require('../utils');
+const LegacyServer = require('./legacy');
 
 const moduleLogger = new LoggerContext({
     module: 'server',
@@ -24,6 +25,7 @@ class UtapiServer extends Process {
         const app = express();
         app.use(bodyParser.json({ strict: false }));
         app.use(middleware.loggerMiddleware);
+        app.use(middleware.apiVersionMiddleware);
         await initializeOasTools(spec, app);
         app.use(middleware.errorMiddleware);
         app.use(middleware.responseLoggerMiddleware);
@@ -47,6 +49,7 @@ class UtapiServer extends Process {
     async _setup() {
         this._app = await UtapiServer._createApp(apiSpec);
         this._server = await UtapiServer._createServer(this._app);
+        LegacyServer.setup();
     }
 
     async _start() {

libV2/server/legacy.js

@@ -0,0 +1,114 @@
+const url = require('url');
+
+const config = require('../config');
+const errors = require('../errors');
+const routes = require('../../router/routes');
+const Route = require('../../router/Route');
+const Router = require('../../router/Router');
+const redisClient = require('../../utils/redisClient');
+const UtapiRequest = require('../../lib/UtapiRequest');
+const Datastore = require('../../lib/Datastore');
+
+const { LoggerContext } = require('../utils');
+
+const moduleLogger = new LoggerContext({
+    module: 'server.legacy',
+});
+
+
+/**
+ * Function to validate a URI component
+ *
+ * @param {string|object} component - path from url.parse of request.url
+ * (pathname plus query) or query from request
+ * @return {string|undefined} If `decodeURIComponent` throws an error,
+ * return the invalid `decodeURIComponent` string, otherwise return
+ * `undefined`
+ */
+function _checkURIComponent(component) {
+    if (typeof component === 'string') {
+        try {
+            decodeURIComponent(component);
+        } catch (err) {
+            return true;
+        }
+    } else {
+        return Object.keys(component).find(x => {
+            try {
+                decodeURIComponent(x);
+                decodeURIComponent(component[x]);
+            } catch (err) {
+                return true;
+            }
+            return false;
+        });
+    }
+    return undefined;
+}
+
+class LegacyServer {
+    constructor() {
+        this.router = null;
+        this.datastore = null;
+    }
+
+    setup() {
+        this.router = new Router(config);
+        routes.forEach(item => this.router.addRoute(new Route(item)));
+
+        const logger = moduleLogger.with({ component: 'redis' });
+        this.datastore = new Datastore().setClient(redisClient(config.redis, logger));
+    }
+
+    handleRequest(req, res, next) {
+        const { query, path, pathname } = url.parse(req.url, true);
+
+        // Sanity check for valid URI component
+        if (_checkURIComponent(query) || _checkURIComponent(path)) {
+            return next(errors.InvalidURI);
+        }
+
+        const utapiRequest = new UtapiRequest()
+            .setRequest(req)
+            .setLog(req.logger)
+            .setResponse(res)
+            .setDatastore(this.datastore)
+            .setRequestQuery(query)
+            .setRequestPath(path)
+            .setRequestPathname(pathname);
+
+        return this.router.doRoute(utapiRequest, (err, data) => {
+            if (err) {
+                // eslint-disable-next-line no-param-reassign
+                err.utapiError = true; // Make sure this error is returned as-is
+                next(err);
+                return;
+            }
+
+            const log = utapiRequest.getLog();
+            const res = utapiRequest.getResponse();
+            req.logger.trace('writing HTTP response', {
+                method: 'UtapiServer.response',
+            });
+            const code = utapiRequest.getStatusCode();
+            /*
+            * Encoding data to binary provides a hot path to write data
+            * directly to the socket, without node.js trying to encode the data
+            * over and over again.
+            */
+            const payload = Buffer.from(JSON.stringify(data), 'utf8');
+            res.writeHead(code, {
+                'server': 'ScalityS3',
+                'x-scal-request-id': log.getSerializedUids(),
+                'content-type': 'application/json',
+                'content-length': payload.length,
+            });
+            res.write(payload);
+            res.end();
+            next();
+        });
+    }
+}
+
+
+module.exports = new LegacyServer();

libV2/server/middleware.js

@@ -1,16 +1,24 @@
-const oasTools = require('oas-tools');
 const path = require('path');
 const { promisify } = require('util');
+const Joi = require('@hapi/joi');
+const oasTools = require('oas-tools');
+const werelogs = require('werelogs');
 const config = require('../config');
-const { logger, buildRequestLogger } = require('../utils');
+const { legacyApiVersion, currentApiVersion } = require('../constants');
 const errors = require('../errors');
+const { buildRequestLogger } = require('../utils');
 const { authenticateRequest } = require('../vault');
+const LegacyServer = require('./legacy');
+
+const oasLogger = new werelogs.Werelogs({
+    level: config.logging.level === 'trace' ? 'debug' : 'info', // oasTools is very verbose
+    dump: config.logging.dumpLevel,
+});
 
 const oasOptions = {
     controllers: path.join(__dirname, './API/'),
     checkControllers: true,
-    loglevel: config.logging.level === 'trace' ? 'debug' : 'info', // oasTools is very verbose
-    customLogger: logger,
+    customLogger: new oasLogger.Logger('Utapi'),
     customErrorHandling: true,
     strict: true,
     router: true,
@@ -38,6 +46,18 @@ function loggerMiddleware(req, res, next) {
     return next();
 }
 
+
+function responseLoggerMiddleware(req, res, next) {
+    const info = {
+        httpCode: res.statusCode,
+        httpMessage: res.statusMessage,
+    };
+    req.logger.end('finished handling request', info);
+    if (next) {
+        next();
+    }
+}
+
 // next is purposely not called as all error responses are handled here
 // eslint-disable-next-line no-unused-vars
 function errorMiddleware(err, req, res, next) {
@@ -64,15 +84,49 @@ function errorMiddleware(err, req, res, next) {
             message,
         },
     });
+    responseLoggerMiddleware(req, {
+        statusCode: code,
+        statusMessage: message,
+    });
 }
+const _versionFormat = Joi.string().pattern(/^\d{4}-\d{2}-\d{2}$/);
+function apiVersionMiddleware(request, response, next) {
+    const apiVersion = request.query.Version;
+    if (!apiVersion) {
+        request.logger.debug('no api version specified, assuming latest');
+        next();
+        return;
+    }
 
-function responseLoggerMiddleware(req, res, next) {
-    const info = {
-        httpCode: res.statusCode,
-        httpMessage: res.statusMessage,
-    };
-    req.logger.end('finished handling request', info);
-    return next();
+    try {
+        Joi.assert(apiVersion, _versionFormat);
+    } catch (err) {
+        request.logger.error('malformed Version parameter', { apiVersion });
+        next(errors.InvalidQueryParameter
+            .customizeDescription('The Version query parameter is malformed.'));
+        return;
+    }
+
+    if (apiVersion === legacyApiVersion) {
+        request.logger.debug('legacy api version specified routing to v1');
+        LegacyServer.handleRequest(request, response, err => {
+            if (err) {
+                return next(err);
+            }
+            responseLoggerMiddleware(request, response);
+            // next is purposefully not called as LegacyServer handles its own response
+        });
+        return;
+    }
+
+    if (apiVersion === currentApiVersion) {
+        request.logger.debug('latest api version specified routing to v2');
+        next();
+        return;
+    }
+
+    next(errors.InvalidQueryParameter
+        .customizeDescription('Invalid value for Version'));
 }
 
 
@@ -116,5 +170,6 @@ module.exports = {
         errorMiddleware,
         responseLoggerMiddleware,
         authV4Middleware,
+        apiVersionMiddleware,
     },
 };

libV2/server/spec.js

@@ -54,6 +54,7 @@ function _getApiOperationMiddleware(routes) {
                     if (op['x-authv4'] === true) {
                         middleware.authv4 = true;
                     }
+
                     optIds[tag][op.operationId] = middleware;
 
                     moduleLogger

openapi.yaml

@@ -98,6 +98,12 @@ components:
       schema:
         type: string
         enum: [ ListMetrics ]
+    version:
+      in: query
+      name: Version
+      required: True
+      schema:
+        type: string
 paths:
   /_/healthcheck:
     get:

router/Router.js

@@ -47,8 +47,17 @@ class Router {
         const reqData = {
             resource,
         };
+
         // assign query params
         Object.assign(reqData, query);
+
+        // This is a redirected request from v2
+        // Reuse the previously parsed body
+        if (req.body) {
+            cb(null, Object.assign(reqData, req.body));
+            return;
+        }
+
         req.on('data', data => body.push(data))
             .on('error', cb)
             .on('end', () => {
@@ -118,7 +127,7 @@ class Router {
         const validator = route.getValidator()(data);
         const validationResult = validator.validate();
         if (!validationResult) {
-            log.trace('input parameters are not well formated or missing', {
+            log.trace('input parameters are not well formatted or missing', {
                 method: 'Router._validateRoute',
             });
             return cb(validator.getValidationError());
@@ -222,8 +231,12 @@ class Router {
         );
         auth.setHandler(this._vault);
         const request = utapiRequest.getRequest();
-        request.path = utapiRequest.getRequestPathname();
-        request.query = utapiRequest.getRequestQuery();
+        // These properties are already defined on a v2 request
+        // And are unable to be overwritten
+        if (!process.env.ENABLE_UTAPI_V2) {
+            request.path = utapiRequest.getRequestPathname();
+            request.query = utapiRequest.getRequestQuery();
+        }
         return auth.server.doAuth(request, log, (err, authResults) => {
             if (err) {
                 return cb(err);

tests/unit/v2/server/testMiddleware.js

@@ -45,7 +45,8 @@ describe('Test middleware', () => {
         });
 
         it('should set a default code and message', () => {
-            middleware.errorMiddleware({}, null, resp);
+            const req = templateRequest();
+            middleware.errorMiddleware({}, req, resp);
             assert.strictEqual(resp._status, 500);
             assert.deepStrictEqual(resp._body, {
                 error: {
@@ -56,7 +57,8 @@ describe('Test middleware', () => {
         });
 
         it('should set the correct info from an error', () => {
-            middleware.errorMiddleware({ code: 123, message: 'Hello World!', utapiError: true }, null, resp);
+            const req = templateRequest();
+            middleware.errorMiddleware({ code: 123, message: 'Hello World!', utapiError: true }, req, resp);
             assert.deepStrictEqual(resp._body, {
                 error: {
                     code: '123',
@@ -66,7 +68,8 @@ describe('Test middleware', () => {
         });
 
         it("should replace an error's message if it's internal and not in development mode", () => {
-            middleware.errorMiddleware({ code: 123, message: 'Hello World!' }, null, resp);
+            const req = templateRequest();
+            middleware.errorMiddleware({ code: 123, message: 'Hello World!' }, req, resp);
             assert.deepStrictEqual(resp._body, {
                 error: {
                     code: '123',