fixup

lib/s3routes/routes.ts

@@ -1,6 +1,7 @@
 import assert from 'assert';
 
 import { RequestLogger } from 'werelogs';
+const opentelemetry = require('@opentelemetry/api');
 
 import errors from '../errors';
 import routeGET from './routes/routeGET';
@@ -13,8 +14,10 @@ import * as routesUtils from './routesUtils';
 import routeWebsite from './routes/routeWebsite';
 import * as http from 'http';
 import StatsClient from '../metrics/StatsClient';
+import { actionMonitoringMapS3 } from '../policyEvaluator/utils/actionMaps';
 
 import * as requestUtils from '../../lib/policyEvaluator/requestUtils';
+import { policies } from '../..';
 
 const routeMap = {
     GET: routeGET,
@@ -172,6 +175,8 @@ export type Params = {
  * data retrieval function
  * @param logger - werelogs logger instance
  * @param [s3config] - s3 configuration
+ * @param [parentSpanFromCloudserver] - server span from cloudserver with context
+ * @param [tracer] - opentelemetry tracer
  */
 export default function routes(
     req: http.IncomingMessage,
@@ -179,9 +184,20 @@ export default function routes(
     params: Params,
     logger: RequestLogger,
     s3config?: any,
+    parentSpanFromCloudserver?: any,
+    tracer?: any,
 ) {
+    parentSpanFromCloudserver.addEvent('Arsenal::routes() Validating and processing request');
+    const ctx = opentelemetry.trace.setSpan(
+        opentelemetry.context.active(),
+        parentSpanFromCloudserver,
+    );
+    const spanOptions = { links: [{ context: parentSpanFromCloudserver.spanContext() }] };
+    return tracer.startActiveSpan('Using Arsenal to validate request', spanOptions, ctx, requestValidatorSpan => {
+        requestValidatorSpan.setAttribute('code.function', 'routes');
+        requestValidatorSpan.setAttribute('code.filepath', 'arsenal/lib/s3routes/routes.ts');
+        requestValidatorSpan.setAttribute('code.lineno', 192);
         checkTypes(req, res, params, logger);
-
         const {
             api,
             internalHandlers,
@@ -259,6 +275,31 @@ export default function routes(
             // @ts-ignore
             bodyLength: parseInt(req.headers['content-length'], 10) || 0,
         });
+        // @ts-ignore
+        parentSpanFromCloudserver.setAttribute('aws.s3.bucket', req.bucketName);
+        // @ts-ignore
+        parentSpanFromCloudserver.setAttribute('aws.s3.key', req.objectKey);
+        // parentSpanFromCloudserver.setAttribute('aws.s3.request_id', reqUids);
+        // @ts-ignore
+        requestValidatorSpan.setAttribute('aws.s3.bucket', req.bucketName);
+        // @ts-ignore
+        requestValidatorSpan.setAttribute("aws.s3.key", req.objectKey);
+        // requestValidatorSpan.setAttribute('aws.s3.request_id', reqUids);
+        // TODO: Use this due to high cardinality
+        // parentSpanFromCloudserver.updateName(`${req.method}`);
+        // @ts-ignore
+        // if(req.objectKey){
+        //     // @ts-ignore
+        //     parentSpanFromCloudserver.updateName(`${req.method} ${req.bucketName}/${req.objectKey}`);
+        //     // @ts-ignore
+        // } else if (req.bucketName){
+        //     // @ts-ignore
+        //     parentSpanFromCloudserver.updateName(`${req.method} ${req.bucketName}`);
+        // } else {
+        //     // @ts-ignore
+        //     parentSpanFromCloudserver.updateName(`${req.method}`);
+        // }
+        // span.setAttribute('aws.s3.upload_id', req.query.uploadId);
 
         // @ts-ignore
         const { error, method } = checkUnsupportedRoutes(req.method, req.query);
@@ -279,12 +320,13 @@ export default function routes(
                 { error: bucketOrKeyError });
             return routesUtils.responseXMLBody(bucketOrKeyError, null, res, log);
         }
-
+        requestValidatorSpan.addEvent('Arsenal::routes() Request Validated Successfully');
+        requestValidatorSpan.end();
         // bucket website request
         // @ts-ignore
         if (websiteEndpoints && websiteEndpoints.indexOf(req.parsedHost) > -1) {
             return routeWebsite(req, res, api, log, statsClient, dataRetrievalParams);
         }
-
-    return method(req, res, api, log, statsClient, dataRetrievalParams);
+        return method(req, res, api, log, statsClient, dataRetrievalParams, tracer, parentSpanFromCloudserver);
+    });
 }

lib/s3routes/routes/routeDELETE.ts

@@ -11,6 +11,9 @@ export default function routeDELETE(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
+    tracer?: any,
+    parentSpanFromCloudserver?: any,
 ) {
     const call = (name: string) => {
         return api.callApiMethod(name, request, response, log, (err, corsHeaders) => {
@@ -20,7 +23,7 @@ export default function routeDELETE(
     }
     log.debug('routing request', { method: 'routeDELETE' });
 
-    const { query, objectKey } = request as any
+    const { query, objectKey, bucketName } = request as any
     if (query?.uploadId) {
         if (objectKey === undefined) {
             const message = 'A key must be specified';
@@ -49,8 +52,14 @@ export default function routeDELETE(
         if (query?.tagging !== undefined) {
             return call('objectDeleteTagging');
         }
+        parentSpanFromCloudserver.addEvent('Detected Object Delete API request');
+        parentSpanFromCloudserver.updateName(`DeleteObject API with bucket: ${bucketName}`);
+        parentSpanFromCloudserver.setAttribute('aws.request_id', log.getUids()[0]);
+        parentSpanFromCloudserver.setAttribute('rpc.method', 'PutObject');
         api.callApiMethod('objectDelete', request, response, log,
             (err, corsHeaders) => {
+                parentSpanFromCloudserver.addEvent('DeleteObject API request completed');
+                parentSpanFromCloudserver.end();
                 /*
               * Since AWS expects a 204 regardless of the existence of
               the object, the errors NoSuchKey and NoSuchVersion should not
@@ -63,6 +72,6 @@ export default function routeDELETE(
                 routesUtils.statsReport500(err, statsClient);
                 return routesUtils.responseNoBody(null, corsHeaders, response,
                     204, log);
-            });
+            }, tracer);
     }
 }

lib/s3routes/routes/routeGET.ts

@@ -4,6 +4,7 @@ import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';
 import StatsClient from '../../metrics/StatsClient';
+import { actionMonitoringMapS3 } from '../../policyEvaluator/utils/actionMaps';
 
 export default function routerGET(
     request: http.IncomingMessage,
@@ -12,16 +13,23 @@ export default function routerGET(
     log: RequestLogger,
     statsClient?: StatsClient,
     dataRetrievalParams?: any,
+    tracer?: any,
+    parentSpanFromCloudserver?: any,
 ) {
     log.debug('routing request', { method: 'routerGET' });
 
     const { bucketName, objectKey, query } = request as any
 
     const call = (name: string) => {
+        const action = actionMonitoringMapS3[name];
+        // @ts-ignore
+        parentSpanFromCloudserver.updateName(`${action} API${request.bucketName ? ` with bucket: ${request.bucketName}` : ''}`);
+        parentSpanFromCloudserver.setAttribute('aws.request_id', log.getUids()[0]);
+        parentSpanFromCloudserver.setAttribute('rpc.method', action);
         api.callApiMethod(name, request, response, log, (err, xml, corsHeaders) => {
             routesUtils.statsReport500(err, statsClient);
             return routesUtils.responseXMLBody(err, xml, response, log, corsHeaders);
-        });
+        }, tracer);
     }
 
     if (bucketName === undefined && objectKey !== undefined) {
@@ -77,20 +85,24 @@ export default function routerGET(
             call('objectGetRetention');
         } else {
             // GET object
+            parentSpanFromCloudserver.updateName(`GetObject API with bucket: ${bucketName}`);
+            parentSpanFromCloudserver.setAttribute('aws.request_id', log.getUids()[0]);
+            parentSpanFromCloudserver.setAttribute('rpc.method', 'GetObject');
             api.callApiMethod('objectGet', request, response, log,
-                (err, dataGetInfo, resMetaHeaders, range) => {
+                (err, dataGetInfo, resMetaHeaders, range, apiSpan, callAPIMethodSpan) => {
                     let contentLength = 0;
                     if (resMetaHeaders && resMetaHeaders['Content-Length']) {
                         contentLength = resMetaHeaders['Content-Length'];
                     }
                     // TODO ARSN-216 Fix logger
+                    apiSpan.addEvent('Located Data')
                     // @ts-ignore
                     log.end().addDefaultFields({ contentLength });
                     routesUtils.statsReport500(err, statsClient);
                     return routesUtils.responseStreamData(err, query,
                         resMetaHeaders, dataGetInfo, dataRetrievalParams, response,
-                        range, log);
-                });
+                        range, log, apiSpan, callAPIMethodSpan, tracer);
+                }, tracer);
         }
     }
 }

lib/s3routes/routes/routeHEAD.ts

@@ -11,6 +11,9 @@ export default function routeHEAD(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
+    tracer?: any,
+    parentSpanFromCloudserver?: any,
 ) {
     log.debug('routing request', { method: 'routeHEAD' });
     const { bucketName, objectKey } = request as any
@@ -20,19 +23,25 @@ export default function routeHEAD(
             null, response, log);
     } else if (objectKey === undefined) {
         // HEAD bucket
+        parentSpanFromCloudserver.updateName(`HeadBucket API with bucket: ${bucketName}`);
+        parentSpanFromCloudserver.setAttribute('aws.request_id', log.getUids()[0]);
+        parentSpanFromCloudserver.setAttribute('rpc.method', 'HeadBucket');
         api.callApiMethod('bucketHead', request, response, log,
             (err, corsHeaders) => {
                 routesUtils.statsReport500(err, statsClient);
                 return routesUtils.responseNoBody(err, corsHeaders, response,
                     200, log);
-            });
+            }, tracer);
     } else {
         // HEAD object
+        parentSpanFromCloudserver.updateName(`HeadObject API with bucket: ${bucketName}`);
+        parentSpanFromCloudserver.setAttribute('aws.request_id', log.getUids()[0]);
+        parentSpanFromCloudserver.setAttribute('rpc.method', 'HeadObject');
         api.callApiMethod('objectHead', request, response, log,
             (err, resHeaders) => {
                 routesUtils.statsReport500(err, statsClient);
                 return routesUtils.responseContentHeaders(err, {}, resHeaders,
                     response, log);
-            });
+            }, tracer);
     }
 }

lib/s3routes/routes/routeOPTIONS.ts

@@ -11,6 +11,7 @@ export default function routeOPTIONS(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    tracer?: any,
 ) {
     log.debug('routing request', { method: 'routeOPTION' });
 

lib/s3routes/routes/routePOST.ts

@@ -10,6 +10,7 @@ export default function routePOST(
     response: http.ServerResponse,
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
+    tracer?: any,
 ) {
     log.debug('routing request', { method: 'routePOST' });
 

lib/s3routes/routes/routePUT.ts

@@ -11,8 +11,12 @@ export default function routePUT(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
+    tracer?: any,
+    parentSpanFromCloudserver?: any,
 ) {
     log.debug('routing request', { method: 'routePUT' });
+    parentSpanFromCloudserver.addEvent('checking which API to call');
 
     const { objectKey, query, bucketName, parsedContentLength } = request as any
 
@@ -221,12 +225,18 @@ export default function routePUT(
             // TODO ARSN-216 What's happening?
             // @ts-ignore
             log.end().addDefaultFields({ contentLength: request.parsedContentLength });
+            parentSpanFromCloudserver.addEvent('Detected Object Put API request');
+            parentSpanFromCloudserver.updateName(`PutObject API with bucket: ${bucketName}`);
+            parentSpanFromCloudserver.setAttribute('aws.request_id', log.getUids()[0]);
+            parentSpanFromCloudserver.setAttribute('rpc.method', 'PutObject');
             api.callApiMethod('objectPut', request, response, log,
                 (err, resHeaders) => {
                     routesUtils.statsReport500(err, statsClient);
+                    parentSpanFromCloudserver.addEvent('PutObject API request completed');
+                    parentSpanFromCloudserver.end();
                     return routesUtils.responseNoBody(err, resHeaders,
                         response, 200, log);
-                });
+                }, tracer);
         }
     }
     return undefined;

lib/s3routes/routesUtils.ts

@@ -1,6 +1,7 @@
 import * as url from 'url';
 import * as http from 'http';
 import { eachSeries } from 'async';
+const opentelemetry = require('@opentelemetry/api');
 
 import { RequestLogger } from 'werelogs';
 
@@ -16,6 +17,7 @@ export type CallApiMethod = (
     response: http.ServerResponse,
     log: RequestLogger,
     callback: (err: ArsenalError | null, ...data: any[]) => void,
+    tracer?: any,
 ) => void;
 
 /**
@@ -350,6 +352,9 @@ function retrieveData(
     retrieveDataParams: any,
     response: http.ServerResponse,
     log: RequestLogger,
+    apiSpan?: any,
+    callApiMethodSpan?: any,
+    tracer?: any,
 ) {
     if (locations.length === 0) {
         return response.end();
@@ -368,6 +373,11 @@ function retrieveData(
     // the S3-client might close the connection while we are processing it
     response.once('close', () => {
         responseDestroyed = true;
+        if (apiSpan) {
+            apiSpan.addEvent('response closed by client request');
+            apiSpan.end();
+            callApiMethodSpan.end();
+        }
         if (currentStream) {
             currentStream.destroy();
         }
@@ -382,8 +392,18 @@ function retrieveData(
         locStorageCheckFn,
         vault,
     } = retrieveDataParams;
+    // need special context for HTTPs requests going externally from cloudserver
+    const ctx = opentelemetry.trace.setSpan(
+        opentelemetry.context.active(),
+        apiSpan,
+    );
     const data = new DataWrapper(
         client, implName, config, kms, metadata, locStorageCheckFn, vault);
+    const spanOptions = { links: [{ context: apiSpan.spanContext() }] };
+    return tracer.startActiveSpan('Getting Data using sproxyd', spanOptions, ctx, dataSpan => {
+        dataSpan.setAttribute('code.function', 'retrieveData');
+        dataSpan.setAttribute('code.filepath', 'lib/s3routes/routesUtils.js');
+        dataSpan.setAttribute('code.lineno', 400);
         return eachSeries(locations,
             (current, next) => data.get(current, response, log,
                 (err: any, readable: http.IncomingMessage) => {
@@ -413,15 +433,28 @@ function retrieveData(
                     // readable stream successfully consumed
                     readable.on('end', () => {
                         currentStream = null;
+                        dataSpan.end();
+                        if (apiSpan) {
+                            apiSpan.addEvent('Readable stream successfully consumed');
+                            apiSpan.end();
+                        }
                         log.debug('readable stream end reached');
                         return next();
                     });
                     // errors on server side with readable stream
                     readable.on('error', err => {
+                        dataSpan.end();
+                        if (apiSpan) {
+                            apiSpan.addEvent('Unable to stream object from Sproxyd');
+                            apiSpan.end();
+                        }
                         log.error('error piping data from source');
                         _destroyResponse();
+                        return process.nextTick(() => {
+                            callApiMethodSpan.end();
                             return next(err);
                         });
+                    });
                     currentStream = readable;
                     return readable.pipe(response, { end: false });
                 }), err => {
@@ -436,6 +469,7 @@ function retrieveData(
                 response.end();
             },
         );
+    })
 }
 
 function _responseBody(
@@ -605,7 +639,11 @@ export function responseStreamData(
     response: http.ServerResponse,
     range: [number, number] | undefined,
     log: RequestLogger,
+    apiSpan?: any,
+    callApiMethodSpan?: any,
+    tracer?: any,
 ) {
+    apiSpan.addEvent('Getting Data from Sproxyd');
     if (errCode && !response.headersSent) {
         return XMLResponseBackend.errorResponse(errCode, response, log,
             resHeaders);
@@ -640,12 +678,14 @@ export function responseStreamData(
     }
     response.on('finish', () => {
         // TODO ARSN-216 Fix logger
+        callApiMethodSpan.addEvent('Sending response to Client');
+        callApiMethodSpan.end();
         // @ts-expect-error
         log.end().info('responded with streamed content', {
             httpCode: response.statusCode,
         });
     });
-    return retrieveData(dataLocations, retrieveDataParams, response, log);
+    return retrieveData(dataLocations, retrieveDataParams, response, log, apiSpan, callApiMethodSpan, tracer);
 }
 
 /**

package.json

@@ -18,6 +18,7 @@
   "homepage": "https://github.com/scality/Arsenal#readme",
   "dependencies": {
     "@js-sdsl/ordered-set": "^4.4.2",
+    "@opentelemetry/api": "^1.9.0",
     "@types/async": "^3.2.12",
     "@types/utf8": "^3.0.1",
     "JSONStream": "^1.0.0",

yarn.lock

@@ -1221,6 +1221,11 @@
     mkdirp "^1.0.4"
     rimraf "^3.0.2"
 
+"@opentelemetry/api@^1.9.0":
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe"
+  integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==
+
 "@sideway/address@^4.1.3":
   version "4.1.4"
   resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0"