updated data message

lib/auth/auth.ts

@@ -58,13 +58,27 @@ function extractParams(
     request: any,
     log: Logger,
     awsService: string,
-    data: { [key: string]: string }
+    data: { [key: string]: string },
+    oTel?: any,
 ) {
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Arsenal:: entered Arsenal.auth.server.extractParams');
+    return tracer.startActiveSpan('Check validity of request parameters to authenticate using Arsenal', undefined, activeTracerContext, extractParamsSpan => {
+        extractParamsSpan.setAttributes({
+            'code.lineno': 75,
+            'code.filename': 'lib/auth/auth.ts',
+            'code.function': 'extractParams',
+            'code.url': 'https://github.com/scality/arsenal/blob/892dee6c1333fcc25c88333ee991f02830cb3c51/lib/auth/auth.ts',
+        });
         log.trace('entered', { method: 'Arsenal.auth.server.extractParams' });
         const authHeader = request.headers.authorization;
         let version: 'v2' |'v4' | null = null;
         let method: 'query' | 'headers' | null = null;
-
+        activeSpan?.addEvent('Arsenal:: Identifying auth version from authentication header');
         // Identify auth version and method to dispatch to the right check function
         if (authHeader) {
             method = 'headers';
@@ -77,6 +91,7 @@ function extractParams(
             } else {
                 log.trace('invalid authorization security header',
                           { header: authHeader });
+                extractParamsSpan.end();
                 return { err: errors.AccessDenied };
             }
         } else if (data.Signature) {
@@ -86,21 +101,28 @@ function extractParams(
             method = 'query';
             version = 'v4';
         }
-
+        activeSpan?.addEvent(`Arsenal::Auth versions identified: ${version}`);
         // Here, either both values are set, or none is set
         if (version !== null && method !== null) {
             if (!checkFunctions[version] || !checkFunctions[version][method]) {
+                activeSpan?.recordException(errors.NotImplemented);
                 log.trace('invalid auth version or method',
                           { version, authMethod: method });
+                extractParamsSpan.end();
                 return { err: errors.NotImplemented };
             }
+            activeSpan?.addEvent(`Arsenal:: Identified auth method version: ${version} and method: ${method}`);
+            activeSpan?.addEvent('Arsenal:: Checking if valid request headers and query are used to make request to vault');
             log.trace('identified auth method', { version, authMethod: method });
-        return checkFunctions[version][method](request, log, data, awsService);
+            return checkFunctions[version][method](request, log, data, awsService, { activeSpan, extractParamsSpan, activeTracerContext, tracer });
         }
 
         // no auth info identified
         log.debug('assuming public user');
+        extractParamsSpan.end();
+        activeSpan?.addEvent(`Arsenal:: Identified as public user`);
         return { err: null, params: publicUserInfo };
+    });
 }
 
 /**
@@ -119,15 +141,30 @@ function doAuth(
     log: Logger,
     cb: (err: Error | null, data?: any) => void,
     awsService: string,
-    requestContexts: any[] | null
+    requestContexts: any[] | null,
+    oTel?: any,
 ) {
-    const res = extractParams(request, log, awsService, request.query);
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Arsenal:: Routing request using doAuth() in arsenal');
+    activeSpan?.addEvent('Arsenal:: Extracting auth parameters and check validity of request parameters to authenticate');
+    const start = process.hrtime.bigint();
+    const res = extractParams(request, log, awsService, request.query, oTel);
+    const end = process.hrtime.bigint();
+    const duration = Number(end - start) / 1e6;
+    activeSpan?.addEvent(`Arsenal:: It took ${duration.toFixed(3)} ms to extract auth parameters and to check validity of request parameters to authenticate`);
     if (res.err) {
+        activeSpan?.recordException(res.err);
         return cb(res.err);
     } else if (res.params instanceof AuthInfo) {
+        activeSpan?.addEvent('Arsenal:: Auth info already in the params, do not need to make a request to cloudserver');
         return cb(null, res.params);
     }
     if (requestContexts) {
+        activeSpan?.addEvent('Arsenal:: Setting auth info in requestContexts');
         requestContexts.forEach((requestContext) => {
             const { params } = res
             if ('data' in params) {
@@ -140,6 +177,7 @@ function doAuth(
                 }
             }
         });
+        activeSpan?.addEvent('Arsenal:: Auth info set in requestContexts');
     }
 
     // Corner cases managed, we're left with normal auth
@@ -147,10 +185,12 @@ function doAuth(
     // @ts-ignore
     res.params.log = log;
     if (res.params.version === 2) {
+        activeSpan?.addEvent('Arsenal:: Sending AuthV2 call to vault');
         // @ts-ignore
         return vault!.authenticateV2Request(res.params, requestContexts, cb);
     }
     if (res.params.version === 4) {
+        activeSpan?.addEvent('Arsenal:: Sending AuthV4 call to vault');
         // @ts-ignore
         return vault!.authenticateV4Request(res.params, requestContexts, cb);
     }
@@ -158,6 +198,7 @@ function doAuth(
     log.error('authentication method not found', {
         method: 'Arsenal.auth.doAuth',
     });
+    activeSpan?.recordException(errors.InternalError);
     return cb(errors.InternalError);
 }
 

lib/auth/v2/headerAuthCheck.ts

@@ -5,64 +5,83 @@ import constructStringToSign from './constructStringToSign';
 import checkRequestExpiry from './checkRequestExpiry';
 import algoCheck from './algoCheck';
 
-export function check(request: any, log: Logger, data: { [key: string]: string }) {
+export function check(request: any, log: Logger, data: { [key: string]: string }, oTel: any) {
+    const { activeSpan, extractParamsSpan, activeTracerContext, tracer } = oTel;
+    activeSpan?.addEvent('Entered V2 header auth check');
     log.trace('running header auth check');
+    activeSpan?.addEvent('Running header auth check');
     const headers = request.headers;
-
+    activeSpan?.addEvent('Extracting security token');
     const token = headers['x-amz-security-token'];
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
-
+    activeSpan?.addEvent('Extracted security token');
+    activeSpan?.addEvent('Checking timestamp');
     // Check to make sure timestamp is within 15 minutes of current time
-    let timestamp = headers['x-amz-date'] ?
+    let timestamp = headers['x-amz-date'] ? headers['x-amz-date'] : headers.date;
-        headers['x-amz-date'] : headers.date;
     timestamp = Date.parse(timestamp);
     if (!timestamp) {
-        log.debug('missing or invalid date header',
+        log.debug('missing or invalid date header', { method: 'auth/v2/headerAuthCheck.check' });
-        { method: 'auth/v2/headerAuthCheck.check' });
+        activeSpan.recordException(errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header'));
-        return { err: errors.AccessDenied.
+        extractParamsSpan.end();
-          customizeDescription('Authentication requires a valid Date or ' +
+        return { err: errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header') };
-          'x-amz-date header') };
     }
-
+    activeSpan?.addEvent('Checked timestamp');
+    activeSpan?.addEvent('Checking request expiry');
     const err = checkRequestExpiry(timestamp, log);
     if (err) {
+        activeSpan.recordException(err);
+        extractParamsSpan.end();
         return { err };
     }
-
+    activeSpan?.addEvent('Checked request expiry');
-    // Authorization Header should be
+    activeSpan?.addEvent('Extracting authorization header');
-    // in the format of 'AWS AccessKey:Signature'
+    // Authorization Header should be in the format of 'AWS AccessKey:Signature'
     const authInfo = headers.authorization;
-
+    activeSpan?.addEvent('Extracted authorization header');
     if (!authInfo) {
         log.debug('missing authorization security header');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     const semicolonIndex = authInfo.indexOf(':');
     if (semicolonIndex < 0) {
         log.debug('invalid authorization header', { authInfo });
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
-    const accessKey = semicolonIndex > 4 ?
+    const accessKey = semicolonIndex > 4 ? authInfo.substring(4, semicolonIndex).trim() : undefined;
-        authInfo.substring(4, semicolonIndex).trim() : undefined;
     if (typeof accessKey !== 'string' || accessKey.length === 0) {
         log.trace('invalid authorization header', { authInfo });
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     // @ts-ignore
     log.addDefaultFields({ accessKey });
-
     const signatureFromRequest = authInfo.substring(semicolonIndex + 1).trim();
     log.trace('signature from request', { signatureFromRequest });
+    activeSpan?.addEvent('Extracting signature from request');
+    activeSpan?.addEvent('Constructing string to sign');
     const stringToSign = constructStringToSign(request, data, log);
     log.trace('constructed string to sign', { stringToSign });
+    activeSpan?.addEvent('Constructed string to sign v2 headers');
     const algo = algoCheck(signatureFromRequest.length);
     log.trace('algo for calculating signature', { algo });
+    activeSpan?.addEvent('Checked algorithm for calculating signature');
     if (algo === undefined) {
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
+    activeSpan?.addEvent('Exiting V2 header auth check');
+    extractParamsSpan.end();
     return {
         err: null,
         params: {
@@ -80,3 +99,4 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
         },
     };
 }
+

lib/auth/v2/queryAuthCheck.ts

@@ -4,19 +4,26 @@ import * as constants from '../../constants';
 import algoCheck from './algoCheck';
 import constructStringToSign from './constructStringToSign';
 
-export function check(request: any, log: Logger, data: { [key: string]: string }) {
+export function check(request: any, log: Logger, data: { [key: string]: string }, oTel: any) {
+    const { activeSpan, extractParamsSpan, activeTracerContext, tracer } = oTel;
+    activeSpan?.addEvent('Entered query auth check');
     log.trace('running query auth check');
+    activeSpan?.addEvent('Running query auth check');
     if (request.method === 'POST') {
         log.debug('query string auth not supported for post requests');
+        activeSpan.recordException(errors.NotImplemented);
+        extractParamsSpan.end();
         return { err: errors.NotImplemented };
     }
-
     const token = data.SecurityToken;
+    activeSpan?.addEvent('Extracting security token');
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
-
+    activeSpan?.addEvent('Extracted security token');
     /*
     Check whether request has expired or if
     expires parameter is more than 604800000 milliseconds
@@ -25,47 +32,57 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     multiply by 1000 to obtain
     milliseconds to compare to Date.now()
     */
+    activeSpan?.addEvent('Checking expiration time');
     const expirationTime = parseInt(data.Expires, 10) * 1000;
     if (Number.isNaN(expirationTime)) {
-        log.debug('invalid expires parameter',
+        log.debug('invalid expires parameter', { expires: data.Expires });
-            { expires: data.Expires });
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
-
+    activeSpan?.addEvent('Checked expiration time');
     const currentTime = Date.now();
-
     const preSignedURLExpiry = process.env.PRE_SIGN_URL_EXPIRY
         && !Number.isNaN(process.env.PRE_SIGN_URL_EXPIRY)
         ? Number.parseInt(process.env.PRE_SIGN_URL_EXPIRY, 10)
         : constants.defaultPreSignedURLExpiry * 1000;
-
     if (expirationTime > currentTime + preSignedURLExpiry) {
-        log.debug('expires parameter too far in future',
+        log.debug('expires parameter too far in future', { expires: request.query.Expires });
-        { expires: request.query.Expires });
+        activeSpan.recordException(errors.AccessDenied);
+        extractParamsSpan.end();
         return { err: errors.AccessDenied };
     }
     if (currentTime > expirationTime) {
-        log.debug('current time exceeds expires time',
+        log.debug('current time exceeds expires time', { expires: request.query.Expires });
-        { expires: request.query.Expires });
+        activeSpan.recordException(errors.RequestTimeTooSkewed);
+        extractParamsSpan.end();
         return { err: errors.RequestTimeTooSkewed };
     }
     const accessKey = data.AWSAccessKeyId;
     // @ts-ignore
     log.addDefaultFields({ accessKey });
-
     const signatureFromRequest = decodeURIComponent(data.Signature);
     log.trace('signature from request', { signatureFromRequest });
+    activeSpan?.addEvent('Extracting signature from request');
     if (!accessKey || !signatureFromRequest) {
         log.debug('invalid access key/signature parameters');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     const stringToSign = constructStringToSign(request, data, log);
     log.trace('constructed string to sign', { stringToSign });
+    activeSpan?.addEvent('Constructed string to sign v2 query');
     const algo = algoCheck(signatureFromRequest.length);
     log.trace('algo for calculating signature', { algo });
+    activeSpan?.addEvent('Checked algorithm for calculating signature');
     if (algo === undefined) {
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
+    activeSpan?.addEvent('Exiting query auth check');
+        extractParamsSpan.end();
     return {
         err: null,
         params: {
@@ -82,3 +99,4 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
         },
     };
 }
+

lib/auth/v4/constructStringToSign.ts

@@ -17,7 +17,7 @@ export default function constructStringToSign(params: {
     log?: Logger;
     proxyPath?: string;
     awsService: string;
-}): string | Error {
+}, oTel?: any,): string | Error {
     const {
         request,
         signedHeaders,
@@ -29,7 +29,12 @@ export default function constructStringToSign(params: {
         proxyPath,
     } = params;
     const path = proxyPath || request.path;
-
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Constructing canonical request for Authv4');
     const canonicalReqResult = createCanonicalRequest({
         pHttpVerb: request.method,
         pResource: path,
@@ -38,8 +43,7 @@ export default function constructStringToSign(params: {
         pSignedHeaders: signedHeaders,
         payloadChecksum,
         service: params.awsService,
-    });
+    }, oTel);
-
     // TODO Why that line?
     // @ts-ignore
     if (canonicalReqResult instanceof Error) {
@@ -51,9 +55,13 @@ export default function constructStringToSign(params: {
     if (log) {
         log.debug('constructed canonicalRequest', { canonicalReqResult });
     }
+    const createSignatureSpan = tracer.startSpan('Creating signature hash for AuthV4 using crypto sha256');
+    activeSpan?.addEvent('Creating signature hash for AuthV4 using crypto sha256');
     const sha256 = crypto.createHash('sha256');
     const canonicalHex = sha256.update(canonicalReqResult, 'binary')
         .digest('hex');
+    activeSpan?.addEvent('Created signature hash for AuthV4 using crypto sha256');
+    createSignatureSpan.end();
     const stringToSign = `AWS4-HMAC-SHA256\n${timestamp}\n` +
     `${credentialScope}\n${canonicalHex}`;
     return stringToSign;

lib/auth/v4/createCanonicalRequest.ts

@@ -19,8 +19,16 @@ export default function createCanonicalRequest(
         pSignedHeaders: any;
         service: string;
         payloadChecksum: string;
-    }
+    },
+    oTel?: any,
 ) {
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Entered createCanonicalRequest');
+
     const pHttpVerb = params.pHttpVerb;
     const pResource = params.pResource;
     const pQuery = params.pQuery;
@@ -28,35 +36,34 @@ export default function createCanonicalRequest(
     const pSignedHeaders = params.pSignedHeaders;
     const service = params.service;
     let payloadChecksum = params.payloadChecksum;
+
+    const payloadChecksumSpan = tracer.startSpan('ComputePayloadChecksum');
     if (!payloadChecksum) {
         if (pHttpVerb === 'GET') {
             payloadChecksum = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b' +
                 '934ca495991b7852b855';
         } else if (pHttpVerb === 'POST') {
             let notEncodeStar = false;
-            // The java sdk does not encode the '*' parameter to compute the
-            // signature, if the user-agent is recognized, we need to keep
-            // the plain '*' as well.
             if (/aws-sdk-java\/[0-9.]+/.test(pHeaders['user-agent'])) {
                 notEncodeStar = true;
             }
             let payload = queryString.stringify(pQuery, undefined, undefined, {
-                encodeURIComponent: input => awsURIencode(input, true,
+                encodeURIComponent: input => awsURIencode(input, true, notEncodeStar),
-                    notEncodeStar),
             });
             payload = payload.replace(/%20/g, '+');
             payloadChecksum = crypto.createHash('sha256')
                 .update(payload, 'binary').digest('hex').toLowerCase();
         }
     }
+    payloadChecksumSpan.end();
 
+    const canonicalURISpan = tracer.startSpan('ComputeCanonicalURI');
     const canonicalURI = !!pResource ? awsURIencode(pResource, false) : '/';
+    canonicalURISpan.end();
 
-    // canonical query string
+    const canonicalQueryStrSpan = tracer.startSpan('ComputeCanonicalQueryStr');
     let canonicalQueryStr = '';
-    if (pQuery && !((service === 'iam' || service === 'ring' ||
+    if (pQuery && !((service === 'iam' || service === 'ring' || service === 'sts') && pHttpVerb === 'POST')) {
-        service === 'sts') &&
-        pHttpVerb === 'POST')) {
         const sortedQueryParams = Object.keys(pQuery).sort().map(key => {
             const encodedKey = awsURIencode(key);
             const value = pQuery[key] ? awsURIencode(pQuery[key]) : '';
@@ -64,32 +71,54 @@ export default function createCanonicalRequest(
         });
         canonicalQueryStr = sortedQueryParams.join('&');
     }
+    canonicalQueryStrSpan.end();
 
-    // signed headers
+    const signedHeadersSpan = tracer.startSpan('SortSignedHeadersAlphabetically');
+    activeSpan?.addEvent('Splitting signed headers using deliminator: ;');
     const signedHeadersList = pSignedHeaders.split(';');
+    activeSpan?.addEvent('Split signed headers using ; as deliminator');
+    activeSpan?.addEvent('Sorting signed headers alphabetically');
     signedHeadersList.sort((a: any, b: any) => a.localeCompare(b));
+    activeSpan?.addEvent('Sorted signed headers alphabetically');
+    activeSpan?.addEvent('Joining signed headers using deliminator: ;');
     const signedHeaders = signedHeadersList.join(';');
+    activeSpan?.addEvent('Joined signed headers using ; as deliminator');
+    activeSpan.setAttributes({
+        'signedHeaders.request': pSignedHeaders,
+        'signedHeaders.request.authv4': signedHeaders,
+    });
+    signedHeadersSpan.setAttributes({
+        'signedHeaders.request': pSignedHeaders,
+        'signedHeaders.request.authv4': signedHeaders,
+        'code.url': 'https://github.com/scality/arsenal/blob/c6bb489adeb7419fdbcdf01db2b46a593747530d/lib/auth/v4/createCanonicalRequest.ts#L76',
+        'code.function': 'createCanonicalRequest',
+        'code.lineno': 76,
+        'code.filename': 'lib/auth/v4/createCanonicalRequest.ts',
+    });
+    signedHeadersSpan.end();
 
-    // canonical headers
+    const canonicalHeadersListSpan = tracer.startSpan('FormatHeadersToMatch CanonicalHeadersList');
     const canonicalHeadersList = signedHeadersList.map((signedHeader: any) => {
         if (pHeaders[signedHeader] !== undefined) {
             const trimmedHeader = pHeaders[signedHeader]
                 .trim().replace(/\s+/g, ' ');
             return `${signedHeader}:${trimmedHeader}\n`;
         }
-        // nginx will strip the actual expect header so add value of
-        // header back here if it was included as a signed header
         if (signedHeader === 'expect') {
             return `${signedHeader}:100-continue\n`;
         }
-        // handle case where signed 'header' is actually query param
         return `${signedHeader}:${pQuery[signedHeader]}\n`;
     });
+    canonicalHeadersListSpan.end();
 
+    const canonicalHeadersSpan = tracer.startSpan('JoinAllCanonicalHeaders using no deliminator');
     const canonicalHeaders = canonicalHeadersList.join('');
+    canonicalHeadersSpan.end();
 
+    const canonicalRequestSpan = tracer.startSpan('ConstructCanonicalRequest');
     const canonicalRequest = `${pHttpVerb}\n${canonicalURI}\n` +
         `${canonicalQueryStr}\n${canonicalHeaders}\n` +
         `${signedHeaders}\n${payloadChecksum}`;
+    canonicalRequestSpan.end();
     return canonicalRequest;
 }

lib/auth/v4/headerAuthCheck.ts

@@ -21,70 +21,78 @@ import {
  *      POST request
  * @param awsService - Aws service ('iam' or 's3')
  */
-export function check(
+export function check(request: any, log: Logger, data: { [key: string]: string }, awsService: string, oTel: any) {
-    request: any,
+    const { activeSpan, extractParamsSpan, activeTracerContext, tracer } = oTel;
-    log: Logger,
+    activeSpan?.addEvent('Entered V4 header auth check');
-    data: { [key: string]: string },
-    awsService: string
-) {
     log.trace('running header auth check');
-
+    activeSpan?.addEvent('Extracting security token');
     const token = request.headers['x-amz-security-token'];
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
-
+    activeSpan?.addEvent('Extracted security token');
-    // authorization header
+    activeSpan?.addEvent('Extracting authorization header');
     const authHeader = request.headers.authorization;
     if (!authHeader) {
         log.debug('missing authorization header');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
-
+     activeSpan?.addEvent('Extracted authorization header');
+    activeSpan?.addEvent('Extracting auth header items');
     const authHeaderItems = extractAuthItems(authHeader, log);
     if (Object.keys(authHeaderItems).length < 3) {
         log.debug('invalid authorization header', { authHeader });
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
-
+    activeSpan?.addEvent('Extracted auth header items');
     const payloadChecksum = request.headers['x-amz-content-sha256'];
     if (!payloadChecksum && awsService !== 'iam') {
         log.debug('missing payload checksum');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     if (payloadChecksum === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
         log.trace('requesting streaming v4 auth');
         if (request.method !== 'PUT') {
-            log.debug('streaming v4 auth for put only',
+            log.debug('streaming v4 auth for put only', { method: 'auth/v4/headerAuthCheck.check' });
-                { method: 'auth/v4/headerAuthCheck.check' });
+            activeSpan.recordException(errors.InvalidArgument);
+            extractParamsSpan.end();
             return { err: errors.InvalidArgument };
         }
         if (!request.headers['x-amz-decoded-content-length']) {
+            activeSpan.recordException(errors.MissingSecurityHeader);
+            extractParamsSpan.end();
             return { err: errors.MissingSecurityHeader };
         }
     }
-
     log.trace('authorization header from request', { authHeader });
-
     const signatureFromRequest = authHeaderItems.signatureFromRequest!;
     const credentialsArr = authHeaderItems.credentialsArr!;
     const signedHeaders = authHeaderItems.signedHeaders!;
-
+    activeSpan.addEvent('Checking if signed headers are complete');
     if (!areSignedHeadersComplete(signedHeaders, request.headers)) {
         log.debug('signedHeaders are incomplete', { signedHeaders });
+        activeSpan.recordException(errors.AccessDenied);
+        extractParamsSpan.end();
         return { err: errors.AccessDenied };
     }
-
+    activeSpan.addEvent('Signed headers are complete');
     let timestamp: string | undefined;
     // check request timestamp
+    activeSpan.addEvent('Checking request timestamp');
     const xAmzDate = request.headers['x-amz-date'];
     if (xAmzDate) {
         const xAmzDateArr = xAmzDate.split('T');
         // check that x-amz- date has the correct format and after epochTime
-        if (xAmzDateArr.length === 2 && xAmzDateArr[0].length === 8
+        if (xAmzDateArr.length === 2 && xAmzDateArr[0].length === 8 && xAmzDateArr[1].length === 7 && Number.parseInt(xAmzDateArr[0], 10) > 19700101) {
-          && xAmzDateArr[1].length === 7
-          && Number.parseInt(xAmzDateArr[0], 10) > 19700101) {
             // format of x-amz- date is ISO 8601: YYYYMMDDTHHMMSSZ
             timestamp = request.headers['x-amz-date'];
         }
@@ -92,27 +100,27 @@ export function check(
         timestamp = convertUTCtoISO8601(request.headers.date);
     }
     if (!timestamp) {
-        log.debug('missing or invalid date header',
+        log.debug('missing or invalid date header', { method: 'auth/v4/headerAuthCheck.check' });
-          { method: 'auth/v4/headerAuthCheck.check' });
+        activeSpan.recordException(errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header'));
-        return { err: errors.AccessDenied.
+        extractParamsSpan.end();
-          customizeDescription('Authentication requires a valid Date or ' +
+        return { err: errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header') };
-          'x-amz-date header') };
     }
-
+    activeSpan.addEvent('Request timestamp is valid');
-    const validationResult = validateCredentials(credentialsArr, timestamp,
+    activeSpan.addEvent('Validating credentials');
-      log);
+    const validationResult = validateCredentials(credentialsArr, timestamp, log);
     if (validationResult instanceof Error) {
-        log.debug('credentials in improper format', { credentialsArr,
+        log.debug('credentials in improper format', { credentialsArr, timestamp, validationResult });
-            timestamp, validationResult });
+        activeSpan.recordException(validationResult);
+        extractParamsSpan.end();
         return { err: validationResult };
     }
+    activeSpan.addEvent('Credentials are valid');
     // credentialsArr is [accessKey, date, region, aws-service, aws4_request]
     const scopeDate = credentialsArr[1];
     const region = credentialsArr[2];
     const service = credentialsArr[3];
     const accessKey = credentialsArr.shift();
     const credentialScope = credentialsArr.join('/');
-
     // In AWS Signature Version 4, the signing key is valid for up to seven days
     // (see Introduction to Signing Requests.
     // Therefore, a signature is also valid for up to seven days or
@@ -124,14 +132,17 @@ export function check(
     // TODO: When implementing bucket policies,
     // note that expiration can be shortened so
     // expiry is as set out in the policy.
-
     // 15 minutes in seconds
+    activeSpan.addEvent('checking if signature is expired')
     const expiry = (15 * 60);
     const isTimeSkewed = checkTimeSkew(timestamp, expiry, log);
     if (isTimeSkewed) {
+        activeSpan.recordException(errors.RequestTimeTooSkewed);
+        extractParamsSpan.end();
         return { err: errors.RequestTimeTooSkewed };
     }
-
+    activeSpan.addEvent('signature is not expired');
+    activeSpan.addEvent('Constructing string to sign');
     const stringToSign = constructStringToSign({
         log,
         request,
@@ -141,13 +152,16 @@ export function check(
         timestamp,
         payloadChecksum,
         awsService: service,
-    });
+    }, oTel);
     log.trace('constructed stringToSign', { stringToSign });
     if (stringToSign instanceof Error) {
+        activeSpan.recordException(stringToSign);
+        extractParamsSpan.end();
         return { err: stringToSign };
     }
-
+    activeSpan.addEvent('Constructed string to sign v4 headers');
-
+    activeSpan.addEvent('Exiting V4 header auth check');
+    extractParamsSpan.end();
     return {
         err: null,
         params: {
@@ -162,8 +176,6 @@ export function check(
                 authType: 'REST-HEADER',
                 signatureVersion: 'AWS4-HMAC-SHA256',
                 signatureAge: Date.now() - convertAmzTimeToMs(timestamp),
-                // credentialScope and timestamp needed for streaming V4
-                // chunk evaluation
                 credentialScope,
                 timestamp,
                 securityToken: token,
@@ -171,3 +183,4 @@ export function check(
         },
     };
 }
+

lib/auth/v4/queryAuthCheck.ts

@@ -12,10 +12,20 @@ import { areSignedHeadersComplete } from './validateInputs';
  * @param log - logging object
  * @param data - Contain authentification params (GET or POST data)
  */
-export function check(request: any, log: Logger, data: { [key: string]: string }) {
+export function check(request: any, log: Logger, data: { [key: string]: string }, oTel: any) {
+    const {
+        activeSpan,
+        extractParamsSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Arsenal:: entered Arsenal.auth.v4.queryAuthCheck');
+    activeSpan?.addEvent('Arsenal:: extracting query parameters')
     const authParams = extractQueryParams(data, log);
-
+    activeSpan?.addEvent('Arsenal:: extracting query params');
     if (Object.keys(authParams).length !== 5) {
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
 
@@ -24,6 +34,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     const token = data['X-Amz-Security-Token'];
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
 
@@ -35,6 +47,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
 
     if (!areSignedHeadersComplete(signedHeaders, request.headers)) {
         log.debug('signedHeaders are incomplete', { signedHeaders });
+        activeSpan.recordException(errors.AccessDenied);
+        extractParamsSpan.end();
         return { err: errors.AccessDenied };
     }
 
@@ -43,6 +57,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     if (validationResult instanceof Error) {
         log.debug('credentials in improper format', { credential,
             timestamp, validationResult });
+        activeSpan.recordException(validationResult);
+        extractParamsSpan.end();
         return { err: validationResult };
     }
     const accessKey = credential[0];
@@ -53,6 +69,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
 
     const isTimeSkewed = checkTimeSkew(timestamp, expiry, log);
     if (isTimeSkewed) {
+        activeSpan.recordException(errors.RequestTimeTooSkewed);
+        extractParamsSpan.end();
         return { err: errors.RequestTimeTooSkewed };
     }
 
@@ -71,6 +89,7 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     // building string to sign
     const payloadChecksum = 'UNSIGNED-PAYLOAD';
 
+    activeSpan?.addEvent('Constructing string to sign');
     const stringToSign = constructStringToSign({
         log,
         request,
@@ -81,11 +100,16 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
         credentialScope:
             `${scopeDate}/${region}/${service}/${requestType}`,
         awsService: service,
-    });
+    }, oTel);
+    activeSpan?.addEvent('Constructed string to sign v4 query');
     if (stringToSign instanceof Error) {
+        activeSpan.recordException(stringToSign);
+        extractParamsSpan.end();
         return { err: stringToSign };
     }
     log.trace('constructed stringToSign', { stringToSign });
+    activeSpan.addEvent('Arsenal:: exiting Arsenal.auth.v4.queryAuthCheck');
+        extractParamsSpan.end();
     return {
         err: null,
         params: {

lib/s3routes/routes.ts

@@ -150,6 +150,7 @@ export type Params = {
     };
     unsupportedQueries: any;
     api: { callApiMethod: routesUtils.CallApiMethod };
+    oTel?: any,
 }
 
 /** routes - route request to appropriate method
@@ -180,6 +181,22 @@ export default function routes(
     logger: RequestLogger,
     s3config?: any,
 ) {
+    const {
+        dataRetrievalParams: {
+            oTel: {
+                tracer,
+                activeSpan,
+                activeTracerContext,
+            }
+        } 
+    } = params;
+    return tracer.startActiveSpan('Validating Request Parameters with Arsenal', undefined, activeTracerContext, arsenalValidatorSpan => {
+        arsenalValidatorSpan.setAttributes({
+            'code.lineno': 176,
+            'code.filename': 'lib/s3routes/routes.ts',
+            'code.function': 'routes()',
+            'code.url': 'https://github.com/scality/Arsenal/blob/development/7.70/lib/s3routes/routes.ts#L176'
+        });
         checkTypes(req, res, params, logger);
 
         const {
@@ -226,6 +243,7 @@ export default function routes(
                 internalServiceName = internalServiceName.slice(0, serviceDelim);
             }
             if (internalHandlers[internalServiceName] === undefined) {
+                activeSpan.recordException(errors.InvalidURI);
                 return routesUtils.responseXMLBody(
                     errors.InvalidURI, null, res, log);
             }
@@ -242,6 +260,8 @@ export default function routes(
             const validHosts = allEndpoints.concat(websiteEndpoints);
             routesUtils.normalizeRequest(req, validHosts);
         } catch (err: any) {
+            activeSpan.recordException(errors.InvalidURI.customizeDescription('Could not parse the ' +
+                'specified URI. Check your restEndpoints configuration.'));
             log.debug('could not normalize request', { error: err.stack });
             return routesUtils.responseXMLBody(
                 errors.InvalidURI.customizeDescription('Could not parse the ' +
@@ -259,11 +279,21 @@ export default function routes(
             // @ts-ignore
             bodyLength: parseInt(req.headers['content-length'], 10) || 0,
         });
+        activeSpan.setAttributes({
+            // @ts-ignore
+            'aws.s3.bucket': req.bucketName,
+            // @ts-ignore
+            'aws.s3.key': req.objectKey,
+            // @ts-ignore
+            'aws.request_id': log.getUids().join(':'),
+            'rpc.service': 'S3',
+        });
 
         // @ts-ignore
         const { error, method } = checkUnsupportedRoutes(req.method, req.query);
 
         if (error) {
+            activeSpan.recordException(error);
             log.trace('error validating route or uri params', { error });
             // @ts-ignore
             return routesUtils.responseXMLBody(error, '', res, log);
@@ -275,6 +305,7 @@ export default function routes(
             req.method, req.query, blacklistedPrefixes, log);
 
         if (bucketOrKeyError) {
+            activeSpan.recordException(bucketOrKeyError)
             log.trace('error with bucket or key value',
                 { error: bucketOrKeyError });
             return routesUtils.responseXMLBody(bucketOrKeyError, null, res, log);
@@ -285,6 +316,7 @@ export default function routes(
         if (websiteEndpoints && websiteEndpoints.indexOf(req.parsedHost) > -1) {
             return routeWebsite(req, res, api, log, statsClient, dataRetrievalParams);
         }
-
+        arsenalValidatorSpan.end();
         return method(req, res, api, log, statsClient, dataRetrievalParams);
+    });
 }

lib/s3routes/routes/routeDELETE.ts

@@ -4,6 +4,7 @@ import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import StatsClient from '../../metrics/StatsClient';
 import * as http from 'http';
+import { actionMonitoringMapS3 } from '../../policyEvaluator/utils/actionMaps';
 
 export default function routeDELETE(
     request: http.IncomingMessage,
@@ -11,58 +12,146 @@ export default function routeDELETE(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Delete API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan  => {
+        activeSpan.addEvent('Request validated, routing request using routeDELETE() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeDELETE.ts',
+            'code.function': 'routeDELETE()',
+        })
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeDELETE()')
         const call = (name: string) => {
             return api.callApiMethod(name, request, response, log, (err, corsHeaders) => {
+                cloudserverApiSpan.end();
+                const action = actionMonitoringMapS3[name];
+                activeSpan.addEvent(`${action} API operation complete`);
+                if (err) {
+                    activeSpan.recordException(err);
+                }
                 routesUtils.statsReport500(err, statsClient);
+                activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                 return routesUtils.responseNoBody(err, corsHeaders, response, 204, log);
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
             });
         }
         log.debug('routing request', { method: 'routeDELETE' });
     
         const { query, objectKey } = request as any
         if (query?.uploadId) {
+            // @ts-ignore
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent('Detected AbortMultipartUpload API request');
+            activeSpan.setAttribute('rpc.method', 'AbortMultipartUpload');
             if (objectKey === undefined) {
                 const message = 'A key must be specified';
                 const err = errors.InvalidRequest.customizeDescription(message);
+                activeSpan.recordException(err);
+                cloudserverApiSpan.end();
                 return routesUtils.responseNoBody(err, null, response, 200, log);
             }
             return call('multipartDelete');
         } else if (objectKey === undefined) {
             if (query?.website !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketWebsite API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketWebsite');
                 return call('bucketDeleteWebsite');
             } else if (query?.cors !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketCors API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketCors');
                 return call('bucketDeleteCors');
             } else if (query?.replication !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketReplication API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketReplication');
                 return call('bucketDeleteReplication');
             } else if (query?.lifecycle !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketLifecycle API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketLifecycle');
                 return call('bucketDeleteLifecycle');
             } else if (query?.policy !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketPolicy API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketPolicy');
                 return call('bucketDeletePolicy');
             } else if (query?.encryption !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketEncryption API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketEncryption');
                 return call('bucketDeleteEncryption');
             } else if (query?.tagging !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketTagging API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketTagging');
                 return call('bucketDeleteTagging');
             }
+            // @ts-ignore
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent('Detected DeleteBucket API request');
+            activeSpan.setAttribute('rpc.method', 'DeleteBucket');
             call('bucketDelete');
         } else {
             if (query?.tagging !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteObjectTagging API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteObjectTagging');
                 return call('objectDeleteTagging');
             }
-        api.callApiMethod('objectDelete', request, response, log,
+            // @ts-ignore
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent('Detected DeleteObject API request');
+            activeSpan.setAttribute('rpc.method', 'DeleteObject');
+            return api.callApiMethod('objectDelete', request, response, log,
                 (err, corsHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('DeleteObject API operation complete')
                     /*
                   * Since AWS expects a 204 regardless of the existence of
                   the object, the errors NoSuchKey and NoSuchVersion should not
                   * be sent back as a response.
                   */
                     if (err && !err.is.NoSuchKey && !err.is.NoSuchVersion) {
+                        activeSpan.recordException(err);
+                        cloudserverApiSpan.end();
                         return routesUtils.responseNoBody(err, corsHeaders,
                             response, undefined, log);
                     }
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
                     routesUtils.statsReport500(err, statsClient);
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                     return routesUtils.responseNoBody(null, corsHeaders, response,
                         204, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
                 });
         }
+    });
 }

lib/s3routes/routes/routeGET.ts

@@ -4,6 +4,7 @@ import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';
 import StatsClient from '../../metrics/StatsClient';
+import { actionMonitoringMapS3 } from '../../policyEvaluator/utils/actionMaps';
 
 export default function routerGET(
     request: http.IncomingMessage,
@@ -13,21 +14,50 @@ export default function routerGET(
     statsClient?: StatsClient,
     dataRetrievalParams?: any,
 ) {
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Get API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routeGET() in arsenal');
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeGET()')
         log.debug('routing request', { method: 'routerGET' });
 
         const { bucketName, objectKey, query } = request as any
 
         const call = (name: string) => {
-        api.callApiMethod(name, request, response, log, (err, xml, corsHeaders) => {
+            const action = actionMonitoringMapS3[name];
+            // @ts-ignore
+            activeSpan.updateName(`S3 API request`);
+            activeSpan.addEvent(`Detected ${action} API request`);
+            activeSpan.setAttribute('rpc.method', action);
+            return api.callApiMethod(name, request, response, log, (err, xml, corsHeaders) => {
+                cloudserverApiSpan.end();
+                activeSpan.addEvent(`${action} API operation complete`);
+                if (err) {
+                    activeSpan.recordException(err);
+                }
                 routesUtils.statsReport500(err, statsClient);
+                activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                 return routesUtils.responseXMLBody(err, xml, response, log, corsHeaders);
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
             });
         }
 
         if (bucketName === undefined && objectKey !== undefined) {
-        routesUtils.responseXMLBody(errors.NoSuchBucket, null, response, log);
+            activeSpan.recordException(errors.NoSuchBucket);
+            cloudserverApiSpan.end();
+            return routesUtils.responseXMLBody(errors.NoSuchBucket, null, response, log);
         } else if (bucketName === undefined && objectKey === undefined) {
             // GET service
+
             call('serviceGet');
         } else if (objectKey === undefined) {
             // GET bucket ACL
@@ -77,8 +107,14 @@ export default function routerGET(
                 call('objectGetRetention');
             } else {
                 // GET object
-            api.callApiMethod('objectGet', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected GetObject API request');
+                activeSpan.setAttribute('aws.request_id', log.getUids()[0]);
+                activeSpan.setAttribute('rpc.method', 'GetObject');
+                return api.callApiMethod('objectGet', request, response, log,
                     (err, dataGetInfo, resMetaHeaders, range) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('Located Data, using arsenal to make GET request')
                         let contentLength = 0;
                         if (resMetaHeaders && resMetaHeaders['Content-Length']) {
                             contentLength = resMetaHeaders['Content-Length'];
@@ -86,11 +122,20 @@ export default function routerGET(
                         // TODO ARSN-216 Fix logger
                         // @ts-ignore
                         log.end().addDefaultFields({ contentLength });
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
                         return routesUtils.responseStreamData(err, query,
                             resMetaHeaders, dataGetInfo, dataRetrievalParams, response,
                             range, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             }
         }
+    });
 }

lib/s3routes/routes/routeHEAD.ts

@@ -11,28 +11,75 @@ export default function routeHEAD(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
+    const{
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Head API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routeHEAD() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeHEAD.ts',
+            'code.function': 'routeHEAD()',
+        });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeHEAD()');
         log.debug('routing request', { method: 'routeHEAD' });
         const { bucketName, objectKey } = request as any
         if (bucketName === undefined) {
             log.trace('head request without bucketName');
+            activeSpan.recordException(errors.MethodNotAllowed);
+            cloudserverApiSpan.end();
             routesUtils.responseXMLBody(errors.MethodNotAllowed,
                 null, response, log);
         } else if (objectKey === undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected HeadBucket API request`);
+            activeSpan.setAttribute('rpc.method', 'HeadBucket');
             // HEAD bucket
             api.callApiMethod('bucketHead', request, response, log,
                 (err, corsHeaders) => {
+                    activeSpan.addEvent('HeadBucket API operation complete')
+                    cloudserverApiSpan.end();
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
                     routesUtils.statsReport500(err, statsClient);
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                     return routesUtils.responseNoBody(err, corsHeaders, response,
                         200, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
                 });
             } else {
             // HEAD object
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected HeadObject API request`);
+            activeSpan.setAttribute('rpc.method', 'HeadObject');
             api.callApiMethod('objectHead', request, response, log,
                 (err, resHeaders) => {
+                    activeSpan.addEvent('HeadObject API operation complete')
+                    cloudserverApiSpan.end();
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
                     routesUtils.statsReport500(err, statsClient);
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                     return routesUtils.responseContentHeaders(err, {}, resHeaders,
                         response, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
                 });
         }
+    });
 }

lib/s3routes/routes/routeOPTIONS.ts

@@ -11,7 +11,23 @@ export default function routeOPTIONS(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing corsPreflight API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routeOPTIONS() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeOPTIONS.ts',
+            'code.function': 'routeOPTIONS()',
+        });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeOPTIONS()');
         log.debug('routing request', { method: 'routeOPTION' });
 
         const corsMethod = request.headers['access-control-request-method'] || null;
@@ -19,21 +35,40 @@ export default function routeOPTIONS(
         if (!request.headers.origin) {
             const msg = 'Insufficient information. Origin request header needed.';
             const err = errors.BadRequest.customizeDescription(msg);
+            activeSpan.recordException(err);
+            cloudserverApiSpan.end();
             log.debug('missing origin', { method: 'routeOPTIONS', error: err });
             return routesUtils.responseXMLBody(err, null, response, log);
         }
         if (['GET', 'PUT', 'HEAD', 'POST', 'DELETE'].indexOf(corsMethod ?? '') < 0) {
             const msg = `Invalid Access-Control-Request-Method: ${corsMethod}`;
             const err = errors.BadRequest.customizeDescription(msg);
+            activeSpan.recordException(err);
+            cloudserverApiSpan.end();
             log.debug('invalid Access-Control-Request-Method',
                 { method: 'routeOPTIONS', error: err });
             return routesUtils.responseXMLBody(err, null, response, log);
         }
-
+        // @ts-ignore
+        activeSpan.updateName('S3 API request');
+        activeSpan.addEvent(`Detected corsPreflight API request`);
+        activeSpan.setAttribute('rpc.method', 'corsPreflight');
         return api.callApiMethod('corsPreflight', request, response, log,
             (err, resHeaders) => {
+                activeSpan.addEvent('corsPreflight API operation complete');
+                cloudserverApiSpan.end();
+                if (err) {
+                    activeSpan.recordException(err);
+                }
                 routesUtils.statsReport500(err, statsClient);
+                activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                 return routesUtils.responseNoBody(err, resHeaders, response, 200,
                     log);
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
     });
 }

lib/s3routes/routes/routePOST.ts

@@ -3,6 +3,7 @@ import { RequestLogger } from 'werelogs';
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';
+import StatsClient from '../../metrics/StatsClient';
 
 /* eslint-disable no-param-reassign */
 export default function routePOST(
@@ -10,7 +11,24 @@ export default function routePOST(
     response: http.ServerResponse,
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
+    statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Post API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routePOST() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 9,
+            'code.filename': 'lib/s3routes/routes/routePOST.ts',
+            'code.function': 'routePOST()',
+        });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routePOST()');
         log.debug('routing request', { method: 'routePOST' });
 
         const { query, bucketName, objectKey } = request as any
@@ -18,6 +36,8 @@ export default function routePOST(
         const invalidMultiObjectDelReq = query.delete !== undefined
             && bucketName === undefined;
         if (invalidMultiObjectDelReq) {
+            activeSpan.recordException(errors.MethodNotAllowed);
+            cloudserverApiSpan.end();
             return routesUtils.responseNoBody(errors.MethodNotAllowed, null,
                 response, undefined, log);
         }
@@ -30,34 +50,76 @@ export default function routePOST(
         const invalidCompleteMpuReq = query.uploadId !== undefined
             && objectKey === undefined;
         if (invalidInitiateMpuReq || invalidCompleteMpuReq) {
+            activeSpan.recordException(errors.InvalidURI);
+            cloudserverApiSpan.end();
             return routesUtils.responseNoBody(errors.InvalidURI, null,
                 response, undefined, log);
         }
 
         // POST initiate multipart upload
         if (query.uploads !== undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected CreateMultipartUpload API request`);
+            activeSpan.setAttribute('rpc.method', 'CreateMultipartUpload');
             return api.callApiMethod('initiateMultipartUpload', request,
-            response, log, (err, result, corsHeaders) =>
+                response, log, (err, result, corsHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('CreateMultipartUpload API operation complete');
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                     routesUtils.responseXMLBody(err, result, response, log,
-                    corsHeaders));
+                        corsHeaders)
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
         }
 
         // POST complete multipart upload
         if (query.uploadId !== undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected CompleteMultipartUpload API request`);
+            activeSpan.setAttribute('rpc.method', 'CompleteMultipartUpload');
             return api.callApiMethod('completeMultipartUpload', request,
-            response, log, (err, result, resHeaders) =>
+                response, log, (err, result, resHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('CompleteMultipartUpload API operation complete');
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                     routesUtils.responseXMLBody(err, result, response, log,
-                    resHeaders));
+                        resHeaders)
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
         }
 
         // POST multiObjectDelete
         if (query.delete !== undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected AbortMultipartUpload API request`);
+            activeSpan.setAttribute('rpc.method', 'AbortMultipartUpload');
             return api.callApiMethod('multiObjectDelete', request, response,
-            log, (err, xml, corsHeaders) =>
+                log, (err, xml, corsHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('AbortMultipartUpload API operation complete');
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                     routesUtils.responseXMLBody(err, xml, response, log,
-                    corsHeaders));
+                        corsHeaders)
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
         }
-
+        activeSpan.recordException(errors.NotImplemented);
+        cloudserverApiSpan.end();
+        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
         return routesUtils.responseNoBody(errors.NotImplemented, null, response,
             200, log);
+    });
+    
 }

lib/s3routes/routes/routePUT.ts

@@ -11,7 +11,23 @@ export default function routePUT(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Put API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan  => {
+        activeSpan.addEvent('Request validated, routing request using routePUT() in arsenal')
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routePUT.ts',
+            'code.function': 'routePUT()',
+        })
+        activeSpan.addEvent('Detecting which API to route to using arsenal routePUT()')
         log.debug('routing request', { method: 'routePUT' });
 
         const { objectKey, query, bucketName, parsedContentLength } = request as any
@@ -29,92 +45,260 @@ export default function routePUT(
             }
             // PUT bucket ACL
             if (query.acl !== undefined) {
-            api.callApiMethod('bucketPutACL', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketAcl API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketAcl');
+                return api.callApiMethod('bucketPutACL', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketAcl API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, corsHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.versioning !== undefined) {
-            api.callApiMethod('bucketPutVersioning', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketVersioning API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketVersioning');
+                return api.callApiMethod('bucketPutVersioning', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketVersioning API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         routesUtils.responseNoBody(err, corsHeaders, response, 200,
                             log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.website !== undefined) {
-            api.callApiMethod('bucketPutWebsite', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketWebsite API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketWebsite');
+                return api.callApiMethod('bucketPutWebsite', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketWebsite API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, corsHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.tagging !== undefined) {
-            api.callApiMethod('bucketPutTagging', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketTagging API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketTagging');
+                return api.callApiMethod('bucketPutTagging', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketTagging API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, corsHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.cors !== undefined) {
-            api.callApiMethod('bucketPutCors', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketCors API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketCors');
+                return api.callApiMethod('bucketPutCors', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketCors API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, corsHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.replication !== undefined) {
-            api.callApiMethod('bucketPutReplication', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketReplication API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketReplication');
+                return api.callApiMethod('bucketPutReplication', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketReplication API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         routesUtils.responseNoBody(err, corsHeaders, response, 200,
                             log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.lifecycle !== undefined) {
-            api.callApiMethod('bucketPutLifecycle', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketLifecycle API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketLifecycle');
+                return api.callApiMethod('bucketPutLifecycle', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketLifecycle API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         routesUtils.responseNoBody(err, corsHeaders, response, 200,
                             log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.policy !== undefined) {
-            api.callApiMethod('bucketPutPolicy', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketPolicy API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketPolicy');
+                return api.callApiMethod('bucketPutPolicy', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketPolicy API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         routesUtils.responseNoBody(err, corsHeaders, response, 200,
                             log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query['object-lock'] !== undefined) {
-            api.callApiMethod('bucketPutObjectLock', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectLockConfiguration API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectLockConfiguration');
+                return api.callApiMethod('bucketPutObjectLock', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectLockConfiguration API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         routesUtils.responseNoBody(err, corsHeaders, response, 200,
                             log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.notification !== undefined) {
-            api.callApiMethod('bucketPutNotification', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketNotificationConfiguration API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketNotificationConfiguration');
+                return api.callApiMethod('bucketPutNotification', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketNotificationConfiguration API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         routesUtils.responseNoBody(err, corsHeaders, response, 200,
                             log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.encryption !== undefined) {
-            api.callApiMethod('bucketPutEncryption', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketEncryption API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketEncryption');
+                return api.callApiMethod('bucketPutEncryption', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketEncryption API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, corsHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else {
                 // PUT bucket
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucket API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucket');
                 return api.callApiMethod('bucketPut', request, response, log,
                     (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucket API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
                         const location = { Location: `/${bucketName}` };
                         const resHeaders = corsHeaders ?
                             Object.assign({}, location, corsHeaders) : location;
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, resHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             }
         } else {
@@ -124,6 +308,8 @@ export default function routePUT(
             // parse content-md5 from meta headers
 
             if (request.headers['content-md5'] === '') {
+                activeSpan.recordException(errors.InvalidDigest);
+                cloudserverApiSpan.end();
                 log.debug('empty content-md5 header', {
                     method: 'routePUT',
                 });
@@ -143,6 +329,8 @@ export default function routePUT(
                 request.contentMD5 = Buffer.from(request.contentMD5, 'base64').toString('hex');
                 // @ts-ignore
                 if (request.contentMD5 && request.contentMD5.length !== 32) {
+                    activeSpan.recordException(errors.InvalidDigest);
+                    cloudserverApiSpan.end();
                     // @ts-ignore
                     log.debug('invalid md5 digest', { contentMD5: request.contentMD5 });
                     return routesUtils
@@ -152,15 +340,34 @@ export default function routePUT(
             }
             if (query.partNumber) {
                 if (request.headers['x-amz-copy-source']) {
-                api.callApiMethod('objectPutCopyPart', request, response, log,
+                    activeSpan.updateName('S3 API request');
+                    activeSpan.addEvent('Detected UploadPartCopy API request');
+                    activeSpan.setAttribute('rpc.method', 'UploadPartCopy');
+                    return api.callApiMethod('objectPutCopyPart', request, response, log,
                         (err, xml, additionalHeaders) => {
+                            cloudserverApiSpan.end();
+                            activeSpan.addEvent('UploadPartCopy API operation complete');
+                            if (err) {
+                                activeSpan.recordException(err);
+                            }
                             routesUtils.statsReport500(err, statsClient);
+                            activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                             return routesUtils.responseXMLBody(err, xml, response, log,
                                 additionalHeaders);
+                        }, {
+                            cloudserverApiSpan,
+                            activeSpan,
+                            activeTracerContext,
+                            tracer,
                         });
                 } else {
-                api.callApiMethod('objectPutPart', request, response, log,
+                    activeSpan.updateName('S3 API request');
+                    activeSpan.addEvent('Detected UploadPart API request');
+                    activeSpan.setAttribute('rpc.method', 'UploadPart');
+                    return api.callApiMethod('objectPutPart', request, response, log,
                         (err, calculatedHash, corsHeaders) => {
+                            cloudserverApiSpan.end();
+                            activeSpan.addEvent('UploadPart API operation complete');
                             if (err) {
                                 return routesUtils.responseNoBody(err, corsHeaders,
                                     response, 200, log);
@@ -168,66 +375,164 @@ export default function routePUT(
                             // ETag's hex should always be enclosed in quotes
                             const resMetaHeaders = corsHeaders || {};
                             resMetaHeaders.ETag = `"${calculatedHash}"`;
+                            if (err) {
+                                activeSpan.recordException(err);
+                            }
                             routesUtils.statsReport500(err, statsClient);
+                            activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                             return routesUtils.responseNoBody(err, resMetaHeaders,
                                 response, 200, log);
+                        }, {
+                            cloudserverApiSpan,
+                            activeSpan,
+                            activeTracerContext,
+                            tracer,
                         });
                 }
             } else if (query.acl !== undefined) {
-            api.callApiMethod('objectPutACL', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectAcl API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectAcl');
+                return api.callApiMethod('objectPutACL', request, response, log,
                     (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectAcl API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, resHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query['legal-hold'] !== undefined) {
-            api.callApiMethod('objectPutLegalHold', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectLegalHold API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectLegalHold');
+                return api.callApiMethod('objectPutLegalHold', request, response, log,
                     (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectLegalHold API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, resHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.tagging !== undefined) {
-            api.callApiMethod('objectPutTagging', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectTagging API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectTagging');
+                return api.callApiMethod('objectPutTagging', request, response, log,
                     (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectTagging API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, resHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (query.retention !== undefined) {
-            api.callApiMethod('objectPutRetention', request, response, log,
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectRetention API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectRetention');
+                return api.callApiMethod('objectPutRetention', request, response, log,
                     (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectRetention API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, resHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else if (request.headers['x-amz-copy-source']) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected CopyObject API request');
+                activeSpan.setAttribute('rpc.method', 'CopyObject');
                 return api.callApiMethod('objectCopy', request, response, log,
                     (err, xml, additionalHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('CopyObject API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         routesUtils.responseXMLBody(err, xml, response, log,
                             additionalHeaders);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else {
                 if (request.headers['content-length'] === undefined &&
                 request.headers['x-amz-decoded-content-length'] === undefined) {
+                    activeSpan.recordException(errors.MissingContentLength);
+                    cloudserverApiSpan.end();
                     return routesUtils.responseNoBody(errors.MissingContentLength,
                         null, response, 411, log);
                 }
                 if (Number.isNaN(parsedContentLength) || parsedContentLength < 0) {
+                    activeSpan.recordException(errors.BadRequest);
+                    cloudserverApiSpan.end();
                     return routesUtils.responseNoBody(errors.BadRequest,
                         null, response, 400, log);
                 }
                 // TODO ARSN-216 What's happening?
                 // @ts-ignore
                 log.end().addDefaultFields({ contentLength: request.parsedContentLength });
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObject API request');
+                activeSpan.setAttribute('rpc.method', 'PutObject');
                 api.callApiMethod('objectPut', request, response, log,
                     (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObject API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                         return routesUtils.responseNoBody(err, resHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             }
         }
         return undefined;
+    });
 }

lib/s3routes/routes/routeWebsite.ts

@@ -13,27 +13,54 @@ export default function routerWebsite(
     statsClient?: StatsClient,
     dataRetrievalFn?: any,
 ) {
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalFn;
+    return tracer.startActiveSpan('Arsenal:: Performing Website API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routerWebsite() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeWebsite.ts',
+            'code.function': 'routerWebsite()',
+        });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routerWebsite()');
         const { bucketName, query } = request as any
         log.debug('routing request', { method: 'routerWebsite' });
         // website endpoint only supports GET and HEAD and must have a bucket
         // http://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteEndpoints.html
         if ((request.method !== 'GET' && request.method !== 'HEAD')
             || !bucketName) {
+            activeSpan.recordException(errors.MethodNotAllowed);
+            cloudserverApiSpan.end();
             return routesUtils.errorHtmlResponse(errors.MethodNotAllowed,
                 false, bucketName, response, null, log);
         }
         if (request.method === 'GET') {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected GetWebsite API request`);
+            activeSpan.setAttribute('rpc.method', 'GetWebsite');
             return api.callApiMethod('websiteGet', request, response, log,
                 (err, userErrorPageFailure, dataGetInfo, resMetaHeaders,
                     redirectInfo, key) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('Located Data')
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
                     routesUtils.statsReport500(err, statsClient);
                     // request being redirected
                     if (redirectInfo) {
                         if (err && redirectInfo.withError) {
+                            activeSpan.recordException(err);
                             return routesUtils.redirectRequestOnError(err,
                                 'GET', redirectInfo, dataGetInfo, dataRetrievalFn,
                                 response, resMetaHeaders, log)
                         }
+                        activeSpan.addEvent('Redirecting request');
                         // note that key might have been modified in websiteGet
                         // api to add index document
                         return routesUtils.redirectRequest(redirectInfo,
@@ -44,11 +71,13 @@ export default function routerWebsite(
                     }
                     // user has their own error page
                     if (err && dataGetInfo) {
+                        activeSpan.recordException(err);
                         return routesUtils.streamUserErrorPage(err, dataGetInfo,
                             dataRetrievalFn, response, resMetaHeaders, log);
                     }
                     // send default error html response
                     if (err) {
+                        activeSpan.recordException(err);
                         return routesUtils.errorHtmlResponse(err,
                             userErrorPageFailure, bucketName,
                             response, resMetaHeaders, log);
@@ -57,18 +86,33 @@ export default function routerWebsite(
                     return routesUtils.responseStreamData(null, query,
                         resMetaHeaders, dataGetInfo, dataRetrievalFn, response,
                         undefined, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
                 });
         }
         if (request.method === 'HEAD') {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected HeadWebsite API request`);
+            activeSpan.setAttribute('rpc.method', 'HeadWebsite');
             return api.callApiMethod('websiteHead', request, response, log,
                 (err, resMetaHeaders, redirectInfo, key) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('HeadWebsite API operation complete')
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
                     routesUtils.statsReport500(err, statsClient);
                     if (redirectInfo) {
                         if (err && redirectInfo.withError) {
+                            activeSpan.recordException(err);
                             return routesUtils.redirectRequestOnError(err,
                                 'HEAD', redirectInfo, null, dataRetrievalFn,
                                 response, resMetaHeaders, log)
                         }
+                        activeSpan.addEvent('Redirecting request');
                         return routesUtils.redirectRequest(redirectInfo,
                             // TODO ARSN-217 encrypted does not exists in request.connection
                             // @ts-ignore
@@ -77,12 +121,21 @@ export default function routerWebsite(
                     }
                     // could redirect on err so check for redirectInfo first
                     if (err) {
+                        activeSpan.recordException(err);
                         return routesUtils.errorHeaderResponse(err, response,
                             resMetaHeaders, log);
                     }
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
                     return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
                         response, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
                 });
         }
         return undefined;
+    });
+    
 }

lib/s3routes/routesUtils.ts

@@ -1,6 +1,7 @@
 import * as url from 'url';
 import * as http from 'http';
 import { eachSeries } from 'async';
+const opentelemetry = require('@opentelemetry/api');
 
 import { RequestLogger } from 'werelogs';
 
@@ -16,6 +17,10 @@ export type CallApiMethod = (
     response: http.ServerResponse,
     log: RequestLogger,
     callback: (err: ArsenalError | null, ...data: any[]) => void,
+    cloudserverApiSpan?: any,
+    activeSpan?: any,
+    activeTracerContext?: any,
+    tracer?: any,
 ) => void;
 
 /**
@@ -350,6 +355,10 @@ function retrieveData(
     retrieveDataParams: any,
     response: http.ServerResponse,
     log: RequestLogger,
+    sproxydSpan?: any,
+    activeSpan?: any,
+    activeTracerContext?: any,
+    tracer?: any,
 ) {
     if (locations.length === 0) {
         return response.end();
@@ -367,6 +376,8 @@ function retrieveData(
     };
     // the S3-client might close the connection while we are processing it
     response.once('close', () => {
+        activeSpan.addEvent('response closed by client request');
+        sproxydSpan?.end();
         responseDestroyed = true;
         if (currentStream) {
             currentStream.destroy();
@@ -384,6 +395,12 @@ function retrieveData(
     } = retrieveDataParams;
     const data = new DataWrapper(
         client, implName, config, kms, metadata, locStorageCheckFn, vault);
+    return tracer.startActiveSpan('Streaming Data Using Sproxyd', dataSpan => {
+        dataSpan.setAttributes({
+            'code.function': 'Arsenal:: retrieveData()',
+            'code.filepath': 'lib/s3routes/routesUtils.js',
+            'code.lineno': 349,
+        });
         return eachSeries(locations,
             (current, next) => data.get(current, response, log,
                 (err: any, readable: http.IncomingMessage) => {
@@ -412,12 +429,18 @@ function retrieveData(
                     }
                     // readable stream successfully consumed
                     readable.on('end', () => {
+                        sproxydSpan?.addEvent('Readable stream successfully consumed');
+                        dataSpan.end();
+                        sproxydSpan?.end();
                         currentStream = null;
                         log.debug('readable stream end reached');
                         return next();
                     });
                     // errors on server side with readable stream
                     readable.on('error', err => {
+                        activeSpan.recordException(err);
+                        dataSpan.end();
+                        sproxydSpan?.end();
                         log.error('error piping data from source');
                         _destroyResponse();
                         return next(err);
@@ -436,6 +459,7 @@ function retrieveData(
                 response.end();
             },
         );
+    });
 }
 
 function _responseBody(
@@ -606,6 +630,20 @@ export function responseStreamData(
     range: [number, number] | undefined,
     log: RequestLogger,
 ) {
+    const{
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    }   = retrieveDataParams;
+    activeSpan.addEvent('Request processed, getting Data from sproxyd');
+    return tracer.startActiveSpan('Getting Object Data from RING', undefined, activeTracerContext, sproxydSpan => {
+        sproxydSpan.setAttributes({
+            'code.function': 'Arsenal:: responseStreamData()',
+            'code.filepath': 'lib/s3routes/routesUtils.js',
+            'code.lineno': 609,
+        })
         if (errCode && !response.headersSent) {
             return XMLResponseBackend.errorResponse(errCode, response, log,
                 resHeaders);
@@ -640,12 +678,14 @@ export function responseStreamData(
         }
         response.on('finish', () => {
             // TODO ARSN-216 Fix logger
+            activeSpan.addEvent('Data retrieved from sproxyd and sending response to client');
             // @ts-expect-error
             log.end().info('responded with streamed content', {
                 httpCode: response.statusCode,
             });
         });
-    return retrieveData(dataLocations, retrieveDataParams, response, log);
+        return retrieveData(dataLocations, retrieveDataParams, response, log, sproxydSpan, activeSpan, activeTracerContext, tracer);
+    });
 }
 
 /**
@@ -666,18 +706,26 @@ export function streamUserErrorPage(
     corsHeaders: { [key: string]: string },
     log: RequestLogger,
 ) {
+    const{
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    }   = retrieveDataParams;
     setCommonResponseHeaders(corsHeaders, response, log);
     response.setHeader('x-amz-error-code', err.message);
     response.setHeader('x-amz-error-message', err.description);
     response.writeHead(err.code, { 'Content-type': 'text/html' });
     response.on('finish', () => {
         // TODO ARSN-216 Fix logger
+        activeSpan.recordException(err);
         // @ts-expect-error
         log.end().info('responded with streamed content', {
             httpCode: response.statusCode,
         });
     });
-    return retrieveData(dataLocations, retrieveDataParams, response, log);
+    return retrieveData(dataLocations, retrieveDataParams, response, log, undefined, activeSpan, activeTracerContext, tracer);
 }
 
 /**