updated data message

lib/auth/auth.ts

@@ -58,49 +58,71 @@ function extractParams(
     request: any,
     log: Logger,
     awsService: string,
-    data: { [key: string]: string }
+    data: { [key: string]: string },
+    oTel?: any,
 ) {
-    log.trace('entered', { method: 'Arsenal.auth.server.extractParams' });
-    const authHeader = request.headers.authorization;
-    let version: 'v2' |'v4' | null = null;
-    let method: 'query' | 'headers' | null = null;
-
-    // Identify auth version and method to dispatch to the right check function
-    if (authHeader) {
-        method = 'headers';
-        // TODO: Check for security token header to handle temporary security
-        // credentials
-        if (authHeader.startsWith('AWS ')) {
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Arsenal:: entered Arsenal.auth.server.extractParams');
+    return tracer.startActiveSpan('Check validity of request parameters to authenticate using Arsenal', undefined, activeTracerContext, extractParamsSpan => {
+        extractParamsSpan.setAttributes({
+            'code.lineno': 75,
+            'code.filename': 'lib/auth/auth.ts',
+            'code.function': 'extractParams',
+            'code.url': 'https://github.com/scality/arsenal/blob/892dee6c1333fcc25c88333ee991f02830cb3c51/lib/auth/auth.ts',
+        });
+        log.trace('entered', { method: 'Arsenal.auth.server.extractParams' });
+        const authHeader = request.headers.authorization;
+        let version: 'v2' |'v4' | null = null;
+        let method: 'query' | 'headers' | null = null;
+        activeSpan?.addEvent('Arsenal:: Identifying auth version from authentication header');
+        // Identify auth version and method to dispatch to the right check function
+        if (authHeader) {
+            method = 'headers';
+            // TODO: Check for security token header to handle temporary security
+            // credentials
+            if (authHeader.startsWith('AWS ')) {
+                version = 'v2';
+            } else if (authHeader.startsWith('AWS4')) {
+                version = 'v4';
+            } else {
+                log.trace('invalid authorization security header',
+                          { header: authHeader });
+                extractParamsSpan.end();
+                return { err: errors.AccessDenied };
+            }
+        } else if (data.Signature) {
+            method = 'query';
             version = 'v2';
-        } else if (authHeader.startsWith('AWS4')) {
+        } else if (data['X-Amz-Algorithm']) {
+            method = 'query';
             version = 'v4';
-        } else {
-            log.trace('invalid authorization security header',
-                      { header: authHeader });
-            return { err: errors.AccessDenied };
         }
-    } else if (data.Signature) {
-        method = 'query';
-        version = 'v2';
-    } else if (data['X-Amz-Algorithm']) {
-        method = 'query';
-        version = 'v4';
-    }
-
-    // Here, either both values are set, or none is set
-    if (version !== null && method !== null) {
-        if (!checkFunctions[version] || !checkFunctions[version][method]) {
-            log.trace('invalid auth version or method',
-                      { version, authMethod: method });
-            return { err: errors.NotImplemented };
+        activeSpan?.addEvent(`Arsenal::Auth versions identified: ${version}`);
+        // Here, either both values are set, or none is set
+        if (version !== null && method !== null) {
+            if (!checkFunctions[version] || !checkFunctions[version][method]) {
+                activeSpan?.recordException(errors.NotImplemented);
+                log.trace('invalid auth version or method',
+                          { version, authMethod: method });
+                extractParamsSpan.end();
+                return { err: errors.NotImplemented };
+            }
+            activeSpan?.addEvent(`Arsenal:: Identified auth method version: ${version} and method: ${method}`);
+            activeSpan?.addEvent('Arsenal:: Checking if valid request headers and query are used to make request to vault');
+            log.trace('identified auth method', { version, authMethod: method });
+            return checkFunctions[version][method](request, log, data, awsService, { activeSpan, extractParamsSpan, activeTracerContext, tracer });
         }
-        log.trace('identified auth method', { version, authMethod: method });
-        return checkFunctions[version][method](request, log, data, awsService);
-    }
 
-    // no auth info identified
-    log.debug('assuming public user');
-    return { err: null, params: publicUserInfo };
+        // no auth info identified
+        log.debug('assuming public user');
+        extractParamsSpan.end();
+        activeSpan?.addEvent(`Arsenal:: Identified as public user`);
+        return { err: null, params: publicUserInfo };
+    });
 }
 
 /**
@@ -119,15 +141,30 @@ function doAuth(
     log: Logger,
     cb: (err: Error | null, data?: any) => void,
     awsService: string,
-    requestContexts: any[] | null
+    requestContexts: any[] | null,
+    oTel?: any,
 ) {
-    const res = extractParams(request, log, awsService, request.query);
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Arsenal:: Routing request using doAuth() in arsenal');
+    activeSpan?.addEvent('Arsenal:: Extracting auth parameters and check validity of request parameters to authenticate');
+    const start = process.hrtime.bigint();
+    const res = extractParams(request, log, awsService, request.query, oTel);
+    const end = process.hrtime.bigint();
+    const duration = Number(end - start) / 1e6;
+    activeSpan?.addEvent(`Arsenal:: It took ${duration.toFixed(3)} ms to extract auth parameters and to check validity of request parameters to authenticate`);
     if (res.err) {
+        activeSpan?.recordException(res.err);
         return cb(res.err);
     } else if (res.params instanceof AuthInfo) {
+        activeSpan?.addEvent('Arsenal:: Auth info already in the params, do not need to make a request to cloudserver');
         return cb(null, res.params);
     }
     if (requestContexts) {
+        activeSpan?.addEvent('Arsenal:: Setting auth info in requestContexts');
         requestContexts.forEach((requestContext) => {
             const { params } = res
             if ('data' in params) {
@@ -140,6 +177,7 @@ function doAuth(
                 }
             }
         });
+        activeSpan?.addEvent('Arsenal:: Auth info set in requestContexts');
     }
 
     // Corner cases managed, we're left with normal auth
@@ -147,10 +185,12 @@ function doAuth(
     // @ts-ignore
     res.params.log = log;
     if (res.params.version === 2) {
+        activeSpan?.addEvent('Arsenal:: Sending AuthV2 call to vault');
         // @ts-ignore
         return vault!.authenticateV2Request(res.params, requestContexts, cb);
     }
     if (res.params.version === 4) {
+        activeSpan?.addEvent('Arsenal:: Sending AuthV4 call to vault');
         // @ts-ignore
         return vault!.authenticateV4Request(res.params, requestContexts, cb);
     }
@@ -158,6 +198,7 @@ function doAuth(
     log.error('authentication method not found', {
         method: 'Arsenal.auth.doAuth',
     });
+    activeSpan?.recordException(errors.InternalError);
     return cb(errors.InternalError);
 }
 

lib/auth/v2/headerAuthCheck.ts

@@ -5,64 +5,83 @@ import constructStringToSign from './constructStringToSign';
 import checkRequestExpiry from './checkRequestExpiry';
 import algoCheck from './algoCheck';
 
-export function check(request: any, log: Logger, data: { [key: string]: string }) {
+export function check(request: any, log: Logger, data: { [key: string]: string }, oTel: any) {
+    const { activeSpan, extractParamsSpan, activeTracerContext, tracer } = oTel;
+    activeSpan?.addEvent('Entered V2 header auth check');
     log.trace('running header auth check');
+    activeSpan?.addEvent('Running header auth check');
     const headers = request.headers;
-
+    activeSpan?.addEvent('Extracting security token');
     const token = headers['x-amz-security-token'];
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
-
+    activeSpan?.addEvent('Extracted security token');
+    activeSpan?.addEvent('Checking timestamp');
     // Check to make sure timestamp is within 15 minutes of current time
-    let timestamp = headers['x-amz-date'] ?
-        headers['x-amz-date'] : headers.date;
+    let timestamp = headers['x-amz-date'] ? headers['x-amz-date'] : headers.date;
     timestamp = Date.parse(timestamp);
     if (!timestamp) {
-        log.debug('missing or invalid date header',
-        { method: 'auth/v2/headerAuthCheck.check' });
-        return { err: errors.AccessDenied.
-          customizeDescription('Authentication requires a valid Date or ' +
-          'x-amz-date header') };
+        log.debug('missing or invalid date header', { method: 'auth/v2/headerAuthCheck.check' });
+        activeSpan.recordException(errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header'));
+        extractParamsSpan.end();
+        return { err: errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header') };
     }
-
+    activeSpan?.addEvent('Checked timestamp');
+    activeSpan?.addEvent('Checking request expiry');
     const err = checkRequestExpiry(timestamp, log);
     if (err) {
+        activeSpan.recordException(err);
+        extractParamsSpan.end();
         return { err };
     }
-
-    // Authorization Header should be
-    // in the format of 'AWS AccessKey:Signature'
+    activeSpan?.addEvent('Checked request expiry');
+    activeSpan?.addEvent('Extracting authorization header');
+    // Authorization Header should be in the format of 'AWS AccessKey:Signature'
     const authInfo = headers.authorization;
-
+    activeSpan?.addEvent('Extracted authorization header');
     if (!authInfo) {
         log.debug('missing authorization security header');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     const semicolonIndex = authInfo.indexOf(':');
     if (semicolonIndex < 0) {
         log.debug('invalid authorization header', { authInfo });
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
-    const accessKey = semicolonIndex > 4 ?
-        authInfo.substring(4, semicolonIndex).trim() : undefined;
+    const accessKey = semicolonIndex > 4 ? authInfo.substring(4, semicolonIndex).trim() : undefined;
     if (typeof accessKey !== 'string' || accessKey.length === 0) {
         log.trace('invalid authorization header', { authInfo });
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     // @ts-ignore
     log.addDefaultFields({ accessKey });
-
     const signatureFromRequest = authInfo.substring(semicolonIndex + 1).trim();
     log.trace('signature from request', { signatureFromRequest });
+    activeSpan?.addEvent('Extracting signature from request');
+    activeSpan?.addEvent('Constructing string to sign');
     const stringToSign = constructStringToSign(request, data, log);
     log.trace('constructed string to sign', { stringToSign });
+    activeSpan?.addEvent('Constructed string to sign v2 headers');
     const algo = algoCheck(signatureFromRequest.length);
     log.trace('algo for calculating signature', { algo });
+    activeSpan?.addEvent('Checked algorithm for calculating signature');
     if (algo === undefined) {
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
+    activeSpan?.addEvent('Exiting V2 header auth check');
+    extractParamsSpan.end();
     return {
         err: null,
         params: {
@@ -80,3 +99,4 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
         },
     };
 }
+

lib/auth/v2/queryAuthCheck.ts

@@ -4,19 +4,26 @@ import * as constants from '../../constants';
 import algoCheck from './algoCheck';
 import constructStringToSign from './constructStringToSign';
 
-export function check(request: any, log: Logger, data: { [key: string]: string }) {
+export function check(request: any, log: Logger, data: { [key: string]: string }, oTel: any) {
+    const { activeSpan, extractParamsSpan, activeTracerContext, tracer } = oTel;
+    activeSpan?.addEvent('Entered query auth check');
     log.trace('running query auth check');
+    activeSpan?.addEvent('Running query auth check');
     if (request.method === 'POST') {
         log.debug('query string auth not supported for post requests');
+        activeSpan.recordException(errors.NotImplemented);
+        extractParamsSpan.end();
         return { err: errors.NotImplemented };
     }
-
     const token = data.SecurityToken;
+    activeSpan?.addEvent('Extracting security token');
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
-
+    activeSpan?.addEvent('Extracted security token');
     /*
     Check whether request has expired or if
     expires parameter is more than 604800000 milliseconds
@@ -25,47 +32,57 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     multiply by 1000 to obtain
     milliseconds to compare to Date.now()
     */
+    activeSpan?.addEvent('Checking expiration time');
     const expirationTime = parseInt(data.Expires, 10) * 1000;
     if (Number.isNaN(expirationTime)) {
-        log.debug('invalid expires parameter',
-            { expires: data.Expires });
+        log.debug('invalid expires parameter', { expires: data.Expires });
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
-
+    activeSpan?.addEvent('Checked expiration time');
     const currentTime = Date.now();
-
     const preSignedURLExpiry = process.env.PRE_SIGN_URL_EXPIRY
         && !Number.isNaN(process.env.PRE_SIGN_URL_EXPIRY)
         ? Number.parseInt(process.env.PRE_SIGN_URL_EXPIRY, 10)
         : constants.defaultPreSignedURLExpiry * 1000;
-
     if (expirationTime > currentTime + preSignedURLExpiry) {
-        log.debug('expires parameter too far in future',
-        { expires: request.query.Expires });
+        log.debug('expires parameter too far in future', { expires: request.query.Expires });
+        activeSpan.recordException(errors.AccessDenied);
+        extractParamsSpan.end();
         return { err: errors.AccessDenied };
     }
     if (currentTime > expirationTime) {
-        log.debug('current time exceeds expires time',
-        { expires: request.query.Expires });
+        log.debug('current time exceeds expires time', { expires: request.query.Expires });
+        activeSpan.recordException(errors.RequestTimeTooSkewed);
+        extractParamsSpan.end();
         return { err: errors.RequestTimeTooSkewed };
     }
     const accessKey = data.AWSAccessKeyId;
     // @ts-ignore
     log.addDefaultFields({ accessKey });
-
     const signatureFromRequest = decodeURIComponent(data.Signature);
     log.trace('signature from request', { signatureFromRequest });
+    activeSpan?.addEvent('Extracting signature from request');
     if (!accessKey || !signatureFromRequest) {
         log.debug('invalid access key/signature parameters');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     const stringToSign = constructStringToSign(request, data, log);
     log.trace('constructed string to sign', { stringToSign });
+    activeSpan?.addEvent('Constructed string to sign v2 query');
     const algo = algoCheck(signatureFromRequest.length);
     log.trace('algo for calculating signature', { algo });
+    activeSpan?.addEvent('Checked algorithm for calculating signature');
     if (algo === undefined) {
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
+    activeSpan?.addEvent('Exiting query auth check');
+        extractParamsSpan.end();
     return {
         err: null,
         params: {
@@ -82,3 +99,4 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
         },
     };
 }
+

lib/auth/v4/constructStringToSign.ts

@@ -17,7 +17,7 @@ export default function constructStringToSign(params: {
     log?: Logger;
     proxyPath?: string;
     awsService: string;
-}): string | Error {
+}, oTel?: any,): string | Error {
     const {
         request,
         signedHeaders,
@@ -29,7 +29,12 @@ export default function constructStringToSign(params: {
         proxyPath,
     } = params;
     const path = proxyPath || request.path;
-
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Constructing canonical request for Authv4');
     const canonicalReqResult = createCanonicalRequest({
         pHttpVerb: request.method,
         pResource: path,
@@ -38,8 +43,7 @@ export default function constructStringToSign(params: {
         pSignedHeaders: signedHeaders,
         payloadChecksum,
         service: params.awsService,
-    });
-
+    }, oTel);
     // TODO Why that line?
     // @ts-ignore
     if (canonicalReqResult instanceof Error) {
@@ -51,9 +55,13 @@ export default function constructStringToSign(params: {
     if (log) {
         log.debug('constructed canonicalRequest', { canonicalReqResult });
     }
+    const createSignatureSpan = tracer.startSpan('Creating signature hash for AuthV4 using crypto sha256');
+    activeSpan?.addEvent('Creating signature hash for AuthV4 using crypto sha256');
     const sha256 = crypto.createHash('sha256');
     const canonicalHex = sha256.update(canonicalReqResult, 'binary')
         .digest('hex');
+    activeSpan?.addEvent('Created signature hash for AuthV4 using crypto sha256');
+    createSignatureSpan.end();
     const stringToSign = `AWS4-HMAC-SHA256\n${timestamp}\n` +
     `${credentialScope}\n${canonicalHex}`;
     return stringToSign;

lib/auth/v4/createCanonicalRequest.ts

@@ -19,8 +19,16 @@ export default function createCanonicalRequest(
         pSignedHeaders: any;
         service: string;
         payloadChecksum: string;
-    }
+    },
+    oTel?: any,
 ) {
+    const {
+        activeSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Entered createCanonicalRequest');
+
     const pHttpVerb = params.pHttpVerb;
     const pResource = params.pResource;
     const pQuery = params.pQuery;
@@ -28,35 +36,34 @@ export default function createCanonicalRequest(
     const pSignedHeaders = params.pSignedHeaders;
     const service = params.service;
     let payloadChecksum = params.payloadChecksum;
+
+    const payloadChecksumSpan = tracer.startSpan('ComputePayloadChecksum');
     if (!payloadChecksum) {
         if (pHttpVerb === 'GET') {
             payloadChecksum = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b' +
                 '934ca495991b7852b855';
         } else if (pHttpVerb === 'POST') {
             let notEncodeStar = false;
-            // The java sdk does not encode the '*' parameter to compute the
-            // signature, if the user-agent is recognized, we need to keep
-            // the plain '*' as well.
             if (/aws-sdk-java\/[0-9.]+/.test(pHeaders['user-agent'])) {
                 notEncodeStar = true;
             }
             let payload = queryString.stringify(pQuery, undefined, undefined, {
-                encodeURIComponent: input => awsURIencode(input, true,
-                    notEncodeStar),
+                encodeURIComponent: input => awsURIencode(input, true, notEncodeStar),
             });
             payload = payload.replace(/%20/g, '+');
             payloadChecksum = crypto.createHash('sha256')
                 .update(payload, 'binary').digest('hex').toLowerCase();
         }
     }
+    payloadChecksumSpan.end();
 
+    const canonicalURISpan = tracer.startSpan('ComputeCanonicalURI');
     const canonicalURI = !!pResource ? awsURIencode(pResource, false) : '/';
+    canonicalURISpan.end();
 
-    // canonical query string
+    const canonicalQueryStrSpan = tracer.startSpan('ComputeCanonicalQueryStr');
     let canonicalQueryStr = '';
-    if (pQuery && !((service === 'iam' || service === 'ring' ||
-        service === 'sts') &&
-        pHttpVerb === 'POST')) {
+    if (pQuery && !((service === 'iam' || service === 'ring' || service === 'sts') && pHttpVerb === 'POST')) {
         const sortedQueryParams = Object.keys(pQuery).sort().map(key => {
             const encodedKey = awsURIencode(key);
             const value = pQuery[key] ? awsURIencode(pQuery[key]) : '';
@@ -64,32 +71,54 @@ export default function createCanonicalRequest(
         });
         canonicalQueryStr = sortedQueryParams.join('&');
     }
+    canonicalQueryStrSpan.end();
 
-    // signed headers
+    const signedHeadersSpan = tracer.startSpan('SortSignedHeadersAlphabetically');
+    activeSpan?.addEvent('Splitting signed headers using deliminator: ;');
     const signedHeadersList = pSignedHeaders.split(';');
+    activeSpan?.addEvent('Split signed headers using ; as deliminator');
+    activeSpan?.addEvent('Sorting signed headers alphabetically');
     signedHeadersList.sort((a: any, b: any) => a.localeCompare(b));
+    activeSpan?.addEvent('Sorted signed headers alphabetically');
+    activeSpan?.addEvent('Joining signed headers using deliminator: ;');
     const signedHeaders = signedHeadersList.join(';');
+    activeSpan?.addEvent('Joined signed headers using ; as deliminator');
+    activeSpan.setAttributes({
+        'signedHeaders.request': pSignedHeaders,
+        'signedHeaders.request.authv4': signedHeaders,
+    });
+    signedHeadersSpan.setAttributes({
+        'signedHeaders.request': pSignedHeaders,
+        'signedHeaders.request.authv4': signedHeaders,
+        'code.url': 'https://github.com/scality/arsenal/blob/c6bb489adeb7419fdbcdf01db2b46a593747530d/lib/auth/v4/createCanonicalRequest.ts#L76',
+        'code.function': 'createCanonicalRequest',
+        'code.lineno': 76,
+        'code.filename': 'lib/auth/v4/createCanonicalRequest.ts',
+    });
+    signedHeadersSpan.end();
 
-    // canonical headers
+    const canonicalHeadersListSpan = tracer.startSpan('FormatHeadersToMatch CanonicalHeadersList');
     const canonicalHeadersList = signedHeadersList.map((signedHeader: any) => {
         if (pHeaders[signedHeader] !== undefined) {
             const trimmedHeader = pHeaders[signedHeader]
                 .trim().replace(/\s+/g, ' ');
             return `${signedHeader}:${trimmedHeader}\n`;
         }
-        // nginx will strip the actual expect header so add value of
-        // header back here if it was included as a signed header
         if (signedHeader === 'expect') {
             return `${signedHeader}:100-continue\n`;
         }
-        // handle case where signed 'header' is actually query param
         return `${signedHeader}:${pQuery[signedHeader]}\n`;
     });
+    canonicalHeadersListSpan.end();
 
+    const canonicalHeadersSpan = tracer.startSpan('JoinAllCanonicalHeaders using no deliminator');
     const canonicalHeaders = canonicalHeadersList.join('');
+    canonicalHeadersSpan.end();
 
+    const canonicalRequestSpan = tracer.startSpan('ConstructCanonicalRequest');
     const canonicalRequest = `${pHttpVerb}\n${canonicalURI}\n` +
         `${canonicalQueryStr}\n${canonicalHeaders}\n` +
         `${signedHeaders}\n${payloadChecksum}`;
+    canonicalRequestSpan.end();
     return canonicalRequest;
 }

lib/auth/v4/headerAuthCheck.ts

@@ -21,70 +21,78 @@ import {
  *      POST request
  * @param awsService - Aws service ('iam' or 's3')
  */
-export function check(
-    request: any,
-    log: Logger,
-    data: { [key: string]: string },
-    awsService: string
-) {
+export function check(request: any, log: Logger, data: { [key: string]: string }, awsService: string, oTel: any) {
+    const { activeSpan, extractParamsSpan, activeTracerContext, tracer } = oTel;
+    activeSpan?.addEvent('Entered V4 header auth check');
     log.trace('running header auth check');
-
+    activeSpan?.addEvent('Extracting security token');
     const token = request.headers['x-amz-security-token'];
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
-
-    // authorization header
+    activeSpan?.addEvent('Extracted security token');
+    activeSpan?.addEvent('Extracting authorization header');
     const authHeader = request.headers.authorization;
     if (!authHeader) {
         log.debug('missing authorization header');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
-
+     activeSpan?.addEvent('Extracted authorization header');
+    activeSpan?.addEvent('Extracting auth header items');
     const authHeaderItems = extractAuthItems(authHeader, log);
     if (Object.keys(authHeaderItems).length < 3) {
         log.debug('invalid authorization header', { authHeader });
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
-
+    activeSpan?.addEvent('Extracted auth header items');
     const payloadChecksum = request.headers['x-amz-content-sha256'];
     if (!payloadChecksum && awsService !== 'iam') {
         log.debug('missing payload checksum');
+        activeSpan.recordException(errors.MissingSecurityHeader);
+        extractParamsSpan.end();
         return { err: errors.MissingSecurityHeader };
     }
     if (payloadChecksum === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
         log.trace('requesting streaming v4 auth');
         if (request.method !== 'PUT') {
-            log.debug('streaming v4 auth for put only',
-                { method: 'auth/v4/headerAuthCheck.check' });
+            log.debug('streaming v4 auth for put only', { method: 'auth/v4/headerAuthCheck.check' });
+            activeSpan.recordException(errors.InvalidArgument);
+            extractParamsSpan.end();
             return { err: errors.InvalidArgument };
         }
         if (!request.headers['x-amz-decoded-content-length']) {
+            activeSpan.recordException(errors.MissingSecurityHeader);
+            extractParamsSpan.end();
             return { err: errors.MissingSecurityHeader };
         }
     }
-
     log.trace('authorization header from request', { authHeader });
-
     const signatureFromRequest = authHeaderItems.signatureFromRequest!;
     const credentialsArr = authHeaderItems.credentialsArr!;
     const signedHeaders = authHeaderItems.signedHeaders!;
-
+    activeSpan.addEvent('Checking if signed headers are complete');
     if (!areSignedHeadersComplete(signedHeaders, request.headers)) {
         log.debug('signedHeaders are incomplete', { signedHeaders });
+        activeSpan.recordException(errors.AccessDenied);
+        extractParamsSpan.end();
         return { err: errors.AccessDenied };
     }
-
+    activeSpan.addEvent('Signed headers are complete');
     let timestamp: string | undefined;
     // check request timestamp
+    activeSpan.addEvent('Checking request timestamp');
     const xAmzDate = request.headers['x-amz-date'];
     if (xAmzDate) {
         const xAmzDateArr = xAmzDate.split('T');
         // check that x-amz- date has the correct format and after epochTime
-        if (xAmzDateArr.length === 2 && xAmzDateArr[0].length === 8
-          && xAmzDateArr[1].length === 7
-          && Number.parseInt(xAmzDateArr[0], 10) > 19700101) {
+        if (xAmzDateArr.length === 2 && xAmzDateArr[0].length === 8 && xAmzDateArr[1].length === 7 && Number.parseInt(xAmzDateArr[0], 10) > 19700101) {
             // format of x-amz- date is ISO 8601: YYYYMMDDTHHMMSSZ
             timestamp = request.headers['x-amz-date'];
         }
@@ -92,27 +100,27 @@ export function check(
         timestamp = convertUTCtoISO8601(request.headers.date);
     }
     if (!timestamp) {
-        log.debug('missing or invalid date header',
-          { method: 'auth/v4/headerAuthCheck.check' });
-        return { err: errors.AccessDenied.
-          customizeDescription('Authentication requires a valid Date or ' +
-          'x-amz-date header') };
+        log.debug('missing or invalid date header', { method: 'auth/v4/headerAuthCheck.check' });
+        activeSpan.recordException(errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header'));
+        extractParamsSpan.end();
+        return { err: errors.AccessDenied.customizeDescription('Authentication requires a valid Date or x-amz-date header') };
     }
-
-    const validationResult = validateCredentials(credentialsArr, timestamp,
-      log);
+    activeSpan.addEvent('Request timestamp is valid');
+    activeSpan.addEvent('Validating credentials');
+    const validationResult = validateCredentials(credentialsArr, timestamp, log);
     if (validationResult instanceof Error) {
-        log.debug('credentials in improper format', { credentialsArr,
-            timestamp, validationResult });
+        log.debug('credentials in improper format', { credentialsArr, timestamp, validationResult });
+        activeSpan.recordException(validationResult);
+        extractParamsSpan.end();
         return { err: validationResult };
     }
+    activeSpan.addEvent('Credentials are valid');
     // credentialsArr is [accessKey, date, region, aws-service, aws4_request]
     const scopeDate = credentialsArr[1];
     const region = credentialsArr[2];
     const service = credentialsArr[3];
     const accessKey = credentialsArr.shift();
     const credentialScope = credentialsArr.join('/');
-
     // In AWS Signature Version 4, the signing key is valid for up to seven days
     // (see Introduction to Signing Requests.
     // Therefore, a signature is also valid for up to seven days or
@@ -124,14 +132,17 @@ export function check(
     // TODO: When implementing bucket policies,
     // note that expiration can be shortened so
     // expiry is as set out in the policy.
-
     // 15 minutes in seconds
+    activeSpan.addEvent('checking if signature is expired')
     const expiry = (15 * 60);
     const isTimeSkewed = checkTimeSkew(timestamp, expiry, log);
     if (isTimeSkewed) {
+        activeSpan.recordException(errors.RequestTimeTooSkewed);
+        extractParamsSpan.end();
         return { err: errors.RequestTimeTooSkewed };
     }
-
+    activeSpan.addEvent('signature is not expired');
+    activeSpan.addEvent('Constructing string to sign');
     const stringToSign = constructStringToSign({
         log,
         request,
@@ -141,13 +152,16 @@ export function check(
         timestamp,
         payloadChecksum,
         awsService: service,
-    });
+    }, oTel);
     log.trace('constructed stringToSign', { stringToSign });
     if (stringToSign instanceof Error) {
+        activeSpan.recordException(stringToSign);
+        extractParamsSpan.end();
         return { err: stringToSign };
     }
-
-
+    activeSpan.addEvent('Constructed string to sign v4 headers');
+    activeSpan.addEvent('Exiting V4 header auth check');
+    extractParamsSpan.end();
     return {
         err: null,
         params: {
@@ -162,8 +176,6 @@ export function check(
                 authType: 'REST-HEADER',
                 signatureVersion: 'AWS4-HMAC-SHA256',
                 signatureAge: Date.now() - convertAmzTimeToMs(timestamp),
-                // credentialScope and timestamp needed for streaming V4
-                // chunk evaluation
                 credentialScope,
                 timestamp,
                 securityToken: token,
@@ -171,3 +183,4 @@ export function check(
         },
     };
 }
+

lib/auth/v4/queryAuthCheck.ts

@@ -12,10 +12,20 @@ import { areSignedHeadersComplete } from './validateInputs';
  * @param log - logging object
  * @param data - Contain authentification params (GET or POST data)
  */
-export function check(request: any, log: Logger, data: { [key: string]: string }) {
+export function check(request: any, log: Logger, data: { [key: string]: string }, oTel: any) {
+    const {
+        activeSpan,
+        extractParamsSpan,
+        activeTracerContext,
+        tracer,
+    } = oTel;
+    activeSpan?.addEvent('Arsenal:: entered Arsenal.auth.v4.queryAuthCheck');
+    activeSpan?.addEvent('Arsenal:: extracting query parameters')
     const authParams = extractQueryParams(data, log);
-
+    activeSpan?.addEvent('Arsenal:: extracting query params');
     if (Object.keys(authParams).length !== 5) {
+        activeSpan.recordException(errors.InvalidArgument);
+        extractParamsSpan.end();
         return { err: errors.InvalidArgument };
     }
 
@@ -24,6 +34,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     const token = data['X-Amz-Security-Token'];
     if (token && !constants.iamSecurityToken.pattern.test(token)) {
         log.debug('invalid security token', { token });
+        activeSpan.recordException(errors.InvalidToken);
+        extractParamsSpan.end();
         return { err: errors.InvalidToken };
     }
 
@@ -35,6 +47,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
 
     if (!areSignedHeadersComplete(signedHeaders, request.headers)) {
         log.debug('signedHeaders are incomplete', { signedHeaders });
+        activeSpan.recordException(errors.AccessDenied);
+        extractParamsSpan.end();
         return { err: errors.AccessDenied };
     }
 
@@ -43,6 +57,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     if (validationResult instanceof Error) {
         log.debug('credentials in improper format', { credential,
             timestamp, validationResult });
+        activeSpan.recordException(validationResult);
+        extractParamsSpan.end();
         return { err: validationResult };
     }
     const accessKey = credential[0];
@@ -53,6 +69,8 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
 
     const isTimeSkewed = checkTimeSkew(timestamp, expiry, log);
     if (isTimeSkewed) {
+        activeSpan.recordException(errors.RequestTimeTooSkewed);
+        extractParamsSpan.end();
         return { err: errors.RequestTimeTooSkewed };
     }
 
@@ -71,6 +89,7 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
     // building string to sign
     const payloadChecksum = 'UNSIGNED-PAYLOAD';
 
+    activeSpan?.addEvent('Constructing string to sign');
     const stringToSign = constructStringToSign({
         log,
         request,
@@ -81,11 +100,16 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
         credentialScope:
             `${scopeDate}/${region}/${service}/${requestType}`,
         awsService: service,
-    });
+    }, oTel);
+    activeSpan?.addEvent('Constructed string to sign v4 query');
     if (stringToSign instanceof Error) {
+        activeSpan.recordException(stringToSign);
+        extractParamsSpan.end();
         return { err: stringToSign };
     }
     log.trace('constructed stringToSign', { stringToSign });
+    activeSpan.addEvent('Arsenal:: exiting Arsenal.auth.v4.queryAuthCheck');
+        extractParamsSpan.end();
     return {
         err: null,
         params: {

lib/s3routes/routes.ts

@@ -150,6 +150,7 @@ export type Params = {
     };
     unsupportedQueries: any;
     api: { callApiMethod: routesUtils.CallApiMethod };
+    oTel?: any,
 }
 
 /** routes - route request to appropriate method
@@ -180,111 +181,142 @@ export default function routes(
     logger: RequestLogger,
     s3config?: any,
 ) {
-    checkTypes(req, res, params, logger);
-
     const {
-        api,
-        internalHandlers,
-        statsClient,
-        allEndpoints,
-        websiteEndpoints,
-        blacklistedPrefixes,
-        dataRetrievalParams,
+        dataRetrievalParams: {
+            oTel: {
+                tracer,
+                activeSpan,
+                activeTracerContext,
+            }
+        } 
     } = params;
+    return tracer.startActiveSpan('Validating Request Parameters with Arsenal', undefined, activeTracerContext, arsenalValidatorSpan => {
+        arsenalValidatorSpan.setAttributes({
+            'code.lineno': 176,
+            'code.filename': 'lib/s3routes/routes.ts',
+            'code.function': 'routes()',
+            'code.url': 'https://github.com/scality/Arsenal/blob/development/7.70/lib/s3routes/routes.ts#L176'
+        });
+        checkTypes(req, res, params, logger);
 
-    const clientInfo = {
-        clientIP: requestUtils.getClientIp(req, s3config),
-        clientPort: req.socket.remotePort,
-        httpMethod: req.method,
-        httpURL: req.url,
-        // @ts-ignore
-        endpoint: req.endpoint,
-    };
+        const {
+            api,
+            internalHandlers,
+            statsClient,
+            allEndpoints,
+            websiteEndpoints,
+            blacklistedPrefixes,
+            dataRetrievalParams,
+        } = params;
 
-    let reqUids = req.headers['x-scal-request-uids'];
-    if (reqUids !== undefined && !isValidReqUids(reqUids)) {
-        // simply ignore invalid id (any user can provide an
-        // invalid request ID through a crafted header)
-        reqUids = undefined;
-    }
-    const log = (reqUids !== undefined ?
-        // @ts-ignore
-        logger.newRequestLoggerFromSerializedUids(reqUids) :
-        // @ts-ignore
-        logger.newRequestLogger());
+        const clientInfo = {
+            clientIP: requestUtils.getClientIp(req, s3config),
+            clientPort: req.socket.remotePort,
+            httpMethod: req.method,
+            httpURL: req.url,
+            // @ts-ignore
+            endpoint: req.endpoint,
+        };
 
-    if (!req.url!.startsWith('/_/healthcheck')) {
-        log.info('received request', clientInfo);
-    }
-
-    log.end().addDefaultFields(clientInfo);
-
-    if (req.url!.startsWith('/_/')) {
-        let internalServiceName = req.url!.slice(3);
-        const serviceDelim = internalServiceName.indexOf('/');
-        if (serviceDelim !== -1) {
-            internalServiceName = internalServiceName.slice(0, serviceDelim);
+        let reqUids = req.headers['x-scal-request-uids'];
+        if (reqUids !== undefined && !isValidReqUids(reqUids)) {
+            // simply ignore invalid id (any user can provide an
+            // invalid request ID through a crafted header)
+            reqUids = undefined;
         }
-        if (internalHandlers[internalServiceName] === undefined) {
+        const log = (reqUids !== undefined ?
+            // @ts-ignore
+            logger.newRequestLoggerFromSerializedUids(reqUids) :
+            // @ts-ignore
+            logger.newRequestLogger());
+
+        if (!req.url!.startsWith('/_/healthcheck')) {
+            log.info('received request', clientInfo);
+        }
+
+        log.end().addDefaultFields(clientInfo);
+
+        if (req.url!.startsWith('/_/')) {
+            let internalServiceName = req.url!.slice(3);
+            const serviceDelim = internalServiceName.indexOf('/');
+            if (serviceDelim !== -1) {
+                internalServiceName = internalServiceName.slice(0, serviceDelim);
+            }
+            if (internalHandlers[internalServiceName] === undefined) {
+                activeSpan.recordException(errors.InvalidURI);
+                return routesUtils.responseXMLBody(
+                    errors.InvalidURI, null, res, log);
+            }
+            return internalHandlers[internalServiceName](
+                clientInfo.clientIP, req, res, log, statsClient);
+        }
+
+        if (statsClient) {
+            // report new request for stats
+            statsClient.reportNewRequest('s3');
+        }
+
+        try {
+            const validHosts = allEndpoints.concat(websiteEndpoints);
+            routesUtils.normalizeRequest(req, validHosts);
+        } catch (err: any) {
+            activeSpan.recordException(errors.InvalidURI.customizeDescription('Could not parse the ' +
+                'specified URI. Check your restEndpoints configuration.'));
+            log.debug('could not normalize request', { error: err.stack });
             return routesUtils.responseXMLBody(
-                errors.InvalidURI, null, res, log);
+                errors.InvalidURI.customizeDescription('Could not parse the ' +
+                    'specified URI. Check your restEndpoints configuration.'),
+                null, res, log);
         }
-        return internalHandlers[internalServiceName](
-            clientInfo.clientIP, req, res, log, statsClient);
-    }
 
-    if (statsClient) {
-        // report new request for stats
-        statsClient.reportNewRequest('s3');
-    }
+        log.addDefaultFields({
+            // @ts-ignore
+            bucketName: req.bucketName,
+            // @ts-ignore
+            objectKey: req.objectKey,
+            // @ts-ignore
+            bytesReceived: req.parsedContentLength || 0,
+            // @ts-ignore
+            bodyLength: parseInt(req.headers['content-length'], 10) || 0,
+        });
+        activeSpan.setAttributes({
+            // @ts-ignore
+            'aws.s3.bucket': req.bucketName,
+            // @ts-ignore
+            'aws.s3.key': req.objectKey,
+            // @ts-ignore
+            'aws.request_id': log.getUids().join(':'),
+            'rpc.service': 'S3',
+        });
 
-    try {
-        const validHosts = allEndpoints.concat(websiteEndpoints);
-        routesUtils.normalizeRequest(req, validHosts);
-    } catch (err: any) {
-        log.debug('could not normalize request', { error: err.stack });
-        return routesUtils.responseXMLBody(
-            errors.InvalidURI.customizeDescription('Could not parse the ' +
-                'specified URI. Check your restEndpoints configuration.'),
-            null, res, log);
-    }
+        // @ts-ignore
+        const { error, method } = checkUnsupportedRoutes(req.method, req.query);
+
+        if (error) {
+            activeSpan.recordException(error);
+            log.trace('error validating route or uri params', { error });
+            // @ts-ignore
+            return routesUtils.responseXMLBody(error, '', res, log);
+        }
 
-    log.addDefaultFields({
         // @ts-ignore
-        bucketName: req.bucketName,
+        const bucketOrKeyError = checkBucketAndKey(req.bucketName, req.objectKey,
+            // @ts-ignore
+            req.method, req.query, blacklistedPrefixes, log);
+
+        if (bucketOrKeyError) {
+            activeSpan.recordException(bucketOrKeyError)
+            log.trace('error with bucket or key value',
+                { error: bucketOrKeyError });
+            return routesUtils.responseXMLBody(bucketOrKeyError, null, res, log);
+        }
+
+        // bucket website request
         // @ts-ignore
-        objectKey: req.objectKey,
-        // @ts-ignore
-        bytesReceived: req.parsedContentLength || 0,
-        // @ts-ignore
-        bodyLength: parseInt(req.headers['content-length'], 10) || 0,
+        if (websiteEndpoints && websiteEndpoints.indexOf(req.parsedHost) > -1) {
+            return routeWebsite(req, res, api, log, statsClient, dataRetrievalParams);
+        }
+        arsenalValidatorSpan.end();
+        return method(req, res, api, log, statsClient, dataRetrievalParams);
     });
-
-    // @ts-ignore
-    const { error, method } = checkUnsupportedRoutes(req.method, req.query);
-
-    if (error) {
-        log.trace('error validating route or uri params', { error });
-        // @ts-ignore
-        return routesUtils.responseXMLBody(error, '', res, log);
-    }
-
-    // @ts-ignore
-    const bucketOrKeyError = checkBucketAndKey(req.bucketName, req.objectKey,
-        // @ts-ignore
-        req.method, req.query, blacklistedPrefixes, log);
-
-    if (bucketOrKeyError) {
-        log.trace('error with bucket or key value',
-            { error: bucketOrKeyError });
-        return routesUtils.responseXMLBody(bucketOrKeyError, null, res, log);
-    }
-
-    // bucket website request
-    // @ts-ignore
-    if (websiteEndpoints && websiteEndpoints.indexOf(req.parsedHost) > -1) {
-        return routeWebsite(req, res, api, log, statsClient, dataRetrievalParams);
-    }
-
-    return method(req, res, api, log, statsClient, dataRetrievalParams);
 }

lib/s3routes/routes/routeDELETE.ts

@@ -4,6 +4,7 @@ import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import StatsClient from '../../metrics/StatsClient';
 import * as http from 'http';
+import { actionMonitoringMapS3 } from '../../policyEvaluator/utils/actionMaps';
 
 export default function routeDELETE(
     request: http.IncomingMessage,
@@ -11,58 +12,146 @@ export default function routeDELETE(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
-    const call = (name: string) => {
-        return api.callApiMethod(name, request, response, log, (err, corsHeaders) => {
-            routesUtils.statsReport500(err, statsClient);
-            return routesUtils.responseNoBody(err, corsHeaders, response, 204, log);
-        });
-    }
-    log.debug('routing request', { method: 'routeDELETE' });
-
-    const { query, objectKey } = request as any
-    if (query?.uploadId) {
-        if (objectKey === undefined) {
-            const message = 'A key must be specified';
-            const err = errors.InvalidRequest.customizeDescription(message);
-            return routesUtils.responseNoBody(err, null, response, 200, log);
-        }
-        return call('multipartDelete');
-    } else if (objectKey === undefined) {
-        if (query?.website !== undefined) {
-            return call('bucketDeleteWebsite');
-        } else if (query?.cors !== undefined) {
-            return call('bucketDeleteCors');
-        } else if (query?.replication !== undefined) {
-            return call('bucketDeleteReplication');
-        } else if (query?.lifecycle !== undefined) {
-            return call('bucketDeleteLifecycle');
-        } else if (query?.policy !== undefined) {
-            return call('bucketDeletePolicy');
-        } else if (query?.encryption !== undefined) {
-            return call('bucketDeleteEncryption');
-        } else if (query?.tagging !== undefined) {
-            return call('bucketDeleteTagging');
-        }
-        call('bucketDelete');
-    } else {
-        if (query?.tagging !== undefined) {
-            return call('objectDeleteTagging');
-        }
-        api.callApiMethod('objectDelete', request, response, log,
-            (err, corsHeaders) => {
-                /*
-              * Since AWS expects a 204 regardless of the existence of
-              the object, the errors NoSuchKey and NoSuchVersion should not
-              * be sent back as a response.
-              */
-                if (err && !err.is.NoSuchKey && !err.is.NoSuchVersion) {
-                    return routesUtils.responseNoBody(err, corsHeaders,
-                        response, undefined, log);
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Delete API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan  => {
+        activeSpan.addEvent('Request validated, routing request using routeDELETE() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeDELETE.ts',
+            'code.function': 'routeDELETE()',
+        })
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeDELETE()')
+        const call = (name: string) => {
+            return api.callApiMethod(name, request, response, log, (err, corsHeaders) => {
+                cloudserverApiSpan.end();
+                const action = actionMonitoringMapS3[name];
+                activeSpan.addEvent(`${action} API operation complete`);
+                if (err) {
+                    activeSpan.recordException(err);
                 }
                 routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(null, corsHeaders, response,
-                    204, log);
+                activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                return routesUtils.responseNoBody(err, corsHeaders, response, 204, log);
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
             });
-    }
+        }
+        log.debug('routing request', { method: 'routeDELETE' });
+    
+        const { query, objectKey } = request as any
+        if (query?.uploadId) {
+            // @ts-ignore
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent('Detected AbortMultipartUpload API request');
+            activeSpan.setAttribute('rpc.method', 'AbortMultipartUpload');
+            if (objectKey === undefined) {
+                const message = 'A key must be specified';
+                const err = errors.InvalidRequest.customizeDescription(message);
+                activeSpan.recordException(err);
+                cloudserverApiSpan.end();
+                return routesUtils.responseNoBody(err, null, response, 200, log);
+            }
+            return call('multipartDelete');
+        } else if (objectKey === undefined) {
+            if (query?.website !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketWebsite API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketWebsite');
+                return call('bucketDeleteWebsite');
+            } else if (query?.cors !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketCors API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketCors');
+                return call('bucketDeleteCors');
+            } else if (query?.replication !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketReplication API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketReplication');
+                return call('bucketDeleteReplication');
+            } else if (query?.lifecycle !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketLifecycle API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketLifecycle');
+                return call('bucketDeleteLifecycle');
+            } else if (query?.policy !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketPolicy API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketPolicy');
+                return call('bucketDeletePolicy');
+            } else if (query?.encryption !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketEncryption API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketEncryption');
+                return call('bucketDeleteEncryption');
+            } else if (query?.tagging !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteBucketTagging API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteBucketTagging');
+                return call('bucketDeleteTagging');
+            }
+            // @ts-ignore
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent('Detected DeleteBucket API request');
+            activeSpan.setAttribute('rpc.method', 'DeleteBucket');
+            call('bucketDelete');
+        } else {
+            if (query?.tagging !== undefined) {
+                // @ts-ignore
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected DeleteObjectTagging API request');
+                activeSpan.setAttribute('rpc.method', 'DeleteObjectTagging');
+                return call('objectDeleteTagging');
+            }
+            // @ts-ignore
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent('Detected DeleteObject API request');
+            activeSpan.setAttribute('rpc.method', 'DeleteObject');
+            return api.callApiMethod('objectDelete', request, response, log,
+                (err, corsHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('DeleteObject API operation complete')
+                    /*
+                  * Since AWS expects a 204 regardless of the existence of
+                  the object, the errors NoSuchKey and NoSuchVersion should not
+                  * be sent back as a response.
+                  */
+                    if (err && !err.is.NoSuchKey && !err.is.NoSuchVersion) {
+                        activeSpan.recordException(err);
+                        cloudserverApiSpan.end();
+                        return routesUtils.responseNoBody(err, corsHeaders,
+                            response, undefined, log);
+                    }
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
+                    routesUtils.statsReport500(err, statsClient);
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                    return routesUtils.responseNoBody(null, corsHeaders, response,
+                        204, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
+                });
+        }
+    });
 }

lib/s3routes/routes/routeGET.ts

@@ -4,6 +4,7 @@ import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';
 import StatsClient from '../../metrics/StatsClient';
+import { actionMonitoringMapS3 } from '../../policyEvaluator/utils/actionMaps';
 
 export default function routerGET(
     request: http.IncomingMessage,
@@ -13,84 +14,128 @@ export default function routerGET(
     statsClient?: StatsClient,
     dataRetrievalParams?: any,
 ) {
-    log.debug('routing request', { method: 'routerGET' });
-
-    const { bucketName, objectKey, query } = request as any
-
-    const call = (name: string) => {
-        api.callApiMethod(name, request, response, log, (err, xml, corsHeaders) => {
-            routesUtils.statsReport500(err, statsClient);
-            return routesUtils.responseXMLBody(err, xml, response, log, corsHeaders);
-        });
-    }
-
-    if (bucketName === undefined && objectKey !== undefined) {
-        routesUtils.responseXMLBody(errors.NoSuchBucket, null, response, log);
-    } else if (bucketName === undefined && objectKey === undefined) {
-        // GET service
-        call('serviceGet');
-    } else if (objectKey === undefined) {
-        // GET bucket ACL
-        if (query.acl !== undefined) {
-            call('bucketGetACL');
-        } else if (query.replication !== undefined) {
-            call('bucketGetReplication');
-        } else if (query.cors !== undefined) {
-            call('bucketGetCors');
-        } else if (query.versioning !== undefined) {
-            call('bucketGetVersioning');
-        } else if (query.website !== undefined) {
-            call('bucketGetWebsite');
-        } else if (query.tagging !== undefined) {
-            call('bucketGetTagging');
-        } else if (query.lifecycle !== undefined) {
-            call('bucketGetLifecycle');
-        } else if (query.uploads !== undefined) {
-            // List MultipartUploads
-            call('listMultipartUploads');
-        } else if (query.location !== undefined) {
-            call('bucketGetLocation');
-        } else if (query.policy !== undefined) {
-            call('bucketGetPolicy');
-        } else if (query['object-lock'] !== undefined) {
-            call('bucketGetObjectLock');
-        } else if (query.notification !== undefined) {
-            call('bucketGetNotification');
-        } else if (query.encryption !== undefined) {
-            call('bucketGetEncryption');
-        } else {
-            // GET bucket
-            call('bucketGet');
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
         }
-    } else {
-        if (query.acl !== undefined) {
-            // GET object ACL
-            call('objectGetACL');
-        } else if (query['legal-hold'] !== undefined) {
-            call('objectGetLegalHold');
-        } else if (query.tagging !== undefined) {
-            call('objectGetTagging');
-            // List parts of an open multipart upload
-        } else if (query.uploadId !== undefined) {
-            call('listParts');
-        } else if (query.retention !== undefined) {
-            call('objectGetRetention');
-        } else {
-            // GET object
-            api.callApiMethod('objectGet', request, response, log,
-                (err, dataGetInfo, resMetaHeaders, range) => {
-                    let contentLength = 0;
-                    if (resMetaHeaders && resMetaHeaders['Content-Length']) {
-                        contentLength = resMetaHeaders['Content-Length'];
-                    }
-                    // TODO ARSN-216 Fix logger
-                    // @ts-ignore
-                    log.end().addDefaultFields({ contentLength });
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseStreamData(err, query,
-                        resMetaHeaders, dataGetInfo, dataRetrievalParams, response,
-                        range, log);
-                });
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Get API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routeGET() in arsenal');
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeGET()')
+        log.debug('routing request', { method: 'routerGET' });
+
+        const { bucketName, objectKey, query } = request as any
+
+        const call = (name: string) => {
+            const action = actionMonitoringMapS3[name];
+            // @ts-ignore
+            activeSpan.updateName(`S3 API request`);
+            activeSpan.addEvent(`Detected ${action} API request`);
+            activeSpan.setAttribute('rpc.method', action);
+            return api.callApiMethod(name, request, response, log, (err, xml, corsHeaders) => {
+                cloudserverApiSpan.end();
+                activeSpan.addEvent(`${action} API operation complete`);
+                if (err) {
+                    activeSpan.recordException(err);
+                }
+                routesUtils.statsReport500(err, statsClient);
+                activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                return routesUtils.responseXMLBody(err, xml, response, log, corsHeaders);
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
         }
-    }
+
+        if (bucketName === undefined && objectKey !== undefined) {
+            activeSpan.recordException(errors.NoSuchBucket);
+            cloudserverApiSpan.end();
+            return routesUtils.responseXMLBody(errors.NoSuchBucket, null, response, log);
+        } else if (bucketName === undefined && objectKey === undefined) {
+            // GET service
+
+            call('serviceGet');
+        } else if (objectKey === undefined) {
+            // GET bucket ACL
+            if (query.acl !== undefined) {
+                call('bucketGetACL');
+            } else if (query.replication !== undefined) {
+                call('bucketGetReplication');
+            } else if (query.cors !== undefined) {
+                call('bucketGetCors');
+            } else if (query.versioning !== undefined) {
+                call('bucketGetVersioning');
+            } else if (query.website !== undefined) {
+                call('bucketGetWebsite');
+            } else if (query.tagging !== undefined) {
+                call('bucketGetTagging');
+            } else if (query.lifecycle !== undefined) {
+                call('bucketGetLifecycle');
+            } else if (query.uploads !== undefined) {
+                // List MultipartUploads
+                call('listMultipartUploads');
+            } else if (query.location !== undefined) {
+                call('bucketGetLocation');
+            } else if (query.policy !== undefined) {
+                call('bucketGetPolicy');
+            } else if (query['object-lock'] !== undefined) {
+                call('bucketGetObjectLock');
+            } else if (query.notification !== undefined) {
+                call('bucketGetNotification');
+            } else if (query.encryption !== undefined) {
+                call('bucketGetEncryption');
+            } else {
+                // GET bucket
+                call('bucketGet');
+            }
+        } else {
+            if (query.acl !== undefined) {
+                // GET object ACL
+                call('objectGetACL');
+            } else if (query['legal-hold'] !== undefined) {
+                call('objectGetLegalHold');
+            } else if (query.tagging !== undefined) {
+                call('objectGetTagging');
+                // List parts of an open multipart upload
+            } else if (query.uploadId !== undefined) {
+                call('listParts');
+            } else if (query.retention !== undefined) {
+                call('objectGetRetention');
+            } else {
+                // GET object
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected GetObject API request');
+                activeSpan.setAttribute('aws.request_id', log.getUids()[0]);
+                activeSpan.setAttribute('rpc.method', 'GetObject');
+                return api.callApiMethod('objectGet', request, response, log,
+                    (err, dataGetInfo, resMetaHeaders, range) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('Located Data, using arsenal to make GET request')
+                        let contentLength = 0;
+                        if (resMetaHeaders && resMetaHeaders['Content-Length']) {
+                            contentLength = resMetaHeaders['Content-Length'];
+                        }
+                        // TODO ARSN-216 Fix logger
+                        // @ts-ignore
+                        log.end().addDefaultFields({ contentLength });
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        return routesUtils.responseStreamData(err, query,
+                            resMetaHeaders, dataGetInfo, dataRetrievalParams, response,
+                            range, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            }
+        }
+    });
 }

lib/s3routes/routes/routeHEAD.ts

@@ -11,28 +11,75 @@ export default function routeHEAD(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
-    log.debug('routing request', { method: 'routeHEAD' });
-    const { bucketName, objectKey } = request as any
-    if (bucketName === undefined) {
-        log.trace('head request without bucketName');
-        routesUtils.responseXMLBody(errors.MethodNotAllowed,
-            null, response, log);
-    } else if (objectKey === undefined) {
-        // HEAD bucket
-        api.callApiMethod('bucketHead', request, response, log,
-            (err, corsHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseNoBody(err, corsHeaders, response,
-                    200, log);
-            });
-    } else {
-        // HEAD object
-        api.callApiMethod('objectHead', request, response, log,
-            (err, resHeaders) => {
-                routesUtils.statsReport500(err, statsClient);
-                return routesUtils.responseContentHeaders(err, {}, resHeaders,
-                    response, log);
-            });
-    }
+    const{
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Head API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routeHEAD() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeHEAD.ts',
+            'code.function': 'routeHEAD()',
+        });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeHEAD()');
+        log.debug('routing request', { method: 'routeHEAD' });
+        const { bucketName, objectKey } = request as any
+        if (bucketName === undefined) {
+            log.trace('head request without bucketName');
+            activeSpan.recordException(errors.MethodNotAllowed);
+            cloudserverApiSpan.end();
+            routesUtils.responseXMLBody(errors.MethodNotAllowed,
+                null, response, log);
+        } else if (objectKey === undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected HeadBucket API request`);
+            activeSpan.setAttribute('rpc.method', 'HeadBucket');
+            // HEAD bucket
+            api.callApiMethod('bucketHead', request, response, log,
+                (err, corsHeaders) => {
+                    activeSpan.addEvent('HeadBucket API operation complete')
+                    cloudserverApiSpan.end();
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
+                    routesUtils.statsReport500(err, statsClient);
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                    return routesUtils.responseNoBody(err, corsHeaders, response,
+                        200, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
+                });
+            } else {
+            // HEAD object
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected HeadObject API request`);
+            activeSpan.setAttribute('rpc.method', 'HeadObject');
+            api.callApiMethod('objectHead', request, response, log,
+                (err, resHeaders) => {
+                    activeSpan.addEvent('HeadObject API operation complete')
+                    cloudserverApiSpan.end();
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
+                    routesUtils.statsReport500(err, statsClient);
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                    return routesUtils.responseContentHeaders(err, {}, resHeaders,
+                        response, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
+                });
+        }
+    });
 }

lib/s3routes/routes/routeOPTIONS.ts

@@ -11,29 +11,64 @@ export default function routeOPTIONS(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
-    log.debug('routing request', { method: 'routeOPTION' });
-
-    const corsMethod = request.headers['access-control-request-method'] || null;
-
-    if (!request.headers.origin) {
-        const msg = 'Insufficient information. Origin request header needed.';
-        const err = errors.BadRequest.customizeDescription(msg);
-        log.debug('missing origin', { method: 'routeOPTIONS', error: err });
-        return routesUtils.responseXMLBody(err, null, response, log);
-    }
-    if (['GET', 'PUT', 'HEAD', 'POST', 'DELETE'].indexOf(corsMethod ?? '') < 0) {
-        const msg = `Invalid Access-Control-Request-Method: ${corsMethod}`;
-        const err = errors.BadRequest.customizeDescription(msg);
-        log.debug('invalid Access-Control-Request-Method',
-            { method: 'routeOPTIONS', error: err });
-        return routesUtils.responseXMLBody(err, null, response, log);
-    }
-
-    return api.callApiMethod('corsPreflight', request, response, log,
-        (err, resHeaders) => {
-            routesUtils.statsReport500(err, statsClient);
-            return routesUtils.responseNoBody(err, resHeaders, response, 200,
-                log);
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing corsPreflight API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routeOPTIONS() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeOPTIONS.ts',
+            'code.function': 'routeOPTIONS()',
         });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routeOPTIONS()');
+        log.debug('routing request', { method: 'routeOPTION' });
+
+        const corsMethod = request.headers['access-control-request-method'] || null;
+
+        if (!request.headers.origin) {
+            const msg = 'Insufficient information. Origin request header needed.';
+            const err = errors.BadRequest.customizeDescription(msg);
+            activeSpan.recordException(err);
+            cloudserverApiSpan.end();
+            log.debug('missing origin', { method: 'routeOPTIONS', error: err });
+            return routesUtils.responseXMLBody(err, null, response, log);
+        }
+        if (['GET', 'PUT', 'HEAD', 'POST', 'DELETE'].indexOf(corsMethod ?? '') < 0) {
+            const msg = `Invalid Access-Control-Request-Method: ${corsMethod}`;
+            const err = errors.BadRequest.customizeDescription(msg);
+            activeSpan.recordException(err);
+            cloudserverApiSpan.end();
+            log.debug('invalid Access-Control-Request-Method',
+                { method: 'routeOPTIONS', error: err });
+            return routesUtils.responseXMLBody(err, null, response, log);
+        }
+        // @ts-ignore
+        activeSpan.updateName('S3 API request');
+        activeSpan.addEvent(`Detected corsPreflight API request`);
+        activeSpan.setAttribute('rpc.method', 'corsPreflight');
+        return api.callApiMethod('corsPreflight', request, response, log,
+            (err, resHeaders) => {
+                activeSpan.addEvent('corsPreflight API operation complete');
+                cloudserverApiSpan.end();
+                if (err) {
+                    activeSpan.recordException(err);
+                }
+                routesUtils.statsReport500(err, statsClient);
+                activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                return routesUtils.responseNoBody(err, resHeaders, response, 200,
+                    log);
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
+    });
 }

lib/s3routes/routes/routePOST.ts

@@ -3,6 +3,7 @@ import { RequestLogger } from 'werelogs';
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';
+import StatsClient from '../../metrics/StatsClient';
 
 /* eslint-disable no-param-reassign */
 export default function routePOST(
@@ -10,54 +11,115 @@ export default function routePOST(
     response: http.ServerResponse,
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
+    statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
-    log.debug('routing request', { method: 'routePOST' });
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Post API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routePOST() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 9,
+            'code.filename': 'lib/s3routes/routes/routePOST.ts',
+            'code.function': 'routePOST()',
+        });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routePOST()');
+        log.debug('routing request', { method: 'routePOST' });
 
-    const { query, bucketName, objectKey } = request as any
+        const { query, bucketName, objectKey } = request as any
 
-    const invalidMultiObjectDelReq = query.delete !== undefined
-        && bucketName === undefined;
-    if (invalidMultiObjectDelReq) {
-        return routesUtils.responseNoBody(errors.MethodNotAllowed, null,
-            response, undefined, log);
-    }
+        const invalidMultiObjectDelReq = query.delete !== undefined
+            && bucketName === undefined;
+        if (invalidMultiObjectDelReq) {
+            activeSpan.recordException(errors.MethodNotAllowed);
+            cloudserverApiSpan.end();
+            return routesUtils.responseNoBody(errors.MethodNotAllowed, null,
+                response, undefined, log);
+        }
 
-    // @ts-ignore
-    request.post = '';
+        // @ts-ignore
+        request.post = '';
 
-    const invalidInitiateMpuReq = query.uploads !== undefined
-        && objectKey === undefined;
-    const invalidCompleteMpuReq = query.uploadId !== undefined
-        && objectKey === undefined;
-    if (invalidInitiateMpuReq || invalidCompleteMpuReq) {
-        return routesUtils.responseNoBody(errors.InvalidURI, null,
-            response, undefined, log);
-    }
+        const invalidInitiateMpuReq = query.uploads !== undefined
+            && objectKey === undefined;
+        const invalidCompleteMpuReq = query.uploadId !== undefined
+            && objectKey === undefined;
+        if (invalidInitiateMpuReq || invalidCompleteMpuReq) {
+            activeSpan.recordException(errors.InvalidURI);
+            cloudserverApiSpan.end();
+            return routesUtils.responseNoBody(errors.InvalidURI, null,
+                response, undefined, log);
+        }
 
-    // POST initiate multipart upload
-    if (query.uploads !== undefined) {
-        return api.callApiMethod('initiateMultipartUpload', request,
-            response, log, (err, result, corsHeaders) =>
-                routesUtils.responseXMLBody(err, result, response, log,
-                    corsHeaders));
-    }
+        // POST initiate multipart upload
+        if (query.uploads !== undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected CreateMultipartUpload API request`);
+            activeSpan.setAttribute('rpc.method', 'CreateMultipartUpload');
+            return api.callApiMethod('initiateMultipartUpload', request,
+                response, log, (err, result, corsHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('CreateMultipartUpload API operation complete');
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                    routesUtils.responseXMLBody(err, result, response, log,
+                        corsHeaders)
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
+        }
 
-    // POST complete multipart upload
-    if (query.uploadId !== undefined) {
-        return api.callApiMethod('completeMultipartUpload', request,
-            response, log, (err, result, resHeaders) =>
-                routesUtils.responseXMLBody(err, result, response, log,
-                    resHeaders));
-    }
+        // POST complete multipart upload
+        if (query.uploadId !== undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected CompleteMultipartUpload API request`);
+            activeSpan.setAttribute('rpc.method', 'CompleteMultipartUpload');
+            return api.callApiMethod('completeMultipartUpload', request,
+                response, log, (err, result, resHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('CompleteMultipartUpload API operation complete');
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                    routesUtils.responseXMLBody(err, result, response, log,
+                        resHeaders)
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
+        }
 
-    // POST multiObjectDelete
-    if (query.delete !== undefined) {
-        return api.callApiMethod('multiObjectDelete', request, response,
-            log, (err, xml, corsHeaders) =>
-                routesUtils.responseXMLBody(err, xml, response, log,
-                    corsHeaders));
-    }
-
-    return routesUtils.responseNoBody(errors.NotImplemented, null, response,
-        200, log);
+        // POST multiObjectDelete
+        if (query.delete !== undefined) {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected AbortMultipartUpload API request`);
+            activeSpan.setAttribute('rpc.method', 'AbortMultipartUpload');
+            return api.callApiMethod('multiObjectDelete', request, response,
+                log, (err, xml, corsHeaders) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('AbortMultipartUpload API operation complete');
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                    routesUtils.responseXMLBody(err, xml, response, log,
+                        corsHeaders)
+            }, {
+                cloudserverApiSpan,
+                activeSpan,
+                activeTracerContext,
+                tracer,
+            });
+        }
+        activeSpan.recordException(errors.NotImplemented);
+        cloudserverApiSpan.end();
+        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+        return routesUtils.responseNoBody(errors.NotImplemented, null, response,
+            200, log);
+    });
+    
 }

lib/s3routes/routes/routePUT.ts

@@ -11,223 +11,528 @@ export default function routePUT(
     api: { callApiMethod: routesUtils.CallApiMethod },
     log: RequestLogger,
     statsClient?: StatsClient,
+    dataRetrievalParams?: any,
 ) {
-    log.debug('routing request', { method: 'routePUT' });
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    } = dataRetrievalParams;
+    return tracer.startActiveSpan('Arsenal:: Performing Put API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan  => {
+        activeSpan.addEvent('Request validated, routing request using routePUT() in arsenal')
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routePUT.ts',
+            'code.function': 'routePUT()',
+        })
+        activeSpan.addEvent('Detecting which API to route to using arsenal routePUT()')
+        log.debug('routing request', { method: 'routePUT' });
 
-    const { objectKey, query, bucketName, parsedContentLength } = request as any
+        const { objectKey, query, bucketName, parsedContentLength } = request as any
 
-    if (objectKey === undefined) {
-        // PUT bucket - PUT bucket ACL
+        if (objectKey === undefined) {
+            // PUT bucket - PUT bucket ACL
 
-        // content-length for object is handled separately below
-        const contentLength = request.headers['content-length'];
-        const len = Number(contentLength);
-        if ((contentLength && (Number.isNaN(len) || len < 0)) || contentLength === '') {
-            log.debug('invalid content-length header');
-            return routesUtils.responseNoBody(
-                errors.BadRequest, null, response, undefined, log);
-        }
-        // PUT bucket ACL
-        if (query.acl !== undefined) {
-            api.callApiMethod('bucketPutACL', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, corsHeaders,
-                        response, 200, log);
-                });
-        } else if (query.versioning !== undefined) {
-            api.callApiMethod('bucketPutVersioning', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    routesUtils.responseNoBody(err, corsHeaders, response, 200,
-                        log);
-                });
-        } else if (query.website !== undefined) {
-            api.callApiMethod('bucketPutWebsite', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, corsHeaders,
-                        response, 200, log);
-                });
-        } else if (query.tagging !== undefined) {
-            api.callApiMethod('bucketPutTagging', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, corsHeaders,
-                        response, 200, log);
-                });
-        } else if (query.cors !== undefined) {
-            api.callApiMethod('bucketPutCors', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, corsHeaders,
-                        response, 200, log);
-                });
-        } else if (query.replication !== undefined) {
-            api.callApiMethod('bucketPutReplication', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    routesUtils.responseNoBody(err, corsHeaders, response, 200,
-                        log);
-                });
-        } else if (query.lifecycle !== undefined) {
-            api.callApiMethod('bucketPutLifecycle', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    routesUtils.responseNoBody(err, corsHeaders, response, 200,
-                        log);
-                });
-        } else if (query.policy !== undefined) {
-            api.callApiMethod('bucketPutPolicy', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    routesUtils.responseNoBody(err, corsHeaders, response, 200,
-                        log);
-                });
-        } else if (query['object-lock'] !== undefined) {
-            api.callApiMethod('bucketPutObjectLock', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    routesUtils.responseNoBody(err, corsHeaders, response, 200,
-                        log);
-                });
-        } else if (query.notification !== undefined) {
-            api.callApiMethod('bucketPutNotification', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    routesUtils.responseNoBody(err, corsHeaders, response, 200,
-                        log);
-                });
-        } else if (query.encryption !== undefined) {
-            api.callApiMethod('bucketPutEncryption', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, corsHeaders,
-                        response, 200, log);
-                });
+            // content-length for object is handled separately below
+            const contentLength = request.headers['content-length'];
+            const len = Number(contentLength);
+            if ((contentLength && (Number.isNaN(len) || len < 0)) || contentLength === '') {
+                log.debug('invalid content-length header');
+                return routesUtils.responseNoBody(
+                    errors.BadRequest, null, response, undefined, log);
+            }
+            // PUT bucket ACL
+            if (query.acl !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketAcl API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketAcl');
+                return api.callApiMethod('bucketPutACL', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketAcl API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, corsHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.versioning !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketVersioning API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketVersioning');
+                return api.callApiMethod('bucketPutVersioning', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketVersioning API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        routesUtils.responseNoBody(err, corsHeaders, response, 200,
+                            log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.website !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketWebsite API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketWebsite');
+                return api.callApiMethod('bucketPutWebsite', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketWebsite API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, corsHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.tagging !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketTagging API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketTagging');
+                return api.callApiMethod('bucketPutTagging', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketTagging API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, corsHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.cors !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketCors API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketCors');
+                return api.callApiMethod('bucketPutCors', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketCors API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, corsHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.replication !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketReplication API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketReplication');
+                return api.callApiMethod('bucketPutReplication', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketReplication API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        routesUtils.responseNoBody(err, corsHeaders, response, 200,
+                            log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.lifecycle !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketLifecycle API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketLifecycle');
+                return api.callApiMethod('bucketPutLifecycle', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketLifecycle API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        routesUtils.responseNoBody(err, corsHeaders, response, 200,
+                            log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.policy !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketPolicy API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketPolicy');
+                return api.callApiMethod('bucketPutPolicy', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketPolicy API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        routesUtils.responseNoBody(err, corsHeaders, response, 200,
+                            log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query['object-lock'] !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectLockConfiguration API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectLockConfiguration');
+                return api.callApiMethod('bucketPutObjectLock', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectLockConfiguration API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        routesUtils.responseNoBody(err, corsHeaders, response, 200,
+                            log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.notification !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketNotificationConfiguration API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketNotificationConfiguration');
+                return api.callApiMethod('bucketPutNotification', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketNotificationConfiguration API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        routesUtils.responseNoBody(err, corsHeaders, response, 200,
+                            log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.encryption !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucketEncryption API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucketEncryption');
+                return api.callApiMethod('bucketPutEncryption', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucketEncryption API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, corsHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else {
+                // PUT bucket
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutBucket API request');
+                activeSpan.setAttribute('rpc.method', 'PutBucket');
+                return api.callApiMethod('bucketPut', request, response, log,
+                    (err, corsHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutBucket API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        const location = { Location: `/${bucketName}` };
+                        const resHeaders = corsHeaders ?
+                            Object.assign({}, location, corsHeaders) : location;
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, resHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            }
         } else {
-            // PUT bucket
-            return api.callApiMethod('bucketPut', request, response, log,
-                (err, corsHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    const location = { Location: `/${bucketName}` };
-                    const resHeaders = corsHeaders ?
-                        Object.assign({}, location, corsHeaders) : location;
-                    return routesUtils.responseNoBody(err, resHeaders,
-                        response, 200, log);
-                });
-        }
-    } else {
-        // PUT object, PUT object ACL, PUT object multipart,
-        // PUT object copy or PUT object legal hold
-        // if content-md5 is not present in the headers, try to
-        // parse content-md5 from meta headers
+            // PUT object, PUT object ACL, PUT object multipart,
+            // PUT object copy or PUT object legal hold
+            // if content-md5 is not present in the headers, try to
+            // parse content-md5 from meta headers
 
-        if (request.headers['content-md5'] === '') {
-            log.debug('empty content-md5 header', {
-                method: 'routePUT',
-            });
-            return routesUtils
-                .responseNoBody(errors.InvalidDigest, null, response, 200, log);
-        }
-        if (request.headers['content-md5']) {
-            // @ts-ignore
-            request.contentMD5 = request.headers['content-md5'];
-        } else {
-            // @ts-ignore
-            request.contentMD5 = routesUtils.parseContentMD5(request.headers);
-        }
-        // @ts-ignore
-        if (request.contentMD5 && request.contentMD5.length !== 32) {
-            // @ts-ignore
-            request.contentMD5 = Buffer.from(request.contentMD5, 'base64').toString('hex');
+            if (request.headers['content-md5'] === '') {
+                activeSpan.recordException(errors.InvalidDigest);
+                cloudserverApiSpan.end();
+                log.debug('empty content-md5 header', {
+                    method: 'routePUT',
+                });
+                return routesUtils
+                    .responseNoBody(errors.InvalidDigest, null, response, 200, log);
+            }
+            if (request.headers['content-md5']) {
+                // @ts-ignore
+                request.contentMD5 = request.headers['content-md5'];
+            } else {
+                // @ts-ignore
+                request.contentMD5 = routesUtils.parseContentMD5(request.headers);
+            }
             // @ts-ignore
             if (request.contentMD5 && request.contentMD5.length !== 32) {
                 // @ts-ignore
-                log.debug('invalid md5 digest', { contentMD5: request.contentMD5 });
-                return routesUtils
-                    .responseNoBody(errors.InvalidDigest, null, response, 200,
-                        log);
+                request.contentMD5 = Buffer.from(request.contentMD5, 'base64').toString('hex');
+                // @ts-ignore
+                if (request.contentMD5 && request.contentMD5.length !== 32) {
+                    activeSpan.recordException(errors.InvalidDigest);
+                    cloudserverApiSpan.end();
+                    // @ts-ignore
+                    log.debug('invalid md5 digest', { contentMD5: request.contentMD5 });
+                    return routesUtils
+                        .responseNoBody(errors.InvalidDigest, null, response, 200,
+                            log);
+                }
             }
-        }
-        if (query.partNumber) {
-            if (request.headers['x-amz-copy-source']) {
-                api.callApiMethod('objectPutCopyPart', request, response, log,
-                    (err, xml, additionalHeaders) => {
+            if (query.partNumber) {
+                if (request.headers['x-amz-copy-source']) {
+                    activeSpan.updateName('S3 API request');
+                    activeSpan.addEvent('Detected UploadPartCopy API request');
+                    activeSpan.setAttribute('rpc.method', 'UploadPartCopy');
+                    return api.callApiMethod('objectPutCopyPart', request, response, log,
+                        (err, xml, additionalHeaders) => {
+                            cloudserverApiSpan.end();
+                            activeSpan.addEvent('UploadPartCopy API operation complete');
+                            if (err) {
+                                activeSpan.recordException(err);
+                            }
+                            routesUtils.statsReport500(err, statsClient);
+                            activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                            return routesUtils.responseXMLBody(err, xml, response, log,
+                                additionalHeaders);
+                        }, {
+                            cloudserverApiSpan,
+                            activeSpan,
+                            activeTracerContext,
+                            tracer,
+                        });
+                } else {
+                    activeSpan.updateName('S3 API request');
+                    activeSpan.addEvent('Detected UploadPart API request');
+                    activeSpan.setAttribute('rpc.method', 'UploadPart');
+                    return api.callApiMethod('objectPutPart', request, response, log,
+                        (err, calculatedHash, corsHeaders) => {
+                            cloudserverApiSpan.end();
+                            activeSpan.addEvent('UploadPart API operation complete');
+                            if (err) {
+                                return routesUtils.responseNoBody(err, corsHeaders,
+                                    response, 200, log);
+                            }
+                            // ETag's hex should always be enclosed in quotes
+                            const resMetaHeaders = corsHeaders || {};
+                            resMetaHeaders.ETag = `"${calculatedHash}"`;
+                            if (err) {
+                                activeSpan.recordException(err);
+                            }
+                            routesUtils.statsReport500(err, statsClient);
+                            activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                            return routesUtils.responseNoBody(err, resMetaHeaders,
+                                response, 200, log);
+                        }, {
+                            cloudserverApiSpan,
+                            activeSpan,
+                            activeTracerContext,
+                            tracer,
+                        });
+                }
+            } else if (query.acl !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectAcl API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectAcl');
+                return api.callApiMethod('objectPutACL', request, response, log,
+                    (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectAcl API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
                         routesUtils.statsReport500(err, statsClient);
-                        return routesUtils.responseXMLBody(err, xml, response, log,
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, resHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query['legal-hold'] !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectLegalHold API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectLegalHold');
+                return api.callApiMethod('objectPutLegalHold', request, response, log,
+                    (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectLegalHold API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, resHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.tagging !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectTagging API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectTagging');
+                return api.callApiMethod('objectPutTagging', request, response, log,
+                    (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectTagging API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, resHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (query.retention !== undefined) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObjectRetention API request');
+                activeSpan.setAttribute('rpc.method', 'PutObjectRetention');
+                return api.callApiMethod('objectPutRetention', request, response, log,
+                    (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObjectRetention API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, resHeaders,
+                            response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
+                    });
+            } else if (request.headers['x-amz-copy-source']) {
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected CopyObject API request');
+                activeSpan.setAttribute('rpc.method', 'CopyObject');
+                return api.callApiMethod('objectCopy', request, response, log,
+                    (err, xml, additionalHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('CopyObject API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
+                        }
+                        routesUtils.statsReport500(err, statsClient);
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        routesUtils.responseXMLBody(err, xml, response, log,
                             additionalHeaders);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             } else {
-                api.callApiMethod('objectPutPart', request, response, log,
-                    (err, calculatedHash, corsHeaders) => {
-                        if (err) {
-                            return routesUtils.responseNoBody(err, corsHeaders,
-                                response, 200, log);
+                if (request.headers['content-length'] === undefined &&
+                request.headers['x-amz-decoded-content-length'] === undefined) {
+                    activeSpan.recordException(errors.MissingContentLength);
+                    cloudserverApiSpan.end();
+                    return routesUtils.responseNoBody(errors.MissingContentLength,
+                        null, response, 411, log);
+                }
+                if (Number.isNaN(parsedContentLength) || parsedContentLength < 0) {
+                    activeSpan.recordException(errors.BadRequest);
+                    cloudserverApiSpan.end();
+                    return routesUtils.responseNoBody(errors.BadRequest,
+                        null, response, 400, log);
+                }
+                // TODO ARSN-216 What's happening?
+                // @ts-ignore
+                log.end().addDefaultFields({ contentLength: request.parsedContentLength });
+                activeSpan.updateName('S3 API request');
+                activeSpan.addEvent('Detected PutObject API request');
+                activeSpan.setAttribute('rpc.method', 'PutObject');
+                api.callApiMethod('objectPut', request, response, log,
+                    (err, resHeaders) => {
+                        cloudserverApiSpan.end();
+                        activeSpan.addEvent('PutObject API operation complete');
+                        if (err?.code === 500) {
+                            activeSpan.recordException(err);
                         }
-                        // ETag's hex should always be enclosed in quotes
-                        const resMetaHeaders = corsHeaders || {};
-                        resMetaHeaders.ETag = `"${calculatedHash}"`;
                         routesUtils.statsReport500(err, statsClient);
-                        return routesUtils.responseNoBody(err, resMetaHeaders,
+                        activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                        return routesUtils.responseNoBody(err, resHeaders,
                             response, 200, log);
+                    }, {
+                        cloudserverApiSpan,
+                        activeSpan,
+                        activeTracerContext,
+                        tracer,
                     });
             }
-        } else if (query.acl !== undefined) {
-            api.callApiMethod('objectPutACL', request, response, log,
-                (err, resHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, resHeaders,
-                        response, 200, log);
-                });
-        } else if (query['legal-hold'] !== undefined) {
-            api.callApiMethod('objectPutLegalHold', request, response, log,
-                (err, resHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, resHeaders,
-                        response, 200, log);
-                });
-        } else if (query.tagging !== undefined) {
-            api.callApiMethod('objectPutTagging', request, response, log,
-                (err, resHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, resHeaders,
-                        response, 200, log);
-                });
-        } else if (query.retention !== undefined) {
-            api.callApiMethod('objectPutRetention', request, response, log,
-                (err, resHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, resHeaders,
-                        response, 200, log);
-                });
-        } else if (request.headers['x-amz-copy-source']) {
-            return api.callApiMethod('objectCopy', request, response, log,
-                (err, xml, additionalHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    routesUtils.responseXMLBody(err, xml, response, log,
-                        additionalHeaders);
-                });
-        } else {
-            if (request.headers['content-length'] === undefined &&
-            request.headers['x-amz-decoded-content-length'] === undefined) {
-                return routesUtils.responseNoBody(errors.MissingContentLength,
-                    null, response, 411, log);
-            }
-            if (Number.isNaN(parsedContentLength) || parsedContentLength < 0) {
-                return routesUtils.responseNoBody(errors.BadRequest,
-                    null, response, 400, log);
-            }
-            // TODO ARSN-216 What's happening?
-            // @ts-ignore
-            log.end().addDefaultFields({ contentLength: request.parsedContentLength });
-            api.callApiMethod('objectPut', request, response, log,
-                (err, resHeaders) => {
-                    routesUtils.statsReport500(err, statsClient);
-                    return routesUtils.responseNoBody(err, resHeaders,
-                        response, 200, log);
-                });
         }
-    }
-    return undefined;
+        return undefined;
+    });
 }

lib/s3routes/routes/routeWebsite.ts

@@ -13,76 +13,129 @@ export default function routerWebsite(
     statsClient?: StatsClient,
     dataRetrievalFn?: any,
 ) {
-    const { bucketName, query } = request as any
-    log.debug('routing request', { method: 'routerWebsite' });
-    // website endpoint only supports GET and HEAD and must have a bucket
-    // http://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteEndpoints.html
-    if ((request.method !== 'GET' && request.method !== 'HEAD')
-        || !bucketName) {
-        return routesUtils.errorHtmlResponse(errors.MethodNotAllowed,
-            false, bucketName, response, null, log);
-    }
-    if (request.method === 'GET') {
-        return api.callApiMethod('websiteGet', request, response, log,
-            (err, userErrorPageFailure, dataGetInfo, resMetaHeaders,
-                redirectInfo, key) => {
-                routesUtils.statsReport500(err, statsClient);
-                // request being redirected
-                if (redirectInfo) {
-                    if (err && redirectInfo.withError) {
-                        return routesUtils.redirectRequestOnError(err,
-                            'GET', redirectInfo, dataGetInfo, dataRetrievalFn,
-                            response, resMetaHeaders, log)
+    const {
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        }
+    } = dataRetrievalFn;
+    return tracer.startActiveSpan('Arsenal:: Performing Website API related operations using Cloudserver, Vault and Metadata', undefined, activeTracerContext, cloudserverApiSpan => {
+        activeSpan.addEvent('Request validated, routing request using routerWebsite() in arsenal');
+        cloudserverApiSpan.setAttributes({
+            'code.lineno': 8,
+            'code.filename': 'lib/s3routes/routes/routeWebsite.ts',
+            'code.function': 'routerWebsite()',
+        });
+        activeSpan.addEvent('Detecting which API to route to using arsenal routerWebsite()');
+        const { bucketName, query } = request as any
+        log.debug('routing request', { method: 'routerWebsite' });
+        // website endpoint only supports GET and HEAD and must have a bucket
+        // http://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteEndpoints.html
+        if ((request.method !== 'GET' && request.method !== 'HEAD')
+            || !bucketName) {
+            activeSpan.recordException(errors.MethodNotAllowed);
+            cloudserverApiSpan.end();
+            return routesUtils.errorHtmlResponse(errors.MethodNotAllowed,
+                false, bucketName, response, null, log);
+        }
+        if (request.method === 'GET') {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected GetWebsite API request`);
+            activeSpan.setAttribute('rpc.method', 'GetWebsite');
+            return api.callApiMethod('websiteGet', request, response, log,
+                (err, userErrorPageFailure, dataGetInfo, resMetaHeaders,
+                    redirectInfo, key) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('Located Data')
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
                     }
-                    // note that key might have been modified in websiteGet
-                    // api to add index document
-                    return routesUtils.redirectRequest(redirectInfo,
-                        // TODO ARSN-217 encrypted does not exists in request.connection
-                        // @ts-ignore
-                        key, request.connection.encrypted,
-                        response, request.headers.host!, resMetaHeaders, log);
-                }
-                // user has their own error page
-                if (err && dataGetInfo) {
-                    return routesUtils.streamUserErrorPage(err, dataGetInfo,
-                        dataRetrievalFn, response, resMetaHeaders, log);
-                }
-                // send default error html response
-                if (err) {
-                    return routesUtils.errorHtmlResponse(err,
-                        userErrorPageFailure, bucketName,
-                        response, resMetaHeaders, log);
-                }
-                // no error, stream data
-                return routesUtils.responseStreamData(null, query,
-                    resMetaHeaders, dataGetInfo, dataRetrievalFn, response,
-                    undefined, log);
-            });
-    }
-    if (request.method === 'HEAD') {
-        return api.callApiMethod('websiteHead', request, response, log,
-            (err, resMetaHeaders, redirectInfo, key) => {
-                routesUtils.statsReport500(err, statsClient);
-                if (redirectInfo) {
-                    if (err && redirectInfo.withError) {
-                        return routesUtils.redirectRequestOnError(err,
-                            'HEAD', redirectInfo, null, dataRetrievalFn,
-                            response, resMetaHeaders, log)
+                    routesUtils.statsReport500(err, statsClient);
+                    // request being redirected
+                    if (redirectInfo) {
+                        if (err && redirectInfo.withError) {
+                            activeSpan.recordException(err);
+                            return routesUtils.redirectRequestOnError(err,
+                                'GET', redirectInfo, dataGetInfo, dataRetrievalFn,
+                                response, resMetaHeaders, log)
+                        }
+                        activeSpan.addEvent('Redirecting request');
+                        // note that key might have been modified in websiteGet
+                        // api to add index document
+                        return routesUtils.redirectRequest(redirectInfo,
+                            // TODO ARSN-217 encrypted does not exists in request.connection
+                            // @ts-ignore
+                            key, request.connection.encrypted,
+                            response, request.headers.host!, resMetaHeaders, log);
                     }
-                    return routesUtils.redirectRequest(redirectInfo,
-                        // TODO ARSN-217 encrypted does not exists in request.connection
-                        // @ts-ignore
-                        key, request.connection.encrypted,
-                        response, request.headers.host!, resMetaHeaders, log);
-                }
-                // could redirect on err so check for redirectInfo first
-                if (err) {
-                    return routesUtils.errorHeaderResponse(err, response,
-                        resMetaHeaders, log);
-                }
-                return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
-                    response, log);
-            });
-    }
-    return undefined;
+                    // user has their own error page
+                    if (err && dataGetInfo) {
+                        activeSpan.recordException(err);
+                        return routesUtils.streamUserErrorPage(err, dataGetInfo,
+                            dataRetrievalFn, response, resMetaHeaders, log);
+                    }
+                    // send default error html response
+                    if (err) {
+                        activeSpan.recordException(err);
+                        return routesUtils.errorHtmlResponse(err,
+                            userErrorPageFailure, bucketName,
+                            response, resMetaHeaders, log);
+                    }
+                    // no error, stream data
+                    return routesUtils.responseStreamData(null, query,
+                        resMetaHeaders, dataGetInfo, dataRetrievalFn, response,
+                        undefined, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
+                });
+        }
+        if (request.method === 'HEAD') {
+            activeSpan.updateName('S3 API request');
+            activeSpan.addEvent(`Detected HeadWebsite API request`);
+            activeSpan.setAttribute('rpc.method', 'HeadWebsite');
+            return api.callApiMethod('websiteHead', request, response, log,
+                (err, resMetaHeaders, redirectInfo, key) => {
+                    cloudserverApiSpan.end();
+                    activeSpan.addEvent('HeadWebsite API operation complete')
+                    if (err?.code === 500) {
+                        activeSpan.recordException(err);
+                    }
+                    routesUtils.statsReport500(err, statsClient);
+                    if (redirectInfo) {
+                        if (err && redirectInfo.withError) {
+                            activeSpan.recordException(err);
+                            return routesUtils.redirectRequestOnError(err,
+                                'HEAD', redirectInfo, null, dataRetrievalFn,
+                                response, resMetaHeaders, log)
+                        }
+                        activeSpan.addEvent('Redirecting request');
+                        return routesUtils.redirectRequest(redirectInfo,
+                            // TODO ARSN-217 encrypted does not exists in request.connection
+                            // @ts-ignore
+                            key, request.connection.encrypted,
+                            response, request.headers.host!, resMetaHeaders, log);
+                    }
+                    // could redirect on err so check for redirectInfo first
+                    if (err) {
+                        activeSpan.recordException(err);
+                        return routesUtils.errorHeaderResponse(err, response,
+                            resMetaHeaders, log);
+                    }
+                    activeSpan.addEvent('Finalizing Response with Content Headers and sending response to client');
+                    return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
+                        response, log);
+                }, {
+                    cloudserverApiSpan,
+                    activeSpan,
+                    activeTracerContext,
+                    tracer,
+                });
+        }
+        return undefined;
+    });
+    
 }

lib/s3routes/routesUtils.ts

@@ -1,6 +1,7 @@
 import * as url from 'url';
 import * as http from 'http';
 import { eachSeries } from 'async';
+const opentelemetry = require('@opentelemetry/api');
 
 import { RequestLogger } from 'werelogs';
 
@@ -16,6 +17,10 @@ export type CallApiMethod = (
     response: http.ServerResponse,
     log: RequestLogger,
     callback: (err: ArsenalError | null, ...data: any[]) => void,
+    cloudserverApiSpan?: any,
+    activeSpan?: any,
+    activeTracerContext?: any,
+    tracer?: any,
 ) => void;
 
 /**
@@ -350,6 +355,10 @@ function retrieveData(
     retrieveDataParams: any,
     response: http.ServerResponse,
     log: RequestLogger,
+    sproxydSpan?: any,
+    activeSpan?: any,
+    activeTracerContext?: any,
+    tracer?: any,
 ) {
     if (locations.length === 0) {
         return response.end();
@@ -367,6 +376,8 @@ function retrieveData(
     };
     // the S3-client might close the connection while we are processing it
     response.once('close', () => {
+        activeSpan.addEvent('response closed by client request');
+        sproxydSpan?.end();
         responseDestroyed = true;
         if (currentStream) {
             currentStream.destroy();
@@ -384,58 +395,71 @@ function retrieveData(
     } = retrieveDataParams;
     const data = new DataWrapper(
         client, implName, config, kms, metadata, locStorageCheckFn, vault);
-    return eachSeries(locations,
-        (current, next) => data.get(current, response, log,
-            (err: any, readable: http.IncomingMessage) => {
-                // NB: readable is of IncomingMessage type
-                if (err) {
-                    log.error('failed to get object', {
-                        error: err,
-                        method: 'retrieveData',
-                    });
-                    _destroyResponse();
-                    return next(err);
-                }
-                // response.isclosed is set by the S3 server. Might happen if
-                // the S3-client closes the connection before the first request
-                // to the backend is started.
-                // @ts-expect-error
-                if (responseDestroyed || response.isclosed) {
-                    log.debug(
-                        'response destroyed before readable could stream');
-                    readable.destroy();
-                    const responseErr = new Error();
-                    // @ts-ignore
-                    responseErr.code = 'ResponseError';
-                    responseErr.message = 'response closed by client request before all data sent';
-                    return next(responseErr);
-                }
-                // readable stream successfully consumed
-                readable.on('end', () => {
-                    currentStream = null;
-                    log.debug('readable stream end reached');
-                    return next();
-                });
-                // errors on server side with readable stream
-                readable.on('error', err => {
-                    log.error('error piping data from source');
-                    _destroyResponse();
-                    return next(err);
-                });
-                currentStream = readable;
-                return readable.pipe(response, { end: false });
-            }), err => {
-            currentStream = null;
-            if (err) {
-                log.debug('abort response due to error', {
+    return tracer.startActiveSpan('Streaming Data Using Sproxyd', dataSpan => {
+        dataSpan.setAttributes({
+            'code.function': 'Arsenal:: retrieveData()',
+            'code.filepath': 'lib/s3routes/routesUtils.js',
+            'code.lineno': 349,
+        });
+        return eachSeries(locations,
+            (current, next) => data.get(current, response, log,
+                (err: any, readable: http.IncomingMessage) => {
+                    // NB: readable is of IncomingMessage type
+                    if (err) {
+                        log.error('failed to get object', {
+                            error: err,
+                            method: 'retrieveData',
+                        });
+                        _destroyResponse();
+                        return next(err);
+                    }
+                    // response.isclosed is set by the S3 server. Might happen if
+                    // the S3-client closes the connection before the first request
+                    // to the backend is started.
                     // @ts-expect-error
-                    error: err.code, errMsg: err.message });
-            }
-            // call end for all cases (error/success) per node.js docs
-            // recommendation
-            response.end();
-        },
-    );
+                    if (responseDestroyed || response.isclosed) {
+                        log.debug(
+                            'response destroyed before readable could stream');
+                        readable.destroy();
+                        const responseErr = new Error();
+                        // @ts-ignore
+                        responseErr.code = 'ResponseError';
+                        responseErr.message = 'response closed by client request before all data sent';
+                        return next(responseErr);
+                    }
+                    // readable stream successfully consumed
+                    readable.on('end', () => {
+                        sproxydSpan?.addEvent('Readable stream successfully consumed');
+                        dataSpan.end();
+                        sproxydSpan?.end();
+                        currentStream = null;
+                        log.debug('readable stream end reached');
+                        return next();
+                    });
+                    // errors on server side with readable stream
+                    readable.on('error', err => {
+                        activeSpan.recordException(err);
+                        dataSpan.end();
+                        sproxydSpan?.end();
+                        log.error('error piping data from source');
+                        _destroyResponse();
+                        return next(err);
+                    });
+                    currentStream = readable;
+                    return readable.pipe(response, { end: false });
+                }), err => {
+                currentStream = null;
+                if (err) {
+                    log.debug('abort response due to error', {
+                        // @ts-expect-error
+                        error: err.code, errMsg: err.message });
+                }
+                // call end for all cases (error/success) per node.js docs
+                // recommendation
+                response.end();
+            },
+        );
+    });
 }
 
 function _responseBody(
@@ -606,46 +630,62 @@ export function responseStreamData(
     range: [number, number] | undefined,
     log: RequestLogger,
 ) {
-    if (errCode && !response.headersSent) {
-        return XMLResponseBackend.errorResponse(errCode, response, log,
-            resHeaders);
-    }
-    if (dataLocations !== null && !response.headersSent) {
-        // sanity check of content length against individual data
-        // locations to fetch
-        const contentLength = resHeaders && resHeaders['Content-Length'];
-        if (contentLength !== undefined &&
-            !_contentLengthMatchesLocations(contentLength,
-                dataLocations)) {
-            log.error('logic error: total length of fetched data ' +
-                      'locations does not match returned content-length',
-            { contentLength, dataLocations });
-            return XMLResponseBackend.errorResponse(errors.InternalError,
-                response, log,
+    const{
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    }   = retrieveDataParams;
+    activeSpan.addEvent('Request processed, getting Data from sproxyd');
+    return tracer.startActiveSpan('Getting Object Data from RING', undefined, activeTracerContext, sproxydSpan => {
+        sproxydSpan.setAttributes({
+            'code.function': 'Arsenal:: responseStreamData()',
+            'code.filepath': 'lib/s3routes/routesUtils.js',
+            'code.lineno': 609,
+        })
+        if (errCode && !response.headersSent) {
+            return XMLResponseBackend.errorResponse(errCode, response, log,
                 resHeaders);
         }
-    }
-    if (!response.headersSent) {
-        okContentHeadersResponse(overrideParams, resHeaders, response,
-            range, log);
-    }
-    if (dataLocations === null || _computeContentLengthFromLocation(dataLocations) === 0) {
-        return response.end(() => {
+        if (dataLocations !== null && !response.headersSent) {
+            // sanity check of content length against individual data
+            // locations to fetch
+            const contentLength = resHeaders && resHeaders['Content-Length'];
+            if (contentLength !== undefined &&
+                !_contentLengthMatchesLocations(contentLength,
+                    dataLocations)) {
+                log.error('logic error: total length of fetched data ' +
+                          'locations does not match returned content-length',
+                { contentLength, dataLocations });
+                return XMLResponseBackend.errorResponse(errors.InternalError,
+                    response, log,
+                    resHeaders);
+            }
+        }
+        if (!response.headersSent) {
+            okContentHeadersResponse(overrideParams, resHeaders, response,
+                range, log);
+        }
+        if (dataLocations === null || _computeContentLengthFromLocation(dataLocations) === 0) {
+            return response.end(() => {
+                // TODO ARSN-216 Fix logger
+                // @ts-expect-error
+                log.end().info('responded with only metadata', {
+                    httpCode: response.statusCode,
+                });
+            });
+        }
+        response.on('finish', () => {
             // TODO ARSN-216 Fix logger
+            activeSpan.addEvent('Data retrieved from sproxyd and sending response to client');
             // @ts-expect-error
-            log.end().info('responded with only metadata', {
+            log.end().info('responded with streamed content', {
                 httpCode: response.statusCode,
             });
         });
-    }
-    response.on('finish', () => {
-        // TODO ARSN-216 Fix logger
-        // @ts-expect-error
-        log.end().info('responded with streamed content', {
-            httpCode: response.statusCode,
-        });
+        return retrieveData(dataLocations, retrieveDataParams, response, log, sproxydSpan, activeSpan, activeTracerContext, tracer);
     });
-    return retrieveData(dataLocations, retrieveDataParams, response, log);
 }
 
 /**
@@ -666,18 +706,26 @@ export function streamUserErrorPage(
     corsHeaders: { [key: string]: string },
     log: RequestLogger,
 ) {
+    const{
+        oTel: {
+            tracer,
+            activeSpan,
+            activeTracerContext,
+        },
+    }   = retrieveDataParams;
     setCommonResponseHeaders(corsHeaders, response, log);
     response.setHeader('x-amz-error-code', err.message);
     response.setHeader('x-amz-error-message', err.description);
     response.writeHead(err.code, { 'Content-type': 'text/html' });
     response.on('finish', () => {
         // TODO ARSN-216 Fix logger
+        activeSpan.recordException(err);
         // @ts-expect-error
         log.end().info('responded with streamed content', {
             httpCode: response.statusCode,
         });
     });
-    return retrieveData(dataLocations, retrieveDataParams, response, log);
+    return retrieveData(dataLocations, retrieveDataParams, response, log, undefined, activeSpan, activeTracerContext, tracer);
 }
 
 /**