testing

Dockerfile

@@ -31,14 +31,16 @@ RUN apt-get update \
         yarn \
         wget \
         libffi-dev \
-        zlib1g-dev \
-    && mkdir -p /root/ssh \
+        zlib1g-dev
+
+RUN mkdir -p /root/ssh \
     && ssh-keyscan -H github.com > /root/ssh/known_hosts
 
 ENV PYTHON=python3
-RUN yarn cache clean \
-    && yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
-    && apt-get autoremove --purge -y python git build-essential \
+RUN yarn cache clean
+RUN yarn install --production --ignore-optional --ignore-engines --network-concurrency 1
+RUN apt-get install htop
+RUN apt-get autoremove --purge -y python git build-essential \
     && rm -rf /var/lib/apt/lists/* \
     && yarn cache clean \
     && rm -rf ~/.node-gyp \
@@ -46,6 +48,30 @@ RUN yarn cache clean \
 
 COPY . /usr/src/app
 
+RUN echo 'fs.file-max = 2097152' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.ip_local_port_range = 2000 65535' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_rfc1337 = 1' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_fin_timeout' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_keepalive_time = 300' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_keepalive_probes = 5' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_keepalive_intvl = 15' >> /etc/sysctl.conf
+RUN echo 'net.core.rmem_default = 31457280' >> /etc/sysctl.conf
+RUN echo 'net.core.rmem_max = 12582912' >> /etc/sysctl.conf
+RUN echo 'net.core.wmem_default = 31457280' >> /etc/sysctl.conf
+RUN echo 'net.core.wmem_max = 12582912' >> /etc/sysctl.conf
+RUN echo 'net.core.somaxconn = 4096' >> /etc/sysctl.conf
+RUN echo 'net.core.netdev_max_backlog = 65536' >> /etc/sysctl.conf
+RUN echo 'net.core.optmem_max = 25165824' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_mem = 65536 131072 262144' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.udp_mem = 65536 131072 262144' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_rmem = 8192 87380 16777216' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.udp_rmem_min = 16384' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_wmem = 8192 65536 16777216' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.udp_wmem_min = 16384' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_max_tw_buckets = 1440000' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_tw_recycle = 1' >> /etc/sysctl.conf
+RUN echo 'net.ipv4.tcp_tw_reuse = 1' >> /etc/sysctl.conf
+
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
 ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]

lib/api/apiUtils/object/createAndStoreObject.js

@@ -18,12 +18,25 @@ const applyZenkoUserMD = require('./applyZenkoUserMD');
 const { externalBackends, versioningNotImplBackends } = constants;
 
 const externalVersioningErrorMessage = 'We do not currently support putting ' +
-'a versioned object to a location-constraint of type Azure or GCP.';
+    'a versioned object to a location-constraint of type Azure or GCP.';
+
+async function fakeMDCall(bucketName, dataGetInfo,
+    cipherBundle, metadataStoreParams, cb) {
+    await new Promise(resolve => setTimeout(resolve, 5 + Math.random() * 20));
+    return cb(null, global.rrresult);
+}
 
 function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
     metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
-    services.metadataStoreObject(bucketName, dataGetInfo,
+    // const rd = Math.random().toString();
+    // console.time(rd+'objectPut (_storeInMDandDeleteData)');
+    let fn = services.metadataStoreObject;
+    if (process.env.NO_MONGO === 'true') {
+        fn = !global.rrresult ? services.metadataStoreObject : fakeMDCall;
+    }
+    fn(bucketName, dataGetInfo,
         cipherBundle, metadataStoreParams, (err, result) => {
+            // console.timeEnd(rd+'objectPut (_storeInMDandDeleteData)');
             if (err) {
                 return callback(err);
             }
@@ -33,6 +46,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
                 return data.batchDelete(dataToDelete, requestMethod,
                     newDataStoreName, deleteLog, err => callback(err, result));
             }
+            global.rrresult = result;
             return callback(null, result);
         });
 }
@@ -243,9 +257,11 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             const prefixedDataStoreETag = dataStoreETag
                 ? `1:${dataStoreETag}`
                 : `1:${calculatedHash}`;
-            const dataGetInfoArr = [{ key, size, start: 0, dataStoreName,
+            const dataGetInfoArr = [{
+                key, size, start: 0, dataStoreName,
                 dataStoreType, dataStoreETag: prefixedDataStoreETag,
-                dataStoreVersionId }];
+                dataStoreVersionId
+            }];
             if (cipherBundle) {
                 dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme;
                 dataGetInfoArr[0].cipheredDataKey =

lib/api/apiUtils/object/storeObject.js

@@ -20,6 +20,7 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) {
     const contentMD5 = stream.contentMD5;
     const completedHash = hashedStream.completedHash;
     if (contentMD5 && completedHash && contentMD5 !== completedHash) {
+        console.log('AZAAAAAAAAAAAAAAAAAAA')
         log.debug('contentMD5 and completedHash do not match, deleting data', {
             method: 'storeObject::dataStore',
             completedHash,

lib/api/objectGet.js

@@ -34,6 +34,8 @@ const validateHeaders = s3middleware.validateConditionalHeaders;
  * @return {undefined}
  */
 function objectGet(authInfo, request, returnTagCount, log, callback) {
+//    const rd = Math.random().toString();
+    // console.time(rd+'objectGet (preparation steps)');
     log.debug('processing request', { method: 'objectGet' });
     const bucketName = request.bucketName;
     const objectKey = request.objectKey;
@@ -67,9 +69,12 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
         requestType: 'objectGet',
         request,
     };
-
+    // console.timeEnd(rd+'objectGet (preparation steps)');
+    // console.time(rd+'objectGet (metadataValidateBucketAndObj)');
     return metadataValidateBucketAndObj(mdValParams, log,
     (err, bucket, objMD) => {
+        // console.timeEnd(rd+'objectGet (metadataValidateBucketAndObj)');
+        // console.time(rd+'objectGet (pre getVersioningConfiguration)');
         const corsHeaders = collectCorsHeaders(request.headers.origin,
             request.method, bucket);
         if (err) {
@@ -87,7 +92,11 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                 'GET', bucketName, err.code, 'getObject');
             return callback(err, null, corsHeaders);
         }
+        // console.timeEnd(rd+'objectGet (pre getVersioningConfiguration)');
+        // console.time(rd+'objectGet (getVersioningConfiguration)');
         const verCfg = bucket.getVersioningConfiguration();
+        // console.timeEnd(rd+'objectGet (getVersioningConfiguration)');
+        // console.time(rd+'objectGet (checks)');
         if (objMD.archive &&
             // Object is in cold backend
             (!objMD.archive.restoreRequestedAt ||
@@ -297,7 +306,10 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                 dataLocator = setPartRanges(dataLocator, byteRange);
             }
         }
+        // console.timeEnd(rd+'objectGet (checks)');
+        // console.time(rd+'objectGet (head)');
         return data.head(dataLocator, log, err => {
+            // console.timeEnd(rd+'objectGet (head)');            
             if (err) {
                 if (!err.is.LocationNotFound) {
                     log.error('error from external backend checking for ' +
@@ -307,6 +319,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
                     'GET', bucketName, err.code, 'getObject');
                 return callback(err);
             }
+            // console.time(rd+'objectGet (metrics)');
             pushMetric('getObject', log, {
                 authInfo,
                 bucket: bucketName,
@@ -318,6 +331,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
             });
             monitoring.promMetrics('GET', bucketName, '200', 'getObject',
                 Number.parseInt(responseMetaHeaders['Content-Length'], 10));
+            // console.timeEnd(rd+'objectGet (metrics)');
             return callback(null, dataLocator, responseMetaHeaders,
                 byteRange);
         });

lib/api/objectPut.js

@@ -20,6 +20,11 @@ const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
 const writeContinue = require('../utilities/writeContinue');
 const versionIdUtils = versioning.VersionID;
 
+
+async function fakeMdCall(a, b, cb) {
+    await new Promise(resolve => setTimeout(resolve, 5 + Math.random() * 20));
+    return cb(null, global.bucket, global.objMD);
+}
 /**
  * PUT Object in the requested bucket. Steps include:
  * validating metadata for authorization, bucket and object existence etc.
@@ -40,6 +45,8 @@ const versionIdUtils = versioning.VersionID;
  * @return {undefined}
  */
 function objectPut(authInfo, request, streamingV4Params, log, callback) {
+    // const rd = Math.random().toString();
+    // console.time(rd+'objectPut (preparation steps)');
     log.debug('processing request', { method: 'objectPut' });
 
     const putVersionId = request.headers['x-scal-s3-version-id'];
@@ -94,8 +101,10 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
     const invalidSSEError = errors.InvalidArgument.customizeDescription(
         'The encryption method specified is not supported');
     const requestType = 'objectPut';
-    const valParams = { authInfo, bucketName, objectKey, versionId,
-        requestType, request };
+    const valParams = {
+        authInfo, bucketName, objectKey, versionId,
+        requestType, request
+    };
     const canonicalID = authInfo.getCanonicalID();
 
     if (hasNonPrintables(objectKey)) {
@@ -105,9 +114,17 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
     }
 
     log.trace('owner canonicalID to send to data', { canonicalID });
-
-    return metadataValidateBucketAndObj(valParams, log,
-    (err, bucket, objMD) => {
+    // console.timeEnd(rd+'objectPut (preparation steps)');
+    // console.time(rd+'objectPut (metadataValidateBucketAndObj)');
+    let fn = metadataValidateBucketAndObj;
+    if (process.env.NO_MONGO === 'true') {
+        fn = !global.bucket ? metadataValidateBucketAndObj : fakeMdCall;
+    }
+    return fn(valParams, log, (err, bucket, objMD) => {
+        global.bucket = bucket;
+        global.objMD = objMD;
+        // console.timeEnd(rd+'objectPut (metadataValidateBucketAndObj)');
+        // console.time(rd+'objectPut (pre waterfall)');
         const responseHeaders = collectCorsHeaders(headers.origin,
             method, bucket);
         if (err) {
@@ -131,6 +148,8 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
                 return callback(error);
             }
         }
+        // console.timeEnd(rd+'objectPut (pre waterfall)');
+        // console.time(rd+'objectPut (waterfall)');
 
         return async.waterfall([
             function handleTransientOrDeleteBuckets(next) {
@@ -169,6 +188,8 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
                     request, false, streamingV4Params, log, next);
             },
         ], (err, storingResult) => {
+            // console.timeEnd(rd+'objectPut (waterfall)');
+            // console.time(rd+'objectPut (cb)');
             if (err) {
                 monitoring.promMetrics('PUT', bucketName, err.code,
                     'putObject');
@@ -234,6 +255,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
             monitoring.promMetrics('PUT', bucketName, '200',
                 'putObject', newByteLength, oldByteLength, isVersionedObj,
                 null, ingestSize);
+            // console.timeEnd(rd+'objectPut (cb)');
             return callback(null, responseHeaders);
         });
     });

lib/auth/streamingV4/V4Transform.js

@@ -51,6 +51,7 @@ class V4Transform extends Transform {
         this.lastPieceDone = false;
         this.lastChunk = false;
         this.clientError = false;
+        console.log('----', this.writableHighWaterMark, this.readableHighWaterMark);
     }
 
     /**
@@ -165,14 +166,15 @@ class V4Transform extends Transform {
                 credentialScope: this.credentialScope,
             },
         };
-        return vault.authenticateV4Request(vaultParams, null, err => {
-            if (err) {
-                this.log.trace('err from vault on streaming v4 auth',
-                    { error: err, paramsSentToVault: vaultParams.data });
-                return done(err);
-            }
         return done();
-        });
+        // return vault.authenticateV4Request(vaultParams, null, err => {
+            // if (err) {
+                // this.log.trace('err from vault on streaming v4 auth',
+                    // { error: err, paramsSentToVault: vaultParams.data });
+                // return done(err);
+            // }
+            // return done();
+        // });
     }
 
 

lib/metadata/metadataUtils.js

@@ -51,8 +51,11 @@ function metadataGetBucketAndObject(requestType, bucketName, objectKey,
         // from most current object md (versionId = undefined)
         versionId: versionId === 'null' ? undefined : versionId,
     };
+//    const rd = Math.random().toString();
+    // console.time(rd+'objectGet (metadataGetBucketAndObject)')
     return metadata.getBucketAndObjectMD(bucketName, objectKey, options, log,
         (err, data) => {
+            // console.timeEnd(rd+'objectGet (metadataGetBucketAndObject)')
             if (err) {
                 log.debug('metadata get failed', { error: err });
                 return cb(err);

lib/server.js

@@ -5,7 +5,8 @@ const { series } = require('async');
 const arsenal = require('arsenal');
 const { RedisClient, StatsClient } = arsenal.metrics;
 const monitoringClient = require('./utilities/monitoringHandler');
-
+const { eventLoopUtilization } = require('perf_hooks').performance;
+const cpuUsage = require('process').cpuUsage;
 const logger = require('./utilities/logger');
 const { internalHandlers } = require('./utilities/internalHandlers');
 const { clientCheck, healthcheckHandler } = require('./utilities/healthcheckHandler');
@@ -339,11 +340,34 @@ class S3Server {
 }
 
 function main() {
+    // var interval = 50;
+    // let lastELU = eventLoopUtilization();
+    // let startUsage = cpuUsage();
+    // var interval = setInterval(function() {
+    //     var last = process.hrtime();          // replace Date.now()        
+    //     setImmediate(function() {
+    //         var delta = process.hrtime(last); // with process.hrtime()
+    //         // console.log('EVENTLOOP', (delta[0] * 1000000000 + delta[1]) / 1000000, process._getActiveRequests().length, process._getActiveHandles().length, process.env.UV_THREADPOOL_SIZE
+    //         //);
+    //     });
+    //     // Store the current ELU so it can be assigned later.
+    //     const tmpELU = eventLoopUtilization();
+    //     // Calculate the diff between the current and last before sending.
+    //     // console.log('EVENTLOOP6', eventLoopUtilization(tmpELU, lastELU));
+    //     // console.log('EVENTLOOP2', eventLoopUtilization(tmpELU, lastELU).idle);
+    //     // console.log('EVENTLOOP3', eventLoopUtilization(tmpELU, lastELU).active);
+    //     // console.log('EVENTLOOP4', eventLoopUtilization(tmpELU, lastELU).utilization);
+    //     // console.log('EVENTLOOP5', cpuUsage(startUsage).user / cpuUsage(startUsage).system);
+    //     startUsage = cpuUsage();
+    //     // Assign over the last value to report the next interval.
+    //     lastELU = tmpELU;
+    // }, interval);
     // TODO: change config to use workers prop. name for clarity
     let workers = _config.clusters || 1;
     if (process.env.S3BACKEND === 'mem') {
         workers = 1;
     }
+    workers = Number(process.env.WORKERS) || workers;
     this.cluster = workers > 1;
     if (!this.cluster) {
         process.env.REPORT_TOKEN = _config.reportToken;

lib/services.js

@@ -99,6 +99,8 @@ const services = {
             tagging, taggingCopy, replicationInfo, defaultRetention,
             dataStoreName, creationTime, retentionMode, retentionDate,
             legalHold, originOp, updateMicroVersionId, archive } = params;
+    //    const rd = Math.random().toString();
+        // console.time(rd+'metadataStoreObject');
         log.trace('storing object in metadata');
         assert.strictEqual(typeof bucketName, 'string');
         const md = new ObjectMD();
@@ -227,6 +229,9 @@ const services = {
         if (legalHold) {
             md.setLegalHold(legalHold);
         }
+        // console.timeEnd(rd+'metadataStoreObject');
+        const rd2 = Math.random().toString();
+        // console.time(rd2+'metadataStoreObject (pre waterfall)');
 
         log.trace('object metadata', { omVal: md.getValue() });
         // If this is not the completion of a multipart upload or
@@ -273,6 +278,8 @@ const services = {
                     versionId = JSON.parse(data).versionId;
                 }
             }
+
+            // console.timeEnd(rd2+'metadataStoreObject (pre waterfall)');
             return cb(err, {
                 lastModified: md.getLastModified(),
                 tags: md.getTags(),

package.json

@@ -20,7 +20,7 @@
   "homepage": "https://github.com/scality/S3#readme",
   "dependencies": {
     "@hapi/joi": "^17.1.0",
-    "arsenal": "git+https://github.com/scality/Arsenal.git#8.1.64",
+    "arsenal": "git+https://github.com/scality/Arsenal#a5bbef3d4ae7724cf3b76927bd3c92a293ef6446",
     "async": "~2.5.0",
     "aws-sdk": "2.905.0",
     "azure-storage": "^2.1.0",
@@ -30,6 +30,7 @@
     "cron-parser": "^2.11.0",
     "diskusage": "1.1.3",
     "google-auto-auth": "^0.9.1",
+    "heapdump": "^0.3.15",
     "http-proxy": "^1.17.0",
     "http-proxy-agent": "^4.0.1",
     "https-proxy-agent": "^2.2.0",
@@ -46,8 +47,8 @@
     "utf-8-validate": "^5.0.8",
     "utf8": "~2.1.1",
     "uuid": "^8.3.2",
-    "vaultclient": "scality/vaultclient#8.3.6",
-    "werelogs": "scality/werelogs#8.1.0",
+    "vaultclient": "scality/vaultclient#1fc8c9e1f240a1efd4f59b6f728dc274dea40c55",
+    "werelogs": "scality/werelogs#89bac782137f9190a5e5dc1eddb3f07576ec9f00",
     "ws": "^5.1.0",
     "xml2js": "~0.4.16"
   },
@@ -98,7 +99,7 @@
     "start_mdserver": "node mdserver.js",
     "start_dataserver": "node dataserver.js",
     "start_pfsserver": "node pfsserver.js",
-    "start_s3server": "node index.js",
+    "start_s3server": "node --max-old-space-size=8192 --max-semi-space-size=128 index.js",
     "start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
     "start_utapi": "node lib/utapi/utapi.js",
     "start_secure_channel_proxy": "node bin/secure_channel_proxy.js",

tests/functional/raw-node/test/badChunkSignatureV4.js

@@ -47,7 +47,7 @@ class HttpChunkedUploadWithBadSignature extends HttpRequestAuthV4 {
     getChunkSignature(chunkData) {
         let signature;
         if (this._chunkId === this._alterSignatureChunkId) {
-            // console.log(
+            // // console.log(
             //     `ALTERING SIGNATURE OF DATA CHUNK #${this._chunkId}`);
             signature = DUMMY_SIGNATURE;
         } else {
@@ -84,7 +84,7 @@ function testChunkedPutWithBadSignature(n, alterSignatureChunkId, cb) {
         assert.ifError(err);
     });
     async.timesSeries(N_DATA_CHUNKS, (chunkIndex, done) => {
-        // console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`);
+        // // console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`);
         if (req.write(CHUNK_DATA)) {
             process.nextTick(done);
         } else {

tests/functional/raw-node/utils/HttpRequestAuthV4.js

@@ -65,7 +65,7 @@ class HttpRequestAuthV4 extends stream.Writable {
         const signingDate = this._timestamp.slice(0, 8);
         const credentialScope =
               `${signingDate}/${REGION}/${SERVICE}/aws4_request`;
-        // console.log(`CREDENTIAL SCOPE: "${credentialScope}"`);
+        // // console.log(`CREDENTIAL SCOPE: "${credentialScope}"`);
         return credentialScope;
     }
 
@@ -118,7 +118,7 @@ class HttpRequestAuthV4 extends stream.Writable {
             signedHeaders['x-amz-content-sha256'],
         ].join('\n');
 
-        // console.log(`CANONICAL REQUEST: "${canonicalRequest}"`);
+        // // console.log(`CANONICAL REQUEST: "${canonicalRequest}"`);
         return canonicalRequest;
     }
 
@@ -127,7 +127,7 @@ class HttpRequestAuthV4 extends stream.Writable {
               crypto.createHash('sha256').update(canonicalReq).digest('hex');
         const stringToSign = `AWS4-HMAC-SHA256\n${this._timestamp}\n` +
               `${this.getCredentialScope()}\n${canonicalReqHash}`;
-        // console.log(`STRING TO SIGN: "${stringToSign}"`);
+        // // console.log(`STRING TO SIGN: "${stringToSign}"`);
         return stringToSign;
     }
 
@@ -158,7 +158,7 @@ class HttpRequestAuthV4 extends stream.Writable {
         const stringToSign = `AWS4-HMAC-SHA256-PAYLOAD\n${this._timestamp}\n` +
               `${this.getCredentialScope()}\n${this._lastSignature}\n` +
               `${EMPTY_STRING_HASH}\n${currentChunkHash}`;
-        // console.log(`CHUNK STRING TO SIGN: "${stringToSign}"`);
+        // // console.log(`CHUNK STRING TO SIGN: "${stringToSign}"`);
         return stringToSign;
     }