Compare commits

...

3 Commits

Author SHA1 Message Date
williamlardier 5f36381b37
testing 2022-09-19 13:48:09 +02:00
williamlardier cfa90708d8
qadl 2022-08-16 11:19:53 +02:00
williamlardier 8257abba4d
qadl 2022-08-16 11:08:09 +02:00
13 changed files with 160 additions and 5586 deletions

View File

@ -31,14 +31,16 @@ RUN apt-get update \
yarn \
wget \
libffi-dev \
zlib1g-dev \
&& mkdir -p /root/ssh \
zlib1g-dev
RUN mkdir -p /root/ssh \
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
ENV PYTHON=python3
RUN yarn cache clean \
&& yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
&& apt-get autoremove --purge -y python git build-essential \
RUN yarn cache clean
RUN yarn install --production --ignore-optional --ignore-engines --network-concurrency 1
RUN apt-get install htop
RUN apt-get autoremove --purge -y python git build-essential \
&& rm -rf /var/lib/apt/lists/* \
&& yarn cache clean \
&& rm -rf ~/.node-gyp \
@ -46,6 +48,30 @@ RUN yarn cache clean \
COPY . /usr/src/app
RUN echo 'fs.file-max = 2097152' >> /etc/sysctl.conf
RUN echo 'net.ipv4.ip_local_port_range = 2000 65535' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_rfc1337 = 1' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_fin_timeout' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_keepalive_time = 300' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_keepalive_probes = 5' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_keepalive_intvl = 15' >> /etc/sysctl.conf
RUN echo 'net.core.rmem_default = 31457280' >> /etc/sysctl.conf
RUN echo 'net.core.rmem_max = 12582912' >> /etc/sysctl.conf
RUN echo 'net.core.wmem_default = 31457280' >> /etc/sysctl.conf
RUN echo 'net.core.wmem_max = 12582912' >> /etc/sysctl.conf
RUN echo 'net.core.somaxconn = 4096' >> /etc/sysctl.conf
RUN echo 'net.core.netdev_max_backlog = 65536' >> /etc/sysctl.conf
RUN echo 'net.core.optmem_max = 25165824' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_mem = 65536 131072 262144' >> /etc/sysctl.conf
RUN echo 'net.ipv4.udp_mem = 65536 131072 262144' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_rmem = 8192 87380 16777216' >> /etc/sysctl.conf
RUN echo 'net.ipv4.udp_rmem_min = 16384' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_wmem = 8192 65536 16777216' >> /etc/sysctl.conf
RUN echo 'net.ipv4.udp_wmem_min = 16384' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_max_tw_buckets = 1440000' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_tw_recycle = 1' >> /etc/sysctl.conf
RUN echo 'net.ipv4.tcp_tw_reuse = 1' >> /etc/sysctl.conf
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]

View File

@ -18,12 +18,25 @@ const applyZenkoUserMD = require('./applyZenkoUserMD');
const { externalBackends, versioningNotImplBackends } = constants;
const externalVersioningErrorMessage = 'We do not currently support putting ' +
'a versioned object to a location-constraint of type Azure or GCP.';
'a versioned object to a location-constraint of type Azure or GCP.';
async function fakeMDCall(bucketName, dataGetInfo,
cipherBundle, metadataStoreParams, cb) {
await new Promise(resolve => setTimeout(resolve, 5 + Math.random() * 20));
return cb(null, global.rrresult);
}
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
services.metadataStoreObject(bucketName, dataGetInfo,
// const rd = Math.random().toString();
// console.time(rd+'objectPut (_storeInMDandDeleteData)');
let fn = services.metadataStoreObject;
if (process.env.NO_MONGO === 'true') {
fn = !global.rrresult ? services.metadataStoreObject : fakeMDCall;
}
fn(bucketName, dataGetInfo,
cipherBundle, metadataStoreParams, (err, result) => {
// console.timeEnd(rd+'objectPut (_storeInMDandDeleteData)');
if (err) {
return callback(err);
}
@ -33,6 +46,7 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
return data.batchDelete(dataToDelete, requestMethod,
newDataStoreName, deleteLog, err => callback(err, result));
}
global.rrresult = result;
return callback(null, result);
});
}
@ -58,8 +72,8 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
* result.versionId - unencrypted versionId returned by metadata
*/
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
log, callback) {
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
log, callback) {
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
@ -179,10 +193,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
if (isVersionedObj) {
log.debug(externalVersioningErrorMessage,
{ method: 'createAndStoreObject', error: errors.NotImplemented });
{ method: 'createAndStoreObject', error: errors.NotImplemented });
return process.nextTick(() => {
callback(errors.NotImplemented.customizeDescription(
externalVersioningErrorMessage));
externalVersioningErrorMessage));
});
}
}
@ -209,7 +223,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
log.debug('metadata only operation x-amz-meta-mdonly');
const md5 = request.headers['x-amz-meta-md5chksum']
? new Buffer(request.headers['x-amz-meta-md5chksum'],
'base64').toString('hex') : null;
'base64').toString('hex') : null;
const numParts = request.headers['x-amz-meta-md5numparts'];
let _md5;
if (numParts === undefined) {
@ -229,7 +243,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
}
}
return dataStore(objectKeyContext, cipherBundle, request, size,
streamingV4Params, backendInfo, log, next);
streamingV4Params, backendInfo, log, next);
},
function processDataResult(dataGetInfo, calculatedHash, next) {
if (dataGetInfo === null || dataGetInfo === undefined) {
@ -241,11 +255,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
const { key, dataStoreName, dataStoreType, dataStoreETag,
dataStoreVersionId } = dataGetInfo;
const prefixedDataStoreETag = dataStoreETag
? `1:${dataStoreETag}`
: `1:${calculatedHash}`;
const dataGetInfoArr = [{ key, size, start: 0, dataStoreName,
? `1:${dataStoreETag}`
: `1:${calculatedHash}`;
const dataGetInfoArr = [{
key, size, start: 0, dataStoreName,
dataStoreType, dataStoreETag: prefixedDataStoreETag,
dataStoreVersionId }];
dataStoreVersionId
}];
if (cipherBundle) {
dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme;
dataGetInfoArr[0].cipheredDataKey =

View File

@ -20,6 +20,7 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) {
const contentMD5 = stream.contentMD5;
const completedHash = hashedStream.completedHash;
if (contentMD5 && completedHash && contentMD5 !== completedHash) {
console.log('AZAAAAAAAAAAAAAAAAAAA')
log.debug('contentMD5 and completedHash do not match, deleting data', {
method: 'storeObject::dataStore',
completedHash,

View File

@ -34,6 +34,8 @@ const validateHeaders = s3middleware.validateConditionalHeaders;
* @return {undefined}
*/
function objectGet(authInfo, request, returnTagCount, log, callback) {
// const rd = Math.random().toString();
// console.time(rd+'objectGet (preparation steps)');
log.debug('processing request', { method: 'objectGet' });
const bucketName = request.bucketName;
const objectKey = request.objectKey;
@ -67,9 +69,12 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
requestType: 'objectGet',
request,
};
// console.timeEnd(rd+'objectGet (preparation steps)');
// console.time(rd+'objectGet (metadataValidateBucketAndObj)');
return metadataValidateBucketAndObj(mdValParams, log,
(err, bucket, objMD) => {
// console.timeEnd(rd+'objectGet (metadataValidateBucketAndObj)');
// console.time(rd+'objectGet (pre getVersioningConfiguration)');
const corsHeaders = collectCorsHeaders(request.headers.origin,
request.method, bucket);
if (err) {
@ -87,7 +92,11 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
'GET', bucketName, err.code, 'getObject');
return callback(err, null, corsHeaders);
}
// console.timeEnd(rd+'objectGet (pre getVersioningConfiguration)');
// console.time(rd+'objectGet (getVersioningConfiguration)');
const verCfg = bucket.getVersioningConfiguration();
// console.timeEnd(rd+'objectGet (getVersioningConfiguration)');
// console.time(rd+'objectGet (checks)');
if (objMD.archive &&
// Object is in cold backend
(!objMD.archive.restoreRequestedAt ||
@ -297,7 +306,10 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
dataLocator = setPartRanges(dataLocator, byteRange);
}
}
// console.timeEnd(rd+'objectGet (checks)');
// console.time(rd+'objectGet (head)');
return data.head(dataLocator, log, err => {
// console.timeEnd(rd+'objectGet (head)');
if (err) {
if (!err.is.LocationNotFound) {
log.error('error from external backend checking for ' +
@ -307,6 +319,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
'GET', bucketName, err.code, 'getObject');
return callback(err);
}
// console.time(rd+'objectGet (metrics)');
pushMetric('getObject', log, {
authInfo,
bucket: bucketName,
@ -318,6 +331,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
});
monitoring.promMetrics('GET', bucketName, '200', 'getObject',
Number.parseInt(responseMetaHeaders['Content-Length'], 10));
// console.timeEnd(rd+'objectGet (metrics)');
return callback(null, dataLocator, responseMetaHeaders,
byteRange);
});

View File

@ -20,6 +20,11 @@ const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
const writeContinue = require('../utilities/writeContinue');
const versionIdUtils = versioning.VersionID;
async function fakeMdCall(a, b, cb) {
await new Promise(resolve => setTimeout(resolve, 5 + Math.random() * 20));
return cb(null, global.bucket, global.objMD);
}
/**
* PUT Object in the requested bucket. Steps include:
* validating metadata for authorization, bucket and object existence etc.
@ -40,6 +45,8 @@ const versionIdUtils = versioning.VersionID;
* @return {undefined}
*/
function objectPut(authInfo, request, streamingV4Params, log, callback) {
// const rd = Math.random().toString();
// console.time(rd+'objectPut (preparation steps)');
log.debug('processing request', { method: 'objectPut' });
const putVersionId = request.headers['x-scal-s3-version-id'];
@ -94,8 +101,10 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
const invalidSSEError = errors.InvalidArgument.customizeDescription(
'The encryption method specified is not supported');
const requestType = 'objectPut';
const valParams = { authInfo, bucketName, objectKey, versionId,
requestType, request };
const valParams = {
authInfo, bucketName, objectKey, versionId,
requestType, request
};
const canonicalID = authInfo.getCanonicalID();
if (hasNonPrintables(objectKey)) {
@ -105,9 +114,17 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
}
log.trace('owner canonicalID to send to data', { canonicalID });
return metadataValidateBucketAndObj(valParams, log,
(err, bucket, objMD) => {
// console.timeEnd(rd+'objectPut (preparation steps)');
// console.time(rd+'objectPut (metadataValidateBucketAndObj)');
let fn = metadataValidateBucketAndObj;
if (process.env.NO_MONGO === 'true') {
fn = !global.bucket ? metadataValidateBucketAndObj : fakeMdCall;
}
return fn(valParams, log, (err, bucket, objMD) => {
global.bucket = bucket;
global.objMD = objMD;
// console.timeEnd(rd+'objectPut (metadataValidateBucketAndObj)');
// console.time(rd+'objectPut (pre waterfall)');
const responseHeaders = collectCorsHeaders(headers.origin,
method, bucket);
if (err) {
@ -131,6 +148,8 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
return callback(error);
}
}
// console.timeEnd(rd+'objectPut (pre waterfall)');
// console.time(rd+'objectPut (waterfall)');
return async.waterfall([
function handleTransientOrDeleteBuckets(next) {
@ -165,10 +184,12 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
}
writeContinue(request, request._response);
return createAndStoreObject(bucketName,
bucket, objectKey, objMD, authInfo, canonicalID, cipherBundle,
request, false, streamingV4Params, log, next);
bucket, objectKey, objMD, authInfo, canonicalID, cipherBundle,
request, false, streamingV4Params, log, next);
},
], (err, storingResult) => {
// console.timeEnd(rd+'objectPut (waterfall)');
// console.time(rd+'objectPut (cb)');
if (err) {
monitoring.promMetrics('PUT', bucketName, err.code,
'putObject');
@ -234,6 +255,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
monitoring.promMetrics('PUT', bucketName, '200',
'putObject', newByteLength, oldByteLength, isVersionedObj,
null, ingestSize);
// console.timeEnd(rd+'objectPut (cb)');
return callback(null, responseHeaders);
});
});

View File

@ -51,6 +51,7 @@ class V4Transform extends Transform {
this.lastPieceDone = false;
this.lastChunk = false;
this.clientError = false;
console.log('----', this.writableHighWaterMark, this.readableHighWaterMark);
}
/**
@ -165,14 +166,15 @@ class V4Transform extends Transform {
credentialScope: this.credentialScope,
},
};
return vault.authenticateV4Request(vaultParams, null, err => {
if (err) {
this.log.trace('err from vault on streaming v4 auth',
{ error: err, paramsSentToVault: vaultParams.data });
return done(err);
}
return done();
});
return done();
// return vault.authenticateV4Request(vaultParams, null, err => {
// if (err) {
// this.log.trace('err from vault on streaming v4 auth',
// { error: err, paramsSentToVault: vaultParams.data });
// return done(err);
// }
// return done();
// });
}

View File

@ -51,8 +51,11 @@ function metadataGetBucketAndObject(requestType, bucketName, objectKey,
// from most current object md (versionId = undefined)
versionId: versionId === 'null' ? undefined : versionId,
};
// const rd = Math.random().toString();
// console.time(rd+'objectGet (metadataGetBucketAndObject)')
return metadata.getBucketAndObjectMD(bucketName, objectKey, options, log,
(err, data) => {
// console.timeEnd(rd+'objectGet (metadataGetBucketAndObject)')
if (err) {
log.debug('metadata get failed', { error: err });
return cb(err);

View File

@ -5,7 +5,8 @@ const { series } = require('async');
const arsenal = require('arsenal');
const { RedisClient, StatsClient } = arsenal.metrics;
const monitoringClient = require('./utilities/monitoringHandler');
const { eventLoopUtilization } = require('perf_hooks').performance;
const cpuUsage = require('process').cpuUsage;
const logger = require('./utilities/logger');
const { internalHandlers } = require('./utilities/internalHandlers');
const { clientCheck, healthcheckHandler } = require('./utilities/healthcheckHandler');
@ -339,11 +340,34 @@ class S3Server {
}
function main() {
// var interval = 50;
// let lastELU = eventLoopUtilization();
// let startUsage = cpuUsage();
// var interval = setInterval(function() {
// var last = process.hrtime(); // replace Date.now()
// setImmediate(function() {
// var delta = process.hrtime(last); // with process.hrtime()
// // console.log('EVENTLOOP', (delta[0] * 1000000000 + delta[1]) / 1000000, process._getActiveRequests().length, process._getActiveHandles().length, process.env.UV_THREADPOOL_SIZE
// //);
// });
// // Store the current ELU so it can be assigned later.
// const tmpELU = eventLoopUtilization();
// // Calculate the diff between the current and last before sending.
// // console.log('EVENTLOOP6', eventLoopUtilization(tmpELU, lastELU));
// // console.log('EVENTLOOP2', eventLoopUtilization(tmpELU, lastELU).idle);
// // console.log('EVENTLOOP3', eventLoopUtilization(tmpELU, lastELU).active);
// // console.log('EVENTLOOP4', eventLoopUtilization(tmpELU, lastELU).utilization);
// // console.log('EVENTLOOP5', cpuUsage(startUsage).user / cpuUsage(startUsage).system);
// startUsage = cpuUsage();
// // Assign over the last value to report the next interval.
// lastELU = tmpELU;
// }, interval);
// TODO: change config to use workers prop. name for clarity
let workers = _config.clusters || 1;
if (process.env.S3BACKEND === 'mem') {
workers = 1;
}
workers = Number(process.env.WORKERS) || workers;
this.cluster = workers > 1;
if (!this.cluster) {
process.env.REPORT_TOKEN = _config.reportToken;

View File

@ -99,6 +99,8 @@ const services = {
tagging, taggingCopy, replicationInfo, defaultRetention,
dataStoreName, creationTime, retentionMode, retentionDate,
legalHold, originOp, updateMicroVersionId, archive } = params;
// const rd = Math.random().toString();
// console.time(rd+'metadataStoreObject');
log.trace('storing object in metadata');
assert.strictEqual(typeof bucketName, 'string');
const md = new ObjectMD();
@ -227,6 +229,9 @@ const services = {
if (legalHold) {
md.setLegalHold(legalHold);
}
// console.timeEnd(rd+'metadataStoreObject');
const rd2 = Math.random().toString();
// console.time(rd2+'metadataStoreObject (pre waterfall)');
log.trace('object metadata', { omVal: md.getValue() });
// If this is not the completion of a multipart upload or
@ -273,6 +278,8 @@ const services = {
versionId = JSON.parse(data).versionId;
}
}
// console.timeEnd(rd2+'metadataStoreObject (pre waterfall)');
return cb(err, {
lastModified: md.getLastModified(),
tags: md.getTags(),

View File

@ -20,7 +20,7 @@
"homepage": "https://github.com/scality/S3#readme",
"dependencies": {
"@hapi/joi": "^17.1.0",
"arsenal": "git+https://github.com/scality/Arsenal.git#8.1.64",
"arsenal": "git+https://github.com/scality/Arsenal#a5bbef3d4ae7724cf3b76927bd3c92a293ef6446",
"async": "~2.5.0",
"aws-sdk": "2.905.0",
"azure-storage": "^2.1.0",
@ -30,6 +30,7 @@
"cron-parser": "^2.11.0",
"diskusage": "1.1.3",
"google-auto-auth": "^0.9.1",
"heapdump": "^0.3.15",
"http-proxy": "^1.17.0",
"http-proxy-agent": "^4.0.1",
"https-proxy-agent": "^2.2.0",
@ -46,8 +47,8 @@
"utf-8-validate": "^5.0.8",
"utf8": "~2.1.1",
"uuid": "^8.3.2",
"vaultclient": "scality/vaultclient#8.3.6",
"werelogs": "scality/werelogs#8.1.0",
"vaultclient": "scality/vaultclient#1fc8c9e1f240a1efd4f59b6f728dc274dea40c55",
"werelogs": "scality/werelogs#89bac782137f9190a5e5dc1eddb3f07576ec9f00",
"ws": "^5.1.0",
"xml2js": "~0.4.16"
},
@ -98,7 +99,7 @@
"start_mdserver": "node mdserver.js",
"start_dataserver": "node dataserver.js",
"start_pfsserver": "node pfsserver.js",
"start_s3server": "node index.js",
"start_s3server": "node --max-old-space-size=8192 --max-semi-space-size=128 index.js",
"start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
"start_utapi": "node lib/utapi/utapi.js",
"start_secure_channel_proxy": "node bin/secure_channel_proxy.js",

View File

@ -47,7 +47,7 @@ class HttpChunkedUploadWithBadSignature extends HttpRequestAuthV4 {
getChunkSignature(chunkData) {
let signature;
if (this._chunkId === this._alterSignatureChunkId) {
// console.log(
// // console.log(
// `ALTERING SIGNATURE OF DATA CHUNK #${this._chunkId}`);
signature = DUMMY_SIGNATURE;
} else {
@ -84,7 +84,7 @@ function testChunkedPutWithBadSignature(n, alterSignatureChunkId, cb) {
assert.ifError(err);
});
async.timesSeries(N_DATA_CHUNKS, (chunkIndex, done) => {
// console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`);
// // console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`);
if (req.write(CHUNK_DATA)) {
process.nextTick(done);
} else {

View File

@ -65,7 +65,7 @@ class HttpRequestAuthV4 extends stream.Writable {
const signingDate = this._timestamp.slice(0, 8);
const credentialScope =
`${signingDate}/${REGION}/${SERVICE}/aws4_request`;
// console.log(`CREDENTIAL SCOPE: "${credentialScope}"`);
// // console.log(`CREDENTIAL SCOPE: "${credentialScope}"`);
return credentialScope;
}
@ -118,7 +118,7 @@ class HttpRequestAuthV4 extends stream.Writable {
signedHeaders['x-amz-content-sha256'],
].join('\n');
// console.log(`CANONICAL REQUEST: "${canonicalRequest}"`);
// // console.log(`CANONICAL REQUEST: "${canonicalRequest}"`);
return canonicalRequest;
}
@ -127,7 +127,7 @@ class HttpRequestAuthV4 extends stream.Writable {
crypto.createHash('sha256').update(canonicalReq).digest('hex');
const stringToSign = `AWS4-HMAC-SHA256\n${this._timestamp}\n` +
`${this.getCredentialScope()}\n${canonicalReqHash}`;
// console.log(`STRING TO SIGN: "${stringToSign}"`);
// // console.log(`STRING TO SIGN: "${stringToSign}"`);
return stringToSign;
}
@ -158,7 +158,7 @@ class HttpRequestAuthV4 extends stream.Writable {
const stringToSign = `AWS4-HMAC-SHA256-PAYLOAD\n${this._timestamp}\n` +
`${this.getCredentialScope()}\n${this._lastSignature}\n` +
`${EMPTY_STRING_HASH}\n${currentChunkHash}`;
// console.log(`CHUNK STRING TO SIGN: "${stringToSign}"`);
// // console.log(`CHUNK STRING TO SIGN: "${stringToSign}"`);
return stringToSign;
}

5542
yarn.lock

File diff suppressed because it is too large Load Diff