Compare commits

...

5 Commits

Author SHA1 Message Date
williamlardier 591a9f669f CLDSRV-553: bump project version 2024-05-16 19:15:43 +02:00
williamlardier e6e1a0efb1 CLDSRV-553: functional restore test to simulate cold backend calls 2024-05-16 19:15:43 +02:00
williamlardier c5b16bcdac CLDSRV-553: unit test the onlyCheckQuota flag 2024-05-16 19:15:43 +02:00
williamlardier 8071eae695 CLDSRV-553: adapt calls to quota evaluation
When the API is being called by a cold backend, the
x-scal-s3-version-id header is set. In this case, the quotas must
be evaluated with a 0 inflight.
2024-05-16 19:15:43 +02:00
williamlardier 8fb87c1b57 CLDSRV-553: conditionnaly force evaluating quotas with 0 inflight
A corner case was found, where any PUT from the cold backend would
fail if the quota is already exceeded, as the storage was reserved
for the restore, but the restore itself requires some more bytes
as inflights when evaluating quotas. By passing a flag in the quota
evaluation function, we ensure that we can, in these cases,
evaluate the quotas with 0 inflight.

Thus, we compare the quota with the current bytes usage minus the
reserved space. This solution is not optimal as it is expected
that the quota will never exceed the limit by a lot: at the end,
we might allow all objects to be restored. This case can happen
only if some inflights were lost due to the limitations of the
metrics update frequency.

A better solution would use the total number of reserved space
bytes, and perfor the following operation:

utilizationMetrics - restoringBytes + currentObjectBytes
    <= quota

In this case, we can better account for the real reserved space.
This will be a second iteration.
2024-05-16 19:15:31 +02:00
10 changed files with 206 additions and 33 deletions

View File

@ -131,8 +131,8 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) {
const actions = Array.isArray(mdValueParams.requestType) ?
mdValueParams.requestType : [mdValueParams.requestType];
const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD);
return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes, log,
err => next(err, bucketMD, objectMD));
return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes,
false, log, err => next(err, bucketMD, objectMD));
},
function updateObjectMD(bucketMD, objectMD, next) {
const params = objectMD.versionId ? { versionId: objectMD.versionId } : {};

View File

@ -81,6 +81,8 @@ function isMetricStale(metric, resourceType, resourceName, action, inflight, log
* @param {number} inflight - The number of inflight requests.
* @param {number} inflightForCheck - The number of inflight requests for checking quotas.
* @param {string} action - The action being performed.
* @param {boolean} isStorageReserved - Flag to check if the current quota, minus
* the incoming bytes, are under the limit.
* @param {object} log - The logger object.
* @param {function} callback - The callback function to be called when evaluation is complete.
* @returns {object} - The result of the evaluation.
@ -93,25 +95,28 @@ function _evaluateQuotas(
inflight,
inflightForCheck,
action,
isStorageReserved,
log,
callback,
) {
let bucketQuotaExceeded = false;
let accountQuotaExceeded = false;
const creationDate = new Date(bucket.getCreationDate()).getTime();
const spaceReservedMultiplier = isStorageReserved ? -1 : 1;
return async.parallel({
bucketQuota: parallelDone => {
if (bucketQuota > 0) {
return QuotaService.getUtilizationMetrics('bucket',
`${bucket.getName()}_${creationDate}`, null, {
action,
inflight,
inflight: isStorageReserved ? 0 : inflight,
}, (err, bucketMetrics) => {
if (err || inflight < 0) {
return parallelDone(err);
}
if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
bucketMetrics.bytesTotal +
(inflightForCheck * spaceReservedMultiplier) > bucketQuota) {
log.debug('Bucket quota exceeded', {
bucket: bucket.getName(),
action,
@ -131,13 +136,14 @@ function _evaluateQuotas(
return QuotaService.getUtilizationMetrics('account',
account.account, null, {
action,
inflight,
inflight: isStorageReserved ? 0 : inflight,
}, (err, accountMetrics) => {
if (err || inflight < 0) {
return parallelDone(err);
}
if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
accountMetrics.bytesTotal +
(inflightForCheck * spaceReservedMultiplier) > accountQuota) {
log.debug('Account quota exceeded', {
accountId: account.account,
action,
@ -189,11 +195,13 @@ function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) {
* @param {array} apiNames - action names: operations to authorize
* @param {string} apiMethod - the main API call
* @param {number} inflight - inflight bytes
* @param {boolean} isStorageReserved - flag to check if the current quota, minus
* the incoming bytes, are under the limit.
* @param {Logger} log - logger
* @param {function} callback - callback function
* @returns {boolean} - true if the quota is valid, false otherwise
*/
function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, log, callback) {
function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, isStorageReserved, log, callback) {
if (!config.isQuotaEnabled() || !inflight) {
return callback(null);
}
@ -248,7 +256,8 @@ function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight,
let _inflights = shouldSendInflights ? inflight : undefined;
const inflightForCheck = shouldSendInflights ? 0 : inflight;
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
inflightForCheck, apiName, log, (err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
inflightForCheck, apiName, isStorageReserved, log,
(err, _bucketQuotaExceeded, _accountQuotaExceeded) => {
if (err) {
return done(err);
}
@ -270,7 +279,7 @@ function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight,
cb => {
if (errorFromAPI) {
return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights,
null, apiName, log, cb);
null, apiName, false, log, cb);
}
return cb();
},

View File

@ -335,7 +335,7 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request,
},
(objMD, versionId, callback) => validateQuotas(
request, bucket, request.accountQuotas, ['objectDelete'], 'objectDelete',
-objMD?.['content-length'] || 0, log, err => callback(err, objMD, versionId)),
-objMD?.['content-length'] || 0, false, log, err => callback(err, objMD, versionId)),
(objMD, versionId, callback) => {
const options = preprocessingVersioningDelete(
bucketName, bucket, objMD, versionId, config.nullVersionCompatMode);

View File

@ -98,7 +98,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
'The encryption method specified is not supported');
const requestType = request.apiMethods || 'objectPut';
const valParams = { authInfo, bucketName, objectKey, versionId,
requestType, request };
requestType, request, withVersionId: isPutVersion };
const canonicalID = authInfo.getCanonicalID();
if (hasNonPrintables(objectKey)) {

View File

@ -199,7 +199,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket,
copyObjectSize, sourceVerId,
sourceLocationConstraintName, sourceObjMD, next) {
return validateQuotas(request, destBucketMD, request.accountQuotas, valPutParams.requestType,
request.apiMethod, sourceObjMD?.['content-length'] || 0, log, err =>
request.apiMethod, sourceObjMD?.['content-length'] || 0, false, log, err =>
next(err, dataLocator, destBucketMD, copyObjectSize, sourceVerId, sourceLocationConstraintName));
},
// get MPU shadow bucket to get splitter based on MD version

View File

@ -61,6 +61,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
log.debug('processing request', { method: 'objectPutPart' });
const size = request.parsedContentLength;
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) {
log.debug('put part size too large', { size });
monitoring.promMetrics('PUT', request.bucketName, 400,
@ -134,7 +137,7 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
return next(null, destinationBucket);
},
(destinationBucket, next) => validateQuotas(request, destinationBucket, request.accountQuotas,
requestType, request.apiMethod, size, log, err => next(err, destinationBucket)),
requestType, request.apiMethod, size, isPutVersion, log, err => next(err, destinationBucket)),
// Get bucket server-side encryption, if it exists.
(destinationBucket, next) => getObjectSSEConfiguration(
request.headers, destinationBucket, log,

View File

@ -184,7 +184,7 @@ function validateBucket(bucket, params, log, actionImplicitDenies = {}) {
* @return {undefined} - and call callback with params err, bucket md
*/
function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log, callback) {
const { authInfo, bucketName, objectKey, versionId, getDeleteMarker, request } = params;
const { authInfo, bucketName, objectKey, versionId, getDeleteMarker, request, withVersionId } = params;
let requestType = params.requestType;
if (!Array.isArray(requestType)) {
requestType = [requestType];
@ -242,13 +242,16 @@ function standardMetadataValidateBucketAndObj(params, actionImplicitDenies, log,
const needQuotaCheck = requestType => requestType.some(type => actionNeedQuotaCheck[type] ||
actionWithDataDeletion[type]);
const checkQuota = params.checkQuota === undefined ? needQuotaCheck(requestType) : params.checkQuota;
if (!checkQuota) {
// withVersionId cover cases when an object is being restored with a specific version ID.
// In this case, the storage space was already accounted for when the RestoreObject API call
// was made, so we don't need to add any inflight, but quota must be evaluated.
if (!checkQuota && !withVersionId) {
return next(null, bucket, objMD);
}
const contentLength = processBytesToWrite(request.apiMethod, bucket, versionId,
request?.parsedContentLength || 0, objMD, params.destObjMD);
return validateQuotas(request, bucket, request.accountQuotas, requestType, request.apiMethod,
contentLength, log, err => next(err, bucket, objMD));
contentLength, withVersionId, log, err => next(err, bucket, objMD));
},
], (err, bucket, objMD) => {
if (err) {

View File

@ -1,6 +1,6 @@
{
"name": "@zenko/cloudserver",
"version": "8.8.23",
"version": "8.8.24",
"description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
"main": "index.js",
"engines": {

View File

@ -85,6 +85,25 @@ function putObject(bucket, key, size, cb) {
});
}
function putObjectWithCustomHeader(bucket, key, size, vID, cb) {
const request = s3Client.putObject({
Bucket: bucket,
Key: key,
Body: Buffer.alloc(size),
});
request.on('build', () => {
request.httpRequest.headers['x-scal-s3-version-id'] = vID;
});
return request.send((err, data) => {
if (!err && !s3Config.isQuotaInflightEnabled()) {
mockScuba.incrementBytesForBucket(bucket, 0);
}
return cb(err, data);
});
}
function copyObject(bucket, key, sourceSize, cb) {
return s3Client.copyObject({
Bucket: bucket,
@ -680,4 +699,87 @@ function multiObjectDelete(bucket, keys, size, callback) {
},
], done);
});
it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header',
done => {
const bucket = 'quota-test-bucket13';
const key = 'quota-test-object';
const size = 100;
let vID = null;
return async.series([
next => createBucket(bucket, true, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, val) => {
assert.ifError(err);
vID = val.VersionId;
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => fakeMetadataArchive(bucket, key, vID, {
archiveInfo: {},
restoreRequestedAt: new Date(0).toISOString(),
restoreRequestedDays: 7,
}, next),
// Simulate the real restore
next => putObjectWithCustomHeader(bucket, key, size, vID, err => {
assert.ifError(err);
return next();
}),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
next => deleteBucket(bucket, next),
], done);
});
it('should allow a restore if the quota is full but the objet fits with its reserved storage space',
done => {
const bucket = 'quota-test-bucket15';
const key = 'quota-test-object';
const size = 1000;
let vID = null;
return async.series([
next => createBucket(bucket, true, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify(quota), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, key, size, (err, val) => {
assert.ifError(err);
vID = val.VersionId;
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => fakeMetadataArchive(bucket, key, vID, {
archiveInfo: {},
restoreRequestedAt: new Date(0).toISOString(),
restoreRequestedDays: 7,
}, next),
// Put an object, the quota should be exceeded
next => putObject(bucket, `${key}-2`, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
// Simulate the real restore
next => putObjectWithCustomHeader(bucket, key, size, vID, err => {
assert.ifError(err);
return next();
}),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
next => deleteBucket(bucket, next),
], done);
});
});

View File

@ -50,7 +50,7 @@ describe('validateQuotas (buckets)', () => {
});
it('should return null if quota is <= 0', done => {
validateQuotas(request, mockBucketNoQuota, {}, [], '', false, mockLog, err => {
validateQuotas(request, mockBucketNoQuota, {}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
@ -59,7 +59,7 @@ describe('validateQuotas (buckets)', () => {
it('should return null if scuba is disabled', done => {
QuotaService.enabled = false;
validateQuotas(request, mockBucket, {}, [], '', false, mockLog, err => {
validateQuotas(request, mockBucket, {}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
@ -71,7 +71,7 @@ describe('validateQuotas (buckets)', () => {
const error = new Error('Failed to get metrics');
QuotaService._getLatestMetricsCallback.yields(error);
validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, mockLog, err => {
validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -97,7 +97,7 @@ describe('validateQuotas (buckets)', () => {
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, mockLog, err => {
validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.strictEqual(err.is.QuotaExceeded, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1);
assert.strictEqual(request.finalizerHooks.length, 1);
@ -124,7 +124,7 @@ describe('validateQuotas (buckets)', () => {
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -50, mockLog, err => {
validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -151,7 +151,7 @@ describe('validateQuotas (buckets)', () => {
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore',
true, mockLog, err => {
true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -179,7 +179,7 @@ describe('validateQuotas (buckets)', () => {
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore',
true, mockLog, err => {
true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -194,6 +194,33 @@ describe('validateQuotas (buckets)', () => {
done();
});
});
it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => {
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {}, ['objectPut'], 'objectPut',
true, true, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectPut',
inflight: 0,
}
), true);
done();
});
});
});
describe('validateQuotas (with accounts)', () => {
@ -224,7 +251,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 0,
}, [], '', false, mockLog, err => {
}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
@ -235,7 +262,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 1000,
}, [], '', false, mockLog, err => {
}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
@ -247,7 +274,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, [], '', false, mockLog, err => {
}, [], '', false, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false);
done();
@ -262,7 +289,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectPut', 'getObject'], 'objectPut', 1, mockLog, err => {
}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -291,7 +318,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 100,
}, ['objectPut', 'getObject'], 'objectPut', 1, mockLog, err => {
}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.strictEqual(err.is.QuotaExceeded, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1);
assert.strictEqual(request.finalizerHooks.length, 1);
@ -321,7 +348,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 1000,
}, ['objectDelete'], 'objectDelete', -50, mockLog, err => {
}, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -350,7 +377,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectRestore', 'objectPut'], 'objectRestore', true, mockLog, err => {
}, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -379,7 +406,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectPut', 'getObject'], 'objectPut', 1, mockLog, err => {
}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => {
assert.strictEqual(err.is.QuotaExceeded, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 2);
assert.strictEqual(request.finalizerHooks.length, 1);
@ -400,7 +427,7 @@ describe('validateQuotas (with accounts)', () => {
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectRestore', 'objectPut'], 'objectRestore', true, mockLog, err => {
}, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
@ -415,6 +442,35 @@ describe('validateQuotas (with accounts)', () => {
done();
});
});
it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => {
const result1 = {
bytesTotal: 80,
};
const result2 = {
bytesTotal: 90,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);
validateQuotas(request, mockBucket, {
account: 'test_1',
quota: 1000,
}, ['objectPut'], 'objectPut', true, true, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'account',
'test_1',
null,
{
action: 'objectPut',
inflight: 0,
}
), true);
done();
});
});
});
describe('processBytesToWrite', () => {