Compare commits

...

5 Commits

Author SHA1 Message Date
Francois Ferrand a8c7d7dbef
Report owner-id when getting archive info
Issue: CLDSRV-545
2024-07-10 19:11:03 +02:00
Francois Ferrand 7c69591b2f
Bump arsenal 8.1.132
Issue: CLDSRV-545
2024-07-10 19:11:03 +02:00
Francois Ferrand 898e9ed29e
Add tests
Issue: CLDSRV-545
2024-07-10 17:18:07 +02:00
Francois Ferrand 39ec6daac5
HeadObject: expose archive info
When the `x-amz-scal-archive-info` header is provider, return a number
of extra details about cold/archive object.

This gives access to the (opaque) archive-info field, as well as
transition and restore details.

Issue: CLDSRV-545
2024-07-10 17:18:07 +02:00
Francois Ferrand be647a9a24
Expose transition-in-progress flag in userMD
This matching the existing `x-amz-meta-scal-s3-transition-attempt`
metadata.

Issue: CLDSRV-545
2024-07-10 17:18:07 +02:00
10 changed files with 289 additions and 9 deletions

View File

@ -140,6 +140,11 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
generateRequestContext(objectHeadVersionAction); generateRequestContext(objectHeadVersionAction);
requestContexts.push(headObjectVersion); requestContexts.push(headObjectVersion);
} }
if (request.headers['x-amz-scal-archive-info']) {
const coldStatus =
generateRequestContext('objectGetArchiveInfo');
requestContexts.push(coldStatus);
}
} else if (apiMethodAfterVersionCheck === 'objectPutTagging') { } else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
const putObjectTaggingRequestContext = const putObjectTaggingRequestContext =
generateRequestContext('objectPutTagging'); generateRequestContext('objectPutTagging');

View File

@ -34,6 +34,42 @@ function getAmzRestoreResHeader(objMD) {
return undefined; return undefined;
} }
/**
* set archiveInfo headers to target header object
* @param {object} objMD - object metadata
* @returns {object} headers - target header object
*/
function setArchiveInfoHeaders(objMD) {
const headers = {};
if (objMD['x-amz-scal-transition-in-progress']) {
headers['x-amz-scal-transition-in-progress'] = true;
headers['x-amz-scal-transition-time'] = new Date(objMD['x-amz-scal-transition-time']).toUTCString();
}
if (objMD.archive) {
headers['x-amz-scal-archive-info'] = JSON.stringify(objMD.archive.archiveInfo);
if (objMD.archive.restoreRequestedAt) {
headers['x-amz-scal-restore-requested-at'] = new Date(objMD.archive.restoreRequestedAt).toUTCString();
headers['x-amz-scal-restore-requested-days'] = objMD.archive.restoreRequestedDays;
}
if (objMD.archive.restoreCompletedAt) {
headers['x-amz-scal-restore-completed-at'] = new Date(objMD.archive.restoreCompletedAt).toUTCString();
headers['x-amz-scal-restore-will-expire-at'] = new Date(objMD.archive.restoreWillExpireAt).toUTCString();
}
}
// Always get the "real" storage class (even when STANDARD) in this case
headers['x-amz-storage-class'] = objMD['x-amz-storage-class'] || objMD.dataStoreName;
// Get the owner-id
headers['x-amz-scal-owner-id'] = objMD['owner-id'];
return headers;
}
/** /**
* Check if restore can be done. * Check if restore can be done.
* *
@ -244,4 +280,5 @@ module.exports = {
getAmzRestoreResHeader, getAmzRestoreResHeader,
validatePutVersionId, validatePutVersionId,
verifyColdObjectAvailable, verifyColdObjectAvailable,
setArchiveInfoHeaders,
}; };

View File

@ -16,6 +16,7 @@ const { locationConstraints } = config;
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { maximumAllowedPartCount } = require('../../constants'); const { maximumAllowedPartCount } = require('../../constants');
const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
const { setArchiveInfoHeaders } = require('./apiUtils/object/coldStorage');
/** /**
* HEAD Object - Same as Get Object but only respond with headers * HEAD Object - Same as Get Object but only respond with headers
@ -110,6 +111,10 @@ function objectHead(authInfo, request, log, callback) {
isVersionedReq: !!versionId, isVersionedReq: !!versionId,
}); });
if (request.headers['x-amz-scal-archive-info']) {
Object.assign(responseHeaders, setArchiveInfoHeaders(objMD));
}
const objLength = (objMD.location === null ? const objLength = (objMD.location === null ?
0 : parseInt(objMD['content-length'], 10)); 0 : parseInt(objMD['content-length'], 10));

View File

@ -52,6 +52,10 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg,
responseMetaHeaders['x-amz-restore'] = restoreHeader; responseMetaHeaders['x-amz-restore'] = restoreHeader;
} }
if (objectMD['x-amz-scal-transition-in-progress']) {
responseMetaHeaders['x-amz-meta-scal-s3-transition-in-progress'] = true;
}
responseMetaHeaders['Accept-Ranges'] = 'bytes'; responseMetaHeaders['Accept-Ranges'] = 'bytes';
if (objectMD['cache-control']) { if (objectMD['cache-control']) {

View File

@ -21,7 +21,7 @@
"dependencies": { "dependencies": {
"@azure/storage-blob": "^12.12.0", "@azure/storage-blob": "^12.12.0",
"@hapi/joi": "^17.1.0", "@hapi/joi": "^17.1.0",
"arsenal": "git+https://github.com/scality/arsenal#8.1.130", "arsenal": "git+https://github.com/scality/arsenal#8.1.132",
"async": "~2.5.0", "async": "~2.5.0",
"aws-sdk": "2.905.0", "aws-sdk": "2.905.0",
"bucketclient": "scality/bucketclient#8.1.9", "bucketclient": "scality/bucketclient#8.1.9",

View File

@ -96,6 +96,60 @@ describe('prepareRequestContexts', () => {
assert.strictEqual(results[1].getAction(), expectedAction2); assert.strictEqual(results[1].getAction(), expectedAction2);
}); });
it('should return s3:GetObject for headObject', () => {
const apiMethod = 'objectHead';
const request = makeRequest({
});
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
sourceObject, sourceVersionId);
assert.strictEqual(results.length, 1);
assert.strictEqual(results[0].getAction(), 's3:GetObject');
});
it('should return s3:GetObject and s3:GetObjectVersion for headObject', () => {
const apiMethod = 'objectHead';
const request = makeRequest({
'x-amz-version-id': '0987654323456789',
});
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
sourceObject, sourceVersionId);
assert.strictEqual(results.length, 2);
assert.strictEqual(results[0].getAction(), 's3:GetObject');
assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion');
});
it('should return s3:GetObject and scality:GetObjectArchiveInfo for headObject ' +
'with x-amz-scal-archive-info header', () => {
const apiMethod = 'objectHead';
const request = makeRequest({
'x-amz-scal-archive-info': 'true',
});
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
sourceObject, sourceVersionId);
assert.strictEqual(results.length, 2);
assert.strictEqual(results[0].getAction(), 's3:GetObject');
assert.strictEqual(results[1].getAction(), 'scality:GetObjectArchiveInfo');
});
it('should return s3:GetObject, s3:GetObjectVersion and scality:GetObjectArchiveInfo ' +
' for headObject with x-amz-scal-archive-info header', () => {
const apiMethod = 'objectHead';
const request = makeRequest({
'x-amz-version-id': '0987654323456789',
'x-amz-scal-archive-info': 'true',
});
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
sourceObject, sourceVersionId);
assert.strictEqual(results.length, 3);
assert.strictEqual(results[0].getAction(), 's3:GetObject');
assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion');
assert.strictEqual(results[2].getAction(), 'scality:GetObjectArchiveInfo');
});
['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'].forEach(apiMethod => { ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'].forEach(apiMethod => {
it(`should return s3:PutObjectVersion request context action for ${apiMethod} method ` + it(`should return s3:PutObjectVersion request context action for ${apiMethod} method ` +
'with x-scal-s3-version-id header', () => { 'with x-scal-s3-version-id header', () => {

View File

@ -337,6 +337,9 @@ describe('objectHead API', () => {
assert.strictEqual(res[userMetadataKey], userMetadataValue); assert.strictEqual(res[userMetadataKey], userMetadataValue);
assert.strictEqual(res.ETag, `"${correctMD5}"`); assert.strictEqual(res.ETag, `"${correctMD5}"`);
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
// Check we do not leak non-standard fields
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
done(); done();
}); });
}); });
@ -378,6 +381,12 @@ describe('objectHead API', () => {
assert.strictEqual(res.ETag, `"${correctMD5}"`); assert.strictEqual(res.ETag, `"${correctMD5}"`);
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
assert.strictEqual(res['x-amz-restore'], 'ongoing-request="true"'); assert.strictEqual(res['x-amz-restore'], 'ongoing-request="true"');
// Check we do not leak non-standard fields
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined);
assert.strictEqual(res['x-amz-scal-restore-requested-days'], undefined);
assert.strictEqual(res['x-amz-scal-owner-id'], undefined);
done(); done();
}); });
}); });
@ -402,9 +411,152 @@ describe('objectHead API', () => {
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
const utcDate = new Date(objectCustomMDFields['x-amz-restore']['expiry-date']).toUTCString(); const utcDate = new Date(objectCustomMDFields['x-amz-restore']['expiry-date']).toUTCString();
assert.strictEqual(res['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`); assert.strictEqual(res['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`);
// Check we do not leak non-standard fields
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined);
assert.strictEqual(res['x-amz-scal-restore-completed-at'], undefined);
assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], undefined);
assert.strictEqual(res['x-amz-scal-owner-id'], undefined);
done(); done();
}); });
}); });
}); });
}); });
it('should report when transition in progress', done => {
const testGetRequest = {
bucketName,
namespace,
objectKey: objectName,
headers: {},
url: `/${bucketName}/${objectName}`,
};
mdColdHelper.putBucketMock(bucketName, null, () => {
const objectCustomMDFields = mdColdHelper.getTransitionInProgressMD();
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
objectHead(authInfo, testGetRequest, log, (err, res) => {
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], true);
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-transition-time'], undefined);
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
assert.strictEqual(res['x-amz-scal-owner-id'], undefined);
done(err);
});
});
});
});
it('should report details when transition in progress', done => {
const testGetRequest = {
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-scal-archive-info': true,
},
url: `/${bucketName}/${objectName}`,
};
mdColdHelper.putBucketMock(bucketName, null, () => {
const objectCustomMDFields = mdColdHelper.getTransitionInProgressMD();
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
objectHead(authInfo, testGetRequest, log, (err, res) => {
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], true);
assert.strictEqual(res['x-amz-scal-transition-in-progress'], true);
assert.strictEqual(res['x-amz-scal-transition-time'],
new Date(objectCustomMDFields['x-amz-scal-transition-time']).toUTCString());
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
done(err);
});
});
});
});
it('should report details when object is archived', done => {
const testGetRequest = {
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-scal-archive-info': true,
},
url: `/${bucketName}/${objectName}`,
};
mdColdHelper.putBucketMock(bucketName, null, () => {
const objectCustomMDFields = mdColdHelper.getArchiveArchivedMD();
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
objectHead(authInfo, testGetRequest, log, (err, res) => {
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}');
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
done(err);
});
});
});
});
it('should report details when restore has been requested', done => {
const testGetRequest = {
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-scal-archive-info': true,
},
url: `/${bucketName}/${objectName}`,
};
mdColdHelper.putBucketMock(bucketName, null, () => {
const objectCustomMDFields = mdColdHelper.getArchiveOngoingRequestMD();
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
objectHead(authInfo, testGetRequest, log, (err, res) => {
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}');
assert.strictEqual(res['x-amz-scal-restore-requested-at'],
new Date(objectCustomMDFields.archive.restoreRequestedAt).toUTCString());
assert.strictEqual(res['x-amz-scal-restore-requested-days'],
objectCustomMDFields.archive.restoreRequestedDays);
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
done(err);
});
});
});
});
it('should report details when object has been restored', done => {
const testGetRequest = {
bucketName,
namespace,
objectKey: objectName,
headers: {
'x-amz-scal-archive-info': true,
},
url: `/${bucketName}/${objectName}`,
};
mdColdHelper.putBucketMock(bucketName, null, () => {
const objectCustomMDFields = mdColdHelper.getArchiveRestoredMD();
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
objectHead(authInfo, testGetRequest, log, (err, res) => {
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}');
assert.strictEqual(res['x-amz-scal-restore-requested-at'],
new Date(objectCustomMDFields.archive.restoreRequestedAt).toUTCString());
assert.strictEqual(res['x-amz-scal-restore-requested-days'],
objectCustomMDFields.archive.restoreRequestedDays);
assert.strictEqual(res['x-amz-scal-restore-completed-at'],
new Date(objectCustomMDFields.archive.restoreCompletedAt).toUTCString());
assert.strictEqual(res['x-amz-scal-restore-will-expire-at'],
new Date(objectCustomMDFields.archive.restoreWillExpireAt).toUTCString());
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
done(err);
});
});
});
});
}); });

View File

@ -7,10 +7,11 @@ const ObjectMDArchive = require('arsenal').models.ObjectMDArchive;
const BucketInfo = require('arsenal').models.BucketInfo; const BucketInfo = require('arsenal').models.BucketInfo;
const defaultLocation = 'location-dmf-v1'; const defaultLocation = 'location-dmf-v1';
const defaultOwnerId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be';
const baseMd = { const baseMd = {
'owner-display-name': 'accessKey1displayName', 'owner-display-name': 'accessKey1displayName',
'owner-id': '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', 'owner-id': defaultOwnerId,
'content-length': 11, 'content-length': 11,
'content-md5': 'be747eb4b75517bf6b3cf7c5fbb62f3a', 'content-md5': 'be747eb4b75517bf6b3cf7c5fbb62f3a',
'content-language': '', 'content-language': '',
@ -105,7 +106,9 @@ function putObjectMock(bucketName, objectName, fields, cb) {
*/ */
function getArchiveArchivedMD() { function getArchiveArchivedMD() {
return { return {
archive: new ObjectMDArchive({}).getValue(), archive: new ObjectMDArchive(
{ foo: 0, bar: 'stuff' }, // opaque, can be anything...
).getValue(),
}; };
} }
@ -115,7 +118,11 @@ function getArchiveArchivedMD() {
*/ */
function getArchiveOngoingRequestMD() { function getArchiveOngoingRequestMD() {
return { return {
archive: new ObjectMDArchive({}, new Date(Date.now() - 60), 5).getValue(), archive: new ObjectMDArchive(
{ foo: 0, bar: 'stuff' }, // opaque, can be anything...
new Date(Date.now() - 60),
5).getValue(),
'x-amz-restore': new ObjectMDAmzRestore(true, new Date(Date.now() + 60 * 60 * 24)),
}; };
} }
@ -126,6 +133,7 @@ function getArchiveOngoingRequestMD() {
function getTransitionInProgressMD() { function getTransitionInProgressMD() {
return { return {
'x-amz-scal-transition-in-progress': true, 'x-amz-scal-transition-in-progress': true,
'x-amz-scal-transition-time': new Date(Date.now() - 60),
}; };
} }
@ -136,7 +144,7 @@ function getTransitionInProgressMD() {
function getArchiveRestoredMD() { function getArchiveRestoredMD() {
return { return {
archive: new ObjectMDArchive( archive: new ObjectMDArchive(
{}, { foo: 0, bar: 'stuff' }, // opaque, can be anything...
new Date(Date.now() - 60000), new Date(Date.now() - 60000),
5, 5,
new Date(Date.now() - 10000), new Date(Date.now() - 10000),
@ -152,7 +160,7 @@ function getArchiveRestoredMD() {
function getArchiveExpiredMD() { function getArchiveExpiredMD() {
return { return {
archive: new ObjectMDArchive( archive: new ObjectMDArchive(
{}, { foo: 0, bar: 'stuff' }, // opaque, can be anything...
new Date(Date.now() - 30000), new Date(Date.now() - 30000),
5, 5,
new Date(Date.now() - 20000), new Date(Date.now() - 20000),
@ -171,4 +179,5 @@ module.exports = {
getTransitionInProgressMD, getTransitionInProgressMD,
putBucketMock, putBucketMock,
defaultLocation, defaultLocation,
defaultOwnerId,
}; };

View File

@ -39,4 +39,18 @@ describe('Middleware: Collect Response Headers', () => {
assert.strictEqual(headers['x-amz-website-redirect-location'], assert.strictEqual(headers['x-amz-website-redirect-location'],
'google.com'); 'google.com');
}); });
it('should not set flag when transition not in progress', () => {
const obj = {};
const headers = collectResponseHeaders(obj);
assert.strictEqual(headers['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(headers['x-amz-meta-scal-s3-transition-in-progress'], undefined);
});
it('should set flag when transition in progress', () => {
const obj = { 'x-amz-scal-transition-in-progress': 'true' };
const headers = collectResponseHeaders(obj);
assert.strictEqual(headers['x-amz-scal-transition-in-progress'], undefined);
assert.strictEqual(headers['x-amz-meta-scal-s3-transition-in-progress'], true);
});
}); });

View File

@ -1040,9 +1040,9 @@ arraybuffer.slice@~0.0.7:
optionalDependencies: optionalDependencies:
ioctl "^2.0.2" ioctl "^2.0.2"
"arsenal@git+https://github.com/scality/arsenal#8.1.130": "arsenal@git+https://github.com/scality/arsenal#8.1.132":
version "8.1.130" version "8.1.132"
resolved "git+https://github.com/scality/arsenal#30eaaf15eb0d6e304c710b0a275410ae7c99d34d" resolved "git+https://github.com/scality/arsenal#9cd72221e88d2acb70b86a1a6a3ba57185a3b180"
dependencies: dependencies:
"@azure/identity" "^3.1.1" "@azure/identity" "^3.1.1"
"@azure/storage-blob" "^12.12.0" "@azure/storage-blob" "^12.12.0"