Compare commits
5 Commits
developmen
...
improvemen
Author | SHA1 | Date |
---|---|---|
Francois Ferrand | a8c7d7dbef | |
Francois Ferrand | 7c69591b2f | |
Francois Ferrand | 898e9ed29e | |
Francois Ferrand | 39ec6daac5 | |
Francois Ferrand | be647a9a24 |
|
@ -140,6 +140,11 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
|
|||
generateRequestContext(objectHeadVersionAction);
|
||||
requestContexts.push(headObjectVersion);
|
||||
}
|
||||
if (request.headers['x-amz-scal-archive-info']) {
|
||||
const coldStatus =
|
||||
generateRequestContext('objectGetArchiveInfo');
|
||||
requestContexts.push(coldStatus);
|
||||
}
|
||||
} else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
|
||||
const putObjectTaggingRequestContext =
|
||||
generateRequestContext('objectPutTagging');
|
||||
|
|
|
@ -34,6 +34,42 @@ function getAmzRestoreResHeader(objMD) {
|
|||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* set archiveInfo headers to target header object
|
||||
* @param {object} objMD - object metadata
|
||||
* @returns {object} headers - target header object
|
||||
*/
|
||||
function setArchiveInfoHeaders(objMD) {
|
||||
const headers = {};
|
||||
|
||||
if (objMD['x-amz-scal-transition-in-progress']) {
|
||||
headers['x-amz-scal-transition-in-progress'] = true;
|
||||
headers['x-amz-scal-transition-time'] = new Date(objMD['x-amz-scal-transition-time']).toUTCString();
|
||||
}
|
||||
|
||||
if (objMD.archive) {
|
||||
headers['x-amz-scal-archive-info'] = JSON.stringify(objMD.archive.archiveInfo);
|
||||
|
||||
if (objMD.archive.restoreRequestedAt) {
|
||||
headers['x-amz-scal-restore-requested-at'] = new Date(objMD.archive.restoreRequestedAt).toUTCString();
|
||||
headers['x-amz-scal-restore-requested-days'] = objMD.archive.restoreRequestedDays;
|
||||
}
|
||||
|
||||
if (objMD.archive.restoreCompletedAt) {
|
||||
headers['x-amz-scal-restore-completed-at'] = new Date(objMD.archive.restoreCompletedAt).toUTCString();
|
||||
headers['x-amz-scal-restore-will-expire-at'] = new Date(objMD.archive.restoreWillExpireAt).toUTCString();
|
||||
}
|
||||
}
|
||||
|
||||
// Always get the "real" storage class (even when STANDARD) in this case
|
||||
headers['x-amz-storage-class'] = objMD['x-amz-storage-class'] || objMD.dataStoreName;
|
||||
|
||||
// Get the owner-id
|
||||
headers['x-amz-scal-owner-id'] = objMD['owner-id'];
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if restore can be done.
|
||||
*
|
||||
|
@ -244,4 +280,5 @@ module.exports = {
|
|||
getAmzRestoreResHeader,
|
||||
validatePutVersionId,
|
||||
verifyColdObjectAvailable,
|
||||
setArchiveInfoHeaders,
|
||||
};
|
||||
|
|
|
@ -16,6 +16,7 @@ const { locationConstraints } = config;
|
|||
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
|
||||
const { maximumAllowedPartCount } = require('../../constants');
|
||||
const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
|
||||
const { setArchiveInfoHeaders } = require('./apiUtils/object/coldStorage');
|
||||
|
||||
/**
|
||||
* HEAD Object - Same as Get Object but only respond with headers
|
||||
|
@ -110,6 +111,10 @@ function objectHead(authInfo, request, log, callback) {
|
|||
isVersionedReq: !!versionId,
|
||||
});
|
||||
|
||||
if (request.headers['x-amz-scal-archive-info']) {
|
||||
Object.assign(responseHeaders, setArchiveInfoHeaders(objMD));
|
||||
}
|
||||
|
||||
const objLength = (objMD.location === null ?
|
||||
0 : parseInt(objMD['content-length'], 10));
|
||||
|
||||
|
|
|
@ -52,6 +52,10 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg,
|
|||
responseMetaHeaders['x-amz-restore'] = restoreHeader;
|
||||
}
|
||||
|
||||
if (objectMD['x-amz-scal-transition-in-progress']) {
|
||||
responseMetaHeaders['x-amz-meta-scal-s3-transition-in-progress'] = true;
|
||||
}
|
||||
|
||||
responseMetaHeaders['Accept-Ranges'] = 'bytes';
|
||||
|
||||
if (objectMD['cache-control']) {
|
||||
|
|
|
@ -21,7 +21,7 @@
|
|||
"dependencies": {
|
||||
"@azure/storage-blob": "^12.12.0",
|
||||
"@hapi/joi": "^17.1.0",
|
||||
"arsenal": "git+https://github.com/scality/arsenal#8.1.130",
|
||||
"arsenal": "git+https://github.com/scality/arsenal#8.1.132",
|
||||
"async": "~2.5.0",
|
||||
"aws-sdk": "2.905.0",
|
||||
"bucketclient": "scality/bucketclient#8.1.9",
|
||||
|
|
|
@ -96,6 +96,60 @@ describe('prepareRequestContexts', () => {
|
|||
assert.strictEqual(results[1].getAction(), expectedAction2);
|
||||
});
|
||||
|
||||
it('should return s3:GetObject for headObject', () => {
|
||||
const apiMethod = 'objectHead';
|
||||
const request = makeRequest({
|
||||
});
|
||||
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||
sourceObject, sourceVersionId);
|
||||
|
||||
assert.strictEqual(results.length, 1);
|
||||
assert.strictEqual(results[0].getAction(), 's3:GetObject');
|
||||
});
|
||||
|
||||
it('should return s3:GetObject and s3:GetObjectVersion for headObject', () => {
|
||||
const apiMethod = 'objectHead';
|
||||
const request = makeRequest({
|
||||
'x-amz-version-id': '0987654323456789',
|
||||
});
|
||||
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||
sourceObject, sourceVersionId);
|
||||
|
||||
assert.strictEqual(results.length, 2);
|
||||
assert.strictEqual(results[0].getAction(), 's3:GetObject');
|
||||
assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion');
|
||||
});
|
||||
|
||||
it('should return s3:GetObject and scality:GetObjectArchiveInfo for headObject ' +
|
||||
'with x-amz-scal-archive-info header', () => {
|
||||
const apiMethod = 'objectHead';
|
||||
const request = makeRequest({
|
||||
'x-amz-scal-archive-info': 'true',
|
||||
});
|
||||
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||
sourceObject, sourceVersionId);
|
||||
|
||||
assert.strictEqual(results.length, 2);
|
||||
assert.strictEqual(results[0].getAction(), 's3:GetObject');
|
||||
assert.strictEqual(results[1].getAction(), 'scality:GetObjectArchiveInfo');
|
||||
});
|
||||
|
||||
it('should return s3:GetObject, s3:GetObjectVersion and scality:GetObjectArchiveInfo ' +
|
||||
' for headObject with x-amz-scal-archive-info header', () => {
|
||||
const apiMethod = 'objectHead';
|
||||
const request = makeRequest({
|
||||
'x-amz-version-id': '0987654323456789',
|
||||
'x-amz-scal-archive-info': 'true',
|
||||
});
|
||||
const results = prepareRequestContexts(apiMethod, request, sourceBucket,
|
||||
sourceObject, sourceVersionId);
|
||||
|
||||
assert.strictEqual(results.length, 3);
|
||||
assert.strictEqual(results[0].getAction(), 's3:GetObject');
|
||||
assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion');
|
||||
assert.strictEqual(results[2].getAction(), 'scality:GetObjectArchiveInfo');
|
||||
});
|
||||
|
||||
['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'].forEach(apiMethod => {
|
||||
it(`should return s3:PutObjectVersion request context action for ${apiMethod} method ` +
|
||||
'with x-scal-s3-version-id header', () => {
|
||||
|
|
|
@ -337,6 +337,9 @@ describe('objectHead API', () => {
|
|||
assert.strictEqual(res[userMetadataKey], userMetadataValue);
|
||||
assert.strictEqual(res.ETag, `"${correctMD5}"`);
|
||||
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
|
||||
// Check we do not leak non-standard fields
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
@ -378,6 +381,12 @@ describe('objectHead API', () => {
|
|||
assert.strictEqual(res.ETag, `"${correctMD5}"`);
|
||||
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
|
||||
assert.strictEqual(res['x-amz-restore'], 'ongoing-request="true"');
|
||||
// Check we do not leak non-standard fields
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-restore-requested-days'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-owner-id'], undefined);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
@ -402,9 +411,152 @@ describe('objectHead API', () => {
|
|||
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
|
||||
const utcDate = new Date(objectCustomMDFields['x-amz-restore']['expiry-date']).toUTCString();
|
||||
assert.strictEqual(res['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`);
|
||||
// Check we do not leak non-standard fields
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-restore-completed-at'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-owner-id'], undefined);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should report when transition in progress', done => {
|
||||
const testGetRequest = {
|
||||
bucketName,
|
||||
namespace,
|
||||
objectKey: objectName,
|
||||
headers: {},
|
||||
url: `/${bucketName}/${objectName}`,
|
||||
};
|
||||
mdColdHelper.putBucketMock(bucketName, null, () => {
|
||||
const objectCustomMDFields = mdColdHelper.getTransitionInProgressMD();
|
||||
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
|
||||
objectHead(authInfo, testGetRequest, log, (err, res) => {
|
||||
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], true);
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-transition-time'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-owner-id'], undefined);
|
||||
done(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should report details when transition in progress', done => {
|
||||
const testGetRequest = {
|
||||
bucketName,
|
||||
namespace,
|
||||
objectKey: objectName,
|
||||
headers: {
|
||||
'x-amz-scal-archive-info': true,
|
||||
},
|
||||
url: `/${bucketName}/${objectName}`,
|
||||
};
|
||||
mdColdHelper.putBucketMock(bucketName, null, () => {
|
||||
const objectCustomMDFields = mdColdHelper.getTransitionInProgressMD();
|
||||
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
|
||||
objectHead(authInfo, testGetRequest, log, (err, res) => {
|
||||
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], true);
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], true);
|
||||
assert.strictEqual(res['x-amz-scal-transition-time'],
|
||||
new Date(objectCustomMDFields['x-amz-scal-transition-time']).toUTCString());
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
|
||||
done(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should report details when object is archived', done => {
|
||||
const testGetRequest = {
|
||||
bucketName,
|
||||
namespace,
|
||||
objectKey: objectName,
|
||||
headers: {
|
||||
'x-amz-scal-archive-info': true,
|
||||
},
|
||||
url: `/${bucketName}/${objectName}`,
|
||||
};
|
||||
mdColdHelper.putBucketMock(bucketName, null, () => {
|
||||
const objectCustomMDFields = mdColdHelper.getArchiveArchivedMD();
|
||||
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
|
||||
objectHead(authInfo, testGetRequest, log, (err, res) => {
|
||||
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}');
|
||||
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
|
||||
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
|
||||
done(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should report details when restore has been requested', done => {
|
||||
const testGetRequest = {
|
||||
bucketName,
|
||||
namespace,
|
||||
objectKey: objectName,
|
||||
headers: {
|
||||
'x-amz-scal-archive-info': true,
|
||||
},
|
||||
url: `/${bucketName}/${objectName}`,
|
||||
};
|
||||
mdColdHelper.putBucketMock(bucketName, null, () => {
|
||||
const objectCustomMDFields = mdColdHelper.getArchiveOngoingRequestMD();
|
||||
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
|
||||
objectHead(authInfo, testGetRequest, log, (err, res) => {
|
||||
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}');
|
||||
assert.strictEqual(res['x-amz-scal-restore-requested-at'],
|
||||
new Date(objectCustomMDFields.archive.restoreRequestedAt).toUTCString());
|
||||
assert.strictEqual(res['x-amz-scal-restore-requested-days'],
|
||||
objectCustomMDFields.archive.restoreRequestedDays);
|
||||
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
|
||||
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
|
||||
done(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should report details when object has been restored', done => {
|
||||
const testGetRequest = {
|
||||
bucketName,
|
||||
namespace,
|
||||
objectKey: objectName,
|
||||
headers: {
|
||||
'x-amz-scal-archive-info': true,
|
||||
},
|
||||
url: `/${bucketName}/${objectName}`,
|
||||
};
|
||||
mdColdHelper.putBucketMock(bucketName, null, () => {
|
||||
const objectCustomMDFields = mdColdHelper.getArchiveRestoredMD();
|
||||
mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => {
|
||||
objectHead(authInfo, testGetRequest, log, (err, res) => {
|
||||
assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}');
|
||||
assert.strictEqual(res['x-amz-scal-restore-requested-at'],
|
||||
new Date(objectCustomMDFields.archive.restoreRequestedAt).toUTCString());
|
||||
assert.strictEqual(res['x-amz-scal-restore-requested-days'],
|
||||
objectCustomMDFields.archive.restoreRequestedDays);
|
||||
assert.strictEqual(res['x-amz-scal-restore-completed-at'],
|
||||
new Date(objectCustomMDFields.archive.restoreCompletedAt).toUTCString());
|
||||
assert.strictEqual(res['x-amz-scal-restore-will-expire-at'],
|
||||
new Date(objectCustomMDFields.archive.restoreWillExpireAt).toUTCString());
|
||||
assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation);
|
||||
assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId);
|
||||
done(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -7,10 +7,11 @@ const ObjectMDArchive = require('arsenal').models.ObjectMDArchive;
|
|||
const BucketInfo = require('arsenal').models.BucketInfo;
|
||||
|
||||
const defaultLocation = 'location-dmf-v1';
|
||||
const defaultOwnerId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be';
|
||||
|
||||
const baseMd = {
|
||||
'owner-display-name': 'accessKey1displayName',
|
||||
'owner-id': '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be',
|
||||
'owner-id': defaultOwnerId,
|
||||
'content-length': 11,
|
||||
'content-md5': 'be747eb4b75517bf6b3cf7c5fbb62f3a',
|
||||
'content-language': '',
|
||||
|
@ -105,7 +106,9 @@ function putObjectMock(bucketName, objectName, fields, cb) {
|
|||
*/
|
||||
function getArchiveArchivedMD() {
|
||||
return {
|
||||
archive: new ObjectMDArchive({}).getValue(),
|
||||
archive: new ObjectMDArchive(
|
||||
{ foo: 0, bar: 'stuff' }, // opaque, can be anything...
|
||||
).getValue(),
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -115,7 +118,11 @@ function getArchiveArchivedMD() {
|
|||
*/
|
||||
function getArchiveOngoingRequestMD() {
|
||||
return {
|
||||
archive: new ObjectMDArchive({}, new Date(Date.now() - 60), 5).getValue(),
|
||||
archive: new ObjectMDArchive(
|
||||
{ foo: 0, bar: 'stuff' }, // opaque, can be anything...
|
||||
new Date(Date.now() - 60),
|
||||
5).getValue(),
|
||||
'x-amz-restore': new ObjectMDAmzRestore(true, new Date(Date.now() + 60 * 60 * 24)),
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -126,6 +133,7 @@ function getArchiveOngoingRequestMD() {
|
|||
function getTransitionInProgressMD() {
|
||||
return {
|
||||
'x-amz-scal-transition-in-progress': true,
|
||||
'x-amz-scal-transition-time': new Date(Date.now() - 60),
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -136,7 +144,7 @@ function getTransitionInProgressMD() {
|
|||
function getArchiveRestoredMD() {
|
||||
return {
|
||||
archive: new ObjectMDArchive(
|
||||
{},
|
||||
{ foo: 0, bar: 'stuff' }, // opaque, can be anything...
|
||||
new Date(Date.now() - 60000),
|
||||
5,
|
||||
new Date(Date.now() - 10000),
|
||||
|
@ -152,7 +160,7 @@ function getArchiveRestoredMD() {
|
|||
function getArchiveExpiredMD() {
|
||||
return {
|
||||
archive: new ObjectMDArchive(
|
||||
{},
|
||||
{ foo: 0, bar: 'stuff' }, // opaque, can be anything...
|
||||
new Date(Date.now() - 30000),
|
||||
5,
|
||||
new Date(Date.now() - 20000),
|
||||
|
@ -171,4 +179,5 @@ module.exports = {
|
|||
getTransitionInProgressMD,
|
||||
putBucketMock,
|
||||
defaultLocation,
|
||||
defaultOwnerId,
|
||||
};
|
||||
|
|
|
@ -39,4 +39,18 @@ describe('Middleware: Collect Response Headers', () => {
|
|||
assert.strictEqual(headers['x-amz-website-redirect-location'],
|
||||
'google.com');
|
||||
});
|
||||
|
||||
it('should not set flag when transition not in progress', () => {
|
||||
const obj = {};
|
||||
const headers = collectResponseHeaders(obj);
|
||||
assert.strictEqual(headers['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(headers['x-amz-meta-scal-s3-transition-in-progress'], undefined);
|
||||
});
|
||||
|
||||
it('should set flag when transition in progress', () => {
|
||||
const obj = { 'x-amz-scal-transition-in-progress': 'true' };
|
||||
const headers = collectResponseHeaders(obj);
|
||||
assert.strictEqual(headers['x-amz-scal-transition-in-progress'], undefined);
|
||||
assert.strictEqual(headers['x-amz-meta-scal-s3-transition-in-progress'], true);
|
||||
});
|
||||
});
|
||||
|
|
|
@ -1040,9 +1040,9 @@ arraybuffer.slice@~0.0.7:
|
|||
optionalDependencies:
|
||||
ioctl "^2.0.2"
|
||||
|
||||
"arsenal@git+https://github.com/scality/arsenal#8.1.130":
|
||||
version "8.1.130"
|
||||
resolved "git+https://github.com/scality/arsenal#30eaaf15eb0d6e304c710b0a275410ae7c99d34d"
|
||||
"arsenal@git+https://github.com/scality/arsenal#8.1.132":
|
||||
version "8.1.132"
|
||||
resolved "git+https://github.com/scality/arsenal#9cd72221e88d2acb70b86a1a6a3ba57185a3b180"
|
||||
dependencies:
|
||||
"@azure/identity" "^3.1.1"
|
||||
"@azure/storage-blob" "^12.12.0"
|
||||
|
|
Loading…
Reference in New Issue