Compare commits

...

1 Commits

Author SHA1 Message Date
Nicolas Humbert 9eb5e9e0c9 CLDSRV-501 fix nullVersionId
Set nullVersionId to the master version if putting a version on top of a null version
2024-03-03 17:30:25 +01:00
3 changed files with 111 additions and 51 deletions

View File

@ -20,7 +20,7 @@
"homepage": "https://github.com/scality/S3#readme", "homepage": "https://github.com/scality/S3#readme",
"dependencies": { "dependencies": {
"@hapi/joi": "^17.1.0", "@hapi/joi": "^17.1.0",
"arsenal": "git+https://github.com/scality/arsenal#7.10.59", "arsenal": "git+https://github.com/scality/arsenal#bugfix/ARSN-392/improve",
"async": "~2.5.0", "async": "~2.5.0",
"aws-sdk": "2.905.0", "aws-sdk": "2.905.0",
"azure-storage": "^2.1.0", "azure-storage": "^2.1.0",

View File

@ -1,9 +1,8 @@
const assert = require('assert'); const assert = require('assert');
const async = require('async'); const async = require('async');
const crypto = require('crypto'); const crypto = require('crypto');
const { models, versioning } = require('arsenal'); const { versioning } = require('arsenal');
const versionIdUtils = versioning.VersionID; const versionIdUtils = versioning.VersionID;
const { ObjectMD } = models;
const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest'); const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest');
const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util'); const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
@ -87,17 +86,15 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
} }
function updateStorageClass(data, storageClass) { function updateStorageClass(data, storageClass) {
let parsedBody; let result;
try { try {
parsedBody = JSON.parse(data.body); const parsedBody = JSON.parse(JSON.parse(data.body).Body);
parsedBody['x-amz-storage-class'] = storageClass;
result = JSON.stringify(parsedBody);
} catch (err) { } catch (err) {
return { error: err }; return { error: err };
} }
const { result, error } = ObjectMD.createFromBlob(parsedBody.Body);
if (error) {
return { error };
}
result.setAmzStorageClass(storageClass);
return { result }; return { result };
} }
@ -202,11 +199,12 @@ describeSkipIfAWS('backbeat routes', () => {
it('should update metadata of a current null version', done => { it('should update metadata of a current null version', done => {
let objMD; let objMD;
return async.series([ let objMDAfter;
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), return async.series({
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, putObject: next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next), enableVersioningSource: next => s3.putBucketVersioning(
next => makeBackbeatRequest({ { Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
getMetadata: next => makeBackbeatRequest({
method: 'GET', method: 'GET',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -226,7 +224,7 @@ describeSkipIfAWS('backbeat routes', () => {
objMD = result; objMD = result;
return next(); return next();
}), }),
next => makeBackbeatRequest({ putMetadata: next => makeBackbeatRequest({
method: 'PUT', method: 'PUT',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -235,25 +233,47 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), headObject: next => s3.headObject(
next => s3.listObjectVersions({ Bucket: bucket }, next), { Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
], (err, data) => { getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, (err, data) => {
if (err) {
return next(err);
}
objMDAfter = JSON.parse(data.body).Body;
return next();
}),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) { if (err) {
return done(err); return done(err);
} }
const headObjectRes = data[4];
const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null'); assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass); assert.strictEqual(headObjectRes.StorageClass, storageClass);
const listObjectVersionsRes = data[5]; const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes; const { Versions } = listObjectVersionsRes;
assert.strictEqual(Versions.length, 1); assert.strictEqual(Versions.length, 1);
const [currentVersion] = Versions; const [currentVersion] = Versions;
assertVersionIsNullAndUpdated(currentVersion); assertVersionIsNullAndUpdated(currentVersion);
const objMDAfterParsed = JSON.parse(objMDAfter);
delete objMDAfterParsed.isNull; // TODO remove the line once CLDSRV-509 is fixed
assert.deepStrictEqual(objMDAfterParsed, JSON.parse(objMD));
return done(); return done();
}); });
}); });
@ -261,18 +281,21 @@ describeSkipIfAWS('backbeat routes', () => {
it('should update metadata of a non-current null version', done => { it('should update metadata of a non-current null version', done => {
let objMD; let objMD;
let expectedVersionId; let expectedVersionId;
return async.series([ let objMDAfter;
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), return async.series({
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, putObjectInitial: next => s3.putObject(
next), { Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { enableVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
putObjectAgain: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
if (err) { if (err) {
return next(err); return next(err);
} }
expectedVersionId = data.VersionId; expectedVersionId = data.VersionId;
return next(); return next();
}), }),
next => makeBackbeatRequest({ getMetadata: next => makeBackbeatRequest({
method: 'GET', method: 'GET',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -292,7 +315,7 @@ describeSkipIfAWS('backbeat routes', () => {
objMD = result; objMD = result;
return next(); return next();
}), }),
next => makeBackbeatRequest({ putMetadata: next => makeBackbeatRequest({
method: 'PUT', method: 'PUT',
resourceType: 'metadata', resourceType: 'metadata',
bucket, bucket,
@ -301,19 +324,35 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), getMetadataAfter: next => makeBackbeatRequest({
], (err, data) => { method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, (err, data) => {
if (err) {
return next(err);
}
objMDAfter = JSON.parse(data.body).Body;
return next();
}),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) { if (err) {
return done(err); return done(err);
} }
const headObjectRes = data[5]; const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null'); assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass); assert.strictEqual(headObjectRes.StorageClass, storageClass);
const listObjectVersionsRes = data[6]; const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes; const { Versions } = listObjectVersionsRes;
assert.strictEqual(Versions.length, 2); assert.strictEqual(Versions.length, 2);
@ -323,12 +362,15 @@ describeSkipIfAWS('backbeat routes', () => {
const nonCurrentVersion = Versions.find(v => !v.IsLatest); const nonCurrentVersion = Versions.find(v => !v.IsLatest);
assertVersionIsNullAndUpdated(nonCurrentVersion); assertVersionIsNullAndUpdated(nonCurrentVersion);
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
return done(); return done();
}); });
}); });
it('should update metadata of a suspended null version', done => { it('should update metadata of a suspended null version', done => {
let objMD; let objMD;
let objMDAfter;
return async.series({ return async.series({
suspendVersioning: next => s3.putBucketVersioning( suspendVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next), { Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, next),
@ -365,9 +407,25 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, (err, data) => {
if (err) {
return next(err);
}
objMDAfter = JSON.parse(data.body).Body;
return next();
}),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next), listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => { }, (err, results) => {
if (err) { if (err) {
@ -384,6 +442,8 @@ describeSkipIfAWS('backbeat routes', () => {
const [currentVersion] = Versions; const [currentVersion] = Versions;
assertVersionIsNullAndUpdated(currentVersion); assertVersionIsNullAndUpdated(currentVersion);
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));
return done(); return done();
}); });
}); });
@ -430,7 +490,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next), listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -487,7 +547,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -549,7 +609,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -612,7 +672,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -683,7 +743,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -748,7 +808,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next), next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -808,7 +868,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -869,7 +929,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
@ -932,7 +992,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next), next),
@ -1012,7 +1072,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -1087,7 +1147,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next), next => s3.listObjectVersions({ Bucket: bucket }, next),
@ -1159,7 +1219,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
@ -1233,7 +1293,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null', versionId: 'null',
}, },
authCredentials: backbeatAuthCredentials, authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(), requestBody: objMD,
}, next), }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next), next),

View File

@ -488,9 +488,9 @@ arraybuffer.slice@~0.0.7:
optionalDependencies: optionalDependencies:
ioctl "^2.0.2" ioctl "^2.0.2"
"arsenal@git+https://github.com/scality/arsenal#7.10.59": "arsenal@git+https://github.com/scality/arsenal#bugfix/ARSN-392/improve":
version "7.10.59" version "7.10.59"
resolved "git+https://github.com/scality/arsenal#f1891851b394f64a6be95c414767d314791884c9" resolved "git+https://github.com/scality/arsenal#bfe3eccbe340d29a4bfd16fb7c14765782ceaab5"
dependencies: dependencies:
"@types/async" "^3.2.12" "@types/async" "^3.2.12"
"@types/utf8" "^3.0.1" "@types/utf8" "^3.0.1"