Compare commits

...

2 Commits

4 changed files with 54 additions and 4 deletions

View File

@ -70,8 +70,6 @@ const constants = {
// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
maximumAllowedPartCount: 10000,
gcpMaximumAllowedPartCount: 1024,
// Max size on put part or copy part is 5GB. For functional
// testing use 110 MB as max
maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :

View File

@ -46,6 +46,19 @@ function completeMPU(params, callback) {
// first compose: in mpu bucket
// max 10,000 => 313 parts
// max component count per object 32
logger.trace('completeMultipartUpload: parttwo',
{ partCount: partList.length });
mpuHelper.splitMerge(params, partList, 'parttwo', next);
},
(numParts, next) => {
// second compose: in mpu bucket
// max 313 => 10 parts
const partList = Array.from(Array(numParts).keys(),
num => ({
PartName: createMpuKey(
params.Key, params.UploadId, num + 1, 'parttwo'),
})
);
logger.trace('completeMultipartUpload: compose',
{ partCount: partList.length });
mpuHelper.splitMerge(params, partList, 'compose', next);

View File

@ -2,7 +2,7 @@ const async = require('async');
const assert = require('assert');
const stream = require('stream');
const { errors } = require('arsenal');
const { minimumAllowedPartSize, gcpMaximumAllowedPartCount } =
const { minimumAllowedPartSize, maximumAllowedPartCount } =
require('../../../../constants');
const { createMpuList, logger } = require('./GcpUtils');
const { logHelper } = require('../utils');
@ -83,7 +83,7 @@ class GcpManagedUpload {
setPartSize() {
const newPartSize =
Math.ceil(this.totalBytes / gcpMaximumAllowedPartCount);
Math.ceil(this.totalBytes / maximumAllowedPartCount);
if (newPartSize > this.partSize) this.partSize = newPartSize;
this.totalParts = Math.ceil(this.totalBytes / this.partSize);
if (this.body instanceof Buffer && this.totalParts > 1) {

View File

@ -22,6 +22,7 @@ const partSize = 10;
const smallMD5 = '583c466f3f31d97b361adc60caea72f5-1';
const bigMD5 = '9c8a62e2c04a512ce348d8280497b49e-1024';
const maxMD5 = '0bd3785d5d1e3c90988917837bbf57fc-10000';
function gcpMpuSetupWrapper(params, callback) {
gcpMpuSetup(params, (err, result) => {
@ -192,4 +193,42 @@ describe('GCP: Complete MPU', function testSuite() {
});
});
});
describe('when MPU has 10000 uploaded parts', () => {
beforeEach(function beforeFn(done) {
this.currentTest.key = `somekey-${genUniqID()}`;
gcpMpuSetupWrapper.call(this, {
gcpClient,
bucketNames,
key: this.currentTest.key,
partCount: 10000, partSize,
}, done);
});
it('should successfully complete MPU',
function testFn(done) {
const parts = GcpUtils.createMpuList({
Key: this.test.key,
UploadId: this.test.uploadId,
}, 'parts', 10000).map(item => {
Object.assign(item, {
ETag: this.test.etagList[item.PartNumber - 1],
});
return item;
});
const params = {
Bucket: bucketNames.main.Name,
MPU: bucketNames.mpu.Name,
Key: this.test.key,
UploadId: this.test.uploadId,
MultipartUpload: { Parts: parts },
};
gcpClient.completeMultipartUpload(params, (err, res) => {
assert.equal(err, null,
`Expected success, but got error ${err}`);
assert.strictEqual(res.ETag, `"${maxMD5}"`);
return done();
});
});
});
});