Merge branch 'feature/ZENKO-869/updateGcpPartLimit' into tmp/octopus/w/8.1/feature/ZENKO-869/updateGcpPartLimit

constants.js

@@ -70,8 +70,6 @@ const constants = {
     // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
     maximumAllowedPartCount: 10000,
 
-    gcpMaximumAllowedPartCount: 1024,
-
     // Max size on put part or copy part is 5GB. For functional
     // testing use 110 MB as max
     maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 :

lib/data/external/GCP/GcpApis/completeMPU.js

@@ -46,6 +46,19 @@ function completeMPU(params, callback) {
             // first compose: in mpu bucket
             // max 10,000 => 313 parts
             // max component count per object 32
+            logger.trace('completeMultipartUpload: parttwo',
+                { partCount: partList.length });
+            mpuHelper.splitMerge(params, partList, 'parttwo', next);
+        },
+        (numParts, next) => {
+            // second compose: in mpu bucket
+            // max 313 => 10 parts
+            const partList = Array.from(Array(numParts).keys(),
+                num => ({
+                    PartName: createMpuKey(
+                        params.Key, params.UploadId, num + 1, 'parttwo'),
+                })
+            );
             logger.trace('completeMultipartUpload: compose',
                 { partCount: partList.length });
             mpuHelper.splitMerge(params, partList, 'compose', next);

lib/data/external/GCP/GcpManagedUpload.js

@@ -2,7 +2,7 @@ const async = require('async');
 const assert = require('assert');
 const stream = require('stream');
 const { errors } = require('arsenal');
-const { minimumAllowedPartSize, gcpMaximumAllowedPartCount } =
+const { minimumAllowedPartSize, maximumAllowedPartCount } =
     require('../../../../constants');
 const { createMpuList, logger } = require('./GcpUtils');
 const { logHelper } = require('../utils');
@@ -83,7 +83,7 @@ class GcpManagedUpload {
 
     setPartSize() {
         const newPartSize =
-            Math.ceil(this.totalBytes / gcpMaximumAllowedPartCount);
+            Math.ceil(this.totalBytes / maximumAllowedPartCount);
         if (newPartSize > this.partSize) this.partSize = newPartSize;
         this.totalParts = Math.ceil(this.totalBytes / this.partSize);
         if (this.body instanceof Buffer && this.totalParts > 1) {

tests/functional/raw-node/test/GCP/object/completeMpu.js

@@ -22,6 +22,7 @@ const partSize = 10;
 
 const smallMD5 = '583c466f3f31d97b361adc60caea72f5-1';
 const bigMD5 = '9c8a62e2c04a512ce348d8280497b49e-1024';
+const maxMD5 = '0bd3785d5d1e3c90988917837bbf57fc-10000';
 
 function gcpMpuSetupWrapper(params, callback) {
     gcpMpuSetup(params, (err, result) => {
@@ -192,4 +193,42 @@ describe('GCP: Complete MPU', function testSuite() {
             });
         });
     });
+
+    describe('when MPU has 10000 uploaded parts', () => {
+        beforeEach(function beforeFn(done) {
+            this.currentTest.key = `somekey-${genUniqID()}`;
+            gcpMpuSetupWrapper.call(this, {
+                gcpClient,
+                bucketNames,
+                key: this.currentTest.key,
+                partCount: 10000, partSize,
+            }, done);
+        });
+
+        it('should successfully complete MPU',
+        function testFn(done) {
+            const parts = GcpUtils.createMpuList({
+                Key: this.test.key,
+                UploadId: this.test.uploadId,
+            }, 'parts', 10000).map(item => {
+                Object.assign(item, {
+                    ETag: this.test.etagList[item.PartNumber - 1],
+                });
+                return item;
+            });
+            const params = {
+                Bucket: bucketNames.main.Name,
+                MPU: bucketNames.mpu.Name,
+                Key: this.test.key,
+                UploadId: this.test.uploadId,
+                MultipartUpload: { Parts: parts },
+            };
+            gcpClient.completeMultipartUpload(params, (err, res) => {
+                assert.equal(err, null,
+                    `Expected success, but got error ${err}`);
+                assert.strictEqual(res.ETag, `"${maxMD5}"`);
+                return done();
+            });
+        });
+    });
 });