ARSN-105 Disjointed reduced locations

(cherry picked from commit 7d254a0556)

lib/models/ObjectMD.js

@@ -575,8 +575,12 @@ class ObjectMD {
         const locations = this.getLocation();
         const reducedLocations = [];
         let partTotal = 0;
+        let start;
         for (let i = 0; i < locations.length; i++) {
             const currPart = new ObjectMDLocation(locations[i]);
+            if (i === 0) {
+                start = currPart.getPartStart();
+            }
             const currPartNum = currPart.getPartNumber();
             let nextPartNum = undefined;
             if (i < locations.length - 1) {
@@ -586,7 +590,9 @@ class ObjectMD {
             partTotal += currPart.getPartSize();
             if (currPartNum !== nextPartNum) {
                 currPart.setPartSize(partTotal);
+                currPart.setPartStart(start);
                 reducedLocations.push(currPart.getValue());
+                start += partTotal;
                 partTotal = 0;
             }
         }

lib/models/ObjectMDLocation.js

@@ -85,6 +85,11 @@ class ObjectMDLocation {
         return this._data.start;
     }
 
+    setPartStart(start) {
+        this._data.start = start;
+        return this;
+    }
+
     getPartSize() {
         return this._data.size;
     }

lib/versioning/VersionID.js

@@ -116,12 +116,19 @@ function generateVersionId(info, replicationGroupId) {
     lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0;
     lastTimestamp = ts;
 
+    // if S3_VERSION_ID_ENCODING_TYPE is "hex", info is used. By default, it is not used.
+    if (process.env.S3_VERSION_ID_ENCODING_TYPE === 'hex') {
+        // info field stays as is
+    } else {
+        info = ''; // eslint-disable-line
+    }
+
     // In the default cases, we reverse the chronological order of the
     // timestamps so that all versions of an object can be retrieved in the
     // reversed chronological order---newest versions first. This is because of
     // the limitation of leveldb for listing keys in the reverse order.
     return padLeft(MAX_TS - lastTimestamp, TEMPLATE_TS) +
-           padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + repGroupId + info;
+        padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + repGroupId + info;
 }
 
 /**
@@ -166,10 +173,11 @@ const B62V_TOTAL = LENGTH_TS + LENGTH_SEQ;
 const B62V_HALF = B62V_TOTAL / 2;
 const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62))));
 const B62V_DPAD = '0'.repeat(B62V_HALF);
+const B62V_STRING_EPAD = '0'.repeat(32 - 2 * B62V_EPAD.length);
 
 /**
  * Encode a versionId to obscure internal information contained
- * in a version ID (less than 40 bytes).
+ * in a version ID (equal to 32 bytes).
  *
  * @param {string} str - the versionId to encode
  * @return {string} - the encoded base62VersionId
@@ -181,9 +189,10 @@ function base62Encode(str) {
     const enc1 = base62Integer.encode(part1);
     const enc2 = base62Integer.encode(part2);
     const enc3 = base62String.encode(part3);
-    return (B62V_EPAD + enc1).slice(-B62V_EPAD.length) +
+    const padded1 = (B62V_EPAD + enc1).slice(-B62V_EPAD.length);
-        (B62V_EPAD + enc2).slice(-B62V_EPAD.length) +
+    const padded2 = (B62V_EPAD + enc2).slice(-B62V_EPAD.length);
-        enc3;
+    const padded3 = (B62V_STRING_EPAD + enc3).slice(-B62V_STRING_EPAD.length);
+    return padded1 + padded2 + padded3;
 }
 
 /**
@@ -202,8 +211,19 @@ function base62Decode(str) {
         const enc2 = str.substring(start, start + B62V_EPAD.length);
         const orig2 = base62Integer.decode(enc2);
         start += B62V_EPAD.length;
-        const enc3 = str.substring(start);
+        let enc3 = str.substring(start);
+        // take off prefix 0s which represent null bytes
+        let idx = 0;
+        while (idx < enc3.length) {
+            if (enc3[idx] === '0') {
+                idx++;
+            } else {
+                break;
+            }
+        }
+        enc3 = enc3.slice(idx);
         const orig3 = base62String.decode(enc3);
+
         return (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) +
             (B62V_DPAD + orig2.toString()).slice(-B62V_DPAD.length) +
             orig3.toString();
@@ -221,13 +241,13 @@ const ENC_TYPE_BASE62 = 1; // new (tiny) encoding
  * in a version ID.
  *
  * @param {string} str - the versionId to encode
- * @param {Number} [encType=ENC_TYPE_HEX] - encode format
  * @return {string} - the encoded versionId
  */
-function encode(str, encType = ENC_TYPE_HEX) {
+function encode(str) {
-    if (encType === ENC_TYPE_BASE62) {
+    // default format without 'info' field will always be 27 characters
+    if (str.length === 27) {
         return base62Encode(str);
-    }
+    } // legacy format
     return hexEncode(str);
 }
 
@@ -240,14 +260,23 @@ function encode(str, encType = ENC_TYPE_HEX) {
  * @return {(string|Error)} - the decoded versionId or an error
  */
 function decode(str) {
-    if (str.length < 40) {
+    // default format is exactly 32 characters when encoded
-        return base62Decode(str);
+    if (str.length === 32) {
+        const decoded = base62Decode(str);
+        if (typeof decoded === 'string' && decoded.length !== 27) {
+            return new Error(`decoded ${str} is not length 27`);
+        }
+        return decoded;
     }
-    return hexDecode(str);
+    // legacy format
+    if (str.length > 32) {
+        return hexDecode(str);
+    }
+    return new Error(`cannot decode str ${str.length}`);
 }
 
 module.exports = { generateVersionId, getInfVid,
-                   hexEncode, hexDecode,
+    hexEncode, hexDecode,
-                   base62Encode, base62Decode,
+    base62Encode, base62Decode,
-                   encode, decode,
+    encode, decode,
-                   ENC_TYPE_HEX, ENC_TYPE_BASE62 };
+    ENC_TYPE_HEX, ENC_TYPE_BASE62 };

tests/unit/models/object.js

@@ -350,3 +350,233 @@ describe('getAttributes static method', () => {
         assert.deepStrictEqual(attributes, expectedResult);
     });
 });
+
+describe('ObjectMD::getReducedLocations', () => {
+    it('should not alter an array when each part has only one element', () => {
+        const md = new ObjectMD();
+        const locations = [
+            {
+                key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
+                size: 1,
+                start: 0,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: '4e67844b674b093a9e109d42172922ea1f32ec12',
+                size: 3,
+                start: 1,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+        ];
+        md.setLocation(locations);
+        assert.deepStrictEqual(md.getReducedLocations(), locations);
+    });
+
+    it('should reduce an array when first part is > 1 element', () => {
+        const md = new ObjectMD();
+        md.setLocation([
+            {
+                key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
+                size: 1,
+                start: 0,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
+                size: 1,
+                start: 1,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: '4e67844b674b093a9e109d42172922ea1f32ec12',
+                size: 3,
+                start: 2,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+        ]);
+        assert.deepStrictEqual(md.getReducedLocations(), [
+            {
+                key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
+                size: 2,
+                start: 0,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: '4e67844b674b093a9e109d42172922ea1f32ec12',
+                size: 3,
+                start: 2,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+        ]);
+    });
+
+    it('should reduce an array when second part is > 1 element', () => {
+        const md = new ObjectMD();
+        md.setLocation([
+            {
+                key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
+                size: 1,
+                start: 0,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
+                size: 1,
+                start: 1,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+            {
+                key: '4e67844b674b093a9e109d42172922ea1f32ec12',
+                size: 3,
+                start: 2,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+        ]);
+        assert.deepStrictEqual(md.getReducedLocations(), [
+            {
+                key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
+                size: 1,
+                start: 0,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: '4e67844b674b093a9e109d42172922ea1f32ec12',
+                size: 4,
+                start: 1,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+        ]);
+    });
+
+    it('should reduce an array when multiple parts are > 1 element', () => {
+        const md = new ObjectMD();
+        md.setLocation([
+            {
+                key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
+                size: 1,
+                start: 0,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: 'c1c1e055b19eb5a61adb8a665e626ff589cff234',
+                size: 2,
+                start: 1,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
+                size: 1,
+                start: 3,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: '8e67844b674b093a9e109d42172922ea1f32ec14',
+                size: 3,
+                start: 4,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+            {
+                key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
+                size: 10,
+                start: 7,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+            {
+                key: '0e67844b674b093a9e109d42172922ea1f32ec11',
+                size: 10,
+                start: 17,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+            {
+                key: '8e67844b674b093a9e109d42172922ea1f32ec14',
+                size: 15,
+                start: 27,
+                dataStoreName: 'file',
+                dataStoreETag: '3:1ca655158ca025aa00a818b6b81f9e4c',
+            },
+            {
+                key: '7e67844b674b093a9e109d42172922ea1f32ec1f',
+                size: 2,
+                start: 42,
+                dataStoreName: 'file',
+                dataStoreETag: '3:1ca655158ca025aa00a818b6b81f9e4c',
+            },
+            {
+                key: '1237844b674b093a9e109d42172922ea1f32ec19',
+                size: 6,
+                start: 44,
+                dataStoreName: 'file',
+                dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
+            },
+            {
+                key: '4567844b674b093a9e109d42172922ea1f32ec00',
+                size: 4,
+                start: 50,
+                dataStoreName: 'file',
+                dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
+            },
+            {
+                key: '53d7844b674b093a9e109d42172922ea1f32ec02',
+                size: 9,
+                start: 54,
+                dataStoreName: 'file',
+                dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
+            },
+            {
+                key: '6f6d7844b674b093a9e109d42172922ea1f32ec01',
+                size: 2,
+                start: 63,
+                dataStoreName: 'file',
+                dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
+            },
+        ]);
+        assert.deepStrictEqual(md.getReducedLocations(), [
+            {
+                key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
+                size: 4,
+                start: 0,
+                dataStoreName: 'file',
+                dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
+            },
+            {
+                key: '0e67844b674b093a9e109d42172922ea1f32ec11',
+                size: 23,
+                start: 4,
+                dataStoreName: 'file',
+                dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
+            },
+            {
+                key: '7e67844b674b093a9e109d42172922ea1f32ec1f',
+                size: 17,
+                start: 27,
+                dataStoreName: 'file',
+                dataStoreETag: '3:1ca655158ca025aa00a818b6b81f9e4c',
+            },
+            {
+                key: '6f6d7844b674b093a9e109d42172922ea1f32ec01',
+                size: 21,
+                start: 44,
+                dataStoreName: 'file',
+                dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
+            },
+        ]);
+    });
+});

tests/unit/versioning/VersionID.js

@@ -1,5 +1,6 @@
 const VID = require('../../../lib/versioning/VersionID.js');
 const assert = require('assert');
+const { env } = require('process');
 
 function randkey(length) {
     let key = '';
@@ -9,83 +10,134 @@ function randkey(length) {
     return key;
 }
 
-describe('test generating versionIds', () => {
+process.env.VID_CRYPTO_PASSWORD = randkey(64);
-    const count = 1000;
+
+function generateRandomVIDs(count) {
     const vids = Array(count).fill(null);
     for (let i = 0; i < count; i++) {
         vids[i] = VID.generateVersionId(randkey(15), 'PARIS');
     }
-    process.env.VID_CRYPTO_PASSWORD = randkey(64);
+    return vids;
+}
 
-    it('sorted in reversed chronological and alphabetical order', () => {
+const count = 1000000;
-        for (let i = 0; i < count; i++) {
+
-            if (i !== 0) {
+describe('test generating versionIds', function() { // eslint-disable-line
+    this.timeout(80000);
+
+    describe('invalid IDs', () => {
+        // A client can use the CLI to send requests with arbitrary version IDs.
+        // These IDs may contain invalid characters and should be handled gracefully.
+        it('should return an error when an ID has unsupported characters', () => {
+            const encoded = 'wHtI53.S4ApsYLRI5VZZ3Iw.7ny4NgQz';
+            const decoded = VID.decode(encoded);
+            assert(decoded instanceof Error);
+            assert.strictEqual(decoded.message, 'Non-base62 character');
+        });
+    });
+
+    describe('legaxy hex encoding', () => {
+        env.S3_VERSION_ID_ENCODING_TYPE = 'hex';
+        const vids = generateRandomVIDs(count);
+
+        it('sorted in reversed chronological and alphabetical order', () => {
+            for (let i = 1; i < count; i++) {
                 assert(vids[i - 1] > vids[i],
                     'previous VersionID is higher than its next');
             }
-        }
+        });
+
+        // nodejs 10 no longer returns error for non-hex string versionIds
+        it.skip('should return error decoding non-hex string versionIds', () => {
+            const encoded = vids.map(vid => VID.hexEncode(vid));
+            const decoded = encoded.map(vid => VID.hexDecode(`${vid}foo`));
+            decoded.forEach(result => assert(result instanceof Error));
+        });
+
+        it('should encode and decode versionIds', () => {
+            const encoded = vids.map(vid => VID.hexEncode(vid));
+            const decoded = encoded.map(vid => VID.hexDecode(vid));
+            assert.deepStrictEqual(vids, decoded);
+        });
+
+        it('should encode and decode correctly with legacy format', () => {
+            const encoded = vids.map(VID.encode);
+            const decoded = encoded.map(VID.decode);
+
+            assert.strictEqual(vids.every(x => x.length > 27), true);
+            assert.strictEqual(encoded.every(x => x.length > 32), true);
+            assert.deepStrictEqual(vids, decoded);
+        });
     });
 
-    // nodejs 10 no longer returns error for non-hex string versionIds
-    it.skip('should return error decoding non-hex string versionIds', () => {
-        const encoded = vids.map(vid => VID.hexEncode(vid));
-        const decoded = encoded.map(vid => VID.hexDecode(`${vid}foo`));
-        decoded.forEach(result => assert(result instanceof Error));
-    });
 
-    it('should encode and decode versionIds', () => {
+    describe('Short IDs', () => {
-        const encoded = vids.map(vid => VID.hexEncode(vid));
+        env.S3_VERSION_ID_ENCODING_TYPE = 'base62';
-        const decoded = encoded.map(vid => VID.hexDecode(vid));
+        const vids = generateRandomVIDs(count);
-        assert.strictEqual(vids.length, count);
-        assert.deepStrictEqual(vids, decoded);
-    });
 
-    it('simple base62 version test', () => {
+        it('sorted in reversed chronological and alphabetical order', () => {
-        const vid = '98376906954349999999RG001  145.20.5';
+            for (let i = 1; i < count; i++) {
-        const encoded = VID.base62Encode(vid);
+                assert(vids[i - 1] > vids[i],
-        assert.strictEqual(encoded, 'aJLWKz4Ko9IjBBgXKj5KQT2G9UHv0g7P');
+                    'previous VersionID is higher than its next');
-        const decoded = VID.base62Decode(encoded);
+            }
-        assert.strictEqual(vid, decoded);
+        },
-    });
+        );
 
-    it('base62 version test with smaller part1 number', () => {
+        it('simple base62 version test', () => {
-        const vid = '00000000054349999999RG001  145.20.5';
+            const vid = '98376906954349999999RG001  145.20.5';
-        const encoded = VID.base62Encode(vid);
+            const encoded = VID.base62Encode(vid);
-        const decoded = VID.base62Decode(encoded);
+            assert.strictEqual(encoded, 'aJLWKz4Ko9IjBBgXKj5KQT2G9UHv0g7P');
-        assert.strictEqual(vid, decoded);
+            const decoded = VID.base62Decode(encoded);
-    });
+            assert.strictEqual(vid, decoded);
+        });
 
-    it('base62 version test with smaller part2 number', () => {
+        it('base62 version test with smaller part1 number', () => {
-        const vid = '98376906950000099999RG001  145.20.5';
+            const vid = '00000000054349999999RG001  145.20.5';
-        const encoded = VID.base62Encode(vid);
+            const encoded = VID.base62Encode(vid);
-        const decoded = VID.base62Decode(encoded);
+            const decoded = VID.base62Decode(encoded);
-        assert.strictEqual(vid, decoded);
+            assert.strictEqual(vid, decoded);
-    });
+        });
 
-    it('base62 version test with smaller part3', () => {
+        it('base62 version test with smaller part2 number', () => {
-        const vid = '98376906950000099999R1  145.20.5';
+            const vid = '98376906950000099999RG001  145.20.5';
-        const encoded = VID.base62Encode(vid);
+            const encoded = VID.base62Encode(vid);
-        const decoded = VID.base62Decode(encoded);
+            const decoded = VID.base62Decode(encoded);
-        assert.strictEqual(vid, decoded);
+            assert.strictEqual(vid, decoded);
-    });
+        });
 
-    it('base62 version test with smaller part3 - 2', () => {
+        it('base62 version test with smaller part3', () => {
-        const vid = '98376906950000099999R1x';
+            const vid = '98376906950000099999R1  145.20.5';
-        const encoded = VID.base62Encode(vid);
+            const encoded = VID.base62Encode(vid);
-        const decoded = VID.base62Decode(encoded);
+            const decoded = VID.base62Decode(encoded);
-        assert.strictEqual(vid, decoded);
+            assert.strictEqual(vid, decoded);
-    });
+        });
 
-    it('error case: when invalid base62 key part 3 has invalid base62 character', () => {
+        it('base62 version test with smaller part3 - 2', () => {
-        const invalidBase62VersionId = 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P';
+            const vid = '98376906950000099999R1x';
-        const decoded = VID.base62Decode(invalidBase62VersionId);
+            const encoded = VID.base62Encode(vid);
-        assert(decoded instanceof Error);
+            const decoded = VID.base62Decode(encoded);
-    });
+            assert.strictEqual(vid, decoded);
+        });
 
-    it('should encode and decode base62 versionIds', () => {
+        it('error case: when invalid base62 key part 3 has invalid base62 character', () => {
-        const encoded = vids.map(vid => VID.base62Encode(vid));
+            const invalidBase62VersionId = 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P';
-        const decoded = encoded.map(vid => VID.base62Decode(vid));
+            const decoded = VID.base62Decode(invalidBase62VersionId);
-        assert.strictEqual(vids.length, count);
+            assert(decoded instanceof Error);
-        assert.deepStrictEqual(vids, decoded);
+        });
+
+        it('should encode and decode base62 versionIds', () => {
+            const encoded = vids.map(vid => VID.base62Encode(vid));
+            const decoded = encoded.map(vid => VID.base62Decode(vid));
+            assert.strictEqual(vids.length, count);
+            assert.deepStrictEqual(vids, decoded);
+        });
+
+        it('should encode and decode correctly with new 32 byte format', () => {
+            const encoded = vids.map(vid => VID.encode(vid));
+            const decoded = encoded.map(vid => VID.decode(vid));
+            assert(vids.every(x => x.length === 27));
+            assert(encoded.every(x => x.length === 32));
+            assert.deepStrictEqual(vids, decoded);
+        });
     });
 });