Compare commits

...

3 Commits

Author SHA1 Message Date
Nicolas Humbert 5805b0b26b ARSN-105 Disjointed reduced locations
(cherry picked from commit 7d254a0556)
2022-06-09 12:27:56 -07:00
Artem Bakalov 77f3a5614a ARSN-212 remove assert in decoder in favor of returning error
(cherry picked from commit 3f26b432b7)
2022-05-19 17:49:36 -07:00
Artem Bakalov fae60cdf3f ARSN-157 short-IDs
(cherry picked from commit 07fd3451ab)
2022-05-10 18:10:43 -07:00
5 changed files with 399 additions and 77 deletions

View File

@ -575,8 +575,12 @@ class ObjectMD {
const locations = this.getLocation();
const reducedLocations = [];
let partTotal = 0;
let start;
for (let i = 0; i < locations.length; i++) {
const currPart = new ObjectMDLocation(locations[i]);
if (i === 0) {
start = currPart.getPartStart();
}
const currPartNum = currPart.getPartNumber();
let nextPartNum = undefined;
if (i < locations.length - 1) {
@ -586,7 +590,9 @@ class ObjectMD {
partTotal += currPart.getPartSize();
if (currPartNum !== nextPartNum) {
currPart.setPartSize(partTotal);
currPart.setPartStart(start);
reducedLocations.push(currPart.getValue());
start += partTotal;
partTotal = 0;
}
}

View File

@ -85,6 +85,11 @@ class ObjectMDLocation {
return this._data.start;
}
setPartStart(start) {
this._data.start = start;
return this;
}
getPartSize() {
return this._data.size;
}

View File

@ -116,6 +116,13 @@ function generateVersionId(info, replicationGroupId) {
lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0;
lastTimestamp = ts;
// if S3_VERSION_ID_ENCODING_TYPE is "hex", info is used. By default, it is not used.
if (process.env.S3_VERSION_ID_ENCODING_TYPE === 'hex') {
// info field stays as is
} else {
info = ''; // eslint-disable-line
}
// In the default cases, we reverse the chronological order of the
// timestamps so that all versions of an object can be retrieved in the
// reversed chronological order---newest versions first. This is because of
@ -166,10 +173,11 @@ const B62V_TOTAL = LENGTH_TS + LENGTH_SEQ;
const B62V_HALF = B62V_TOTAL / 2;
const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62))));
const B62V_DPAD = '0'.repeat(B62V_HALF);
const B62V_STRING_EPAD = '0'.repeat(32 - 2 * B62V_EPAD.length);
/**
* Encode a versionId to obscure internal information contained
* in a version ID (less than 40 bytes).
* in a version ID (equal to 32 bytes).
*
* @param {string} str - the versionId to encode
* @return {string} - the encoded base62VersionId
@ -181,9 +189,10 @@ function base62Encode(str) {
const enc1 = base62Integer.encode(part1);
const enc2 = base62Integer.encode(part2);
const enc3 = base62String.encode(part3);
return (B62V_EPAD + enc1).slice(-B62V_EPAD.length) +
(B62V_EPAD + enc2).slice(-B62V_EPAD.length) +
enc3;
const padded1 = (B62V_EPAD + enc1).slice(-B62V_EPAD.length);
const padded2 = (B62V_EPAD + enc2).slice(-B62V_EPAD.length);
const padded3 = (B62V_STRING_EPAD + enc3).slice(-B62V_STRING_EPAD.length);
return padded1 + padded2 + padded3;
}
/**
@ -202,8 +211,19 @@ function base62Decode(str) {
const enc2 = str.substring(start, start + B62V_EPAD.length);
const orig2 = base62Integer.decode(enc2);
start += B62V_EPAD.length;
const enc3 = str.substring(start);
let enc3 = str.substring(start);
// take off prefix 0s which represent null bytes
let idx = 0;
while (idx < enc3.length) {
if (enc3[idx] === '0') {
idx++;
} else {
break;
}
}
enc3 = enc3.slice(idx);
const orig3 = base62String.decode(enc3);
return (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) +
(B62V_DPAD + orig2.toString()).slice(-B62V_DPAD.length) +
orig3.toString();
@ -221,13 +241,13 @@ const ENC_TYPE_BASE62 = 1; // new (tiny) encoding
* in a version ID.
*
* @param {string} str - the versionId to encode
* @param {Number} [encType=ENC_TYPE_HEX] - encode format
* @return {string} - the encoded versionId
*/
function encode(str, encType = ENC_TYPE_HEX) {
if (encType === ENC_TYPE_BASE62) {
function encode(str) {
// default format without 'info' field will always be 27 characters
if (str.length === 27) {
return base62Encode(str);
}
} // legacy format
return hexEncode(str);
}
@ -240,11 +260,20 @@ function encode(str, encType = ENC_TYPE_HEX) {
* @return {(string|Error)} - the decoded versionId or an error
*/
function decode(str) {
if (str.length < 40) {
return base62Decode(str);
// default format is exactly 32 characters when encoded
if (str.length === 32) {
const decoded = base62Decode(str);
if (typeof decoded === 'string' && decoded.length !== 27) {
return new Error(`decoded ${str} is not length 27`);
}
return decoded;
}
// legacy format
if (str.length > 32) {
return hexDecode(str);
}
return new Error(`cannot decode str ${str.length}`);
}
module.exports = { generateVersionId, getInfVid,
hexEncode, hexDecode,

View File

@ -350,3 +350,233 @@ describe('getAttributes static method', () => {
assert.deepStrictEqual(attributes, expectedResult);
});
});
describe('ObjectMD::getReducedLocations', () => {
it('should not alter an array when each part has only one element', () => {
const md = new ObjectMD();
const locations = [
{
key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
size: 1,
start: 0,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: '4e67844b674b093a9e109d42172922ea1f32ec12',
size: 3,
start: 1,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
];
md.setLocation(locations);
assert.deepStrictEqual(md.getReducedLocations(), locations);
});
it('should reduce an array when first part is > 1 element', () => {
const md = new ObjectMD();
md.setLocation([
{
key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
size: 1,
start: 0,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
size: 1,
start: 1,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: '4e67844b674b093a9e109d42172922ea1f32ec12',
size: 3,
start: 2,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
]);
assert.deepStrictEqual(md.getReducedLocations(), [
{
key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
size: 2,
start: 0,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: '4e67844b674b093a9e109d42172922ea1f32ec12',
size: 3,
start: 2,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
]);
});
it('should reduce an array when second part is > 1 element', () => {
const md = new ObjectMD();
md.setLocation([
{
key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
size: 1,
start: 0,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
size: 1,
start: 1,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
{
key: '4e67844b674b093a9e109d42172922ea1f32ec12',
size: 3,
start: 2,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
]);
assert.deepStrictEqual(md.getReducedLocations(), [
{
key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
size: 1,
start: 0,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: '4e67844b674b093a9e109d42172922ea1f32ec12',
size: 4,
start: 1,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
]);
});
it('should reduce an array when multiple parts are > 1 element', () => {
const md = new ObjectMD();
md.setLocation([
{
key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
size: 1,
start: 0,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: 'c1c1e055b19eb5a61adb8a665e626ff589cff234',
size: 2,
start: 1,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
size: 1,
start: 3,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: '8e67844b674b093a9e109d42172922ea1f32ec14',
size: 3,
start: 4,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
{
key: 'd1d1e055b19eb5a61adb8a665e626ff589cff233',
size: 10,
start: 7,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
{
key: '0e67844b674b093a9e109d42172922ea1f32ec11',
size: 10,
start: 17,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
{
key: '8e67844b674b093a9e109d42172922ea1f32ec14',
size: 15,
start: 27,
dataStoreName: 'file',
dataStoreETag: '3:1ca655158ca025aa00a818b6b81f9e4c',
},
{
key: '7e67844b674b093a9e109d42172922ea1f32ec1f',
size: 2,
start: 42,
dataStoreName: 'file',
dataStoreETag: '3:1ca655158ca025aa00a818b6b81f9e4c',
},
{
key: '1237844b674b093a9e109d42172922ea1f32ec19',
size: 6,
start: 44,
dataStoreName: 'file',
dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
},
{
key: '4567844b674b093a9e109d42172922ea1f32ec00',
size: 4,
start: 50,
dataStoreName: 'file',
dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
},
{
key: '53d7844b674b093a9e109d42172922ea1f32ec02',
size: 9,
start: 54,
dataStoreName: 'file',
dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
},
{
key: '6f6d7844b674b093a9e109d42172922ea1f32ec01',
size: 2,
start: 63,
dataStoreName: 'file',
dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
},
]);
assert.deepStrictEqual(md.getReducedLocations(), [
{
key: 'deebfb287cfcee1d137b0136562d2d776ba491e1',
size: 4,
start: 0,
dataStoreName: 'file',
dataStoreETag: '1:0e5a6f42662652d44fcf978399ef5709',
},
{
key: '0e67844b674b093a9e109d42172922ea1f32ec11',
size: 23,
start: 4,
dataStoreName: 'file',
dataStoreETag: '2:9ca655158ca025aa00a818b6b81f9e48',
},
{
key: '7e67844b674b093a9e109d42172922ea1f32ec1f',
size: 17,
start: 27,
dataStoreName: 'file',
dataStoreETag: '3:1ca655158ca025aa00a818b6b81f9e4c',
},
{
key: '6f6d7844b674b093a9e109d42172922ea1f32ec01',
size: 21,
start: 44,
dataStoreName: 'file',
dataStoreETag: '4:afa655158ca025aa00a818b6b81f9e4d',
},
]);
});
});

View File

@ -1,5 +1,6 @@
const VID = require('../../../lib/versioning/VersionID.js');
const assert = require('assert');
const { env } = require('process');
function randkey(length) {
let key = '';
@ -9,21 +10,41 @@ function randkey(length) {
return key;
}
describe('test generating versionIds', () => {
const count = 1000;
process.env.VID_CRYPTO_PASSWORD = randkey(64);
function generateRandomVIDs(count) {
const vids = Array(count).fill(null);
for (let i = 0; i < count; i++) {
vids[i] = VID.generateVersionId(randkey(15), 'PARIS');
}
process.env.VID_CRYPTO_PASSWORD = randkey(64);
return vids;
}
const count = 1000000;
describe('test generating versionIds', function() { // eslint-disable-line
this.timeout(80000);
describe('invalid IDs', () => {
// A client can use the CLI to send requests with arbitrary version IDs.
// These IDs may contain invalid characters and should be handled gracefully.
it('should return an error when an ID has unsupported characters', () => {
const encoded = 'wHtI53.S4ApsYLRI5VZZ3Iw.7ny4NgQz';
const decoded = VID.decode(encoded);
assert(decoded instanceof Error);
assert.strictEqual(decoded.message, 'Non-base62 character');
});
});
describe('legaxy hex encoding', () => {
env.S3_VERSION_ID_ENCODING_TYPE = 'hex';
const vids = generateRandomVIDs(count);
it('sorted in reversed chronological and alphabetical order', () => {
for (let i = 0; i < count; i++) {
if (i !== 0) {
for (let i = 1; i < count; i++) {
assert(vids[i - 1] > vids[i],
'previous VersionID is higher than its next');
}
}
});
// nodejs 10 no longer returns error for non-hex string versionIds
@ -36,10 +57,32 @@ describe('test generating versionIds', () => {
it('should encode and decode versionIds', () => {
const encoded = vids.map(vid => VID.hexEncode(vid));
const decoded = encoded.map(vid => VID.hexDecode(vid));
assert.strictEqual(vids.length, count);
assert.deepStrictEqual(vids, decoded);
});
it('should encode and decode correctly with legacy format', () => {
const encoded = vids.map(VID.encode);
const decoded = encoded.map(VID.decode);
assert.strictEqual(vids.every(x => x.length > 27), true);
assert.strictEqual(encoded.every(x => x.length > 32), true);
assert.deepStrictEqual(vids, decoded);
});
});
describe('Short IDs', () => {
env.S3_VERSION_ID_ENCODING_TYPE = 'base62';
const vids = generateRandomVIDs(count);
it('sorted in reversed chronological and alphabetical order', () => {
for (let i = 1; i < count; i++) {
assert(vids[i - 1] > vids[i],
'previous VersionID is higher than its next');
}
},
);
it('simple base62 version test', () => {
const vid = '98376906954349999999RG001 145.20.5';
const encoded = VID.base62Encode(vid);
@ -88,4 +131,13 @@ describe('test generating versionIds', () => {
assert.strictEqual(vids.length, count);
assert.deepStrictEqual(vids, decoded);
});
it('should encode and decode correctly with new 32 byte format', () => {
const encoded = vids.map(vid => VID.encode(vid));
const decoded = encoded.map(vid => VID.decode(vid));
assert(vids.every(x => x.length === 27));
assert(encoded.every(x => x.length === 32));
assert.deepStrictEqual(vids, decoded);
});
});
});