Compare commits

...

2 Commits

Author SHA1 Message Date
Xin LI 0244d8b4d9 improvement: lint 2022-05-06 18:35:03 +02:00
Xin LI 72c293ebc2 Revert "ARSN-157 short-IDs"
This reverts commit 07fd3451ab.
2022-05-06 18:31:30 +02:00
2 changed files with 73 additions and 140 deletions

View File

@ -9,7 +9,6 @@
const base62Integer = require('base62'); const base62Integer = require('base62');
const BASE62 = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; const BASE62 = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
const base62String = require('base-x')(BASE62); const base62String = require('base-x')(BASE62);
const assert = require('assert/strict');
// the lengths of the components in bytes // the lengths of the components in bytes
const LENGTH_TS = 14; // timestamp: epoch in ms const LENGTH_TS = 14; // timestamp: epoch in ms
@ -117,13 +116,6 @@ function generateVersionId(info, replicationGroupId) {
lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0; lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0;
lastTimestamp = ts; lastTimestamp = ts;
// if S3_VERSION_ID_ENCODING_TYPE is "hex", info is used. By default, it is not used.
if (process.env.S3_VERSION_ID_ENCODING_TYPE === 'hex') {
// info field stays as is
} else {
info = ''; // eslint-disable-line
}
// In the default cases, we reverse the chronological order of the // In the default cases, we reverse the chronological order of the
// timestamps so that all versions of an object can be retrieved in the // timestamps so that all versions of an object can be retrieved in the
// reversed chronological order---newest versions first. This is because of // reversed chronological order---newest versions first. This is because of
@ -174,11 +166,10 @@ const B62V_TOTAL = LENGTH_TS + LENGTH_SEQ;
const B62V_HALF = B62V_TOTAL / 2; const B62V_HALF = B62V_TOTAL / 2;
const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62)))); const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62))));
const B62V_DPAD = '0'.repeat(B62V_HALF); const B62V_DPAD = '0'.repeat(B62V_HALF);
const B62V_STRING_EPAD = '0'.repeat(32 - 2 * B62V_EPAD.length);
/** /**
* Encode a versionId to obscure internal information contained * Encode a versionId to obscure internal information contained
* in a version ID (equal to 32 bytes). * in a version ID (less than 40 bytes).
* *
* @param {string} str - the versionId to encode * @param {string} str - the versionId to encode
* @return {string} - the encoded base62VersionId * @return {string} - the encoded base62VersionId
@ -190,10 +181,9 @@ function base62Encode(str) {
const enc1 = base62Integer.encode(part1); const enc1 = base62Integer.encode(part1);
const enc2 = base62Integer.encode(part2); const enc2 = base62Integer.encode(part2);
const enc3 = base62String.encode(part3); const enc3 = base62String.encode(part3);
const padded1 = (B62V_EPAD + enc1).slice(-B62V_EPAD.length); return (B62V_EPAD + enc1).slice(-B62V_EPAD.length) +
const padded2 = (B62V_EPAD + enc2).slice(-B62V_EPAD.length); (B62V_EPAD + enc2).slice(-B62V_EPAD.length) +
const padded3 = (B62V_STRING_EPAD + enc3).slice(-B62V_STRING_EPAD.length); enc3;
return padded1 + padded2 + padded3;
} }
/** /**
@ -212,19 +202,8 @@ function base62Decode(str) {
const enc2 = str.substring(start, start + B62V_EPAD.length); const enc2 = str.substring(start, start + B62V_EPAD.length);
const orig2 = base62Integer.decode(enc2); const orig2 = base62Integer.decode(enc2);
start += B62V_EPAD.length; start += B62V_EPAD.length;
let enc3 = str.substring(start); const enc3 = str.substring(start);
// take off prefix 0s which represent null bytes
let idx = 0;
while (idx < enc3.length) {
if (enc3[idx] === '0') {
idx++;
} else {
break;
}
}
enc3 = enc3.slice(idx);
const orig3 = base62String.decode(enc3); const orig3 = base62String.decode(enc3);
return (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) + return (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) +
(B62V_DPAD + orig2.toString()).slice(-B62V_DPAD.length) + (B62V_DPAD + orig2.toString()).slice(-B62V_DPAD.length) +
orig3.toString(); orig3.toString();
@ -242,13 +221,13 @@ const ENC_TYPE_BASE62 = 1; // new (tiny) encoding
* in a version ID. * in a version ID.
* *
* @param {string} str - the versionId to encode * @param {string} str - the versionId to encode
* @param {Number} [encType=ENC_TYPE_HEX] - encode format
* @return {string} - the encoded versionId * @return {string} - the encoded versionId
*/ */
function encode(str) { function encode(str, encType = ENC_TYPE_HEX) {
// default format without 'info' field will always be 27 characters if (encType === ENC_TYPE_BASE62) {
if (str.length === 27) {
return base62Encode(str); return base62Encode(str);
} // legacy format }
return hexEncode(str); return hexEncode(str);
} }
@ -261,18 +240,11 @@ function encode(str) {
* @return {(string|Error)} - the decoded versionId or an error * @return {(string|Error)} - the decoded versionId or an error
*/ */
function decode(str) { function decode(str) {
// default format is exactly 32 characters when encoded if (str.length < 40) {
if (str.length === 32) { return base62Decode(str);
const decoded = base62Decode(str);
assert.strictEqual(decoded.length, 27);
return decoded;
} }
// legacy format
if (str.length > 32) {
return hexDecode(str); return hexDecode(str);
} }
return new Error(`cannot decode str ${str.length}`);
}
module.exports = { generateVersionId, getInfVid, module.exports = { generateVersionId, getInfVid,
hexEncode, hexDecode, hexEncode, hexDecode,

View File

@ -1,6 +1,5 @@
const VID = require('../../../lib/versioning/VersionID.js'); const VID = require('../../../lib/versioning/VersionID.js');
const assert = require('assert'); const assert = require('assert');
const { env } = require('process');
function randkey(length) { function randkey(length) {
let key = ''; let key = '';
@ -10,28 +9,21 @@ function randkey(length) {
return key; return key;
} }
process.env.VID_CRYPTO_PASSWORD = randkey(64); describe('test generating versionIds', () => {
const count = 1000;
function generateRandomVIDs(count) {
const vids = Array(count).fill(null); const vids = Array(count).fill(null);
for (let i = 0; i < count; i++) { for (let i = 0; i < count; i++) {
vids[i] = VID.generateVersionId(randkey(15), 'PARIS'); vids[i] = VID.generateVersionId(randkey(15), 'PARIS');
} }
return vids; process.env.VID_CRYPTO_PASSWORD = randkey(64);
}
const count = 1000000;
describe('test generating versionIds', () => {
describe('legaxy hex encoding', () => {
env.S3_VERSION_ID_ENCODING_TYPE = 'hex';
const vids = generateRandomVIDs(count);
it('sorted in reversed chronological and alphabetical order', () => { it('sorted in reversed chronological and alphabetical order', () => {
for (let i = 1; i < count; i++) { for (let i = 0; i < count; i++) {
if (i !== 0) {
assert(vids[i - 1] > vids[i], assert(vids[i - 1] > vids[i],
'previous VersionID is higher than its next'); 'previous VersionID is higher than its next');
} }
}
}); });
// nodejs 10 no longer returns error for non-hex string versionIds // nodejs 10 no longer returns error for non-hex string versionIds
@ -44,32 +36,10 @@ describe('test generating versionIds', () => {
it('should encode and decode versionIds', () => { it('should encode and decode versionIds', () => {
const encoded = vids.map(vid => VID.hexEncode(vid)); const encoded = vids.map(vid => VID.hexEncode(vid));
const decoded = encoded.map(vid => VID.hexDecode(vid)); const decoded = encoded.map(vid => VID.hexDecode(vid));
assert.strictEqual(vids.length, count);
assert.deepStrictEqual(vids, decoded); assert.deepStrictEqual(vids, decoded);
}); });
it('should encode and decode correctly with legacy format', () => {
const encoded = vids.map(VID.encode);
const decoded = encoded.map(VID.decode);
assert.strictEqual(vids.every(x => x.length > 27), true);
assert.strictEqual(encoded.every(x => x.length > 32), true);
assert.deepStrictEqual(vids, decoded);
});
});
describe('Short IDs', () => {
env.S3_VERSION_ID_ENCODING_TYPE = 'base62';
const vids = generateRandomVIDs(count);
it('sorted in reversed chronological and alphabetical order', () => {
for (let i = 1; i < count; i++) {
assert(vids[i - 1] > vids[i],
'previous VersionID is higher than its next');
}
},
);
it('simple base62 version test', () => { it('simple base62 version test', () => {
const vid = '98376906954349999999RG001 145.20.5'; const vid = '98376906954349999999RG001 145.20.5';
const encoded = VID.base62Encode(vid); const encoded = VID.base62Encode(vid);
@ -118,13 +88,4 @@ describe('test generating versionIds', () => {
assert.strictEqual(vids.length, count); assert.strictEqual(vids.length, count);
assert.deepStrictEqual(vids, decoded); assert.deepStrictEqual(vids, decoded);
}); });
it('should encode and decode correctly with new 32 byte format', () => {
const encoded = vids.map(vid => VID.encode(vid));
const decoded = encoded.map(vid => VID.decode(vid));
assert(vids.every(x => x.length === 27));
assert(encoded.every(x => x.length === 32));
assert.deepStrictEqual(vids, decoded);
});
});
}); });