Compare commits
2 Commits
developmen
...
improvemen
Author | SHA1 | Date |
---|---|---|
Xin LI | 0244d8b4d9 | |
Xin LI | 72c293ebc2 |
|
@ -9,7 +9,6 @@
|
||||||
const base62Integer = require('base62');
|
const base62Integer = require('base62');
|
||||||
const BASE62 = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
|
const BASE62 = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
|
||||||
const base62String = require('base-x')(BASE62);
|
const base62String = require('base-x')(BASE62);
|
||||||
const assert = require('assert/strict');
|
|
||||||
|
|
||||||
// the lengths of the components in bytes
|
// the lengths of the components in bytes
|
||||||
const LENGTH_TS = 14; // timestamp: epoch in ms
|
const LENGTH_TS = 14; // timestamp: epoch in ms
|
||||||
|
@ -117,19 +116,12 @@ function generateVersionId(info, replicationGroupId) {
|
||||||
lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0;
|
lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0;
|
||||||
lastTimestamp = ts;
|
lastTimestamp = ts;
|
||||||
|
|
||||||
// if S3_VERSION_ID_ENCODING_TYPE is "hex", info is used. By default, it is not used.
|
|
||||||
if (process.env.S3_VERSION_ID_ENCODING_TYPE === 'hex') {
|
|
||||||
// info field stays as is
|
|
||||||
} else {
|
|
||||||
info = ''; // eslint-disable-line
|
|
||||||
}
|
|
||||||
|
|
||||||
// In the default cases, we reverse the chronological order of the
|
// In the default cases, we reverse the chronological order of the
|
||||||
// timestamps so that all versions of an object can be retrieved in the
|
// timestamps so that all versions of an object can be retrieved in the
|
||||||
// reversed chronological order---newest versions first. This is because of
|
// reversed chronological order---newest versions first. This is because of
|
||||||
// the limitation of leveldb for listing keys in the reverse order.
|
// the limitation of leveldb for listing keys in the reverse order.
|
||||||
return padLeft(MAX_TS - lastTimestamp, TEMPLATE_TS) +
|
return padLeft(MAX_TS - lastTimestamp, TEMPLATE_TS) +
|
||||||
padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + repGroupId + info;
|
padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + repGroupId + info;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -174,11 +166,10 @@ const B62V_TOTAL = LENGTH_TS + LENGTH_SEQ;
|
||||||
const B62V_HALF = B62V_TOTAL / 2;
|
const B62V_HALF = B62V_TOTAL / 2;
|
||||||
const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62))));
|
const B62V_EPAD = '0'.repeat(Math.ceil(B62V_HALF * (Math.log(10) / Math.log(62))));
|
||||||
const B62V_DPAD = '0'.repeat(B62V_HALF);
|
const B62V_DPAD = '0'.repeat(B62V_HALF);
|
||||||
const B62V_STRING_EPAD = '0'.repeat(32 - 2 * B62V_EPAD.length);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Encode a versionId to obscure internal information contained
|
* Encode a versionId to obscure internal information contained
|
||||||
* in a version ID (equal to 32 bytes).
|
* in a version ID (less than 40 bytes).
|
||||||
*
|
*
|
||||||
* @param {string} str - the versionId to encode
|
* @param {string} str - the versionId to encode
|
||||||
* @return {string} - the encoded base62VersionId
|
* @return {string} - the encoded base62VersionId
|
||||||
|
@ -190,10 +181,9 @@ function base62Encode(str) {
|
||||||
const enc1 = base62Integer.encode(part1);
|
const enc1 = base62Integer.encode(part1);
|
||||||
const enc2 = base62Integer.encode(part2);
|
const enc2 = base62Integer.encode(part2);
|
||||||
const enc3 = base62String.encode(part3);
|
const enc3 = base62String.encode(part3);
|
||||||
const padded1 = (B62V_EPAD + enc1).slice(-B62V_EPAD.length);
|
return (B62V_EPAD + enc1).slice(-B62V_EPAD.length) +
|
||||||
const padded2 = (B62V_EPAD + enc2).slice(-B62V_EPAD.length);
|
(B62V_EPAD + enc2).slice(-B62V_EPAD.length) +
|
||||||
const padded3 = (B62V_STRING_EPAD + enc3).slice(-B62V_STRING_EPAD.length);
|
enc3;
|
||||||
return padded1 + padded2 + padded3;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -212,19 +202,8 @@ function base62Decode(str) {
|
||||||
const enc2 = str.substring(start, start + B62V_EPAD.length);
|
const enc2 = str.substring(start, start + B62V_EPAD.length);
|
||||||
const orig2 = base62Integer.decode(enc2);
|
const orig2 = base62Integer.decode(enc2);
|
||||||
start += B62V_EPAD.length;
|
start += B62V_EPAD.length;
|
||||||
let enc3 = str.substring(start);
|
const enc3 = str.substring(start);
|
||||||
// take off prefix 0s which represent null bytes
|
|
||||||
let idx = 0;
|
|
||||||
while (idx < enc3.length) {
|
|
||||||
if (enc3[idx] === '0') {
|
|
||||||
idx++;
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
enc3 = enc3.slice(idx);
|
|
||||||
const orig3 = base62String.decode(enc3);
|
const orig3 = base62String.decode(enc3);
|
||||||
|
|
||||||
return (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) +
|
return (B62V_DPAD + orig1.toString()).slice(-B62V_DPAD.length) +
|
||||||
(B62V_DPAD + orig2.toString()).slice(-B62V_DPAD.length) +
|
(B62V_DPAD + orig2.toString()).slice(-B62V_DPAD.length) +
|
||||||
orig3.toString();
|
orig3.toString();
|
||||||
|
@ -242,13 +221,13 @@ const ENC_TYPE_BASE62 = 1; // new (tiny) encoding
|
||||||
* in a version ID.
|
* in a version ID.
|
||||||
*
|
*
|
||||||
* @param {string} str - the versionId to encode
|
* @param {string} str - the versionId to encode
|
||||||
|
* @param {Number} [encType=ENC_TYPE_HEX] - encode format
|
||||||
* @return {string} - the encoded versionId
|
* @return {string} - the encoded versionId
|
||||||
*/
|
*/
|
||||||
function encode(str) {
|
function encode(str, encType = ENC_TYPE_HEX) {
|
||||||
// default format without 'info' field will always be 27 characters
|
if (encType === ENC_TYPE_BASE62) {
|
||||||
if (str.length === 27) {
|
|
||||||
return base62Encode(str);
|
return base62Encode(str);
|
||||||
} // legacy format
|
}
|
||||||
return hexEncode(str);
|
return hexEncode(str);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -261,17 +240,10 @@ function encode(str) {
|
||||||
* @return {(string|Error)} - the decoded versionId or an error
|
* @return {(string|Error)} - the decoded versionId or an error
|
||||||
*/
|
*/
|
||||||
function decode(str) {
|
function decode(str) {
|
||||||
// default format is exactly 32 characters when encoded
|
if (str.length < 40) {
|
||||||
if (str.length === 32) {
|
return base62Decode(str);
|
||||||
const decoded = base62Decode(str);
|
|
||||||
assert.strictEqual(decoded.length, 27);
|
|
||||||
return decoded;
|
|
||||||
}
|
}
|
||||||
// legacy format
|
return hexDecode(str);
|
||||||
if (str.length > 32) {
|
|
||||||
return hexDecode(str);
|
|
||||||
}
|
|
||||||
return new Error(`cannot decode str ${str.length}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { generateVersionId, getInfVid,
|
module.exports = { generateVersionId, getInfVid,
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
const VID = require('../../../lib/versioning/VersionID.js');
|
const VID = require('../../../lib/versioning/VersionID.js');
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const { env } = require('process');
|
|
||||||
|
|
||||||
function randkey(length) {
|
function randkey(length) {
|
||||||
let key = '';
|
let key = '';
|
||||||
|
@ -10,121 +9,83 @@ function randkey(length) {
|
||||||
return key;
|
return key;
|
||||||
}
|
}
|
||||||
|
|
||||||
process.env.VID_CRYPTO_PASSWORD = randkey(64);
|
describe('test generating versionIds', () => {
|
||||||
|
const count = 1000;
|
||||||
function generateRandomVIDs(count) {
|
|
||||||
const vids = Array(count).fill(null);
|
const vids = Array(count).fill(null);
|
||||||
for (let i = 0; i < count; i++) {
|
for (let i = 0; i < count; i++) {
|
||||||
vids[i] = VID.generateVersionId(randkey(15), 'PARIS');
|
vids[i] = VID.generateVersionId(randkey(15), 'PARIS');
|
||||||
}
|
}
|
||||||
return vids;
|
process.env.VID_CRYPTO_PASSWORD = randkey(64);
|
||||||
}
|
|
||||||
|
|
||||||
const count = 1000000;
|
it('sorted in reversed chronological and alphabetical order', () => {
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
describe('test generating versionIds', () => {
|
if (i !== 0) {
|
||||||
describe('legaxy hex encoding', () => {
|
|
||||||
env.S3_VERSION_ID_ENCODING_TYPE = 'hex';
|
|
||||||
const vids = generateRandomVIDs(count);
|
|
||||||
|
|
||||||
it('sorted in reversed chronological and alphabetical order', () => {
|
|
||||||
for (let i = 1; i < count; i++) {
|
|
||||||
assert(vids[i - 1] > vids[i],
|
assert(vids[i - 1] > vids[i],
|
||||||
'previous VersionID is higher than its next');
|
'previous VersionID is higher than its next');
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
|
||||||
// nodejs 10 no longer returns error for non-hex string versionIds
|
|
||||||
it.skip('should return error decoding non-hex string versionIds', () => {
|
|
||||||
const encoded = vids.map(vid => VID.hexEncode(vid));
|
|
||||||
const decoded = encoded.map(vid => VID.hexDecode(`${vid}foo`));
|
|
||||||
decoded.forEach(result => assert(result instanceof Error));
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should encode and decode versionIds', () => {
|
|
||||||
const encoded = vids.map(vid => VID.hexEncode(vid));
|
|
||||||
const decoded = encoded.map(vid => VID.hexDecode(vid));
|
|
||||||
assert.deepStrictEqual(vids, decoded);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should encode and decode correctly with legacy format', () => {
|
|
||||||
const encoded = vids.map(VID.encode);
|
|
||||||
const decoded = encoded.map(VID.decode);
|
|
||||||
|
|
||||||
assert.strictEqual(vids.every(x => x.length > 27), true);
|
|
||||||
assert.strictEqual(encoded.every(x => x.length > 32), true);
|
|
||||||
assert.deepStrictEqual(vids, decoded);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// nodejs 10 no longer returns error for non-hex string versionIds
|
||||||
|
it.skip('should return error decoding non-hex string versionIds', () => {
|
||||||
|
const encoded = vids.map(vid => VID.hexEncode(vid));
|
||||||
|
const decoded = encoded.map(vid => VID.hexDecode(`${vid}foo`));
|
||||||
|
decoded.forEach(result => assert(result instanceof Error));
|
||||||
|
});
|
||||||
|
|
||||||
describe('Short IDs', () => {
|
it('should encode and decode versionIds', () => {
|
||||||
env.S3_VERSION_ID_ENCODING_TYPE = 'base62';
|
const encoded = vids.map(vid => VID.hexEncode(vid));
|
||||||
const vids = generateRandomVIDs(count);
|
const decoded = encoded.map(vid => VID.hexDecode(vid));
|
||||||
|
assert.strictEqual(vids.length, count);
|
||||||
|
assert.deepStrictEqual(vids, decoded);
|
||||||
|
});
|
||||||
|
|
||||||
it('sorted in reversed chronological and alphabetical order', () => {
|
it('simple base62 version test', () => {
|
||||||
for (let i = 1; i < count; i++) {
|
const vid = '98376906954349999999RG001 145.20.5';
|
||||||
assert(vids[i - 1] > vids[i],
|
const encoded = VID.base62Encode(vid);
|
||||||
'previous VersionID is higher than its next');
|
assert.strictEqual(encoded, 'aJLWKz4Ko9IjBBgXKj5KQT2G9UHv0g7P');
|
||||||
}
|
const decoded = VID.base62Decode(encoded);
|
||||||
},
|
assert.strictEqual(vid, decoded);
|
||||||
);
|
});
|
||||||
|
|
||||||
it('simple base62 version test', () => {
|
it('base62 version test with smaller part1 number', () => {
|
||||||
const vid = '98376906954349999999RG001 145.20.5';
|
const vid = '00000000054349999999RG001 145.20.5';
|
||||||
const encoded = VID.base62Encode(vid);
|
const encoded = VID.base62Encode(vid);
|
||||||
assert.strictEqual(encoded, 'aJLWKz4Ko9IjBBgXKj5KQT2G9UHv0g7P');
|
const decoded = VID.base62Decode(encoded);
|
||||||
const decoded = VID.base62Decode(encoded);
|
assert.strictEqual(vid, decoded);
|
||||||
assert.strictEqual(vid, decoded);
|
});
|
||||||
});
|
|
||||||
|
|
||||||
it('base62 version test with smaller part1 number', () => {
|
it('base62 version test with smaller part2 number', () => {
|
||||||
const vid = '00000000054349999999RG001 145.20.5';
|
const vid = '98376906950000099999RG001 145.20.5';
|
||||||
const encoded = VID.base62Encode(vid);
|
const encoded = VID.base62Encode(vid);
|
||||||
const decoded = VID.base62Decode(encoded);
|
const decoded = VID.base62Decode(encoded);
|
||||||
assert.strictEqual(vid, decoded);
|
assert.strictEqual(vid, decoded);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('base62 version test with smaller part2 number', () => {
|
it('base62 version test with smaller part3', () => {
|
||||||
const vid = '98376906950000099999RG001 145.20.5';
|
const vid = '98376906950000099999R1 145.20.5';
|
||||||
const encoded = VID.base62Encode(vid);
|
const encoded = VID.base62Encode(vid);
|
||||||
const decoded = VID.base62Decode(encoded);
|
const decoded = VID.base62Decode(encoded);
|
||||||
assert.strictEqual(vid, decoded);
|
assert.strictEqual(vid, decoded);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('base62 version test with smaller part3', () => {
|
it('base62 version test with smaller part3 - 2', () => {
|
||||||
const vid = '98376906950000099999R1 145.20.5';
|
const vid = '98376906950000099999R1x';
|
||||||
const encoded = VID.base62Encode(vid);
|
const encoded = VID.base62Encode(vid);
|
||||||
const decoded = VID.base62Decode(encoded);
|
const decoded = VID.base62Decode(encoded);
|
||||||
assert.strictEqual(vid, decoded);
|
assert.strictEqual(vid, decoded);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('base62 version test with smaller part3 - 2', () => {
|
it('error case: when invalid base62 key part 3 has invalid base62 character', () => {
|
||||||
const vid = '98376906950000099999R1x';
|
const invalidBase62VersionId = 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P';
|
||||||
const encoded = VID.base62Encode(vid);
|
const decoded = VID.base62Decode(invalidBase62VersionId);
|
||||||
const decoded = VID.base62Decode(encoded);
|
assert(decoded instanceof Error);
|
||||||
assert.strictEqual(vid, decoded);
|
});
|
||||||
});
|
|
||||||
|
|
||||||
it('error case: when invalid base62 key part 3 has invalid base62 character', () => {
|
it('should encode and decode base62 versionIds', () => {
|
||||||
const invalidBase62VersionId = 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P';
|
const encoded = vids.map(vid => VID.base62Encode(vid));
|
||||||
const decoded = VID.base62Decode(invalidBase62VersionId);
|
const decoded = encoded.map(vid => VID.base62Decode(vid));
|
||||||
assert(decoded instanceof Error);
|
assert.strictEqual(vids.length, count);
|
||||||
});
|
assert.deepStrictEqual(vids, decoded);
|
||||||
|
|
||||||
it('should encode and decode base62 versionIds', () => {
|
|
||||||
const encoded = vids.map(vid => VID.base62Encode(vid));
|
|
||||||
const decoded = encoded.map(vid => VID.base62Decode(vid));
|
|
||||||
assert.strictEqual(vids.length, count);
|
|
||||||
assert.deepStrictEqual(vids, decoded);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should encode and decode correctly with new 32 byte format', () => {
|
|
||||||
const encoded = vids.map(vid => VID.encode(vid));
|
|
||||||
const decoded = encoded.map(vid => VID.decode(vid));
|
|
||||||
assert(vids.every(x => x.length === 27));
|
|
||||||
assert(encoded.every(x => x.length === 32));
|
|
||||||
assert.deepStrictEqual(vids, decoded);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
Loading…
Reference in New Issue