Compare commits

...

1 Commits

Author SHA1 Message Date
Vinh Tao 066ef5e9a4 [ci skip] wip refactoring siteId 2017-04-26 20:31:03 +09:00
2 changed files with 115 additions and 81 deletions

View File

@ -35,7 +35,7 @@ module.exports = {
VersioningConstants: require('./lib/versioning/constants.js') VersioningConstants: require('./lib/versioning/constants.js')
.VersioningConstants, .VersioningConstants,
Version: require('./lib/versioning/Version.js').Version, Version: require('./lib/versioning/Version.js').Version,
VersionID: require('./lib/versioning/VersionID.js'), VersionID: require('./lib/versioning/VersionID.js').instance,
}, },
network: { network: {
http: { http: {

View File

@ -40,21 +40,6 @@ function padRight(value, template) {
return `${value}${template}`.slice(0, template.length); return `${value}${template}`.slice(0, template.length);
} }
// site identifier, like PARIS, TOKYO; will be trimmed if exceeding max length
const SITE_ID = padRight(process.env.SITE_ID, TEMPLATE_SITE);
// constants for max epoch and max sequential number in the same epoch
const MAX_TS = Math.pow(10, LENGTH_TS) - 1; // good until 16 Nov 5138
const MAX_SEQ = Math.pow(10, LENGTH_SEQ) - 1; // good for 1 billion ops
// the earliest versionId, used for versions before versioning
const VID_INF = (padLeft(MAX_TS, TEMPLATE_TS) +
padLeft(MAX_SEQ, TEMPLATE_SEQ) + SITE_ID);
// internal state of the module
let lastTimestamp = 0; // epoch of the last versionId
let lastSeq = 0; // sequential number of the last versionId
/** /**
* This function ACTIVELY (wastes CPU cycles and) waits for an amount of time * This function ACTIVELY (wastes CPU cycles and) waits for an amount of time
* before returning to the caller. This should not be used frequently. * before returning to the caller. This should not be used frequently.
@ -72,75 +57,124 @@ function wait(span) {
} }
} }
/** // constants for max epoch and max sequential number in the same epoch
* This function returns a "versionId" string indicating the current time as a const MAX_TS = Math.pow(10, LENGTH_TS) - 1; // good until 16 Nov 5138
* combination of the current time in millisecond, the position of the request const MAX_SEQ = Math.pow(10, LENGTH_SEQ) - 1; // good for 1 billion ops
* in that millisecond, and the identifier of the local site (which could be
* datacenter, region, or server depending on the notion of geographics). This // default site name in CRR context for single/bucketfile/membackend deployments
* function is stateful which means it keeps some values in the memory and the const SITE = 'PARIS';
* next call depends on the previous call.
* class VersionID {
* @param {string} info - the additional info to ensure uniqueness if desired
* @return {string} - the formated versionId string /**
*/ * Instantiate VersionID; the result should be initalized before use.
function generateVersionId(info) { */
// Need to wait for the millisecond slot got "flushed". We wait for constructor() {
// only a single millisecond when the module is restarted, which is // some configurable variables
// necessary for the correctness of the system. This is therefore cheap. this.SITE_ID = padRight(SITE_ID, TEMPLATE_SITE); // site identifier
if (lastTimestamp === 0) { this.VID_INF = padLeft(MAX_TS, TEMPLATE_TS) +
wait(1000000); padLeft(MAX_SEQ, TEMPLATE_SEQ) + this.SITE_ID; // earliest versionId
// internal state of the module
this.lastTimestamp = 0; // epoch of the last versionId
this.lastSeq = 0; // sequential number of the last versionId
} }
// get the present epoch (in millisecond)
const ts = Date.now();
// A bit more rationale: why do we use a sequence number instead of using
// process.hrtime which gives us time in nanoseconds? The idea is that at
// any time resolution, some concurrent requests may have the same time due
// to the way the OS is queueing requests or getting clock cycles. Our
// approach however will give the time based on the position of a request
// in the queue for the same millisecond which is supposed to be unique.
// increase the position if this request is in the same epoch /**
lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0; * Initialize the VersionID instance.
lastTimestamp = ts; *
* @param {object} params - input parameters for initalization
* @param {string} params.siteId - region identifier in CRR context
* @return {VersionID} - this instance
*/
init(params) {
assert(params && params.siteId, 'input parameters is required');
// formated site identifier, like PARIS, TOKYO, SF000
this.SITE_ID = padRight(params.siteId, TEMPLATE_SITE);
// the earliest versionId, used for versions before versioning
this.VID_INF = padLeft(MAX_TS, TEMPLATE_TS) +
padLeft(MAX_SEQ, TEMPLATE_SEQ) + this.SITE_ID;
return this;
}
// In the default cases, we reverse the chronological order of the /**
// timestamps so that all versions of an object can be retrieved in the * This function returns a "versionId" string indicating the current time as a
// reversed chronological order---newest versions first. This is because of * combination of the current time in millisecond, the position of the request
// the limitation of leveldb for listing keys in the reverse order. * in that millisecond, and the identifier of the local site (which could be
return padLeft(MAX_TS - lastTimestamp, TEMPLATE_TS) + * datacenter, region, or server depending on the notion of geographics). This
padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + SITE_ID + info; * function is stateful which means it keeps some values in the memory and the
} * next call depends on the previous call.
*
/** * @param {string} info - the additional info to ensure uniqueness if desired
* Encode a versionId to obscure internal information contained * @return {string} - the formated versionId string
* in a version ID. */
* generateVersionId(info) {
* @param {string} str - the versionId to encode // Need to wait for the millisecond slot got "flushed". We wait for
* @return {string} - the encoded versionId // only a single millisecond when the module is restarted, which is
*/ // necessary for the correctness of the system. This is therefore cheap.
function encode(str) { if (this.lastTimestamp === 0) {
return Buffer.from(str, 'utf8').toString('hex'); wait(1000000);
} }
// get the present epoch (in millisecond)
/** const ts = Date.now();
* Decode a versionId. May return an error if the input string is // A bit more rationale: why do we use a sequence number instead of using
* invalid hex string or results in an invalid value. // process.hrtime which gives us time in nanoseconds? The idea is that at
* // any time resolution, some concurrent requests may have the same time due
* @param {string} str - the encoded versionId to decode // to the way the OS is queueing requests or getting clock cycles. Our
* @return {(string|Error)} - the decoded versionId or an error // approach however will give the time based on the position of a request
*/ // in the queue for the same millisecond which is supposed to be unique.
function decode(str) {
try { // increase the position if this request is in the same epoch
const result = Buffer.from(str, 'hex').toString('utf8'); this.lastSeq = (this.lastTimestamp === ts) ? this.lastSeq + 1 : 0;
if (result === '') { this.lastTimestamp = ts;
return new Error('invalid decoded value');
// In the default cases, we reverse the chronological order of the
// timestamps so that all versions of an object can be retrieved in the
// reversed chronological order---newest versions first. This is because of
// the limitation of leveldb for listing keys in the reverse order.
return padLeft(MAX_TS - this.lastTimestamp, TEMPLATE_TS) +
padLeft(MAX_SEQ - this.lastSeq, TEMPLATE_SEQ) + this.SITE_ID + info;
}
/**
* Get the predefined-earliest versionId of this site.
*
* @return {string} - the predefined-earliest versionId
*/
getINF() {
return this.VID_INF;
}
/**
* Encode a versionId to obscure internal information contained
* in a version ID.
*
* @param {string} str - the versionId to encode
* @return {string} - the encoded versionId
*/
static encode(str) {
return Buffer.from(str, 'utf8').toString('hex');
}
/**
* Decode a versionId. May return an error if the input string is
* invalid hex string or results in an invalid value.
*
* @param {string} str - the encoded versionId to decode
* @return {(string|Error)} - the decoded versionId or an error
*/
static decode(str) {
try {
const result = Buffer.from(str, 'hex').toString('utf8');
if (result === '') {
return new Error('invalid decoded value');
}
return result;
} catch (err) {
// Buffer.from() may throw TypeError if invalid input, e.g. non-string
// or string with inappropriate charlength
return err;
} }
return result;
} catch (err) {
// Buffer.from() may throw TypeError if invalid input, e.g. non-string
// or string with inappropriate charlength
return err;
} }
} }
module.exports = { generateVersionId, VID_INF, encode, decode }; module.exports = { instance: new VersionID() };