Compare commits

...

1 Commits

Author SHA1 Message Date
Vinh Tao 066ef5e9a4 [ci skip] wip refactoring siteId 2017-04-26 20:31:03 +09:00
2 changed files with 115 additions and 81 deletions

View File

@ -35,7 +35,7 @@ module.exports = {
VersioningConstants: require('./lib/versioning/constants.js')
.VersioningConstants,
Version: require('./lib/versioning/Version.js').Version,
VersionID: require('./lib/versioning/VersionID.js'),
VersionID: require('./lib/versioning/VersionID.js').instance,
},
network: {
http: {

View File

@ -40,21 +40,6 @@ function padRight(value, template) {
return `${value}${template}`.slice(0, template.length);
}
// site identifier, like PARIS, TOKYO; will be trimmed if exceeding max length
const SITE_ID = padRight(process.env.SITE_ID, TEMPLATE_SITE);
// constants for max epoch and max sequential number in the same epoch
const MAX_TS = Math.pow(10, LENGTH_TS) - 1; // good until 16 Nov 5138
const MAX_SEQ = Math.pow(10, LENGTH_SEQ) - 1; // good for 1 billion ops
// the earliest versionId, used for versions before versioning
const VID_INF = (padLeft(MAX_TS, TEMPLATE_TS) +
padLeft(MAX_SEQ, TEMPLATE_SEQ) + SITE_ID);
// internal state of the module
let lastTimestamp = 0; // epoch of the last versionId
let lastSeq = 0; // sequential number of the last versionId
/**
* This function ACTIVELY (wastes CPU cycles and) waits for an amount of time
* before returning to the caller. This should not be used frequently.
@ -72,75 +57,124 @@ function wait(span) {
}
}
/**
* This function returns a "versionId" string indicating the current time as a
* combination of the current time in millisecond, the position of the request
* in that millisecond, and the identifier of the local site (which could be
* datacenter, region, or server depending on the notion of geographics). This
* function is stateful which means it keeps some values in the memory and the
* next call depends on the previous call.
*
* @param {string} info - the additional info to ensure uniqueness if desired
* @return {string} - the formated versionId string
*/
function generateVersionId(info) {
// Need to wait for the millisecond slot got "flushed". We wait for
// only a single millisecond when the module is restarted, which is
// necessary for the correctness of the system. This is therefore cheap.
if (lastTimestamp === 0) {
wait(1000000);
// constants for max epoch and max sequential number in the same epoch
const MAX_TS = Math.pow(10, LENGTH_TS) - 1; // good until 16 Nov 5138
const MAX_SEQ = Math.pow(10, LENGTH_SEQ) - 1; // good for 1 billion ops
// default site name in CRR context for single/bucketfile/membackend deployments
const SITE = 'PARIS';
class VersionID {
/**
* Instantiate VersionID; the result should be initalized before use.
*/
constructor() {
// some configurable variables
this.SITE_ID = padRight(SITE_ID, TEMPLATE_SITE); // site identifier
this.VID_INF = padLeft(MAX_TS, TEMPLATE_TS) +
padLeft(MAX_SEQ, TEMPLATE_SEQ) + this.SITE_ID; // earliest versionId
// internal state of the module
this.lastTimestamp = 0; // epoch of the last versionId
this.lastSeq = 0; // sequential number of the last versionId
}
// get the present epoch (in millisecond)
const ts = Date.now();
// A bit more rationale: why do we use a sequence number instead of using
// process.hrtime which gives us time in nanoseconds? The idea is that at
// any time resolution, some concurrent requests may have the same time due
// to the way the OS is queueing requests or getting clock cycles. Our
// approach however will give the time based on the position of a request
// in the queue for the same millisecond which is supposed to be unique.
// increase the position if this request is in the same epoch
lastSeq = (lastTimestamp === ts) ? lastSeq + 1 : 0;
lastTimestamp = ts;
/**
* Initialize the VersionID instance.
*
* @param {object} params - input parameters for initalization
* @param {string} params.siteId - region identifier in CRR context
* @return {VersionID} - this instance
*/
init(params) {
assert(params && params.siteId, 'input parameters is required');
// formated site identifier, like PARIS, TOKYO, SF000
this.SITE_ID = padRight(params.siteId, TEMPLATE_SITE);
// the earliest versionId, used for versions before versioning
this.VID_INF = padLeft(MAX_TS, TEMPLATE_TS) +
padLeft(MAX_SEQ, TEMPLATE_SEQ) + this.SITE_ID;
return this;
}
// In the default cases, we reverse the chronological order of the
// timestamps so that all versions of an object can be retrieved in the
// reversed chronological order---newest versions first. This is because of
// the limitation of leveldb for listing keys in the reverse order.
return padLeft(MAX_TS - lastTimestamp, TEMPLATE_TS) +
padLeft(MAX_SEQ - lastSeq, TEMPLATE_SEQ) + SITE_ID + info;
}
/**
* Encode a versionId to obscure internal information contained
* in a version ID.
*
* @param {string} str - the versionId to encode
* @return {string} - the encoded versionId
*/
function encode(str) {
return Buffer.from(str, 'utf8').toString('hex');
}
/**
* Decode a versionId. May return an error if the input string is
* invalid hex string or results in an invalid value.
*
* @param {string} str - the encoded versionId to decode
* @return {(string|Error)} - the decoded versionId or an error
*/
function decode(str) {
try {
const result = Buffer.from(str, 'hex').toString('utf8');
if (result === '') {
return new Error('invalid decoded value');
/**
* This function returns a "versionId" string indicating the current time as a
* combination of the current time in millisecond, the position of the request
* in that millisecond, and the identifier of the local site (which could be
* datacenter, region, or server depending on the notion of geographics). This
* function is stateful which means it keeps some values in the memory and the
* next call depends on the previous call.
*
* @param {string} info - the additional info to ensure uniqueness if desired
* @return {string} - the formated versionId string
*/
generateVersionId(info) {
// Need to wait for the millisecond slot got "flushed". We wait for
// only a single millisecond when the module is restarted, which is
// necessary for the correctness of the system. This is therefore cheap.
if (this.lastTimestamp === 0) {
wait(1000000);
}
// get the present epoch (in millisecond)
const ts = Date.now();
// A bit more rationale: why do we use a sequence number instead of using
// process.hrtime which gives us time in nanoseconds? The idea is that at
// any time resolution, some concurrent requests may have the same time due
// to the way the OS is queueing requests or getting clock cycles. Our
// approach however will give the time based on the position of a request
// in the queue for the same millisecond which is supposed to be unique.
// increase the position if this request is in the same epoch
this.lastSeq = (this.lastTimestamp === ts) ? this.lastSeq + 1 : 0;
this.lastTimestamp = ts;
// In the default cases, we reverse the chronological order of the
// timestamps so that all versions of an object can be retrieved in the
// reversed chronological order---newest versions first. This is because of
// the limitation of leveldb for listing keys in the reverse order.
return padLeft(MAX_TS - this.lastTimestamp, TEMPLATE_TS) +
padLeft(MAX_SEQ - this.lastSeq, TEMPLATE_SEQ) + this.SITE_ID + info;
}
/**
* Get the predefined-earliest versionId of this site.
*
* @return {string} - the predefined-earliest versionId
*/
getINF() {
return this.VID_INF;
}
/**
* Encode a versionId to obscure internal information contained
* in a version ID.
*
* @param {string} str - the versionId to encode
* @return {string} - the encoded versionId
*/
static encode(str) {
return Buffer.from(str, 'utf8').toString('hex');
}
/**
* Decode a versionId. May return an error if the input string is
* invalid hex string or results in an invalid value.
*
* @param {string} str - the encoded versionId to decode
* @return {(string|Error)} - the decoded versionId or an error
*/
static decode(str) {
try {
const result = Buffer.from(str, 'hex').toString('utf8');
if (result === '') {
return new Error('invalid decoded value');
}
return result;
} catch (err) {
// Buffer.from() may throw TypeError if invalid input, e.g. non-string
// or string with inappropriate charlength
return err;
}
return result;
} catch (err) {
// Buffer.from() may throw TypeError if invalid input, e.g. non-string
// or string with inappropriate charlength
return err;
}
}
module.exports = { generateVersionId, VID_INF, encode, decode };
module.exports = { instance: new VersionID() };