Compare commits
7 Commits
developmen
...
wip/philz-
Author | SHA1 | Date |
---|---|---|
philipyoo | bc72f8d05f | |
Bennett Buchanan | 395d50399e | |
philipyoo | 8e5eb410b0 | |
philipyoo | d67b073050 | |
philipyoo | 6e1b49f9c1 | |
Bennett Buchanan | d622bdd86b | |
philipyoo | 6a78bc51ef |
|
@ -22,6 +22,28 @@ class RedisClient {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* scan a pattern and return matching keys
|
||||
* @param {string} pattern - string pattern to match with all existing keys
|
||||
* @param {number} [count=10] - scan count
|
||||
* @param {callback} cb - callback (error, result)
|
||||
* @return {undefined}
|
||||
*/
|
||||
scan(pattern, count = 10, cb) {
|
||||
const params = { match: pattern, count };
|
||||
const keys = [];
|
||||
|
||||
const stream = this._client.scanStream(params);
|
||||
stream.on('data', resultKeys => {
|
||||
for (let i = 0; i < resultKeys.length; i++) {
|
||||
keys.push(resultKeys[i]);
|
||||
}
|
||||
});
|
||||
stream.on('end', () => {
|
||||
cb(null, keys);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* increment value of a key by 1 and set a ttl
|
||||
* @param {string} key - key holding the value
|
||||
|
@ -35,6 +57,38 @@ class RedisClient {
|
|||
.exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* increment value of a key by a given amount
|
||||
* @param {string} key - key holding the value
|
||||
* @param {number} amount - amount to increase by
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
incrby(key, amount, cb) {
|
||||
return this._client.incrby(key, amount, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* decrement value of a key by a given amount
|
||||
* @param {string} key - key holding the value
|
||||
* @param {number} amount - amount to increase by
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
decrby(key, amount, cb) {
|
||||
return this._client.decrby(key, amount, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* get value stored at key
|
||||
* @param {string} key - key holding the value
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
get(key, cb) {
|
||||
return this._client.get(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* increment value of a key by a given amount and set a ttl
|
||||
* @param {string} key - key holding the value
|
||||
|
@ -62,6 +116,14 @@ class RedisClient {
|
|||
clear(cb) {
|
||||
return this._client.flushdb(cb);
|
||||
}
|
||||
|
||||
disconnect(cb) {
|
||||
return this._client.quit(cb);
|
||||
}
|
||||
|
||||
listClients(cb) {
|
||||
return this._client.client('list', cb);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RedisClient;
|
||||
|
|
|
@ -41,11 +41,11 @@ class StatsClient {
|
|||
/**
|
||||
* build redis key to get total number of occurrences on the server
|
||||
* @param {string} name - key name identifier
|
||||
* @param {object} d - Date instance
|
||||
* @param {Date} date - Date instance
|
||||
* @return {string} key - key for redis
|
||||
*/
|
||||
_buildKey(name, d) {
|
||||
return `${name}:${this._normalizeTimestamp(d)}`;
|
||||
buildKey(name, date) {
|
||||
return `${name}:${this._normalizeTimestamp(date)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -85,11 +85,35 @@ class StatsClient {
|
|||
amount = (typeof incr === 'number') ? incr : 1;
|
||||
}
|
||||
|
||||
const key = this._buildKey(`${id}:requests`, new Date());
|
||||
const key = this.buildKey(`${id}:requests`, new Date());
|
||||
|
||||
return this._redis.incrbyEx(key, amount, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment the given key by the given value.
|
||||
* @param {String} key - The Redis key to increment
|
||||
* @param {Number} incr - The value to increment by
|
||||
* @param {function} [cb] - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
incrementKey(key, incr, cb) {
|
||||
const callback = cb || this._noop;
|
||||
return this._redis.incrby(key, incr, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement the given key by the given value.
|
||||
* @param {String} key - The Redis key to decrement
|
||||
* @param {Number} decr - The value to decrement by
|
||||
* @param {function} [cb] - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
decrementKey(key, decr, cb) {
|
||||
const callback = cb || this._noop;
|
||||
return this._redis.decrby(key, decr, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* report/record a request that ended up being a 500 on the server
|
||||
* @param {string} id - service identifier
|
||||
|
@ -101,10 +125,54 @@ class StatsClient {
|
|||
return undefined;
|
||||
}
|
||||
const callback = cb || this._noop;
|
||||
const key = this._buildKey(`${id}:500s`, new Date());
|
||||
const key = this.buildKey(`${id}:500s`, new Date());
|
||||
return this._redis.incrEx(key, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* wrapper on `getStats` that handles a list of keys
|
||||
* @param {object} log - Werelogs request logger
|
||||
* @param {array} ids - service identifiers
|
||||
* @param {callback} cb - callback to call with the err/result
|
||||
* @return {undefined}
|
||||
*/
|
||||
getAllStats(log, ids, cb) {
|
||||
if (!this._redis) {
|
||||
return cb(null, {});
|
||||
}
|
||||
|
||||
const statsRes = {
|
||||
'requests': 0,
|
||||
'500s': 0,
|
||||
'sampleDuration': this._expiry,
|
||||
};
|
||||
let requests = 0;
|
||||
let errors = 0;
|
||||
|
||||
// for now set concurrency to default of 10
|
||||
return async.eachLimit(ids, 10, (id, done) => {
|
||||
this.getStats(log, id, (err, res) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
requests += res.requests;
|
||||
errors += res['500s'];
|
||||
return done();
|
||||
});
|
||||
}, error => {
|
||||
if (error) {
|
||||
log.error('error getting stats', {
|
||||
error,
|
||||
method: 'StatsClient.getAllStats',
|
||||
});
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
statsRes.requests = requests;
|
||||
statsRes['500s'] = errors;
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* get stats for the last x seconds, x being the sampling duration
|
||||
* @param {object} log - Werelogs request logger
|
||||
|
@ -121,8 +189,8 @@ class StatsClient {
|
|||
const reqsKeys = [];
|
||||
const req500sKeys = [];
|
||||
for (let i = 0; i < totalKeys; i++) {
|
||||
reqsKeys.push(['get', this._buildKey(`${id}:requests`, d)]);
|
||||
req500sKeys.push(['get', this._buildKey(`${id}:500s`, d)]);
|
||||
reqsKeys.push(['get', this.buildKey(`${id}:requests`, d)]);
|
||||
req500sKeys.push(['get', this.buildKey(`${id}:500s`, d)]);
|
||||
this._setPrevInterval(d);
|
||||
}
|
||||
return async.parallel([
|
||||
|
|
|
@ -775,6 +775,16 @@ class ObjectMD {
|
|||
return undefined;
|
||||
}
|
||||
|
||||
setReplicationBackends(backends) {
|
||||
this._data.replicationInfo.backends = backends;
|
||||
return this;
|
||||
}
|
||||
|
||||
setReplicationStorageClass(storageClass) {
|
||||
this._data.replicationInfo.storageClass = storageClass;
|
||||
return this;
|
||||
}
|
||||
|
||||
getReplicationDataStoreVersionId() {
|
||||
return this._data.replicationInfo.dataStoreVersionId;
|
||||
}
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
"engines": {
|
||||
"node": ">=6.9.5"
|
||||
},
|
||||
"version": "7.4.0",
|
||||
"version": "7.4.0-bump",
|
||||
"description": "Common utilities for the S3 project components",
|
||||
"main": "index.js",
|
||||
"repository": {
|
||||
|
|
|
@ -136,6 +136,24 @@ describe('ObjectMD class setters/getters', () => {
|
|||
}]);
|
||||
});
|
||||
|
||||
it('ObjectMD::setReplicationBackends', () => {
|
||||
md.setReplicationBackends([{
|
||||
site: 'a',
|
||||
status: 'b',
|
||||
dataStoreVersionId: 'c',
|
||||
}]);
|
||||
assert.deepStrictEqual(md.getReplicationBackends(), [{
|
||||
site: 'a',
|
||||
status: 'b',
|
||||
dataStoreVersionId: 'c',
|
||||
}]);
|
||||
});
|
||||
|
||||
it('ObjectMD::setReplicationStorageClass', () => {
|
||||
md.setReplicationStorageClass('a');
|
||||
assert.strictEqual(md.getReplicationStorageClass(), 'a');
|
||||
});
|
||||
|
||||
it('ObjectMD::getReplicationSiteStatus', () => {
|
||||
md.setReplicationInfo({
|
||||
backends: [{
|
||||
|
|
Loading…
Reference in New Issue