Compare commits

...

7 Commits

Author SHA1 Message Date
philipyoo bc72f8d05f DROP: bump 2018-10-15 10:20:42 -07:00
Bennett Buchanan 395d50399e FT: Add objectMD setters for replicationInfo 2018-10-15 09:57:38 -07:00
philipyoo 8e5eb410b0 bugfix: ZENKO-1024 Add pending counters
partial backport
PR#555
2018-10-12 14:17:39 -07:00
philipyoo d67b073050 bf: ZENKO-922 add redis disconnect method
Partial backport
PR#548
2018-10-12 14:12:13 -07:00
philipyoo 6e1b49f9c1 improvement: move metrics tests to functional
partial backport PR#545
2018-10-12 14:09:47 -07:00
Bennett Buchanan d622bdd86b bugfix: ZENKO-621 Make _buildKey public method 2018-10-12 14:06:00 -07:00
philipyoo 6a78bc51ef ft: extend statsclient to query by list of ids
Extend support for querying a list of ids and
returning a total sum of results for that list.

Also add wrapper for redis method `keys`
2018-10-12 13:55:25 -07:00
6 changed files with 166 additions and 8 deletions

View File

@ -22,6 +22,28 @@ class RedisClient {
return this;
}
/**
* scan a pattern and return matching keys
* @param {string} pattern - string pattern to match with all existing keys
* @param {number} [count=10] - scan count
* @param {callback} cb - callback (error, result)
* @return {undefined}
*/
scan(pattern, count = 10, cb) {
const params = { match: pattern, count };
const keys = [];
const stream = this._client.scanStream(params);
stream.on('data', resultKeys => {
for (let i = 0; i < resultKeys.length; i++) {
keys.push(resultKeys[i]);
}
});
stream.on('end', () => {
cb(null, keys);
});
}
/**
* increment value of a key by 1 and set a ttl
* @param {string} key - key holding the value
@ -35,6 +57,38 @@ class RedisClient {
.exec(cb);
}
/**
* increment value of a key by a given amount
* @param {string} key - key holding the value
* @param {number} amount - amount to increase by
* @param {callback} cb - callback
* @return {undefined}
*/
incrby(key, amount, cb) {
return this._client.incrby(key, amount, cb);
}
/**
* decrement value of a key by a given amount
* @param {string} key - key holding the value
* @param {number} amount - amount to increase by
* @param {callback} cb - callback
* @return {undefined}
*/
decrby(key, amount, cb) {
return this._client.decrby(key, amount, cb);
}
/**
* get value stored at key
* @param {string} key - key holding the value
* @param {callback} cb - callback
* @return {undefined}
*/
get(key, cb) {
return this._client.get(key, cb);
}
/**
* increment value of a key by a given amount and set a ttl
* @param {string} key - key holding the value
@ -62,6 +116,14 @@ class RedisClient {
clear(cb) {
return this._client.flushdb(cb);
}
disconnect(cb) {
return this._client.quit(cb);
}
listClients(cb) {
return this._client.client('list', cb);
}
}
module.exports = RedisClient;

View File

@ -41,11 +41,11 @@ class StatsClient {
/**
* build redis key to get total number of occurrences on the server
* @param {string} name - key name identifier
* @param {object} d - Date instance
* @param {Date} date - Date instance
* @return {string} key - key for redis
*/
_buildKey(name, d) {
return `${name}:${this._normalizeTimestamp(d)}`;
buildKey(name, date) {
return `${name}:${this._normalizeTimestamp(date)}`;
}
/**
@ -85,11 +85,35 @@ class StatsClient {
amount = (typeof incr === 'number') ? incr : 1;
}
const key = this._buildKey(`${id}:requests`, new Date());
const key = this.buildKey(`${id}:requests`, new Date());
return this._redis.incrbyEx(key, amount, this._expiry, callback);
}
/**
* Increment the given key by the given value.
* @param {String} key - The Redis key to increment
* @param {Number} incr - The value to increment by
* @param {function} [cb] - callback
* @return {undefined}
*/
incrementKey(key, incr, cb) {
const callback = cb || this._noop;
return this._redis.incrby(key, incr, callback);
}
/**
* Decrement the given key by the given value.
* @param {String} key - The Redis key to decrement
* @param {Number} decr - The value to decrement by
* @param {function} [cb] - callback
* @return {undefined}
*/
decrementKey(key, decr, cb) {
const callback = cb || this._noop;
return this._redis.decrby(key, decr, callback);
}
/**
* report/record a request that ended up being a 500 on the server
* @param {string} id - service identifier
@ -101,10 +125,54 @@ class StatsClient {
return undefined;
}
const callback = cb || this._noop;
const key = this._buildKey(`${id}:500s`, new Date());
const key = this.buildKey(`${id}:500s`, new Date());
return this._redis.incrEx(key, this._expiry, callback);
}
/**
* wrapper on `getStats` that handles a list of keys
* @param {object} log - Werelogs request logger
* @param {array} ids - service identifiers
* @param {callback} cb - callback to call with the err/result
* @return {undefined}
*/
getAllStats(log, ids, cb) {
if (!this._redis) {
return cb(null, {});
}
const statsRes = {
'requests': 0,
'500s': 0,
'sampleDuration': this._expiry,
};
let requests = 0;
let errors = 0;
// for now set concurrency to default of 10
return async.eachLimit(ids, 10, (id, done) => {
this.getStats(log, id, (err, res) => {
if (err) {
return done(err);
}
requests += res.requests;
errors += res['500s'];
return done();
});
}, error => {
if (error) {
log.error('error getting stats', {
error,
method: 'StatsClient.getAllStats',
});
return cb(null, statsRes);
}
statsRes.requests = requests;
statsRes['500s'] = errors;
return cb(null, statsRes);
});
}
/**
* get stats for the last x seconds, x being the sampling duration
* @param {object} log - Werelogs request logger
@ -121,8 +189,8 @@ class StatsClient {
const reqsKeys = [];
const req500sKeys = [];
for (let i = 0; i < totalKeys; i++) {
reqsKeys.push(['get', this._buildKey(`${id}:requests`, d)]);
req500sKeys.push(['get', this._buildKey(`${id}:500s`, d)]);
reqsKeys.push(['get', this.buildKey(`${id}:requests`, d)]);
req500sKeys.push(['get', this.buildKey(`${id}:500s`, d)]);
this._setPrevInterval(d);
}
return async.parallel([

View File

@ -775,6 +775,16 @@ class ObjectMD {
return undefined;
}
setReplicationBackends(backends) {
this._data.replicationInfo.backends = backends;
return this;
}
setReplicationStorageClass(storageClass) {
this._data.replicationInfo.storageClass = storageClass;
return this;
}
getReplicationDataStoreVersionId() {
return this._data.replicationInfo.dataStoreVersionId;
}

View File

@ -3,7 +3,7 @@
"engines": {
"node": ">=6.9.5"
},
"version": "7.4.0",
"version": "7.4.0-bump",
"description": "Common utilities for the S3 project components",
"main": "index.js",
"repository": {

View File

@ -136,6 +136,24 @@ describe('ObjectMD class setters/getters', () => {
}]);
});
it('ObjectMD::setReplicationBackends', () => {
md.setReplicationBackends([{
site: 'a',
status: 'b',
dataStoreVersionId: 'c',
}]);
assert.deepStrictEqual(md.getReplicationBackends(), [{
site: 'a',
status: 'b',
dataStoreVersionId: 'c',
}]);
});
it('ObjectMD::setReplicationStorageClass', () => {
md.setReplicationStorageClass('a');
assert.strictEqual(md.getReplicationStorageClass(), 'a');
});
it('ObjectMD::getReplicationSiteStatus', () => {
md.setReplicationInfo({
backends: [{