DROP: bump

partial backport
PR#555

lib/metrics/RedisClient.js

@@ -22,6 +22,28 @@ class RedisClient {
         return this;
     }
 
+    /**
+    * scan a pattern and return matching keys
+    * @param {string} pattern - string pattern to match with all existing keys
+    * @param {number} [count=10] - scan count
+    * @param {callback} cb - callback (error, result)
+    * @return {undefined}
+    */
+    scan(pattern, count = 10, cb) {
+        const params = { match: pattern, count };
+        const keys = [];
+
+        const stream = this._client.scanStream(params);
+        stream.on('data', resultKeys => {
+            for (let i = 0; i < resultKeys.length; i++) {
+                keys.push(resultKeys[i]);
+            }
+        });
+        stream.on('end', () => {
+            cb(null, keys);
+        });
+    }
+
     /**
     * increment value of a key by 1 and set a ttl
     * @param {string} key - key holding the value
@@ -35,6 +57,38 @@ class RedisClient {
             .exec(cb);
     }
 
+    /**
+     * increment value of a key by a given amount
+     * @param {string} key - key holding the value
+     * @param {number} amount - amount to increase by
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    incrby(key, amount, cb) {
+        return this._client.incrby(key, amount, cb);
+    }
+
+    /**
+     * decrement value of a key by a given amount
+     * @param {string} key - key holding the value
+     * @param {number} amount - amount to increase by
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    decrby(key, amount, cb) {
+        return this._client.decrby(key, amount, cb);
+    }
+
+    /**
+     * get value stored at key
+     * @param {string} key - key holding the value
+     * @param {callback} cb - callback
+     * @return {undefined}
+     */
+    get(key, cb) {
+        return this._client.get(key, cb);
+    }
+
     /**
      * increment value of a key by a given amount and set a ttl
      * @param {string} key - key holding the value
@@ -62,6 +116,14 @@ class RedisClient {
     clear(cb) {
         return this._client.flushdb(cb);
     }
+
+    disconnect(cb) {
+        return this._client.quit(cb);
+    }
+
+    listClients(cb) {
+        return this._client.client('list', cb);
+    }
 }
 
 module.exports = RedisClient;

lib/metrics/StatsClient.js

@@ -41,11 +41,11 @@ class StatsClient {
     /**
     * build redis key to get total number of occurrences on the server
     * @param {string} name - key name identifier
-    * @param {object} d - Date instance
+    * @param {Date} date - Date instance
     * @return {string} key - key for redis
     */
-    _buildKey(name, d) {
+    buildKey(name, date) {
-        return `${name}:${this._normalizeTimestamp(d)}`;
+        return `${name}:${this._normalizeTimestamp(date)}`;
     }
 
     /**
@@ -85,11 +85,35 @@ class StatsClient {
             amount = (typeof incr === 'number') ? incr : 1;
         }
 
-        const key = this._buildKey(`${id}:requests`, new Date());
+        const key = this.buildKey(`${id}:requests`, new Date());
 
         return this._redis.incrbyEx(key, amount, this._expiry, callback);
     }
 
+    /**
+     * Increment the given key by the given value.
+     * @param {String} key - The Redis key to increment
+     * @param {Number} incr - The value to increment by
+     * @param {function} [cb] - callback
+     * @return {undefined}
+     */
+    incrementKey(key, incr, cb) {
+        const callback = cb || this._noop;
+        return this._redis.incrby(key, incr, callback);
+    }
+
+    /**
+     * Decrement the given key by the given value.
+     * @param {String} key - The Redis key to decrement
+     * @param {Number} decr - The value to decrement by
+     * @param {function} [cb] - callback
+     * @return {undefined}
+     */
+    decrementKey(key, decr, cb) {
+        const callback = cb || this._noop;
+        return this._redis.decrby(key, decr, callback);
+    }
+
     /**
     * report/record a request that ended up being a 500 on the server
     * @param {string} id - service identifier
@@ -101,10 +125,54 @@ class StatsClient {
             return undefined;
         }
         const callback = cb || this._noop;
-        const key = this._buildKey(`${id}:500s`, new Date());
+        const key = this.buildKey(`${id}:500s`, new Date());
         return this._redis.incrEx(key, this._expiry, callback);
     }
 
+    /**
+    * wrapper on `getStats` that handles a list of keys
+    * @param {object} log - Werelogs request logger
+    * @param {array} ids - service identifiers
+    * @param {callback} cb - callback to call with the err/result
+    * @return {undefined}
+    */
+    getAllStats(log, ids, cb) {
+        if (!this._redis) {
+            return cb(null, {});
+        }
+
+        const statsRes = {
+            'requests': 0,
+            '500s': 0,
+            'sampleDuration': this._expiry,
+        };
+        let requests = 0;
+        let errors = 0;
+
+        // for now set concurrency to default of 10
+        return async.eachLimit(ids, 10, (id, done) => {
+            this.getStats(log, id, (err, res) => {
+                if (err) {
+                    return done(err);
+                }
+                requests += res.requests;
+                errors += res['500s'];
+                return done();
+            });
+        }, error => {
+            if (error) {
+                log.error('error getting stats', {
+                    error,
+                    method: 'StatsClient.getAllStats',
+                });
+                return cb(null, statsRes);
+            }
+            statsRes.requests = requests;
+            statsRes['500s'] = errors;
+            return cb(null, statsRes);
+        });
+    }
+
     /**
     * get stats for the last x seconds, x being the sampling duration
     * @param {object} log - Werelogs request logger
@@ -121,8 +189,8 @@ class StatsClient {
         const reqsKeys = [];
         const req500sKeys = [];
         for (let i = 0; i < totalKeys; i++) {
-            reqsKeys.push(['get', this._buildKey(`${id}:requests`, d)]);
+            reqsKeys.push(['get', this.buildKey(`${id}:requests`, d)]);
-            req500sKeys.push(['get', this._buildKey(`${id}:500s`, d)]);
+            req500sKeys.push(['get', this.buildKey(`${id}:500s`, d)]);
             this._setPrevInterval(d);
         }
         return async.parallel([

lib/models/ObjectMD.js

@@ -775,6 +775,16 @@ class ObjectMD {
         return undefined;
     }
 
+    setReplicationBackends(backends) {
+        this._data.replicationInfo.backends = backends;
+        return this;
+    }
+
+    setReplicationStorageClass(storageClass) {
+        this._data.replicationInfo.storageClass = storageClass;
+        return this;
+    }
+
     getReplicationDataStoreVersionId() {
         return this._data.replicationInfo.dataStoreVersionId;
     }

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=6.9.5"
   },
-  "version": "7.4.0",
+  "version": "7.4.0-bump",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/models/object.js

@@ -136,6 +136,24 @@ describe('ObjectMD class setters/getters', () => {
         }]);
     });
 
+    it('ObjectMD::setReplicationBackends', () => {
+        md.setReplicationBackends([{
+            site: 'a',
+            status: 'b',
+            dataStoreVersionId: 'c',
+        }]);
+        assert.deepStrictEqual(md.getReplicationBackends(), [{
+            site: 'a',
+            status: 'b',
+            dataStoreVersionId: 'c',
+        }]);
+    });
+
+    it('ObjectMD::setReplicationStorageClass', () => {
+        md.setReplicationStorageClass('a');
+        assert.strictEqual(md.getReplicationStorageClass(), 'a');
+    });
+
     it('ObjectMD::getReplicationSiteStatus', () => {
         md.setReplicationInfo({
             backends: [{