ft: S3C-2065 site-level metrics backport

114cbf5 bf: use expired interval to avg out throughput
9e234e2 bf: zero-fill response for getAllStats

lib/metrics/StatsModel.js

@@ -1,4 +1,7 @@
+const async = require('async');
+
 const StatsClient = require('./StatsClient');
+
  /**
  * @class StatsModel
  *
@@ -6,22 +9,18 @@ const StatsClient = require('./StatsClient');
  * rather than by seconds
  */
 class StatsModel extends StatsClient {
-    /**
-     * normalize date timestamp to the nearest hour
-     * @param {Date} d - Date instance
-     * @return {number} timestamp - normalized to the nearest hour
-     */
-    normalizeTimestampByHour(d) {
-        return d.setMinutes(0, 0, 0);
-    }
 
     /**
-     * get previous hour to date given
+    * Utility method to convert 2d array rows to columns, and vice versa
-     * @param {Date} d - Date instance
+    * See also: https://docs.ruby-lang.org/en/2.0.0/Array.html#method-i-zip
-     * @return {number} timestamp - one hour prior to date passed
+    * @param {array} arrays - 2d array of integers
+    * @return {array} converted array
     */
-    _getDatePreviousHour(d) {
+    _zip(arrays) {
-        return d.setHours(d.getHours() - 1);
+        if (arrays.length > 0 && arrays.every(a => Array.isArray(a))) {
+            return arrays[0].map((_, i) => arrays.map(a => a[i]));
+        }
+        return [];
     }
 
     /**
@@ -44,12 +43,90 @@ class StatsModel extends StatsClient {
      *   oldest interval
      */
     _getCount(arr) {
-        return arr.reduce((store, i) => {
+        const size = Math.floor(this._expiry / this._interval);
+        const array = arr.reduce((store, i) => {
             let num = parseInt(i[1], 10);
             num = Number.isNaN(num) ? 0 : num;
             store.push(num);
             return store;
         }, []);
+
+        if (array.length < size) {
+            array.push(...Array(size - array.length).fill(0));
+        }
+        return array;
+    }
+
+    /**
+    * wrapper on `getStats` that handles a list of keys
+    * override the method to reduce the returned 2d array from `_getCount`
+    * @param {object} log - Werelogs request logger
+    * @param {array} ids - service identifiers
+    * @param {callback} cb - callback to call with the err/result
+    * @return {undefined}
+    */
+    getAllStats(log, ids, cb) {
+        if (!this._redis) {
+            return cb(null, {});
+        }
+
+        const size = Math.floor(this._expiry / this._interval);
+        const statsRes = {
+            'requests': Array(size).fill(0),
+            '500s': Array(size).fill(0),
+            'sampleDuration': this._expiry,
+        };
+        const requests = [];
+        const errors = [];
+
+        if (ids.length === 0) {
+            return cb(null, statsRes);
+        }
+
+        // for now set concurrency to default of 10
+        return async.eachLimit(ids, 10, (id, done) => {
+            this.getStats(log, id, (err, res) => {
+                if (err) {
+                    return done(err);
+                }
+                requests.push(res.requests);
+                errors.push(res['500s']);
+                return done();
+            });
+        }, error => {
+            if (error) {
+                log.error('error getting stats', {
+                    error,
+                    method: 'StatsModel.getAllStats',
+                });
+                return cb(null, statsRes);
+            }
+
+            statsRes.requests = this._zip(requests).map(arr =>
+                arr.reduce((acc, i) => acc + i), 0);
+            statsRes['500s'] = this._zip(errors).map(arr =>
+                arr.reduce((acc, i) => acc + i), 0);
+
+            return cb(null, statsRes);
+        });
+    }
+
+    /**
+     * normalize date timestamp to the nearest hour
+     * @param {Date} d - Date instance
+     * @return {number} timestamp - normalized to the nearest hour
+     */
+    normalizeTimestampByHour(d) {
+        return d.setMinutes(0, 0, 0);
+    }
+
+    /**
+     * get previous hour to date given
+     * @param {Date} d - Date instance
+     * @return {number} timestamp - one hour prior to date passed
+     */
+    _getDatePreviousHour(d) {
+        return d.setHours(d.getHours() - 1);
     }
 
     /**