lib/backbeat/Metrics.js

@@ -4,8 +4,7 @@ const errors = require('../../lib/errors');
 const RedisClient = require('../../lib/metrics/RedisClient');
 const StatsModel = require('../../lib/metrics/StatsModel');
 const INTERVAL = 300; // 5 minutes
-const EXPIRY = 86400; // 24 hours
+const EXPIRY = 900; // 15 minutes
-const THROUGHPUT_EXPIRY = 900; // 15 minutes
 const OBJECT_MONITORING_EXPIRY = 86400; // 24 hours.
 
 class Metrics {
@@ -15,7 +14,8 @@ class Metrics {
         this._redisClient = new RedisClient(redisConfig, this._logger);
         // Redis expiry increased by an additional interval so we can reference
         // the immediate older data for average throughput calculation
-        this._statsClient = new StatsModel(this._redisClient, INTERVAL, EXPIRY);
+        this._statsClient = new StatsModel(this._redisClient, INTERVAL,
+            (EXPIRY + INTERVAL));
         this._validSites = validSites;
         this._internalStart = internalStart;
     }
@@ -96,16 +96,6 @@ class Metrics {
         return cb(null, data);
     }
 
-    /**
-     * Uptime of server based on this._internalStart up to max of expiry
-     * @param {number} expiry - max expiry
-     * @return {number} uptime of server up to expiry time
-     */
-    _getMaxUptime(expiry) {
-        const timeSinceStart = (Date.now() - this._internalStart) / 1000;
-        return timeSinceStart < expiry ? (timeSinceStart || 1) : expiry;
-    }
-
     /**
      * Get replication backlog in ops count and size in bytes
      * @param {object} details - route details from lib/backbeat/routes.js
@@ -129,16 +119,13 @@ class Metrics {
                 });
                 return cb(errors.InternalError);
             }
-            const uptime = this._getMaxUptime(EXPIRY);
+            const d = res.map(r => (
-            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+                r.requests.slice(0, 3).reduce((acc, i) => acc + i, 0)
-            const [ops, opsDone, bytes, bytesDone] = res.map(r => (
-                r.requests.slice(0, numOfIntervals).reduce((acc, i) =>
-                    acc + i, 0)
             ));
 
-            let opsBacklog = ops - opsDone;
+            let opsBacklog = d[0] - d[1];
             if (opsBacklog < 0) opsBacklog = 0;
-            let bytesBacklog = bytes - bytesDone;
+            let bytesBacklog = d[2] - d[3];
             if (bytesBacklog < 0) bytesBacklog = 0;
             const response = {
                 backlog: {
@@ -179,9 +166,13 @@ class Metrics {
                 return cb(errors.InternalError);
             }
 
-            const uptime = this._getMaxUptime(EXPIRY);
+            // Find if time since start is less than EXPIRY time
-            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const timeSinceStart = (Date.now() - this._internalStart) / 1000;
-            const [opsDone, bytesDone] = res.map(r => (
+            const timeDisplay = timeSinceStart < EXPIRY ?
+                timeSinceStart : EXPIRY;
+            const numOfIntervals = Math.ceil(timeDisplay / INTERVAL);
+
+            const d = res.map(r => (
                 r.requests.slice(0, numOfIntervals).reduce((acc, i) =>
                     acc + i, 0)
             ));
@@ -190,10 +181,10 @@ class Metrics {
                 completions: {
                     description: 'Number of completed replication operations ' +
                         '(count) and number of bytes transferred (size) in ' +
-                        `the last ${Math.floor(uptime)} seconds`,
+                        `the last ${Math.floor(timeDisplay)} seconds`,
                     results: {
-                        count: opsDone,
+                        count: d[0],
-                        size: bytesDone,
+                        size: d[1],
                     },
                 },
             };
@@ -225,9 +216,13 @@ class Metrics {
                 return cb(errors.InternalError);
             }
 
-            const uptime = this._getMaxUptime(EXPIRY);
+            // Find if time since start is less than EXPIRY time
-            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const timeSinceStart = (Date.now() - this._internalStart) / 1000;
-            const [opsFail, bytesFail] = res.map(r => (
+            const timeDisplay = timeSinceStart < EXPIRY ?
+                timeSinceStart : EXPIRY;
+            const numOfIntervals = Math.ceil(timeDisplay / INTERVAL);
+
+            const d = res.map(r => (
                 r.requests.slice(0, numOfIntervals).reduce((acc, i) =>
                     acc + i, 0)
             ));
@@ -236,10 +231,10 @@ class Metrics {
                 failures: {
                     description: 'Number of failed replication operations ' +
                         '(count) and bytes (size) in the last ' +
-                        `${Math.floor(uptime)} seconds`,
+                        `${Math.floor(timeDisplay)} seconds`,
                     results: {
-                        count: opsFail,
+                        count: d[0],
-                        size: bytesFail,
+                        size: d[1],
                     },
                 },
             };
@@ -248,7 +243,7 @@ class Metrics {
     }
 
     /**
-     * Get current throughput in ops/sec and bytes/sec up to max of 15 minutes
+     * Get current throughput in ops/sec and bytes/sec
      * Throughput is the number of units processed in a given time
      * @param {object} details - route details from lib/backbeat/routes.js
      * @param {function} cb - callback(error, data)
@@ -271,37 +266,42 @@ class Metrics {
                 });
                 return cb(errors.InternalError);
             }
+
             const now = new Date();
-            const uptime = this._getMaxUptime(THROUGHPUT_EXPIRY);
+            const timeSinceStart = (now - this._internalStart) / 1000;
-            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            // Seconds up to a max of EXPIRY seconds
+            const timeDisplay = timeSinceStart < EXPIRY ?
+                (timeSinceStart || 1) : EXPIRY;
+            const numOfIntervals = Math.ceil(timeDisplay / INTERVAL);
 
             const [opsThroughput, bytesThroughput] = res.map(r => {
                 let total = r.requests.slice(0, numOfIntervals).reduce(
                     (acc, i) => acc + i, 0);
 
-                // if timeDisplay !== THROUGHPUT_EXPIRY, use internal timer and
+                // if timeDisplay !== EXPIRY, use internal timer and do not
-                // do not include the extra 4th interval
+                // include the extra 4th interval
-                if (uptime === THROUGHPUT_EXPIRY) {
+                if (timeDisplay === EXPIRY) {
                     // all intervals apply, including 4th interval
                     const lastInterval =
-                        this._statsClient._normalizeTimestamp(now);
+                        this._statsClient._normalizeTimestamp(new Date(now));
                     // in seconds
                     const diff = (now - lastInterval) / 1000;
+
                     // Get average for last interval depending on time
                     // surpassed so far for newest interval
                     total += ((INTERVAL - diff) / INTERVAL) *
                         r.requests[numOfIntervals];
                 }
 
-                // Divide total by uptime to determine data per second
+                // Divide total by timeDisplay to determine data per second
-                return (total / uptime);
+                return (total / timeDisplay);
             });
 
             const response = {
                 throughput: {
                     description: 'Current throughput for replication ' +
                         'operations in ops/sec (count) and bytes/sec (size) ' +
-                        `in the last ${Math.floor(uptime)} seconds`,
+                        `in the last ${Math.floor(timeDisplay)} seconds`,
                     results: {
                         count: opsThroughput.toFixed(2),
                         size: bytesThroughput.toFixed(2),
@@ -336,18 +336,20 @@ class Metrics {
                 return cb(errors.InternalError);
             }
             const now = new Date();
-            const uptime = this._getMaxUptime(THROUGHPUT_EXPIRY);
+            const timeSinceStart = (now - this._internalStart) / 1000;
-            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            // Seconds up to a max of EXPIRY seconds
-
+            const timeDisplay = timeSinceStart < EXPIRY ?
+                (timeSinceStart || 1) : EXPIRY;
+            const numOfIntervals = Math.ceil(timeDisplay / INTERVAL);
             const { requests } = res[0]; // Bytes done
             let total = requests.slice(0, numOfIntervals)
                 .reduce((acc, i) => acc + i, 0);
-            // if timeDisplay !== THROUGHPUT_EXPIRY, use internal timer
+            // if timeDisplay !== OBJECT_MONITORING_EXPIRY, use internal timer
             // and do not include the extra 4th interval
-            if (uptime === THROUGHPUT_EXPIRY) {
+            if (timeDisplay === EXPIRY) {
                 // all intervals apply, including 4th interval
                 const lastInterval =
-                    this._statsClient._normalizeTimestamp(now);
+                    this._statsClient._normalizeTimestamp(new Date(now));
                 // in seconds
                 const diff = (now - lastInterval) / 1000;
                 // Get average for last interval depending on time passed so
@@ -359,7 +361,7 @@ class Metrics {
             const response = {
                 description: 'Current throughput for object replication in ' +
                     'bytes/sec (throughput)',
-                throughput: (total / uptime).toFixed(2),
+                throughput: (total / timeDisplay).toFixed(2),
             };
             return cb(null, response);
         });
@@ -390,8 +392,10 @@ class Metrics {
             }
             // Find if time since start is less than OBJECT_MONITORING_EXPIRY
             // time
-            const uptime = this._getMaxUptime(OBJECT_MONITORING_EXPIRY);
+            const timeSinceStart = (Date.now() - this._internalStart) / 1000;
-            const numOfIntervals = Math.ceil(uptime / INTERVAL);
+            const timeDisplay = timeSinceStart < OBJECT_MONITORING_EXPIRY ?
+                timeSinceStart : OBJECT_MONITORING_EXPIRY;
+            const numOfIntervals = Math.ceil(timeDisplay / INTERVAL);
             const [totalBytesToComplete, bytesComplete] = res.map(r => (
                 r.requests.slice(0, numOfIntervals).reduce((acc, i) =>
                     acc + i, 0)

tests/functional/backbeat/Metrics.js

@@ -5,10 +5,6 @@ const assert = require('assert');
 const RedisClient = require('../../../lib/metrics/RedisClient');
 const { backbeat } = require('../../../');
 
-// expirations
-const EXPIRY = 86400; // 24 hours
-const THROUGHPUT_EXPIRY = 900; // 15 minutes
-
 // setup redis client
 const config = {
     host: '127.0.0.1',
@@ -26,7 +22,7 @@ const sites = ['site1', 'site2'];
 const metrics = new backbeat.Metrics({
     redisConfig: config,
     validSites: ['site1', 'site2', 'all'],
-    internalStart: Date.now() - (EXPIRY * 1000), // 24 hours ago.
+    internalStart: Date.now() - 900000, // 15 minutes ago.
 }, fakeLogger);
 
 // Since many methods were overwritten, these tests should validate the changes
@@ -61,7 +57,7 @@ describe('Metrics class', () => {
                 completions: {
                     description: 'Number of completed replication operations' +
                         ' (count) and number of bytes transferred (size) in ' +
-                        `the last ${EXPIRY} seconds`,
+                        'the last 900 seconds',
                     results: {
                         count: 0,
                         size: 0,
@@ -69,8 +65,7 @@ describe('Metrics class', () => {
                 },
                 failures: {
                     description: 'Number of failed replication operations ' +
-                        `(count) and bytes (size) in the last ${EXPIRY} ` +
+                        '(count) and bytes (size) in the last 900 seconds',
-                        'seconds',
                     results: {
                         count: 0,
                         size: 0,
@@ -79,7 +74,7 @@ describe('Metrics class', () => {
                 throughput: {
                     description: 'Current throughput for replication' +
                         ' operations in ops/sec (count) and bytes/sec (size) ' +
-                        `in the last ${THROUGHPUT_EXPIRY} seconds`,
+                        'in the last 900 seconds',
                     results: {
                         count: '0.00',
                         size: '0.00',

tests/functional/metrics/StatsModel.js

@@ -20,14 +20,9 @@ const redisClient = new RedisClient(config, fakeLogger);
 
 // setup stats model
 const STATS_INTERVAL = 300; // 5 minutes
-const STATS_EXPIRY = 86400; // 24 hours
+const STATS_EXPIRY = 900; // 15 minutes
 const statsModel = new StatsModel(redisClient, STATS_INTERVAL, STATS_EXPIRY);
 
-function setExpectedStats(expected) {
-    return expected.concat(
-        Array((STATS_EXPIRY / STATS_INTERVAL) - expected.length).fill(0));
-}
-
 // Since many methods were overwritten, these tests should validate the changes
 // made to the original methods
 describe('StatsModel class', () => {
@@ -70,7 +65,7 @@ describe('StatsModel class', () => {
             [null, '2'],
             [null, null],
         ]);
-        assert.deepStrictEqual(res, setExpectedStats([1, 2]));
+        assert.deepStrictEqual(res, [1, 2, 0]);
     });
 
     it('should correctly record a new request by default one increment',
@@ -106,7 +101,7 @@ describe('StatsModel class', () => {
                 statsModel.getStats(fakeLogger, id, (err, res) => {
                     assert.ifError(err);
 
-                    assert.deepStrictEqual(res.requests, setExpectedStats([9]));
+                    assert.deepStrictEqual(res.requests, [9, 0, 0]);
                     next();
                 });
             },
@@ -115,8 +110,7 @@ describe('StatsModel class', () => {
                 statsModel.getStats(fakeLogger, id, (err, res) => {
                     assert.ifError(err);
 
-                    assert.deepStrictEqual(res.requests,
+                    assert.deepStrictEqual(res.requests, [10, 0, 0]);
-                        setExpectedStats([10]));
                     next();
                 });
             },
@@ -125,8 +119,7 @@ describe('StatsModel class', () => {
                 statsModel.getStats(fakeLogger, id, (err, res) => {
                     assert.ifError(err);
 
-                    assert.deepStrictEqual(res.requests,
+                    assert.deepStrictEqual(res.requests, [11, 0, 0]);
-                        setExpectedStats([11]));
                     next();
                 });
             },
@@ -162,8 +155,8 @@ describe('StatsModel class', () => {
                     assert.ifError(err);
 
                     const expected = {
-                        'requests': setExpectedStats([1]),
+                        'requests': [1, 0, 0],
-                        '500s': setExpectedStats([1]),
+                        '500s': [1, 0, 0],
                         'sampleDuration': STATS_EXPIRY,
                     };
                     assert.deepStrictEqual(res, expected);
@@ -179,8 +172,8 @@ describe('StatsModel class', () => {
                 statsModel.getStats(fakeLogger, id, (err, res) => {
                     assert.ifError(err);
                     const expected = {
-                        'requests': setExpectedStats([]),
+                        'requests': [0, 0, 0],
-                        '500s': setExpectedStats([]),
+                        '500s': [0, 0, 0],
                         'sampleDuration': STATS_EXPIRY,
                     };
                     assert.deepStrictEqual(res, expected);
@@ -191,8 +184,8 @@ describe('StatsModel class', () => {
                 statsModel.getAllStats(fakeLogger, id, (err, res) => {
                     assert.ifError(err);
                     const expected = {
-                        'requests': setExpectedStats([]),
+                        'requests': [0, 0, 0],
-                        '500s': setExpectedStats([]),
+                        '500s': [0, 0, 0],
                         'sampleDuration': STATS_EXPIRY,
                     };
                     assert.deepStrictEqual(res, expected);
@@ -207,8 +200,10 @@ describe('StatsModel class', () => {
         statsModel.getAllStats(fakeLogger, [], (err, res) => {
             assert.ifError(err);
 
-            assert.deepStrictEqual(res.requests, setExpectedStats([]));
+            const expected = Array(STATS_EXPIRY / STATS_INTERVAL).fill(0);
-            assert.deepStrictEqual(res['500s'], setExpectedStats([]));
+
+            assert.deepStrictEqual(res.requests, expected);
+            assert.deepStrictEqual(res['500s'], expected);
             done();
         });
     });
@@ -236,8 +231,7 @@ describe('StatsModel class', () => {
                     assert.ifError(err);
 
                     assert.equal(res.requests[0], 14);
-                    assert.deepStrictEqual(res.requests,
+                    assert.deepStrictEqual(res.requests, [14, 0, 0]);
-                        setExpectedStats([14]));
                     next();
                 });
             },