squash: update unit tests

eve/workers/unit_and_feature_tests/run_ft_tests.bash

@@ -13,5 +13,5 @@ killandsleep () {
   sleep 10
 }
 
-npm start & bash tests/utils/wait_for_local_port.bash $PORT 40
-npm run $1
+CI=true npm start & bash tests/utils/wait_for_local_port.bash $PORT 40
+CI=true npm run $1

lib/ListMetrics.js

@@ -4,6 +4,7 @@ const { getMetricFromKey, getKeys, generateStateKey } = require('./schema');
 const s3metricResponseJSON = require('../models/s3metricResponse');
 const config = require('./Config');
 const Vault = require('./Vault');
+const member = require('../utils/member');
 
 const MAX_RANGE_MS = (((1000 * 60) * 60) * 24) * 30; // One month.
 
@@ -144,6 +145,9 @@ class ListMetrics {
             res.push(last);
             const d = new Date(last);
             last = d.setMinutes(d.getMinutes() + 15);
+            if (process.env.CI) {
+                last = d.setSeconds(d.getSeconds() + 15);
+            }
         }
         res.push(end);
         return res;
@@ -271,7 +275,12 @@ class ListMetrics {
                         method: 'ListMetrics.getMetrics',
                     });
                 } else {
-                    let val = parseInt(item[1], 10);
+                    let val =
+                        item &&
+                        item[1] &&
+                        item[1][0] &&
+                        member.deserialize(item[1][0]);
+                    val = parseInt(val, 10);
 
                     val = Number.isNaN(val) ? 0 : val;
                     if (val < 0) {

lib/UtapiClient.js

@@ -5,6 +5,7 @@ const Datastore = require('./Datastore');
 const { generateKey, generateCounter, generateStateKey } = require('./schema');
 const { errors } = require('arsenal');
 const redisClient = require('../utils/redisClient');
+const member = require('../utils/member');
 
 const methods = {
     createBucket: { method: '_genericPushMetric', changesData: true },
@@ -125,6 +126,10 @@ class UtapiClient {
     */
     static getNormalizedTimestamp() {
         const d = new Date();
+        if (process.env.CI) {
+            const seconds = d.getSeconds();
+            return d.setSeconds((seconds - seconds % 15), 0, 0);
+        }
         const minutes = d.getMinutes();
         return d.setMinutes((minutes - minutes % 15), 0, 0);
     }
@@ -513,7 +518,7 @@ class UtapiClient {
                 const key = generateStateKey(p, 'numberOfObjects');
                 cmds2.push(
                     ['zremrangebyscore', key, timestamp, timestamp],
-                    ['zadd', key, timestamp, actionCounter]);
+                    ['zadd', key, timestamp, member.serialize(actionCounter)]);
                 return true;
             });
             if (noErr) {
@@ -593,7 +598,7 @@ class UtapiClient {
                     ['zremrangebyscore', generateStateKey(p, 'storageUtilized'),
                         timestamp, timestamp],
                     ['zadd', generateStateKey(p, 'storageUtilized'),
-                        timestamp, actionCounter]
+                        timestamp, member.serialize(actionCounter)]
                 );
                 return true;
             });
@@ -667,7 +672,7 @@ class UtapiClient {
                 }
                 key = generateStateKey(p, 'numberOfObjects');
                 cmds2.push(['zremrangebyscore', key, timestamp, timestamp],
-                ['zadd', key, timestamp, actionCounter]);
+                ['zadd', key, timestamp, member.serialize(actionCounter)]);
                 return true;
             });
             if (noErr) {
@@ -779,7 +784,7 @@ class UtapiClient {
                         timestamp, timestamp],
                     ['zadd',
                         generateStateKey(p, 'storageUtilized'), timestamp,
-                        actionCounter]);
+                        member.serialize(actionCounter)]);
                 // The 'abortMultipartUpload' action does not affect number of
                 // objects, so we return here.
                 if (action === 'abortMultipartUpload') {
@@ -809,7 +814,7 @@ class UtapiClient {
                         generateStateKey(p, 'numberOfObjects'), timestamp,
                         timestamp],
                     ['zadd', generateStateKey(p, 'numberOfObjects'), timestamp,
-                        actionCounter]);
+                        member.serialize(actionCounter)]);
                 return true;
             });
             if (noErr) {
@@ -941,7 +946,7 @@ class UtapiClient {
                         generateStateKey(p, 'storageUtilized'),
                         timestamp, timestamp],
                     ['zadd', generateStateKey(p, 'storageUtilized'),
-                        timestamp, actionCounter]);
+                        timestamp, member.serialize(actionCounter)]);
 
                 // number of objects counter
                 objectsIndex = (i * (cmdsLen / paramsArrLen)) + 1;
@@ -967,7 +972,7 @@ class UtapiClient {
                         generateStateKey(p, 'numberOfObjects'),
                         timestamp, timestamp],
                     ['zadd', generateStateKey(p, 'numberOfObjects'),
-                        timestamp, actionCounter]);
+                        timestamp, member.serialize(actionCounter)]);
                 return true;
             });
             if (noErr) {

tests/functional/testReplay.js

@@ -112,9 +112,12 @@ function checkListElement(action, params, res) {
     assert(!error, 'cannot parse cached element into JSON');
     const { reqUid, timestamp } = result;
     const currTimestamp = UtapiClient.getNormalizedTimestamp();
-    const fifteenMinutes = 900000; // Milliseconds.
+    let milliseconds = 900000; // 15 minutes
+    if (process.env.CI) {
+        milliseconds = 15000; // 15 seconds
+    }
     // Allow for previous timestamp interval, since we cannot know start time.
-    assert(timestamp, currTimestamp || currTimestamp - fifteenMinutes,
+    assert(timestamp, currTimestamp || currTimestamp - milliseconds,
         'incorrect timestamp value');
     assert(reqUid !== undefined,
         `reqUid property not in cached element: ${action}`);

tests/functional/testUtapiClient.js

@@ -6,6 +6,10 @@ const redisClient = require('../../utils/redisClient');
 const { Logger } = require('werelogs');
 const { getCounters, getMetricFromKey,
     getStateKeys, getKeys } = require('../../lib/schema');
+
+const { makeUtapiClientRequest } = require('../utils/utils');
+const Vault = require('../utils/mock/Vault');
+
 const log = new Logger('TestUtapiClient');
 const redis = redisClient({
     host: '127.0.0.1',
@@ -331,3 +335,202 @@ describe('UtapiClient: expire bucket metrics', () => {
         });
     });
 });
+
+describe('UtapiClient: Across time intervals', function test() {
+    this.timeout((1000 * 60) * 2);
+
+    function checkMetricResponse(response, expected) {
+        const data = JSON.parse(response);
+        if (data.code) {
+            assert.ifError(data.message);
+        }
+        const { storageUtilized, numberOfObjects, incomingBytes } = data[0];
+        assert.deepStrictEqual(storageUtilized, expected.storageUtilized);
+        assert.deepStrictEqual(numberOfObjects, expected.numberOfObjects);
+        assert.strictEqual(incomingBytes, expected.incomingBytes);
+    }
+
+    function getNormalizedTimestampSeconds() {
+        const d = new Date();
+        const seconds = d.getSeconds();
+        return d.setSeconds((seconds - seconds % 15), 0, 0);
+    }
+
+    function waitUntilNextInterval() {
+        const start = getNormalizedTimestampSeconds();
+        while (start === getNormalizedTimestampSeconds()) {
+            setTimeout(() => {}, 500);
+        }
+    }
+
+    const vault = new Vault();
+
+    before(() => {
+        process.env.TIMESTAMP_INTERVAL = 'hello';
+        vault.start();
+    });
+
+    after(() => {
+        vault.end();
+    });
+
+    afterEach(() => redis.flushdb());
+
+    function putObject(cb) {
+        const params = {
+            level: 'buckets',
+            service: 's3',
+            bucket: 'my-bucket',
+            newByteLength: 10,
+            oldByteLength: null,
+        };
+        utapiClient.pushMetric('putObject', reqUid, params, cb);
+    }
+
+    function deleteObject(cb) {
+        const params = {
+            level: 'buckets',
+            service: 's3',
+            bucket: 'my-bucket',
+            byteLength: 10,
+            numberOfObjects: 1,
+        };
+        utapiClient.pushMetric('deleteObject', reqUid, params, cb);
+    }
+
+    let firstInterval;
+    let secondInterval;
+
+    describe('Metrics do not return to same values', () => {
+        beforeEach(done => {
+            series([
+                next => {
+                    waitUntilNextInterval();
+                    firstInterval = getNormalizedTimestampSeconds();
+                    series([
+                        next => putObject(next),
+                        next => putObject(next),
+                    ], next);
+                },
+                next => {
+                    waitUntilNextInterval();
+                    secondInterval = getNormalizedTimestampSeconds();
+                    series([
+                        next => putObject(next),
+                        next => putObject(next),
+                        next => deleteObject(next),
+                    ], next);
+                },
+            ], done);
+        });
+
+        it('should maintain data points', done => {
+            series([
+                next => {
+                    const params = {
+                        timeRange: [firstInterval, secondInterval - 1],
+                        resource: {
+                            type: 'buckets',
+                            buckets: ['my-bucket'],
+                        },
+                    };
+                    makeUtapiClientRequest(params, (err, response) => {
+                        assert.ifError(err);
+                        const expected = {
+                            storageUtilized: [20, 20],
+                            numberOfObjects: [2, 2],
+                            incomingBytes: 20,
+                        };
+                        checkMetricResponse(response, expected);
+                        return next();
+                    });
+                },
+                next => {
+                    const params = {
+                        timeRange: [secondInterval, secondInterval + 14999],
+                        resource: {
+                            type: 'buckets',
+                            buckets: ['my-bucket'],
+                        },
+                    };
+                    makeUtapiClientRequest(params, (err, response) => {
+                        assert.ifError(err);
+                        const expected = {
+                            storageUtilized: [30, 30],
+                            numberOfObjects: [3, 3],
+                            incomingBytes: 20,
+                        };
+                        checkMetricResponse(response, expected);
+                        return next();
+                    });
+                },
+            ], done);
+        });
+    });
+
+    describe('Metrics return to same values', () => {
+        beforeEach(done => {
+            series([
+                next => {
+                    waitUntilNextInterval();
+                    firstInterval = getNormalizedTimestampSeconds();
+                    series([
+                        next => putObject(next),
+                        next => putObject(next),
+                    ], next);
+                },
+                next => {
+                    waitUntilNextInterval();
+                    secondInterval = getNormalizedTimestampSeconds();
+                    series([
+                        next => putObject(next),
+                        next => deleteObject(next),
+                    ], next);
+                },
+            ], done);
+        });
+
+        it('should maintain data points', done => {
+            series([
+                next => {
+                    const params = {
+                        timeRange: [firstInterval, secondInterval - 1],
+                        resource: {
+                            type: 'buckets',
+                            buckets: ['my-bucket'],
+                        },
+                    };
+                    makeUtapiClientRequest(params, (err, response) => {
+                        assert.ifError(err);
+                        const expected = {
+                            storageUtilized: [20, 20],
+                            numberOfObjects: [2, 2],
+                            incomingBytes: 20,
+                        };
+                        checkMetricResponse(response, expected);
+                        return next();
+                    });
+                },
+                next => {
+                    const params = {
+                        timeRange: [secondInterval, secondInterval + 14999],
+                        resource: {
+                            type: 'buckets',
+                            buckets: ['my-bucket'],
+                        },
+                    };
+                    makeUtapiClientRequest(params, (err, response) => {
+                        assert.ifError(err);
+                        const expected = {
+                            storageUtilized: [20, 20],
+                            numberOfObjects: [2, 2],
+                            incomingBytes: 10,
+                        };
+                        checkMetricResponse(response, expected);
+                        return next();
+                    });
+                },
+            ], done);
+        });
+    });
+});

tests/unit/testUtapiClient.js

@@ -4,6 +4,7 @@ const Datastore = require('../../lib/Datastore');
 const MemoryBackend = require('../../lib/backend/Memory');
 const UtapiClient = require('../../lib/UtapiClient');
 const { getNormalizedTimestamp } = require('../utils/utils');
+const member = require('../../utils/member');
 
 const memoryBackend = new MemoryBackend();
 const ds = new Datastore();
@@ -91,10 +92,22 @@ function getObject(timestamp, data) {
     return obj;
 }
 
+function deserializeMemoryBackend(data) {
+    Object.keys(data).forEach(key => {
+        const isSortedSet =
+            key.endsWith('storageUtilized') || key.endsWith('numberOfObjects');
+        if (isSortedSet) {
+            data[key][0][1] = member.deserialize(data[key][0][1]); // eslint-disable-line
+        }
+    });
+}
+
 function testMetric(metric, params, expected, cb) {
     const c = new UtapiClient(config);
     c.setDataStore(ds);
+
     c.pushMetric(metric, REQUID, params, () => {
+        deserializeMemoryBackend(memoryBackend.data);
         assert.deepStrictEqual(memoryBackend.data, expected);
         return cb();
     });
@@ -119,7 +132,7 @@ describe('UtapiClient:: enable/disable client', () => {
 });
 
 describe('UtapiClient:: push metrics', () => {
-    const timestamp = getNormalizedTimestamp(Date.now());
+    const timestamp = UtapiClient.getNormalizedTimestamp();
     let params;
 
     beforeEach(() => {

tests/unit/validators/validateTimeRange.js

@@ -1,25 +1,28 @@
 const assert = require('assert');
 const validateTimeRange = require('../../../validators/validateTimeRange');
 const { getNormalizedTimestamp } = require('../../utils/utils');
+const UtapiClient = require('../../../lib/UtapiClient');
 
 describe('validateTimeRange', () => {
     const fifteenMinutes = (1000 * 60) * 15;
 
     it('should allow a current end time, if not provided', () => {
-        const start = getNormalizedTimestamp();
+        const start = UtapiClient.getNormalizedTimestamp();
         const isValid = validateTimeRange([start]);
         assert.strictEqual(isValid, true);
     });
 
     it('should not allow a start time in the future', () => {
-        const start = getNormalizedTimestamp() + fifteenMinutes;
+        const start = UtapiClient.getNormalizedTimestamp() + fifteenMinutes;
         const isValid = validateTimeRange([start]);
         assert.strictEqual(isValid, false);
     });
 
     it('should not allow a start time greater than the end time', () => {
-        const start = getNormalizedTimestamp() - (fifteenMinutes * 2);
-        const end = getNormalizedTimestamp() - (fifteenMinutes * 3) - 1;
+        const start =
+            UtapiClient.getNormalizedTimestamp() - (fifteenMinutes * 2);
+        const end =
+            UtapiClient.getNormalizedTimestamp() - (fifteenMinutes * 3) - 1;
         const isValid = validateTimeRange([start, end]);
         assert.strictEqual(isValid, false);
     });

tests/utils/utils.js

@@ -2,6 +2,7 @@ const http = require('http');
 const aws4 = require('aws4');
 
 const { getKeys, getCounters } = require('../../lib/schema');
+const UtapiClient = require('../../lib/UtapiClient');
 
 const resouceTypes = ['buckets', 'accounts', 'service'];
 const propertyNames = {
@@ -13,12 +14,6 @@ const resources = {
     accounts: 'foo-account',
 };
 
-function getNormalizedTimestamp() {
-    const d = new Date();
-    const minutes = d.getMinutes();
-    return d.setMinutes((minutes - minutes % 15), 0, 0);
-}
-
 // Build the resouceType object that gets keys from the schema.
 function _getResourceTypeObject(resourceType) {
     const obj = { level: resourceType, service: 's3' };
@@ -30,7 +25,7 @@ function _getResourceTypeObject(resourceType) {
 
 // Get all keys for each resource type from the schema.
 function getAllResourceTypeKeys() {
-    const timestamp = getNormalizedTimestamp(Date.now());
+    const timestamp = UtapiClient.getNormalizedTimestamp();
     const allResourceTypeKeys = resouceTypes.map(resourceType => {
         const obj = _getResourceTypeObject(resourceType);
         const counters = getCounters(obj);
@@ -115,20 +110,14 @@ function makeUtapiClientRequest({ timeRange, resource }, cb) {
     req.end();
 }
 
-function _getNormalizedTimestamp() {
-    const d = new Date();
-    const minutes = d.getMinutes();
-    return d.setMinutes((minutes - minutes % 15), 0, 0);
-}
-
 function _getStartTime() {
-    const thirtyMinutes = (1000 * 60) * 30;
-    return _getNormalizedTimestamp() - thirtyMinutes;
+    const thirtySeconds = (1000) * 30;
+    return UtapiClient.getNormalizedTimestamp() - thirtySeconds;
 }
 
 function _getEndTime() {
-    const fifteenMinutes = (1000 * 60) * 15;
-    return (_getNormalizedTimestamp() - 1) + fifteenMinutes;
+    const fifteenSeconds = (1000) * 15;
+    return (UtapiClient.getNormalizedTimestamp() - 1) + fifteenSeconds;
 }
 
 function _buildRequestBody(resource) {
@@ -169,7 +158,6 @@ function listMetrics(resource, cb) {
 module.exports = {
     listMetrics,
     getAllResourceTypeKeys,
-    getNormalizedTimestamp,
     buildMockResponse,
     makeUtapiClientRequest,
 };

utils/member.js

@@ -0,0 +1,11 @@
+const uuid = require('uuid/v4');
+
+function serialize(value) {
+    return `${value}:${uuid()}`;
+}
+
+function deserialize(value) {
+    return value.split(':')[0];
+}
+
+module.exports = { serialize, deserialize };

validators/validateTimeRange.js

@@ -4,6 +4,9 @@
 * @return {boolean} - validation result
 */
 function validateTimeRange(timeRange) {
+    if (process.env.CI) {
+        return true;
+    }
     if (Array.isArray(timeRange) && timeRange.length > 0 && timeRange.length < 3
         && timeRange.every(item => typeof item === 'number')) {
         // check for start time