eve/workers/unit_and_feature_tests/run_ft_tests.bash

@@ -13,5 +13,5 @@ killandsleep () {
   sleep 10
 }
 
-CI=true npm start & bash tests/utils/wait_for_local_port.bash $PORT 40
+npm start & bash tests/utils/wait_for_local_port.bash $PORT 40
-CI=true npm run $1
+npm run $1

lib/ListMetrics.js

@@ -4,7 +4,6 @@ const { getMetricFromKey, getKeys, generateStateKey } = require('./schema');
 const s3metricResponseJSON = require('../models/s3metricResponse');
 const config = require('./Config');
 const Vault = require('./Vault');
-const member = require('../utils/member');
 
 const MAX_RANGE_MS = (((1000 * 60) * 60) * 24) * 30; // One month.
 
@@ -145,9 +144,6 @@ class ListMetrics {
             res.push(last);
             const d = new Date(last);
             last = d.setMinutes(d.getMinutes() + 15);
-            if (process.env.CI) {
-                last = d.setSeconds(d.getSeconds() + 15);
-            }
         }
         res.push(end);
         return res;
@@ -275,12 +271,7 @@ class ListMetrics {
                         method: 'ListMetrics.getMetrics',
                     });
                 } else {
-                    let val =
+                    let val = parseInt(item[1], 10);
-                        item &&
-                        item[1] &&
-                        item[1][0] &&
-                        member.deserialize(item[1][0]);
-                    val = parseInt(val, 10);
 
                     val = Number.isNaN(val) ? 0 : val;
                     if (val < 0) {

lib/UtapiClient.js

@@ -5,7 +5,6 @@ const Datastore = require('./Datastore');
 const { generateKey, generateCounter, generateStateKey } = require('./schema');
 const { errors } = require('arsenal');
 const redisClient = require('../utils/redisClient');
-const member = require('../utils/member');
 
 const methods = {
     createBucket: { method: '_genericPushMetric', changesData: true },
@@ -126,10 +125,6 @@ class UtapiClient {
     */
     static getNormalizedTimestamp() {
         const d = new Date();
-        if (process.env.CI) {
-            const seconds = d.getSeconds();
-            return d.setSeconds((seconds - seconds % 15), 0, 0);
-        }
         const minutes = d.getMinutes();
         return d.setMinutes((minutes - minutes % 15), 0, 0);
     }
@@ -518,7 +513,7 @@ class UtapiClient {
                 const key = generateStateKey(p, 'numberOfObjects');
                 cmds2.push(
                     ['zremrangebyscore', key, timestamp, timestamp],
-                    ['zadd', key, timestamp, member.serialize(actionCounter)]);
+                    ['zadd', key, timestamp, actionCounter]);
                 return true;
             });
             if (noErr) {
@@ -598,7 +593,7 @@ class UtapiClient {
                     ['zremrangebyscore', generateStateKey(p, 'storageUtilized'),
                         timestamp, timestamp],
                     ['zadd', generateStateKey(p, 'storageUtilized'),
-                        timestamp, member.serialize(actionCounter)]
+                        timestamp, actionCounter]
                 );
                 return true;
             });
@@ -672,7 +667,7 @@ class UtapiClient {
                 }
                 key = generateStateKey(p, 'numberOfObjects');
                 cmds2.push(['zremrangebyscore', key, timestamp, timestamp],
-                ['zadd', key, timestamp, member.serialize(actionCounter)]);
+                ['zadd', key, timestamp, actionCounter]);
                 return true;
             });
             if (noErr) {
@@ -784,7 +779,7 @@ class UtapiClient {
                         timestamp, timestamp],
                     ['zadd',
                         generateStateKey(p, 'storageUtilized'), timestamp,
-                        member.serialize(actionCounter)]);
+                        actionCounter]);
                 // The 'abortMultipartUpload' action does not affect number of
                 // objects, so we return here.
                 if (action === 'abortMultipartUpload') {
@@ -814,7 +809,7 @@ class UtapiClient {
                         generateStateKey(p, 'numberOfObjects'), timestamp,
                         timestamp],
                     ['zadd', generateStateKey(p, 'numberOfObjects'), timestamp,
-                        member.serialize(actionCounter)]);
+                        actionCounter]);
                 return true;
             });
             if (noErr) {
@@ -946,7 +941,7 @@ class UtapiClient {
                         generateStateKey(p, 'storageUtilized'),
                         timestamp, timestamp],
                     ['zadd', generateStateKey(p, 'storageUtilized'),
-                        timestamp, member.serialize(actionCounter)]);
+                        timestamp, actionCounter]);
 
                 // number of objects counter
                 objectsIndex = (i * (cmdsLen / paramsArrLen)) + 1;
@@ -972,7 +967,7 @@ class UtapiClient {
                         generateStateKey(p, 'numberOfObjects'),
                         timestamp, timestamp],
                     ['zadd', generateStateKey(p, 'numberOfObjects'),
-                        timestamp, member.serialize(actionCounter)]);
+                        timestamp, actionCounter]);
                 return true;
             });
             if (noErr) {

tests/functional/testReplay.js

@@ -112,12 +112,9 @@ function checkListElement(action, params, res) {
     assert(!error, 'cannot parse cached element into JSON');
     const { reqUid, timestamp } = result;
     const currTimestamp = UtapiClient.getNormalizedTimestamp();
-    let milliseconds = 900000; // 15 minutes
+    const fifteenMinutes = 900000; // Milliseconds.
-    if (process.env.CI) {
-        milliseconds = 15000; // 15 seconds
-    }
     // Allow for previous timestamp interval, since we cannot know start time.
-    assert(timestamp, currTimestamp || currTimestamp - milliseconds,
+    assert(timestamp, currTimestamp || currTimestamp - fifteenMinutes,
         'incorrect timestamp value');
     assert(reqUid !== undefined,
         `reqUid property not in cached element: ${action}`);

tests/functional/testUtapiClient.js

@@ -6,10 +6,6 @@ const redisClient = require('../../utils/redisClient');
 const { Logger } = require('werelogs');
 const { getCounters, getMetricFromKey,
     getStateKeys, getKeys } = require('../../lib/schema');
-
-const { makeUtapiClientRequest } = require('../utils/utils');
-const Vault = require('../utils/mock/Vault');
-
 const log = new Logger('TestUtapiClient');
 const redis = redisClient({
     host: '127.0.0.1',
@@ -335,202 +331,3 @@ describe('UtapiClient: expire bucket metrics', () => {
         });
     });
 });
-
-describe('UtapiClient: Across time intervals', function test() {
-    this.timeout((1000 * 60) * 2);
-
-    function checkMetricResponse(response, expected) {
-        const data = JSON.parse(response);
-        if (data.code) {
-            assert.ifError(data.message);
-        }
-        const { storageUtilized, numberOfObjects, incomingBytes } = data[0];
-        assert.deepStrictEqual(storageUtilized, expected.storageUtilized);
-        assert.deepStrictEqual(numberOfObjects, expected.numberOfObjects);
-        assert.strictEqual(incomingBytes, expected.incomingBytes);
-    }
-
-    function getNormalizedTimestampSeconds() {
-        const d = new Date();
-        const seconds = d.getSeconds();
-        return d.setSeconds((seconds - seconds % 15), 0, 0);
-    }
-
-    function waitUntilNextInterval() {
-        const start = getNormalizedTimestampSeconds();
-        while (start === getNormalizedTimestampSeconds()) {
-            setTimeout(() => {}, 500);
-        }
-    }
-
-    const vault = new Vault();
-
-    before(() => {
-        process.env.TIMESTAMP_INTERVAL = 'hello';
-        vault.start();
-    });
-
-    after(() => {
-        vault.end();
-    });
-
-    afterEach(() => redis.flushdb());
-
-    function putObject(cb) {
-        const params = {
-            level: 'buckets',
-            service: 's3',
-            bucket: 'my-bucket',
-            newByteLength: 10,
-            oldByteLength: null,
-        };
-        utapiClient.pushMetric('putObject', reqUid, params, cb);
-    }
-
-    function deleteObject(cb) {
-        const params = {
-            level: 'buckets',
-            service: 's3',
-            bucket: 'my-bucket',
-            byteLength: 10,
-            numberOfObjects: 1,
-        };
-        utapiClient.pushMetric('deleteObject', reqUid, params, cb);
-    }
-
-    let firstInterval;
-    let secondInterval;
-
-    describe('Metrics do not return to same values', () => {
-        beforeEach(done => {
-            series([
-                next => {
-                    waitUntilNextInterval();
-                    firstInterval = getNormalizedTimestampSeconds();
-                    series([
-                        next => putObject(next),
-                        next => putObject(next),
-                    ], next);
-                },
-                next => {
-                    waitUntilNextInterval();
-                    secondInterval = getNormalizedTimestampSeconds();
-                    series([
-                        next => putObject(next),
-                        next => putObject(next),
-                        next => deleteObject(next),
-                    ], next);
-                },
-            ], done);
-        });
-
-        it('should maintain data points', done => {
-            series([
-                next => {
-                    const params = {
-                        timeRange: [firstInterval, secondInterval - 1],
-                        resource: {
-                            type: 'buckets',
-                            buckets: ['my-bucket'],
-                        },
-                    };
-                    makeUtapiClientRequest(params, (err, response) => {
-                        assert.ifError(err);
-                        const expected = {
-                            storageUtilized: [20, 20],
-                            numberOfObjects: [2, 2],
-                            incomingBytes: 20,
-                        };
-                        checkMetricResponse(response, expected);
-                        return next();
-                    });
-                },
-                next => {
-                    const params = {
-                        timeRange: [secondInterval, secondInterval + 14999],
-                        resource: {
-                            type: 'buckets',
-                            buckets: ['my-bucket'],
-                        },
-                    };
-                    makeUtapiClientRequest(params, (err, response) => {
-                        assert.ifError(err);
-                        const expected = {
-                            storageUtilized: [30, 30],
-                            numberOfObjects: [3, 3],
-                            incomingBytes: 20,
-                        };
-                        checkMetricResponse(response, expected);
-                        return next();
-                    });
-                },
-            ], done);
-        });
-    });
-
-    describe('Metrics return to same values', () => {
-        beforeEach(done => {
-            series([
-                next => {
-                    waitUntilNextInterval();
-                    firstInterval = getNormalizedTimestampSeconds();
-                    series([
-                        next => putObject(next),
-                        next => putObject(next),
-                    ], next);
-                },
-                next => {
-                    waitUntilNextInterval();
-                    secondInterval = getNormalizedTimestampSeconds();
-                    series([
-                        next => putObject(next),
-                        next => deleteObject(next),
-                    ], next);
-                },
-            ], done);
-        });
-
-        it('should maintain data points', done => {
-            series([
-                next => {
-                    const params = {
-                        timeRange: [firstInterval, secondInterval - 1],
-                        resource: {
-                            type: 'buckets',
-                            buckets: ['my-bucket'],
-                        },
-                    };
-                    makeUtapiClientRequest(params, (err, response) => {
-                        assert.ifError(err);
-                        const expected = {
-                            storageUtilized: [20, 20],
-                            numberOfObjects: [2, 2],
-                            incomingBytes: 20,
-                        };
-                        checkMetricResponse(response, expected);
-                        return next();
-                    });
-                },
-                next => {
-                    const params = {
-                        timeRange: [secondInterval, secondInterval + 14999],
-                        resource: {
-                            type: 'buckets',
-                            buckets: ['my-bucket'],
-                        },
-                    };
-                    makeUtapiClientRequest(params, (err, response) => {
-                        assert.ifError(err);
-                        const expected = {
-                            storageUtilized: [20, 20],
-                            numberOfObjects: [2, 2],
-                            incomingBytes: 10,
-                        };
-                        checkMetricResponse(response, expected);
-                        return next();
-                    });
-                },
-            ], done);
-        });
-    });
-});

tests/unit/testUtapiClient.js

@@ -4,7 +4,6 @@ const Datastore = require('../../lib/Datastore');
 const MemoryBackend = require('../../lib/backend/Memory');
 const UtapiClient = require('../../lib/UtapiClient');
 const { getNormalizedTimestamp } = require('../utils/utils');
-const member = require('../../utils/member');
 
 const memoryBackend = new MemoryBackend();
 const ds = new Datastore();
@@ -92,22 +91,10 @@ function getObject(timestamp, data) {
     return obj;
 }
 
-function deserializeMemoryBackend(data) {
-    Object.keys(data).forEach(key => {
-        const isSortedSet =
-            key.endsWith('storageUtilized') || key.endsWith('numberOfObjects');
-        if (isSortedSet) {
-            data[key][0][1] = member.deserialize(data[key][0][1]); // eslint-disable-line
-        }
-    });
-}
-
 function testMetric(metric, params, expected, cb) {
     const c = new UtapiClient(config);
     c.setDataStore(ds);
-
     c.pushMetric(metric, REQUID, params, () => {
-        deserializeMemoryBackend(memoryBackend.data);
         assert.deepStrictEqual(memoryBackend.data, expected);
         return cb();
     });
@@ -132,7 +119,7 @@ describe('UtapiClient:: enable/disable client', () => {
 });
 
 describe('UtapiClient:: push metrics', () => {
-    const timestamp = UtapiClient.getNormalizedTimestamp();
+    const timestamp = getNormalizedTimestamp(Date.now());
     let params;
 
     beforeEach(() => {

tests/unit/validators/validateTimeRange.js

@@ -1,28 +1,25 @@
 const assert = require('assert');
 const validateTimeRange = require('../../../validators/validateTimeRange');
 const { getNormalizedTimestamp } = require('../../utils/utils');
-const UtapiClient = require('../../../lib/UtapiClient');
 
 describe('validateTimeRange', () => {
     const fifteenMinutes = (1000 * 60) * 15;
 
     it('should allow a current end time, if not provided', () => {
-        const start = UtapiClient.getNormalizedTimestamp();
+        const start = getNormalizedTimestamp();
         const isValid = validateTimeRange([start]);
         assert.strictEqual(isValid, true);
     });
 
     it('should not allow a start time in the future', () => {
-        const start = UtapiClient.getNormalizedTimestamp() + fifteenMinutes;
+        const start = getNormalizedTimestamp() + fifteenMinutes;
         const isValid = validateTimeRange([start]);
         assert.strictEqual(isValid, false);
     });
 
     it('should not allow a start time greater than the end time', () => {
-        const start =
+        const start = getNormalizedTimestamp() - (fifteenMinutes * 2);
-            UtapiClient.getNormalizedTimestamp() - (fifteenMinutes * 2);
+        const end = getNormalizedTimestamp() - (fifteenMinutes * 3) - 1;
-        const end =
-            UtapiClient.getNormalizedTimestamp() - (fifteenMinutes * 3) - 1;
         const isValid = validateTimeRange([start, end]);
         assert.strictEqual(isValid, false);
     });

tests/utils/utils.js

@@ -2,7 +2,6 @@ const http = require('http');
 const aws4 = require('aws4');
 
 const { getKeys, getCounters } = require('../../lib/schema');
-const UtapiClient = require('../../lib/UtapiClient');
 
 const resouceTypes = ['buckets', 'accounts', 'service'];
 const propertyNames = {
@@ -14,6 +13,12 @@ const resources = {
     accounts: 'foo-account',
 };
 
+function getNormalizedTimestamp() {
+    const d = new Date();
+    const minutes = d.getMinutes();
+    return d.setMinutes((minutes - minutes % 15), 0, 0);
+}
+
 // Build the resouceType object that gets keys from the schema.
 function _getResourceTypeObject(resourceType) {
     const obj = { level: resourceType, service: 's3' };
@@ -25,7 +30,7 @@ function _getResourceTypeObject(resourceType) {
 
 // Get all keys for each resource type from the schema.
 function getAllResourceTypeKeys() {
-    const timestamp = UtapiClient.getNormalizedTimestamp();
+    const timestamp = getNormalizedTimestamp(Date.now());
     const allResourceTypeKeys = resouceTypes.map(resourceType => {
         const obj = _getResourceTypeObject(resourceType);
         const counters = getCounters(obj);
@@ -110,14 +115,20 @@ function makeUtapiClientRequest({ timeRange, resource }, cb) {
     req.end();
 }
 
+function _getNormalizedTimestamp() {
+    const d = new Date();
+    const minutes = d.getMinutes();
+    return d.setMinutes((minutes - minutes % 15), 0, 0);
+}
+
 function _getStartTime() {
-    const thirtySeconds = (1000) * 30;
+    const thirtyMinutes = (1000 * 60) * 30;
-    return UtapiClient.getNormalizedTimestamp() - thirtySeconds;
+    return _getNormalizedTimestamp() - thirtyMinutes;
 }
 
 function _getEndTime() {
-    const fifteenSeconds = (1000) * 15;
+    const fifteenMinutes = (1000 * 60) * 15;
-    return (UtapiClient.getNormalizedTimestamp() - 1) + fifteenSeconds;
+    return (_getNormalizedTimestamp() - 1) + fifteenMinutes;
 }
 
 function _buildRequestBody(resource) {
@@ -158,6 +169,7 @@ function listMetrics(resource, cb) {
 module.exports = {
     listMetrics,
     getAllResourceTypeKeys,
+    getNormalizedTimestamp,
     buildMockResponse,
     makeUtapiClientRequest,
 };

utils/member.js

@@ -1,11 +0,0 @@
-const uuid = require('uuid/v4');
-
-function serialize(value) {
-    return `${value}:${uuid()}`;
-}
-
-function deserialize(value) {
-    return value.split(':')[0];
-}
-
-module.exports = { serialize, deserialize };

validators/validateTimeRange.js

@@ -4,9 +4,6 @@
 * @return {boolean} - validation result
 */
 function validateTimeRange(timeRange) {
-    if (process.env.CI) {
-        return true;
-    }
     if (Array.isArray(timeRange) && timeRange.length > 0 && timeRange.length < 3
         && timeRange.every(item => typeof item === 'number')) {
         // check for start time