stash

eve/workers/unit_and_feature_tests/run_ft_tests.bash

@@ -17,6 +17,6 @@ if [ -z "$SETUP_CMD" ]; then
   SETUP_CMD="start"
 fi
 
-UTAPI_INTERVAL_TEST_MODE=$1 npm $SETUP_CMD 2>&1 | tee -a "/artifacts/setup_$2.log" &
+UTAPI_FILTER_BUCKET_DENY='deny-this-bucket' UTAPI_INTERVAL_TEST_MODE=$1 npm $SETUP_CMD 2>&1 | tee -a "/artifacts/setup_$2.log" &
 bash tests/utils/wait_for_local_port.bash $PORT 40
 UTAPI_INTERVAL_TEST_MODE=$1 npm run $2 | tee -a "/artifacts/test_$2.log"

libV2/config/defaults.json

@@ -48,5 +48,9 @@
     "serviceUser": {
         "arn": "arn:aws:iam::000000000000:user/service-utapi-user",
         "enabled": false
+    },
+    "filter": {
+        "allow": {},
+        "deny": {}
     }
 }

libV2/config/index.js

@@ -3,7 +3,9 @@ const path = require('path');
 const Joi = require('@hapi/joi');
 const assert = require('assert');
 
-const { truthy, envNamespace } = require('../constants');
+const {
+    truthy, envNamespace, allowedFilterFields, allowedFilterStates,
+} = require('../constants');
 const configSchema = require('./schema');
 // We need to require the specific file rather than the parent module to avoid a circular require
 const { parseDiskSizeSpec } = require('../utils/disk');
@@ -225,6 +227,28 @@ class Config {
         return certs;
     }
 
+    static _parseResourceFilters(config) {
+        const resourceFilters = {};
+
+        allowedFilterFields.forEach(
+            field => allowedFilterStates.forEach(
+                state => {
+                    const configResources = (config[state] && config[state][field]) || null;
+                    const envVar = `FILTER_${field.toUpperCase()}_${state.toUpperCase()}`;
+                    const resources = _loadFromEnv(envVar, configResources, _typeCasts.list);
+                    if (resources) {
+                        if (resourceFilters[field]) {
+                            throw new Error('You can not define both an allow and a deny list for an event field.');
+                        }
+                        resourceFilters[field] = { [state]: new Set(resources) };
+                    }
+                },
+            ),
+        );
+
+        return resourceFilters;
+    }
+
     _parseConfig(config) {
         const parsedConfig = {};
 
@@ -345,6 +369,8 @@ class Config {
             enabled: _loadFromEnv('SERVICE_USER_ENABLED', config.serviceUser.enabled, _typeCasts.bool),
         };
 
+        parsedConfig.filter = Config._parseResourceFilters(config.filter);
+
         return parsedConfig;
     }
 

libV2/config/schema.js

@@ -1,4 +1,5 @@
 const Joi = require('@hapi/joi');
+const { allowedFilterFields, allowedFilterStates } = require('../constants');
 
 const redisServerSchema = Joi.object({
     host: Joi.string(),
@@ -89,6 +90,17 @@ const schema = Joi.object({
         arn: Joi.string(),
         enabled: Joi.boolean(),
     }),
+    filter: Joi.object(allowedFilterStates.reduce(
+        (filterObj, state) => {
+            filterObj[state] = allowedFilterFields.reduce(
+                (stateObj, field) => {
+                    stateObj[field] = Joi.string();
+                    return stateObj;
+                }, {},
+            );
+            return filterObj;
+        }, {},
+    )),
 });
 
 module.exports = schema;

libV2/constants.js

@@ -111,6 +111,15 @@ const constants = {
         putDeleteMarkerObject: 'deleteObject',
     },
     expirationChunkDuration: 900000000, // 15 minutes in microseconds
+    allowedFilterFields: [
+        'operationId',
+        'location',
+        'account',
+        'user',
+        'bucket',
+        'object',
+    ],
+    allowedFilterStates: ['allow', 'deny'],
 };
 
 constants.operationToResponse = constants.operations

libV2/tasks/IngestShard.js

@@ -5,7 +5,7 @@ const { UtapiMetric } = require('../models');
 const config = require('../config');
 const { checkpointLagSecs } = require('../constants');
 const {
-    LoggerContext, shardFromTimestamp, convertTimestamp, InterpolatedClock, now,
+    LoggerContext, shardFromTimestamp, convertTimestamp, InterpolatedClock, now, filterObject,
 } = require('../utils');
 
 const logger = new LoggerContext({
@@ -20,6 +20,11 @@ class IngestShardTask extends BaseTask {
         this._defaultSchedule = config.ingestionSchedule;
         this._defaultLag = config.ingestionLagSeconds;
         this._stripEventUUID = options.stripEventUUID !== undefined ? options.stripEventUUID : true;
+        this._eventFilter = Object.entries(config.filter)
+            .reduce((chain, [level, filter]) => {
+                const filterFunc = filterObject(level, filter);
+                return obj => filterFunc(obj) && chain(obj);
+            }, () => true);
     }
 
     _hydrateEvent(data, stripTimestamp = false) {
@@ -61,7 +66,9 @@ class IngestShardTask extends BaseTask {
                             logger.info('Detected slow records, ingesting as repair');
                         }
 
-                        const records = metrics.map(m => this._hydrateEvent(m, areSlowEvents));
+                        const records = metrics
+                            .map(m => this._hydrateEvent(m, areSlowEvents))
+                            .filter(m => this._eventFilter(m));
 
                         records.sort((a, b) => a.timestamp - b.timestamp);
 

libV2/utils/filter.js

@@ -0,0 +1,14 @@
+function filterObject(key, { allow, deny }) {
+    if (allow && deny) {
+        throw new Error('You can not define both an allow and a deny list.');
+    }
+    if (!allow && !deny) {
+        throw new Error('You must define either an allow or a deny list.');
+    }
+    if (allow) {
+        return obj => (obj[key] === undefined) || allow.has(obj[key]);
+    }
+    return obj => (obj[key] === undefined) || !deny.has(obj[key]);
+}
+
+module.exports = { filterObject };

libV2/utils/index.js

@@ -3,6 +3,7 @@ const shard = require('./shard');
 const timestamp = require('./timestamp');
 const func = require('./func');
 const disk = require('./disk');
+const filter = require('./filter');
 
 module.exports = {
     ...log,
@@ -10,4 +11,5 @@ module.exports = {
     ...timestamp,
     ...func,
     ...disk,
+    ...filter,
 };

tests/unit/v2/utils/testfilterObject.js

@@ -0,0 +1,52 @@
+const assert = require('assert');
+const { filterObject } = require('../../../../libV2/utils');
+
+const testCases = [
+    {
+        filter: { allow: new Set(['foo', 'bar']) },
+        value: 'foo',
+        expected: true,
+    },
+    {
+        filter: { allow: new Set(['foo', 'bar']) },
+        value: 'baz',
+        expected: false,
+    },
+    {
+        filter: { deny: new Set(['foo', 'bar']) },
+        value: 'foo',
+        expected: false,
+    },
+    {
+        filter: { deny: new Set(['foo', 'bar']) },
+        value: 'baz',
+        expected: true,
+    },
+];
+
+describe'Test filterObject', () => {
+    testCases.forEach(testCase => {
+        const { value, expected, filter } = testCase;
+        const successMsg = expected ? 'should not filter' : 'should filter';
+        const state = (expected && filter.allow) || (!expected && filter.deny) ? '' : ' not';
+        const ruleType = Object.keys(filter)[0];
+        const msg = `${successMsg} object if value is${state} present in ${ruleType} list`;
+        it(msg, () => {
+            const func = filterObject('value', filter);
+            assert.strictEqual(func({ value }), expected);
+        });
+    });
+
+    it('should not filter an object if the filter key in undefined', () => {
+        const func = filterObject('value', { allow: ['foo'] });
+        assert.strictEqual(func({}), true);
+    });
+
+    it('should throw if creating a filter with both allow and deny lists', () => {
+        assert.throws(() => filterObject('value', { allow: ['foo'], deny: ['bar'] }));
+    });
+
+    it('should throw if creating a filter without an allow or deny lists', () => {
+        assert.throws(() => filterObject('value', {}));
+    });
+});