eve/workers/unit_and_feature_tests/run_ft_tests.bash

@@ -17,6 +17,6 @@ if [ -z "$SETUP_CMD" ]; then
   SETUP_CMD="start"
 fi
 
-UTAPI_FILTER_BUCKET_DENY='deny-this-bucket' UTAPI_INTERVAL_TEST_MODE=$1 npm $SETUP_CMD 2>&1 | tee -a "/artifacts/setup_$2.log" &
+UTAPI_INTERVAL_TEST_MODE=$1 npm $SETUP_CMD 2>&1 | tee -a "/artifacts/setup_$2.log" &
 bash tests/utils/wait_for_local_port.bash $PORT 40
 UTAPI_INTERVAL_TEST_MODE=$1 npm run $2 | tee -a "/artifacts/test_$2.log"

libV2/config/defaults.json

@@ -48,9 +48,5 @@
     "serviceUser": {
         "arn": "arn:aws:iam::000000000000:user/service-utapi-user",
         "enabled": false
-    },
-    "filter": {
-        "allow": {},
-        "deny": {}
     }
 }

libV2/config/index.js

@@ -3,9 +3,7 @@ const path = require('path');
 const Joi = require('@hapi/joi');
 const assert = require('assert');
 
-const {
+const { truthy, envNamespace } = require('../constants');
-    truthy, envNamespace, allowedFilterFields, allowedFilterStates,
-} = require('../constants');
 const configSchema = require('./schema');
 // We need to require the specific file rather than the parent module to avoid a circular require
 const { parseDiskSizeSpec } = require('../utils/disk');
@@ -227,28 +225,6 @@ class Config {
         return certs;
     }
 
-    static _parseResourceFilters(config) {
-        const resourceFilters = {};
-
-        allowedFilterFields.forEach(
-            field => allowedFilterStates.forEach(
-                state => {
-                    const configResources = (config[state] && config[state][field]) || null;
-                    const envVar = `FILTER_${field.toUpperCase()}_${state.toUpperCase()}`;
-                    const resources = _loadFromEnv(envVar, configResources, _typeCasts.list);
-                    if (resources) {
-                        if (resourceFilters[field]) {
-                            throw new Error('You can not define both an allow and a deny list for an event field.');
-                        }
-                        resourceFilters[field] = { [state]: new Set(resources) };
-                    }
-                },
-            ),
-        );
-
-        return resourceFilters;
-    }
-
     _parseConfig(config) {
         const parsedConfig = {};
 
@@ -369,8 +345,6 @@ class Config {
             enabled: _loadFromEnv('SERVICE_USER_ENABLED', config.serviceUser.enabled, _typeCasts.bool),
         };
 
-        parsedConfig.filter = Config._parseResourceFilters(config.filter);
-
         return parsedConfig;
     }
 

libV2/config/schema.js

@@ -1,5 +1,4 @@
 const Joi = require('@hapi/joi');
-const { allowedFilterFields, allowedFilterStates } = require('../constants');
 
 const redisServerSchema = Joi.object({
     host: Joi.string(),
@@ -90,17 +89,6 @@ const schema = Joi.object({
         arn: Joi.string(),
         enabled: Joi.boolean(),
     }),
-    filter: Joi.object(allowedFilterStates.reduce(
-        (filterObj, state) => {
-            filterObj[state] = allowedFilterFields.reduce(
-                (stateObj, field) => {
-                    stateObj[field] = Joi.string();
-                    return stateObj;
-                }, {},
-            );
-            return filterObj;
-        }, {},
-    )),
 });
 
 module.exports = schema;

libV2/constants.js

@@ -111,15 +111,6 @@ const constants = {
         putDeleteMarkerObject: 'deleteObject',
     },
     expirationChunkDuration: 900000000, // 15 minutes in microseconds
-    allowedFilterFields: [
-        'operationId',
-        'location',
-        'account',
-        'user',
-        'bucket',
-        'object',
-    ],
-    allowedFilterStates: ['allow', 'deny'],
 };
 
 constants.operationToResponse = constants.operations

libV2/tasks/IngestShard.js

@@ -5,7 +5,7 @@ const { UtapiMetric } = require('../models');
 const config = require('../config');
 const { checkpointLagSecs } = require('../constants');
 const {
-    LoggerContext, shardFromTimestamp, convertTimestamp, InterpolatedClock, now, filterObject,
+    LoggerContext, shardFromTimestamp, convertTimestamp, InterpolatedClock, now,
 } = require('../utils');
 
 const logger = new LoggerContext({
@@ -20,11 +20,6 @@ class IngestShardTask extends BaseTask {
         this._defaultSchedule = config.ingestionSchedule;
         this._defaultLag = config.ingestionLagSeconds;
         this._stripEventUUID = options.stripEventUUID !== undefined ? options.stripEventUUID : true;
-        this._eventFilter = Object.entries(config.filter)
-            .reduce((chain, [level, filter]) => {
-                const filterFunc = filterObject(level, filter);
-                return obj => filterFunc(obj) && chain(obj);
-            }, () => true);
     }
 
     _hydrateEvent(data, stripTimestamp = false) {
@@ -66,9 +61,7 @@ class IngestShardTask extends BaseTask {
                             logger.info('Detected slow records, ingesting as repair');
                         }
 
-                        const records = metrics
+                        const records = metrics.map(m => this._hydrateEvent(m, areSlowEvents));
-                            .map(m => this._hydrateEvent(m, areSlowEvents))
-                            .filter(m => this._eventFilter(m));
 
                         records.sort((a, b) => a.timestamp - b.timestamp);
 

libV2/utils/filter.js

@@ -1,14 +0,0 @@
-function filterObject(key, { allow, deny }) {
-    if (allow && deny) {
-        throw new Error('You can not define both an allow and a deny list.');
-    }
-    if (!allow && !deny) {
-        throw new Error('You must define either an allow or a deny list.');
-    }
-    if (allow) {
-        return obj => (obj[key] === undefined) || allow.has(obj[key]);
-    }
-    return obj => (obj[key] === undefined) || !deny.has(obj[key]);
-}
-
-module.exports = { filterObject };

libV2/utils/index.js

@@ -3,7 +3,6 @@ const shard = require('./shard');
 const timestamp = require('./timestamp');
 const func = require('./func');
 const disk = require('./disk');
-const filter = require('./filter');
 
 module.exports = {
     ...log,
@@ -11,5 +10,4 @@ module.exports = {
     ...timestamp,
     ...func,
     ...disk,
-    ...filter,
 };

tests/unit/v2/utils/testfilterObject.js

@@ -1,52 +0,0 @@
-const assert = require('assert');
-const { filterObject } = require('../../../../libV2/utils');
-
-const testCases = [
-    {
-        filter: { allow: new Set(['foo', 'bar']) },
-        value: 'foo',
-        expected: true,
-    },
-    {
-        filter: { allow: new Set(['foo', 'bar']) },
-        value: 'baz',
-        expected: false,
-    },
-    {
-        filter: { deny: new Set(['foo', 'bar']) },
-        value: 'foo',
-        expected: false,
-    },
-    {
-        filter: { deny: new Set(['foo', 'bar']) },
-        value: 'baz',
-        expected: true,
-    },
-];
-
-describe'Test filterObject', () => {
-    testCases.forEach(testCase => {
-        const { value, expected, filter } = testCase;
-        const successMsg = expected ? 'should not filter' : 'should filter';
-        const state = (expected && filter.allow) || (!expected && filter.deny) ? '' : ' not';
-        const ruleType = Object.keys(filter)[0];
-        const msg = `${successMsg} object if value is${state} present in ${ruleType} list`;
-        it(msg, () => {
-            const func = filterObject('value', filter);
-            assert.strictEqual(func({ value }), expected);
-        });
-    });
-
-    it('should not filter an object if the filter key in undefined', () => {
-        const func = filterObject('value', { allow: ['foo'] });
-        assert.strictEqual(func({}), true);
-    });
-
-    it('should throw if creating a filter with both allow and deny lists', () => {
-        assert.throws(() => filterObject('value', { allow: ['foo'], deny: ['bar'] }));
-    });
-
-    it('should throw if creating a filter without an allow or deny lists', () => {
-        assert.throws(() => filterObject('value', {}));
-    });
-});