index.js

@@ -24,9 +24,6 @@ module.exports = {
         listTools: {
             DelimiterTools: require('./lib/algos/list/tools'),
         },
-        cache: {
-            LRUCache: require('./lib/algos/cache/LRUCache'),
-        },
     },
     policies: {
         evaluators: require('./lib/policyEvaluator/evaluator.js'),
@@ -34,7 +31,6 @@ module.exports = {
             .validateUserPolicy,
         evaluatePrincipal: require('./lib/policyEvaluator/principal'),
         RequestContext: require('./lib/policyEvaluator/RequestContext.js'),
-        requestUtils: require('./lib/policyEvaluator/requestUtils'),
     },
     Clustering: require('./lib/Clustering'),
     testing: {

lib/algos/cache/LRUCache.js

@@ -1,167 +0,0 @@
-const assert = require('assert');
-
-/**
- * @class
- * @classdesc Implements a key-value in-memory cache with a capped
- * number of items and a Least Recently Used (LRU) strategy for
- * eviction.
- */
-class LRUCache {
-    /**
-     * @constructor
-     * @param {number} maxEntries - maximum number of entries kept in
-     * the cache
-     */
-    constructor(maxEntries) {
-        assert(maxEntries >= 1);
-        this._maxEntries = maxEntries;
-        this.clear();
-    }
-
-    /**
-     * Add or update the value associated to a key in the cache,
-     * making it the most recently accessed for eviction purpose.
-     *
-     * @param {string} key - key to add
-     * @param {object} value - associated value (can be of any type)
-     * @return {boolean} true if the cache contained an entry with
-     * this key, false if it did not
-     */
-    add(key, value) {
-        let entry = this._entryMap[key];
-        if (entry) {
-            entry.value = value;
-            // make the entry the most recently used by re-pushing it
-            // to the head of the LRU list
-            this._lruRemoveEntry(entry);
-            this._lruPushEntry(entry);
-            return true;
-        }
-        if (this._entryCount === this._maxEntries) {
-            // if the cache is already full, abide by the LRU strategy
-            // and remove the least recently used entry from the cache
-            // before pushing the new entry
-            this._removeEntry(this._lruTail);
-        }
-        entry = { key, value };
-        this._entryMap[key] = entry;
-        this._entryCount += 1;
-        this._lruPushEntry(entry);
-        return false;
-    }
-
-    /**
-     * Get the value associated to a key in the cache, making it the
-     * most recently accessed for eviction purpose.
-     *
-     * @param {string} key - key of which to fetch the associated value
-     * @return {object|undefined} - returns the associated value if
-     * exists in the cache, or undefined if not found - either if the
-     * key was never added or if it has been evicted from the cache.
-     */
-    get(key) {
-        const entry = this._entryMap[key];
-        if (entry) {
-            // make the entry the most recently used by re-pushing it
-            // to the head of the LRU list
-            this._lruRemoveEntry(entry);
-            this._lruPushEntry(entry);
-            return entry.value;
-        }
-        return undefined;
-    }
-
-    /**
-     * Remove an entry from the cache if exists
-     *
-     * @param {string} key - key to remove
-     * @return {boolean} true if an entry has been removed, false if
-     * there was no entry with this key in the cache - either if the
-     * key was never added or if it has been evicted from the cache.
-     */
-    remove(key) {
-        const entry = this._entryMap[key];
-        if (entry) {
-            this._removeEntry(entry);
-            return true;
-        }
-        return false;
-    }
-
-    /**
-     * Get the current number of cached entries
-     *
-     * @return {number} current number of cached entries
-     */
-    count() {
-        return this._entryCount;
-    }
-
-    /**
-     * Remove all entries from the cache
-     *
-     * @return {undefined}
-     */
-    clear() {
-        this._entryMap = {};
-        this._entryCount = 0;
-        this._lruHead = null;
-        this._lruTail = null;
-    }
-
-    /**
-     * Push an entry to the front of the LRU list, making it the most
-     * recently accessed
-     *
-     * @param {object} entry - entry to push
-     * @return {undefined}
-     */
-    _lruPushEntry(entry) {
-        /* eslint-disable no-param-reassign */
-        entry._lruNext = this._lruHead;
-        entry._lruPrev = null;
-        if (this._lruHead) {
-            this._lruHead._lruPrev = entry;
-        }
-        this._lruHead = entry;
-        if (!this._lruTail) {
-            this._lruTail = entry;
-        }
-        /* eslint-enable no-param-reassign */
-    }
-
-    /**
-     * Remove an entry from the LRU list
-     *
-     * @param {object} entry - entry to remove
-     * @return {undefined}
-     */
-    _lruRemoveEntry(entry) {
-        /* eslint-disable no-param-reassign */
-        if (entry._lruPrev) {
-            entry._lruPrev._lruNext = entry._lruNext;
-        } else {
-            this._lruHead = entry._lruNext;
-        }
-        if (entry._lruNext) {
-            entry._lruNext._lruPrev = entry._lruPrev;
-        } else {
-            this._lruTail = entry._lruPrev;
-        }
-        /* eslint-enable no-param-reassign */
-    }
-
-    /**
-     * Helper function to remove an existing entry from the cache
-     *
-     * @param {object} entry - cache entry to remove
-     * @return {undefined}
-     */
-    _removeEntry(entry) {
-        this._lruRemoveEntry(entry);
-        delete this._entryMap[entry.key];
-        this._entryCount -= 1;
-    }
-}
-
-module.exports = LRUCache;

lib/policyEvaluator/requestUtils.js

@@ -1,34 +0,0 @@
-const ipCheck = require('../ipCheck');
-
-/**
- * getClientIp - Gets the client IP from the request
- * @param {object} request - http request object
- * @param {object} s3config - s3 config
- * @return {string} - returns client IP from the request
- */
-function getClientIp(request, s3config) {
-    const clientIp = request.socket.remoteAddress;
-    const requestConfig = s3config ? s3config.requests : {};
-    if (requestConfig && requestConfig.viaProxy) {
-        /**
-         * if requests are configured to come via proxy,
-         * check from config which proxies are to be trusted and
-         * which header to be used to extract client IP
-         */
-        if (ipCheck.ipMatchCidrList(requestConfig.trustedProxyCIDRs,
-            clientIp)) {
-            const ipFromHeader
-            // eslint-disable-next-line operator-linebreak
-                = request.headers[requestConfig.extractClientIPFromHeader];
-            if (ipFromHeader && ipFromHeader.trim().length) {
-                return ipFromHeader.split(',')[0].trim();
-            }
-        }
-    }
-
-    return clientIp;
-}
-
-module.exports = {
-    getClientIp,
-};

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=6.9.5"
   },
-  "version": "7.5.0",
+  "version": "7.4.3",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/algos/cache/LRUCache.spec.js

@@ -1,123 +0,0 @@
-const assert = require('assert');
-
-const LRUCache = require('../../../../lib/algos/cache/LRUCache');
-
-describe('LRUCache', () => {
-    it('max 1 entry', () => {
-        const lru = new LRUCache(1);
-        assert.strictEqual(lru.count(), 0);
-
-        assert.strictEqual(lru.add('a', 1), false);
-        assert.strictEqual(lru.add('b', 2), false);
-        assert.strictEqual(lru.add('b', 3), true);
-        assert.strictEqual(lru.count(), 1);
-
-        assert.strictEqual(lru.get('b'), 3);
-        // a has been evicted when b was inserted
-        assert.strictEqual(lru.get('a'), undefined);
-
-        assert.strictEqual(lru.remove('a'), false);
-        assert.strictEqual(lru.remove('b'), true);
-        assert.strictEqual(lru.remove('c'), false);
-        assert.strictEqual(lru.remove('b'), false);
-        assert.strictEqual(lru.count(), 0);
-        assert.strictEqual(lru.get('b'), undefined);
-    });
-
-    it('max 3 entries', () => {
-        const lru = new LRUCache(3);
-
-        assert.strictEqual(lru.add('a', 1), false);
-        assert.strictEqual(lru.add('b', 2), false);
-        assert.strictEqual(lru.add('b', 3), true);
-        assert.strictEqual(lru.count(), 2);
-
-        assert.strictEqual(lru.get('b'), 3);
-        assert.strictEqual(lru.get('a'), 1);
-        assert.strictEqual(lru.add('c', 4), false);
-        assert.strictEqual(lru.count(), 3);
-
-        assert.strictEqual(lru.get('b'), 3);
-
-        // a is the least recently accessed item at the time of
-        // insertion of d, so will be evicted first
-        assert.strictEqual(lru.add('d', 5), false);
-        assert.strictEqual(lru.get('a'), undefined);
-        assert.strictEqual(lru.get('b'), 3);
-        assert.strictEqual(lru.get('c'), 4);
-        assert.strictEqual(lru.get('d'), 5);
-
-        assert.strictEqual(lru.remove('d'), true);
-        assert.strictEqual(lru.remove('c'), true);
-        assert.strictEqual(lru.count(), 1);
-        assert.strictEqual(lru.remove('b'), true);
-        assert.strictEqual(lru.count(), 0);
-    });
-
-    it('max 1000 entries', () => {
-        const lru = new LRUCache(1000);
-
-        for (let i = 0; i < 1000; ++i) {
-            assert.strictEqual(lru.add(`${i}`, i), false);
-        }
-        assert.strictEqual(lru.count(), 1000);
-        for (let i = 0; i < 1000; ++i) {
-            assert.strictEqual(lru.get(`${i}`), i);
-        }
-        for (let i = 999; i >= 0; --i) {
-            assert.strictEqual(lru.get(`${i}`), i);
-        }
-        // this shall evict the least recently accessed items, which
-        // are in the range [500..1000)
-        for (let i = 1000; i < 1500; ++i) {
-            assert.strictEqual(lru.add(`${i}`, i), false);
-        }
-        for (let i = 0; i < 500; ++i) {
-            assert.strictEqual(lru.get(`${i}`), i);
-        }
-        // check evicted items
-        for (let i = 500; i < 1000; ++i) {
-            assert.strictEqual(lru.get(`${i}`), undefined);
-        }
-
-        lru.clear();
-        assert.strictEqual(lru.count(), 0);
-        assert.strictEqual(lru.get(100), undefined);
-    });
-
-    it('max 1000000 entries', function lru1M() {
-        // this test takes ~1-2 seconds on a laptop, nevertheless set a
-        // large timeout to reduce the potential of flakiness on possibly
-        // slower CI environment.
-        this.timeout(30000);
-
-        const lru = new LRUCache(1000000);
-
-        for (let i = 0; i < 1000000; ++i) {
-            assert.strictEqual(lru.add(`${i}`, i), false);
-        }
-        assert.strictEqual(lru.count(), 1000000);
-        // access all even-numbered items to make them the most
-        // recently accessed
-        for (let i = 0; i < 1000000; i += 2) {
-            assert.strictEqual(lru.get(`${i}`), i);
-        }
-        // this shall evict the 500K least recently accessed items,
-        // which are all odd-numbered items
-        for (let i = 1000000; i < 1500000; ++i) {
-            assert.strictEqual(lru.add(`${i}`, i), false);
-        }
-        assert.strictEqual(lru.count(), 1000000);
-        // check present (even) and evicted (odd) items
-        for (let i = 0; i < 1000000; ++i) {
-            assert.strictEqual(lru.get(`${i}`),
-                               i % 2 === 0 ? i : undefined);
-            assert.strictEqual(lru.remove(`${i}`), i % 2 === 0);
-        }
-        assert.strictEqual(lru.count(), 500000);
-        for (let i = 1499999; i >= 1000000; --i) {
-            assert.strictEqual(lru.remove(`${i}`), true);
-        }
-        assert.strictEqual(lru.count(), 0);
-    });
-});

tests/unit/policyEvaluator/requestUtils.js

@@ -1,64 +0,0 @@
-const assert = require('assert');
-const DummyRequest = require('../../utils/DummyRequest');
-const requestUtils = require('../../../lib/policyEvaluator/requestUtils');
-
-describe('requestUtils.getClientIp', () => {
-    // s3 config with 'requests.viaProxy` enabled
-    const configWithProxy
-        = require('../../utils/dummyS3ConfigProxy.json');
-    // s3 config with 'requests.viaProxy` disabled
-    const configWithoutProxy = require('../../utils/dummyS3Config.json');
-    const testClientIp1 = '192.168.100.1';
-    const testClientIp2 = '192.168.104.0';
-    const testProxyIp = '192.168.100.2';
-
-    it('should return client Ip address from header ' +
-        'if the request comes via proxies', () => {
-        const request = new DummyRequest({
-            headers: {
-                'x-forwarded-for': [testClientIp1, testProxyIp].join(','),
-            },
-            url: '/',
-            parsedHost: 'localhost',
-            socket: {
-                remoteAddress: testProxyIp,
-            },
-        });
-        const result = requestUtils.getClientIp(request, configWithProxy);
-        assert.strictEqual(result, testClientIp1);
-    });
-
-    it('should not return client Ip address from header ' +
-        'if the request is not forwarded from proxies or ' +
-        'fails ip check', () => {
-        const request = new DummyRequest({
-            headers: {
-                'x-forwarded-for': [testClientIp1, testProxyIp].join(','),
-            },
-            url: '/',
-            parsedHost: 'localhost',
-            socket: {
-                remoteAddress: testClientIp2,
-            },
-        });
-        const result = requestUtils.getClientIp(request, configWithoutProxy);
-        assert.strictEqual(result, testClientIp2);
-    });
-
-    it('should not return client Ip address from header ' +
-        'if the request is forwarded from proxies, but the request' +
-        'has no expected header or the header value is empty', () => {
-        const request = new DummyRequest({
-            headers: {
-                'x-forwarded-for': ' ',
-            },
-            url: '/',
-            parsedHost: 'localhost',
-            socket: {
-                remoteAddress: testClientIp2,
-            },
-        });
-        const result = requestUtils.getClientIp(request, configWithProxy);
-        assert.strictEqual(result, testClientIp2);
-    });
-});

tests/utils/dummyS3Config.json

@@ -1,81 +0,0 @@
-{
-    "port": 8000,
-    "listenOn": [],
-    "replicationGroupId": "RG001",
-    "restEndpoints": {
-        "localhost": "us-east-1",
-        "127.0.0.1": "us-east-1",
-        "cloudserver-front": "us-east-1",
-        "s3.docker.test": "us-east-1",
-        "127.0.0.2": "us-east-1",
-        "s3.amazonaws.com": "us-east-1"
-    },
-    "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
-                        "s3-website.us-east-2.amazonaws.com",
-                        "s3-website-us-west-1.amazonaws.com",
-                        "s3-website-us-west-2.amazonaws.com",
-                        "s3-website.ap-south-1.amazonaws.com",
-                        "s3-website.ap-northeast-2.amazonaws.com",
-                        "s3-website-ap-southeast-1.amazonaws.com",
-                        "s3-website-ap-southeast-2.amazonaws.com",
-                        "s3-website-ap-northeast-1.amazonaws.com",
-                        "s3-website.eu-central-1.amazonaws.com",
-                        "s3-website-eu-west-1.amazonaws.com",
-                        "s3-website-sa-east-1.amazonaws.com",
-                        "s3-website.localhost",
-                        "s3-website.scality.test"],
-    "replicationEndpoints": [{
-        "site": "zenko",
-        "servers": ["127.0.0.1:8000"],
-        "default": true
-    }, {
-        "site": "us-east-2",
-        "type": "aws_s3"
-    }],
-    "cdmi": {
-        "host": "localhost",
-        "port": 81,
-        "path": "/dewpoint",
-        "readonly": true
-    },
-    "bucketd": {
-        "bootstrap": ["localhost:9000"]
-    },
-    "vaultd": {
-        "host": "localhost",
-        "port": 8500
-    },
-    "clusters": 10,
-    "log": {
-        "logLevel": "info",
-        "dumpLevel": "error"
-    },
-    "healthChecks": {
-        "allowFrom": ["127.0.0.1/8", "::1"]
-    },
-    "metadataClient": {
-        "host": "localhost",
-        "port": 9990
-    },
-    "dataClient": {
-        "host": "localhost",
-        "port": 9991
-    },
-    "metadataDaemon": {
-        "bindAddress": "localhost",
-        "port": 9990
-    },
-    "dataDaemon": {
-        "bindAddress": "localhost",
-        "port": 9991
-    },
-    "recordLog": {
-        "enabled": false,
-        "recordLogName": "s3-recordlog"
-    },
-    "mongodb": {
-	"host": "localhost",
-	"port": 27018,
-	"database": "metadata"
-    }
-}

tests/utils/dummyS3ConfigProxy.json

@@ -1,86 +0,0 @@
-{
-    "port": 8000,
-    "listenOn": [],
-    "replicationGroupId": "RG001",
-    "restEndpoints": {
-        "localhost": "us-east-1",
-        "127.0.0.1": "us-east-1",
-        "cloudserver-front": "us-east-1",
-        "s3.docker.test": "us-east-1",
-        "127.0.0.2": "us-east-1",
-        "s3.amazonaws.com": "us-east-1"
-    },
-    "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
-                        "s3-website.us-east-2.amazonaws.com",
-                        "s3-website-us-west-1.amazonaws.com",
-                        "s3-website-us-west-2.amazonaws.com",
-                        "s3-website.ap-south-1.amazonaws.com",
-                        "s3-website.ap-northeast-2.amazonaws.com",
-                        "s3-website-ap-southeast-1.amazonaws.com",
-                        "s3-website-ap-southeast-2.amazonaws.com",
-                        "s3-website-ap-northeast-1.amazonaws.com",
-                        "s3-website.eu-central-1.amazonaws.com",
-                        "s3-website-eu-west-1.amazonaws.com",
-                        "s3-website-sa-east-1.amazonaws.com",
-                        "s3-website.localhost",
-                        "s3-website.scality.test"],
-    "replicationEndpoints": [{
-        "site": "zenko",
-        "servers": ["127.0.0.1:8000"],
-        "default": true
-    }, {
-        "site": "us-east-2",
-        "type": "aws_s3"
-    }],
-    "cdmi": {
-        "host": "localhost",
-        "port": 81,
-        "path": "/dewpoint",
-        "readonly": true
-    },
-    "bucketd": {
-        "bootstrap": ["localhost:9000"]
-    },
-    "vaultd": {
-        "host": "localhost",
-        "port": 8500
-    },
-    "clusters": 10,
-    "log": {
-        "logLevel": "info",
-        "dumpLevel": "error"
-    },
-    "healthChecks": {
-        "allowFrom": ["127.0.0.1/8", "::1"]
-    },
-    "metadataClient": {
-        "host": "localhost",
-        "port": 9990
-    },
-    "dataClient": {
-        "host": "localhost",
-        "port": 9991
-    },
-    "metadataDaemon": {
-        "bindAddress": "localhost",
-        "port": 9990
-    },
-    "dataDaemon": {
-        "bindAddress": "localhost",
-        "port": 9991
-    },
-    "recordLog": {
-        "enabled": false,
-        "recordLogName": "s3-recordlog"
-    },
-    "mongodb": {
-	"host": "localhost",
-	"port": 27018,
-	"database": "metadata"
-    },
-    "requests": {
-        "viaProxy": true,
-        "trustedProxyCIDRs": ["192.168.100.0/22"],
-        "extractClientIPFromHeader": "x-forwarded-for"
-    }
-}