Merge pull request #957 from scality/bugfix/S3C-2502-vault-header-port

bf: S3C-2502 move ip util to arsenal

index.js

@@ -24,6 +24,9 @@ module.exports = {
         listTools: {
             DelimiterTools: require('./lib/algos/list/tools'),
         },
+        cache: {
+            LRUCache: require('./lib/algos/cache/LRUCache'),
+        },
     },
     policies: {
         evaluators: require('./lib/policyEvaluator/evaluator.js'),
@@ -31,6 +34,7 @@ module.exports = {
             .validateUserPolicy,
         evaluatePrincipal: require('./lib/policyEvaluator/principal'),
         RequestContext: require('./lib/policyEvaluator/RequestContext.js'),
+        requestUtils: require('./lib/policyEvaluator/requestUtils'),
     },
     Clustering: require('./lib/Clustering'),
     testing: {

lib/algos/cache/LRUCache.js

@@ -0,0 +1,167 @@
+const assert = require('assert');
+
+/**
+ * @class
+ * @classdesc Implements a key-value in-memory cache with a capped
+ * number of items and a Least Recently Used (LRU) strategy for
+ * eviction.
+ */
+class LRUCache {
+    /**
+     * @constructor
+     * @param {number} maxEntries - maximum number of entries kept in
+     * the cache
+     */
+    constructor(maxEntries) {
+        assert(maxEntries >= 1);
+        this._maxEntries = maxEntries;
+        this.clear();
+    }
+
+    /**
+     * Add or update the value associated to a key in the cache,
+     * making it the most recently accessed for eviction purpose.
+     *
+     * @param {string} key - key to add
+     * @param {object} value - associated value (can be of any type)
+     * @return {boolean} true if the cache contained an entry with
+     * this key, false if it did not
+     */
+    add(key, value) {
+        let entry = this._entryMap[key];
+        if (entry) {
+            entry.value = value;
+            // make the entry the most recently used by re-pushing it
+            // to the head of the LRU list
+            this._lruRemoveEntry(entry);
+            this._lruPushEntry(entry);
+            return true;
+        }
+        if (this._entryCount === this._maxEntries) {
+            // if the cache is already full, abide by the LRU strategy
+            // and remove the least recently used entry from the cache
+            // before pushing the new entry
+            this._removeEntry(this._lruTail);
+        }
+        entry = { key, value };
+        this._entryMap[key] = entry;
+        this._entryCount += 1;
+        this._lruPushEntry(entry);
+        return false;
+    }
+
+    /**
+     * Get the value associated to a key in the cache, making it the
+     * most recently accessed for eviction purpose.
+     *
+     * @param {string} key - key of which to fetch the associated value
+     * @return {object|undefined} - returns the associated value if
+     * exists in the cache, or undefined if not found - either if the
+     * key was never added or if it has been evicted from the cache.
+     */
+    get(key) {
+        const entry = this._entryMap[key];
+        if (entry) {
+            // make the entry the most recently used by re-pushing it
+            // to the head of the LRU list
+            this._lruRemoveEntry(entry);
+            this._lruPushEntry(entry);
+            return entry.value;
+        }
+        return undefined;
+    }
+
+    /**
+     * Remove an entry from the cache if exists
+     *
+     * @param {string} key - key to remove
+     * @return {boolean} true if an entry has been removed, false if
+     * there was no entry with this key in the cache - either if the
+     * key was never added or if it has been evicted from the cache.
+     */
+    remove(key) {
+        const entry = this._entryMap[key];
+        if (entry) {
+            this._removeEntry(entry);
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Get the current number of cached entries
+     *
+     * @return {number} current number of cached entries
+     */
+    count() {
+        return this._entryCount;
+    }
+
+    /**
+     * Remove all entries from the cache
+     *
+     * @return {undefined}
+     */
+    clear() {
+        this._entryMap = {};
+        this._entryCount = 0;
+        this._lruHead = null;
+        this._lruTail = null;
+    }
+
+    /**
+     * Push an entry to the front of the LRU list, making it the most
+     * recently accessed
+     *
+     * @param {object} entry - entry to push
+     * @return {undefined}
+     */
+    _lruPushEntry(entry) {
+        /* eslint-disable no-param-reassign */
+        entry._lruNext = this._lruHead;
+        entry._lruPrev = null;
+        if (this._lruHead) {
+            this._lruHead._lruPrev = entry;
+        }
+        this._lruHead = entry;
+        if (!this._lruTail) {
+            this._lruTail = entry;
+        }
+        /* eslint-enable no-param-reassign */
+    }
+
+    /**
+     * Remove an entry from the LRU list
+     *
+     * @param {object} entry - entry to remove
+     * @return {undefined}
+     */
+    _lruRemoveEntry(entry) {
+        /* eslint-disable no-param-reassign */
+        if (entry._lruPrev) {
+            entry._lruPrev._lruNext = entry._lruNext;
+        } else {
+            this._lruHead = entry._lruNext;
+        }
+        if (entry._lruNext) {
+            entry._lruNext._lruPrev = entry._lruPrev;
+        } else {
+            this._lruTail = entry._lruPrev;
+        }
+        /* eslint-enable no-param-reassign */
+    }
+
+    /**
+     * Helper function to remove an existing entry from the cache
+     *
+     * @param {object} entry - cache entry to remove
+     * @return {undefined}
+     */
+    _removeEntry(entry) {
+        this._lruRemoveEntry(entry);
+        delete this._entryMap[entry.key];
+        this._entryCount -= 1;
+    }
+}
+
+module.exports = LRUCache;

lib/policyEvaluator/requestUtils.js

@@ -0,0 +1,34 @@
+const ipCheck = require('../ipCheck');
+
+/**
+ * getClientIp - Gets the client IP from the request
+ * @param {object} request - http request object
+ * @param {object} s3config - s3 config
+ * @return {string} - returns client IP from the request
+ */
+function getClientIp(request, s3config) {
+    const clientIp = request.socket.remoteAddress;
+    const requestConfig = s3config ? s3config.requests : {};
+    if (requestConfig && requestConfig.viaProxy) {
+        /**
+         * if requests are configured to come via proxy,
+         * check from config which proxies are to be trusted and
+         * which header to be used to extract client IP
+         */
+        if (ipCheck.ipMatchCidrList(requestConfig.trustedProxyCIDRs,
+            clientIp)) {
+            const ipFromHeader
+            // eslint-disable-next-line operator-linebreak
+                = request.headers[requestConfig.extractClientIPFromHeader];
+            if (ipFromHeader && ipFromHeader.trim().length) {
+                return ipFromHeader.split(',')[0].trim();
+            }
+        }
+    }
+
+    return clientIp;
+}
+
+module.exports = {
+    getClientIp,
+};

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=6.9.5"
   },
-  "version": "7.4.3",
+  "version": "7.5.0",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/algos/cache/LRUCache.spec.js

@@ -0,0 +1,123 @@
+const assert = require('assert');
+
+const LRUCache = require('../../../../lib/algos/cache/LRUCache');
+
+describe('LRUCache', () => {
+    it('max 1 entry', () => {
+        const lru = new LRUCache(1);
+        assert.strictEqual(lru.count(), 0);
+
+        assert.strictEqual(lru.add('a', 1), false);
+        assert.strictEqual(lru.add('b', 2), false);
+        assert.strictEqual(lru.add('b', 3), true);
+        assert.strictEqual(lru.count(), 1);
+
+        assert.strictEqual(lru.get('b'), 3);
+        // a has been evicted when b was inserted
+        assert.strictEqual(lru.get('a'), undefined);
+
+        assert.strictEqual(lru.remove('a'), false);
+        assert.strictEqual(lru.remove('b'), true);
+        assert.strictEqual(lru.remove('c'), false);
+        assert.strictEqual(lru.remove('b'), false);
+        assert.strictEqual(lru.count(), 0);
+        assert.strictEqual(lru.get('b'), undefined);
+    });
+
+    it('max 3 entries', () => {
+        const lru = new LRUCache(3);
+
+        assert.strictEqual(lru.add('a', 1), false);
+        assert.strictEqual(lru.add('b', 2), false);
+        assert.strictEqual(lru.add('b', 3), true);
+        assert.strictEqual(lru.count(), 2);
+
+        assert.strictEqual(lru.get('b'), 3);
+        assert.strictEqual(lru.get('a'), 1);
+        assert.strictEqual(lru.add('c', 4), false);
+        assert.strictEqual(lru.count(), 3);
+
+        assert.strictEqual(lru.get('b'), 3);
+
+        // a is the least recently accessed item at the time of
+        // insertion of d, so will be evicted first
+        assert.strictEqual(lru.add('d', 5), false);
+        assert.strictEqual(lru.get('a'), undefined);
+        assert.strictEqual(lru.get('b'), 3);
+        assert.strictEqual(lru.get('c'), 4);
+        assert.strictEqual(lru.get('d'), 5);
+
+        assert.strictEqual(lru.remove('d'), true);
+        assert.strictEqual(lru.remove('c'), true);
+        assert.strictEqual(lru.count(), 1);
+        assert.strictEqual(lru.remove('b'), true);
+        assert.strictEqual(lru.count(), 0);
+    });
+
+    it('max 1000 entries', () => {
+        const lru = new LRUCache(1000);
+
+        for (let i = 0; i < 1000; ++i) {
+            assert.strictEqual(lru.add(`${i}`, i), false);
+        }
+        assert.strictEqual(lru.count(), 1000);
+        for (let i = 0; i < 1000; ++i) {
+            assert.strictEqual(lru.get(`${i}`), i);
+        }
+        for (let i = 999; i >= 0; --i) {
+            assert.strictEqual(lru.get(`${i}`), i);
+        }
+        // this shall evict the least recently accessed items, which
+        // are in the range [500..1000)
+        for (let i = 1000; i < 1500; ++i) {
+            assert.strictEqual(lru.add(`${i}`, i), false);
+        }
+        for (let i = 0; i < 500; ++i) {
+            assert.strictEqual(lru.get(`${i}`), i);
+        }
+        // check evicted items
+        for (let i = 500; i < 1000; ++i) {
+            assert.strictEqual(lru.get(`${i}`), undefined);
+        }
+
+        lru.clear();
+        assert.strictEqual(lru.count(), 0);
+        assert.strictEqual(lru.get(100), undefined);
+    });
+
+    it('max 1000000 entries', function lru1M() {
+        // this test takes ~1-2 seconds on a laptop, nevertheless set a
+        // large timeout to reduce the potential of flakiness on possibly
+        // slower CI environment.
+        this.timeout(30000);
+
+        const lru = new LRUCache(1000000);
+
+        for (let i = 0; i < 1000000; ++i) {
+            assert.strictEqual(lru.add(`${i}`, i), false);
+        }
+        assert.strictEqual(lru.count(), 1000000);
+        // access all even-numbered items to make them the most
+        // recently accessed
+        for (let i = 0; i < 1000000; i += 2) {
+            assert.strictEqual(lru.get(`${i}`), i);
+        }
+        // this shall evict the 500K least recently accessed items,
+        // which are all odd-numbered items
+        for (let i = 1000000; i < 1500000; ++i) {
+            assert.strictEqual(lru.add(`${i}`, i), false);
+        }
+        assert.strictEqual(lru.count(), 1000000);
+        // check present (even) and evicted (odd) items
+        for (let i = 0; i < 1000000; ++i) {
+            assert.strictEqual(lru.get(`${i}`),
+                               i % 2 === 0 ? i : undefined);
+            assert.strictEqual(lru.remove(`${i}`), i % 2 === 0);
+        }
+        assert.strictEqual(lru.count(), 500000);
+        for (let i = 1499999; i >= 1000000; --i) {
+            assert.strictEqual(lru.remove(`${i}`), true);
+        }
+        assert.strictEqual(lru.count(), 0);
+    });
+});

tests/unit/policyEvaluator/requestUtils.js

@@ -0,0 +1,64 @@
+const assert = require('assert');
+const DummyRequest = require('../../utils/DummyRequest');
+const requestUtils = require('../../../lib/policyEvaluator/requestUtils');
+
+describe('requestUtils.getClientIp', () => {
+    // s3 config with 'requests.viaProxy` enabled
+    const configWithProxy
+        = require('../../utils/dummyS3ConfigProxy.json');
+    // s3 config with 'requests.viaProxy` disabled
+    const configWithoutProxy = require('../../utils/dummyS3Config.json');
+    const testClientIp1 = '192.168.100.1';
+    const testClientIp2 = '192.168.104.0';
+    const testProxyIp = '192.168.100.2';
+
+    it('should return client Ip address from header ' +
+        'if the request comes via proxies', () => {
+        const request = new DummyRequest({
+            headers: {
+                'x-forwarded-for': [testClientIp1, testProxyIp].join(','),
+            },
+            url: '/',
+            parsedHost: 'localhost',
+            socket: {
+                remoteAddress: testProxyIp,
+            },
+        });
+        const result = requestUtils.getClientIp(request, configWithProxy);
+        assert.strictEqual(result, testClientIp1);
+    });
+
+    it('should not return client Ip address from header ' +
+        'if the request is not forwarded from proxies or ' +
+        'fails ip check', () => {
+        const request = new DummyRequest({
+            headers: {
+                'x-forwarded-for': [testClientIp1, testProxyIp].join(','),
+            },
+            url: '/',
+            parsedHost: 'localhost',
+            socket: {
+                remoteAddress: testClientIp2,
+            },
+        });
+        const result = requestUtils.getClientIp(request, configWithoutProxy);
+        assert.strictEqual(result, testClientIp2);
+    });
+
+    it('should not return client Ip address from header ' +
+        'if the request is forwarded from proxies, but the request' +
+        'has no expected header or the header value is empty', () => {
+        const request = new DummyRequest({
+            headers: {
+                'x-forwarded-for': ' ',
+            },
+            url: '/',
+            parsedHost: 'localhost',
+            socket: {
+                remoteAddress: testClientIp2,
+            },
+        });
+        const result = requestUtils.getClientIp(request, configWithProxy);
+        assert.strictEqual(result, testClientIp2);
+    });
+});

tests/utils/dummyS3Config.json

@@ -0,0 +1,81 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "replicationGroupId": "RG001",
+    "restEndpoints": {
+        "localhost": "us-east-1",
+        "127.0.0.1": "us-east-1",
+        "cloudserver-front": "us-east-1",
+        "s3.docker.test": "us-east-1",
+        "127.0.0.2": "us-east-1",
+        "s3.amazonaws.com": "us-east-1"
+    },
+    "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
+                        "s3-website.us-east-2.amazonaws.com",
+                        "s3-website-us-west-1.amazonaws.com",
+                        "s3-website-us-west-2.amazonaws.com",
+                        "s3-website.ap-south-1.amazonaws.com",
+                        "s3-website.ap-northeast-2.amazonaws.com",
+                        "s3-website-ap-southeast-1.amazonaws.com",
+                        "s3-website-ap-southeast-2.amazonaws.com",
+                        "s3-website-ap-northeast-1.amazonaws.com",
+                        "s3-website.eu-central-1.amazonaws.com",
+                        "s3-website-eu-west-1.amazonaws.com",
+                        "s3-website-sa-east-1.amazonaws.com",
+                        "s3-website.localhost",
+                        "s3-website.scality.test"],
+    "replicationEndpoints": [{
+        "site": "zenko",
+        "servers": ["127.0.0.1:8000"],
+        "default": true
+    }, {
+        "site": "us-east-2",
+        "type": "aws_s3"
+    }],
+    "cdmi": {
+        "host": "localhost",
+        "port": 81,
+        "path": "/dewpoint",
+        "readonly": true
+    },
+    "bucketd": {
+        "bootstrap": ["localhost:9000"]
+    },
+    "vaultd": {
+        "host": "localhost",
+        "port": 8500
+    },
+    "clusters": 10,
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "metadataClient": {
+        "host": "localhost",
+        "port": 9990
+    },
+    "dataClient": {
+        "host": "localhost",
+        "port": 9991
+    },
+    "metadataDaemon": {
+        "bindAddress": "localhost",
+        "port": 9990
+    },
+    "dataDaemon": {
+        "bindAddress": "localhost",
+        "port": 9991
+    },
+    "recordLog": {
+        "enabled": false,
+        "recordLogName": "s3-recordlog"
+    },
+    "mongodb": {
+	"host": "localhost",
+	"port": 27018,
+	"database": "metadata"
+    }
+}

tests/utils/dummyS3ConfigProxy.json

@@ -0,0 +1,86 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "replicationGroupId": "RG001",
+    "restEndpoints": {
+        "localhost": "us-east-1",
+        "127.0.0.1": "us-east-1",
+        "cloudserver-front": "us-east-1",
+        "s3.docker.test": "us-east-1",
+        "127.0.0.2": "us-east-1",
+        "s3.amazonaws.com": "us-east-1"
+    },
+    "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com",
+                        "s3-website.us-east-2.amazonaws.com",
+                        "s3-website-us-west-1.amazonaws.com",
+                        "s3-website-us-west-2.amazonaws.com",
+                        "s3-website.ap-south-1.amazonaws.com",
+                        "s3-website.ap-northeast-2.amazonaws.com",
+                        "s3-website-ap-southeast-1.amazonaws.com",
+                        "s3-website-ap-southeast-2.amazonaws.com",
+                        "s3-website-ap-northeast-1.amazonaws.com",
+                        "s3-website.eu-central-1.amazonaws.com",
+                        "s3-website-eu-west-1.amazonaws.com",
+                        "s3-website-sa-east-1.amazonaws.com",
+                        "s3-website.localhost",
+                        "s3-website.scality.test"],
+    "replicationEndpoints": [{
+        "site": "zenko",
+        "servers": ["127.0.0.1:8000"],
+        "default": true
+    }, {
+        "site": "us-east-2",
+        "type": "aws_s3"
+    }],
+    "cdmi": {
+        "host": "localhost",
+        "port": 81,
+        "path": "/dewpoint",
+        "readonly": true
+    },
+    "bucketd": {
+        "bootstrap": ["localhost:9000"]
+    },
+    "vaultd": {
+        "host": "localhost",
+        "port": 8500
+    },
+    "clusters": 10,
+    "log": {
+        "logLevel": "info",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "metadataClient": {
+        "host": "localhost",
+        "port": 9990
+    },
+    "dataClient": {
+        "host": "localhost",
+        "port": 9991
+    },
+    "metadataDaemon": {
+        "bindAddress": "localhost",
+        "port": 9990
+    },
+    "dataDaemon": {
+        "bindAddress": "localhost",
+        "port": 9991
+    },
+    "recordLog": {
+        "enabled": false,
+        "recordLogName": "s3-recordlog"
+    },
+    "mongodb": {
+	"host": "localhost",
+	"port": 27018,
+	"database": "metadata"
+    },
+    "requests": {
+        "viaProxy": true,
+        "trustedProxyCIDRs": ["192.168.100.0/22"],
+        "extractClientIPFromHeader": "x-forwarded-for"
+    }
+}