added create header methods

index.js

@@ -60,6 +60,7 @@ module.exports = {
         },
         kmip: require('./lib/network/kmip'),
         kmipClient: require('./lib/network/kmip/Client'),
+        request: require('./lib/network/request'),
     },
     s3routes: {
         routes: require('./lib/s3routes/routes'),

lib/network/request.js

@@ -0,0 +1,137 @@
+const url = require('url');
+const http = require('http');
+const https = require('https');
+const HttpProxyAgent = require('http-proxy-agent');
+const HttpsProxyAgent = require('https-proxy-agent');
+
+const { proxyCompareUrl } = require('../storage/data/external/utils');
+const jsutil = require('../jsutil');
+
+const validVerbs = new Set(['HEAD', 'GET', 'POST', 'PUT', 'DELETE']);
+const updateVerbs = new Set(['POST', 'PUT']);
+
+/*
+ * create a new header object from an existing header object. Similar keys
+ * will be ignored if a value has been set for theirlower-cased form
+ */
+function createHeaders(headers) {
+    if (typeof headers !== 'object') {
+        return {};
+    }
+    const out = {};
+    Object.entries(headers).forEach(([key, value]) => {
+        const lowKey = key.toLowerCase();
+        if (out[lowKey] === undefined) {
+            out[lowKey] = value;
+        }
+    });
+    return out;
+}
+
+
+function request(endpoint, options, callback) {
+    if (!endpoint || typeof endpoint === 'function') {
+        throw new Error('Missing target endpoint');
+    }
+
+    let cb;
+    let opts = {};
+    if (typeof options === 'function') {
+        cb = jsutil.once(options);
+    } else if (typeof options === 'object') {
+        opts = JSON.parse(JSON.stringify(options)); // deep-copy
+        if (typeof callback === 'function') {
+            cb = jsutil.once(callback);
+        }
+    }
+
+    if (typeof cb !== 'function') {
+        throw new Error('Missing request callback');
+    }
+
+    if (!(endpoint instanceof url.URL || typeof endpoint === 'string')) {
+        return cb(new Error(`Invalid URI ${endpoint}`));
+    }
+
+    if (!opts.method) {
+        opts.method = 'GET';
+    } else if (!validVerbs.has(opts.method)) {
+        return cb(new Error(`Invalid Method ${opts.method}`));
+    }
+
+    let reqParams;
+    if (typeof endpoint === 'string') {
+        try {
+            reqParams = new url.URL(endpoint);
+        } catch (error) {
+            return cb(error);
+        }
+    } else {
+        reqParams = new url.URL(endpoint.href);
+    }
+    reqParams.method = opts.method;
+    reqParams.headers = createHeaders(opts.headers || {});
+
+    let request;
+    if (reqParams.protocol === 'http:') {
+        request = http;
+        const httpProxy = process.env.HTTP_PROXY || process.env.http_proxy;
+        if (!proxyCompareUrl(reqParams.hostname) && httpProxy) {
+            reqParams.agent = new HttpProxyAgent(new url.URL(httpProxy));
+        }
+    } else if (reqParams.protocol === 'https:') {
+        request = https;
+        const httpsProxy = process.env.HTTPS_PROXY || process.env.https_proxy;
+        if (!proxyCompareUrl(reqParams.hostname) && httpsProxy) {
+            reqParams.agent = new HttpsProxyAgent(new url.URL(httpsProxy));
+        }
+    } else {
+        return cb(new Error(`Invalid Protocol ${reqParams.protocol}`));
+    }
+
+    let data;
+    if (opts.body) {
+        if (typeof opts.body === 'object') {
+            data = JSON.stringify(opts.body);
+            if (!reqParams.headers['content-type']) {
+                reqParams.headers['content-type'] = 'application/json';
+            }
+        } else {
+            data = opts.body;
+        }
+        reqParams.headers['content-length'] = Buffer.byteLength(data);
+    }
+
+    const req = request.request(reqParams);
+    req.on('error', cb);
+    req.on('response', res => {
+        const rawData = [];
+        res.on('data', chunk => { rawData.push(chunk); });
+        res.on('end', () => {
+            const data = rawData.join('');
+            if (res.statusCode >= 400) {
+                return cb(new Error(res.statusMessage), res, data);
+            }
+
+            if (opts.json && data) {
+                try {
+                    const parsed = JSON.parse(data);
+                    return cb(null, res, parsed);
+                } catch (err) {
+                    // invalid json response
+                    return cb(err, res, null);
+                }
+            }
+            return cb(null, res, data);
+        });
+    });
+    if (data !== undefined && updateVerbs.has(opts.method)) {
+        req.write(data);
+    }
+    req.end();
+    return req;
+}
+
+module.exports = {
+    request,
+};

package.json

@@ -29,7 +29,8 @@
     "diskusage": "^1.1.1",
     "fcntl": "github:scality/node-fcntl",
     "hdclient": "scality/hdclient#5145e04e5ed33e85106765b1caa90cd245ef482b",
-    "https-proxy-agent": "^2.2.0",
+    "http-proxy-agent": "^4.0.1",
+    "https-proxy-agent": "^5.0.0",
     "ioredis": "4.9.5",
     "ipaddr.js": "1.9.1",
     "level": "~5.0.1",

yarn.lock

@@ -75,6 +75,11 @@
   resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz#8da5c6530915653f3a1f38fd5f101d8c3f8079c5"
   integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==
 
+"@tootallnate/once@1":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
+  integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
+
 JSONStream@^1.0.0:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0"
@@ -128,6 +133,13 @@ after@0.8.2:
   resolved "https://registry.yarnpkg.com/after/-/after-0.8.2.tgz#fedb394f9f0e02aa9768e702bda23b505fae7e1f"
   integrity sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8=
 
+agent-base@6:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
+  integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==
+  dependencies:
+    debug "4"
+
 agent-base@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.3.0.tgz#8165f01c436009bccad0b1d122f05ed770efc6ee"
@@ -640,6 +652,13 @@ debug@3.1.0, debug@~3.1.0:
   dependencies:
     ms "2.0.0"
 
+debug@4:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee"
+  integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==
+  dependencies:
+    ms "2.1.2"
+
 debug@^2.1.1:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -1331,6 +1350,15 @@ he@1.1.1:
   resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd"
   integrity sha1-k0EP0hsAlzUVH4howvJx80J+I/0=
 
+http-proxy-agent@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a"
+  integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==
+  dependencies:
+    "@tootallnate/once" "1"
+    agent-base "6"
+    debug "4"
+
 http-signature@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
@@ -1340,14 +1368,6 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-https-proxy-agent@^2.2.0:
-  version "2.2.2"
-  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.2.tgz#271ea8e90f836ac9f119daccd39c19ff7dfb0793"
-  integrity sha512-c8Ndjc9Bkpfx/vCJueCPy0jlP4ccCCSNDp8xwCZzPjKJUm+B+u9WX2x98Qx4n1PiMNTWo3D7KK5ifNV/yJyRzg==
-  dependencies:
-    agent-base "^4.3.0"
-    debug "^3.1.0"
-
 https-proxy-agent@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz#b8c286433e87602311b01c8ea34413d856a4af81"
@@ -1356,6 +1376,14 @@ https-proxy-agent@^3.0.0:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
+https-proxy-agent@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2"
+  integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==
+  dependencies:
+    agent-base "6"
+    debug "4"
+
 ieee754@^1.1.4:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
@@ -1966,7 +1994,7 @@ ms@2.0.0:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
   integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=
 
-ms@^2.1.1:
+ms@2.1.2, ms@^2.1.1:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==