feature: ZENKO-1520 expose prometheus metrics in HealthProbeServer

Add a new route '/_/monitoring/metrics' exposed by the
HealthProbeServer that runs on pods, to expose default metrics (nodejs
etc) and custom metrics in prometheus format. Then, prometheus will be
able to scrape them for each pod.

Ideally the class should be renamed, maybe to MonitoringServer, kept
it for later as it involves a larger refactor.

lib/network/probe/HealthProbeServer.js

@@ -1,6 +1,7 @@
 const httpServer = require('../http/server');
 const werelogs = require('werelogs');
 const errors = require('../../errors');
+const promClient = require('prom-client');
 
 function sendError(res, log, error, optMessage) {
     res.writeHead(error.code);
@@ -24,10 +25,6 @@ function sendSuccess(res, log, msg) {
     res.end(message);
 }
 
-function constructEndpoints(ns, path) {
-    return `/${ns}/${path}`;
-}
-
 function checkStub(log) { // eslint-disable-line
     return true;
 }
@@ -38,19 +35,20 @@ class HealthProbeServer extends httpServer {
         super(params.port, logging);
         this.logging = logging;
         this.setBindAddress(params.bindAddress || 'localhost');
-        this._namespace = params.namespace || '_/health';
-        const livenessURI = constructEndpoints(this._namespace,
-            params.livenessURI || 'liveness');
-        const readinessURI = constructEndpoints(this._namespace,
-            params.readinessURI || 'readiness');
         // hooking our request processing function by calling the
         // parent's method for that
         this.onRequest(this._onRequest);
-        this._reqHandlers = {};
+        this._reqHandlers = {
-        this._reqHandlers[livenessURI] = this._onLiveness.bind(this);
+            '/_/health/liveness': this._onLiveness.bind(this),
-        this._reqHandlers[readinessURI] = this._onReadiness.bind(this);
+            '/_/health/readiness': this._onReadiness.bind(this),
+            '/_/monitoring/metrics': this._onMetrics.bind(this),
+        };
         this._livenessCheck = params.livenessCheck || checkStub;
         this._readinessCheck = params.readinessCheck || checkStub;
+
+        this._metrics = {};
+
+        this._initMetrics();
     }
 
     onLiveCheck(f) {
@@ -68,8 +66,7 @@ class HealthProbeServer extends httpServer {
         if (req.method !== 'GET') {
             sendError(res, log, errors.MethodNotAllowed);
         }
-        if (req.url.startsWith(`/${this._namespace}`) &&
+        if (req.url in this._reqHandlers) {
-            req.url in this._reqHandlers) {
             this._reqHandlers[req.url](req, res, log);
         } else {
             sendError(res, log, errors.InvalidURI);
@@ -92,6 +89,30 @@ class HealthProbeServer extends httpServer {
         }
     }
 
+    createCounter(name, help) {
+        const counter = new promClient.Counter({ name, help });
+        this._metrics[name] = counter;
+        return counter;
+    }
+
+    createGauge(name, help) {
+        const gauge = new promClient.Gauge({ name, help });
+        this._metrics[name] = gauge;
+        return gauge;
+    }
+
+    getMetric(name) {
+        return this._metrics[name];
+    }
+
+    _initMetrics() {
+        promClient.collectDefaultMetrics({ timeout: 5000 });
+    }
+
+    _onMetrics(req, res) {
+        res.writeHead(200);
+        res.end(promClient.register.metrics());
+    }
 }
 
 module.exports = HealthProbeServer;

package.json

@@ -35,6 +35,7 @@
     "level-sublevel": "~6.6.1",
     "mongodb": "^3.0.1",
     "node-forge": "^0.7.1",
+    "prom-client": "10.2.3",
     "simple-glob": "^0.2.0",
     "socket.io": "~2.2.0",
     "socket.io-client": "~2.2.0",

tests/unit/network/probe/HealthProbeServer.spec.js

@@ -15,12 +15,12 @@ function makeRequest(meth, uri) {
     return req;
 }
 
-const endpoints = [
+const healthcheckEndpoints = [
     '/_/health/liveness',
     '/_/health/readiness',
 ];
 
-const badEndpoints = [
+const badHealthcheckEndpoints = [
     '/_/health/liveness_thisiswrong',
     '/_/health/readiness_thisiswrong',
 ];
@@ -42,7 +42,7 @@ describe('network.probe.HealthProbeServer', () => {
             server.stop();
             done();
         });
-        endpoints.forEach(ep => {
+        healthcheckEndpoints.forEach(ep => {
             it('should perform a GET and ' +
                 'return 200 OK', done => {
                 makeRequest('GET', ep)
@@ -82,7 +82,7 @@ describe('network.probe.HealthProbeServer', () => {
             done();
         });
 
-        endpoints.forEach(ep => {
+        healthcheckEndpoints.forEach(ep => {
             it('should perform a GET and ' +
                 'return 503 ServiceUnavailable', done => {
                 makeRequest('GET', ep)
@@ -117,7 +117,7 @@ describe('network.probe.HealthProbeServer', () => {
             done();
         });
 
-        endpoints.forEach(ep => {
+        healthcheckEndpoints.forEach(ep => {
             it('should perform a POST and ' +
                 'return 405 MethodNotAllowed', done => {
                 makeRequest('POST', ep)
@@ -152,7 +152,7 @@ describe('network.probe.HealthProbeServer', () => {
             done();
         });
 
-        badEndpoints.forEach(ep => {
+        badHealthcheckEndpoints.forEach(ep => {
             it('should perform a GET and ' +
                 'return 400 InvalidURI', done => {
                 makeRequest('GET', ep)
@@ -167,4 +167,85 @@ describe('network.probe.HealthProbeServer', () => {
             });
         });
     });
+
+    describe('metrics route', () => {
+        let server;
+        function setup(done) {
+            server = new HealthProbeServer({ port: 4042 });
+            server._cbOnListening = done;
+            server.start();
+        }
+
+        before(done => {
+            setup(done);
+        });
+
+        after(done => {
+            server.stop();
+            done();
+        });
+        it('should expose metrics in prometheus format', done => {
+            const counter = server.createCounter(
+                'gizmo_counter', 'Count gizmos');
+            counter.inc();
+            counter.inc(10);
+            const gauge = server.createGauge(
+                'gizmo_gauge', 'Measure gizmos');
+            gauge.set(42);
+            gauge.inc();
+            gauge.dec(10);
+
+            const savedCounter = server.getMetric('gizmo_counter');
+            // check we get the same original counter object
+            assert.strictEqual(savedCounter, counter);
+
+            const savedGauge = server.getMetric('gizmo_gauge');
+            // check we get the same original gauge object
+            assert.strictEqual(savedGauge, gauge);
+
+            assert.strictEqual(server.getMetric('does_not_exist'), undefined);
+
+            const expectedLines = [
+                '# HELP gizmo_counter Count gizmos',
+                '# TYPE gizmo_counter counter',
+                'gizmo_counter 11',
+                '# HELP gizmo_gauge Measure gizmos',
+                '# TYPE gizmo_gauge gauge',
+                'gizmo_gauge 33',
+            ];
+
+            makeRequest('GET', '/_/monitoring/metrics')
+                .on('response', res => {
+                    assert(res.statusCode === 200);
+                    const respBufs = [];
+                    res.on('data', data => {
+                        respBufs.push(data);
+                    });
+                    res.on('end', () => {
+                        const respContents = respBufs.join('');
+                        // check that each expected line is present in
+                        // the response
+                        const respLines = {};
+                        respContents.split('\n').forEach(line => {
+                            respLines[line.trimRight()] = true;
+                        });
+                        expectedLines.forEach(expectedLine => {
+                            assert.notStrictEqual(
+                                respLines[expectedLine], undefined,
+                                'missing expected line in response ' +
+                                    `'${expectedLine}'`);
+                        });
+                        done();
+                    });
+                    res.on('error', err => {
+                        assert.ifError(err);
+                        done();
+                    });
+                })
+                .on('error', err => {
+                    assert.ifError(err);
+                    done();
+                }).end();
+        });
+    });
 });