Compare commits

...

1 Commits

Author SHA1 Message Date
Jonathan Gramain 00cfbafa55 feature: ZENKO-1520 expose prometheus metrics in HealthProbeServer
Add a new route '/_/monitoring/metrics' exposed by the
HealthProbeServer that runs on pods, to expose default metrics (nodejs
etc) and custom metrics in prometheus format. Then, prometheus will be
able to scrape them for each pod.

Ideally the class should be renamed, maybe to MonitoringServer, kept
it for later as it involves a larger refactor.
2019-04-25 11:57:01 -07:00
4 changed files with 968 additions and 914 deletions

View File

@ -1,6 +1,7 @@
const httpServer = require('../http/server');
const werelogs = require('werelogs');
const errors = require('../../errors');
const promClient = require('prom-client');
function sendError(res, log, error, optMessage) {
res.writeHead(error.code);
@ -24,10 +25,6 @@ function sendSuccess(res, log, msg) {
res.end(message);
}
function constructEndpoints(ns, path) {
return `/${ns}/${path}`;
}
function checkStub(log) { // eslint-disable-line
return true;
}
@ -38,19 +35,20 @@ class HealthProbeServer extends httpServer {
super(params.port, logging);
this.logging = logging;
this.setBindAddress(params.bindAddress || 'localhost');
this._namespace = params.namespace || '_/health';
const livenessURI = constructEndpoints(this._namespace,
params.livenessURI || 'liveness');
const readinessURI = constructEndpoints(this._namespace,
params.readinessURI || 'readiness');
// hooking our request processing function by calling the
// parent's method for that
this.onRequest(this._onRequest);
this._reqHandlers = {};
this._reqHandlers[livenessURI] = this._onLiveness.bind(this);
this._reqHandlers[readinessURI] = this._onReadiness.bind(this);
this._reqHandlers = {
'/_/health/liveness': this._onLiveness.bind(this),
'/_/health/readiness': this._onReadiness.bind(this),
'/_/monitoring/metrics': this._onMetrics.bind(this),
};
this._livenessCheck = params.livenessCheck || checkStub;
this._readinessCheck = params.readinessCheck || checkStub;
this._metrics = {};
this._initMetrics();
}
onLiveCheck(f) {
@ -68,8 +66,7 @@ class HealthProbeServer extends httpServer {
if (req.method !== 'GET') {
sendError(res, log, errors.MethodNotAllowed);
}
if (req.url.startsWith(`/${this._namespace}`) &&
req.url in this._reqHandlers) {
if (req.url in this._reqHandlers) {
this._reqHandlers[req.url](req, res, log);
} else {
sendError(res, log, errors.InvalidURI);
@ -92,6 +89,30 @@ class HealthProbeServer extends httpServer {
}
}
createCounter(name, help) {
const counter = new promClient.Counter({ name, help });
this._metrics[name] = counter;
return counter;
}
createGauge(name, help) {
const gauge = new promClient.Gauge({ name, help });
this._metrics[name] = gauge;
return gauge;
}
getMetric(name) {
return this._metrics[name];
}
_initMetrics() {
promClient.collectDefaultMetrics({ timeout: 5000 });
}
_onMetrics(req, res) {
res.writeHead(200);
res.end(promClient.register.metrics());
}
}
module.exports = HealthProbeServer;

1739
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -35,6 +35,7 @@
"level-sublevel": "~6.6.1",
"mongodb": "^3.0.1",
"node-forge": "^0.7.1",
"prom-client": "10.2.3",
"simple-glob": "^0.2.0",
"socket.io": "~2.2.0",
"socket.io-client": "~2.2.0",

View File

@ -15,12 +15,12 @@ function makeRequest(meth, uri) {
return req;
}
const endpoints = [
const healthcheckEndpoints = [
'/_/health/liveness',
'/_/health/readiness',
];
const badEndpoints = [
const badHealthcheckEndpoints = [
'/_/health/liveness_thisiswrong',
'/_/health/readiness_thisiswrong',
];
@ -42,7 +42,7 @@ describe('network.probe.HealthProbeServer', () => {
server.stop();
done();
});
endpoints.forEach(ep => {
healthcheckEndpoints.forEach(ep => {
it('should perform a GET and ' +
'return 200 OK', done => {
makeRequest('GET', ep)
@ -82,7 +82,7 @@ describe('network.probe.HealthProbeServer', () => {
done();
});
endpoints.forEach(ep => {
healthcheckEndpoints.forEach(ep => {
it('should perform a GET and ' +
'return 503 ServiceUnavailable', done => {
makeRequest('GET', ep)
@ -117,7 +117,7 @@ describe('network.probe.HealthProbeServer', () => {
done();
});
endpoints.forEach(ep => {
healthcheckEndpoints.forEach(ep => {
it('should perform a POST and ' +
'return 405 MethodNotAllowed', done => {
makeRequest('POST', ep)
@ -152,7 +152,7 @@ describe('network.probe.HealthProbeServer', () => {
done();
});
badEndpoints.forEach(ep => {
badHealthcheckEndpoints.forEach(ep => {
it('should perform a GET and ' +
'return 400 InvalidURI', done => {
makeRequest('GET', ep)
@ -167,4 +167,85 @@ describe('network.probe.HealthProbeServer', () => {
});
});
});
describe('metrics route', () => {
let server;
function setup(done) {
server = new HealthProbeServer({ port: 4042 });
server._cbOnListening = done;
server.start();
}
before(done => {
setup(done);
});
after(done => {
server.stop();
done();
});
it('should expose metrics in prometheus format', done => {
const counter = server.createCounter(
'gizmo_counter', 'Count gizmos');
counter.inc();
counter.inc(10);
const gauge = server.createGauge(
'gizmo_gauge', 'Measure gizmos');
gauge.set(42);
gauge.inc();
gauge.dec(10);
const savedCounter = server.getMetric('gizmo_counter');
// check we get the same original counter object
assert.strictEqual(savedCounter, counter);
const savedGauge = server.getMetric('gizmo_gauge');
// check we get the same original gauge object
assert.strictEqual(savedGauge, gauge);
assert.strictEqual(server.getMetric('does_not_exist'), undefined);
const expectedLines = [
'# HELP gizmo_counter Count gizmos',
'# TYPE gizmo_counter counter',
'gizmo_counter 11',
'# HELP gizmo_gauge Measure gizmos',
'# TYPE gizmo_gauge gauge',
'gizmo_gauge 33',
];
makeRequest('GET', '/_/monitoring/metrics')
.on('response', res => {
assert(res.statusCode === 200);
const respBufs = [];
res.on('data', data => {
respBufs.push(data);
});
res.on('end', () => {
const respContents = respBufs.join('');
// check that each expected line is present in
// the response
const respLines = {};
respContents.split('\n').forEach(line => {
respLines[line.trimRight()] = true;
});
expectedLines.forEach(expectedLine => {
assert.notStrictEqual(
respLines[expectedLine], undefined,
'missing expected line in response ' +
`'${expectedLine}'`);
});
done();
});
res.on('error', err => {
assert.ifError(err);
done();
});
})
.on('error', err => {
assert.ifError(err);
done();
}).end();
});
});
});