add images changes

docker-compose.yaml

@@ -0,0 +1,47 @@
+version: '3.8'
+
+x-models:
+  warp10: &warp10
+    build:
+      context: .
+      dockerfile: ./images/warp10/Dockerfile
+    volumes: [ $PWD/warpscript:/usr/local/share/warpscript ]
+
+  warp10_env: &warp10_env
+    ENABLE_WARPSTUDIO: 'true'
+    ENABLE_SENSISION: 'true'
+    warpscript.repository.refresh: 1000
+    warpscript.maxops: 1000000000
+    warpscript.maxops.hard: 1000000000
+    warpscript.maxfetch: 1000000000
+    warpscript.maxfetch.hard: 1000000000
+    warpscript.extension.debug: io.warp10.script.ext.debug.DebugWarpScriptExtension
+    warpscript.maxrecursion: 1000
+    warpscript.repository.directory: /usr/local/share/warpscript
+    warpscript.extension.logEvent: io.warp10.script.ext.logging.LoggingWarpScriptExtension
+  
+  redis: &redis
+      build:
+        context: .
+        dockerfile: ./images/redis/Dockerfile
+
+services:
+  redis:
+    << : *redis
+    ports:
+      - 6379:6379
+      - 9121:9121
+
+  warp10:
+    << : *warp10
+    environment:
+      << : *warp10_env
+
+    ports:
+      - 4802:4802
+      - 8081:8081
+      - 9718:9718
+
+    volumes:
+      - /tmp/warp10:/data
+      - '${PWD}/warpscript:/usr/local/share/warpscript'

images/redis/Dockerfile

@@ -0,0 +1,23 @@
+FROM redis:alpine
+
+ENV S6_VERSION 2.0.0.1
+ENV EXPORTER_VERSION 1.24.0
+ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2
+
+USER root
+RUN apk update && apk add wget
+
+RUN wget https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-amd64.tar.gz -O /tmp/s6-overlay-amd64.tar.gz \
+    && tar xzf /tmp/s6-overlay-amd64.tar.gz -C / \
+    && rm -rf /tmp/s6-overlay-amd64.tar.gz 
+
+RUN wget https://github.com/oliver006/redis_exporter/releases/download/v${EXPORTER_VERSION}/redis_exporter-v${EXPORTER_VERSION}.linux-amd64.tar.gz -O redis_exporter.tar.gz \
+    && tar xzf redis_exporter.tar.gz -C / 
+
+EXPOSE 6379
+EXPOSE 9121
+
+WORKDIR /..
+
+ADD ./images/redis/s6 /etc
+CMD /init

images/redis/s6/cont-init.d/00-configure-exporter

@@ -0,0 +1,3 @@
+#!/usr/bin/with-contenv sh
+
+mv /redis_exporter-v${EXPORTER_VERSION}.linux-amd64 /run/s6/services/exporter

images/redis/s6/services.d/exporter/run

@@ -0,0 +1,11 @@
+#!/usr/bin/with-contenv sh
+
+./redis_exporter
+status=$?
+if [ $status -ne 0 ]; 
+then
+  echo "Failed to start exporter: $status"
+  exit $status
+else
+  echo "redis exporter started"
+fi

images/redis/s6/services.d/redis/run

@@ -0,0 +1,10 @@
+#!/usr/bin/with-contenv sh
+
+redis-server
+status=$?
+if [ $status -ne 0 ]; then
+  echo "Failed to start redis: $status"
+  exit $status
+else
+  echo "redis started"
+fi

images/warp10/Dockerfile

@@ -1,10 +1,96 @@
-FROM warp10io/warp10:2.7.5
+FROM openjdk:8-jdk-buster as warp10
+
+ENV THRIFT_HOME=/usr
+
+ARG WARP10_VERSION=2.8.1-95-g73e7de80
+
+RUN apt-get update \
+    && apt-get install -y thrift-compiler \
+    && git clone https://github.com/senx/warp10-platform/ /tmp/warp10 \
+    && cd /tmp/warp10 \
+    && ./gradlew clean build createTarArchive \
+    && cp warp10/build/libs/warp10-${WARP10_VERSION}.tar.gz /tmp/warp10.tar.gz
+
+FROM golang:1.14-alpine as builder
+
+ENV WARP10_EXPORTER_VERSION 2.8.1
+
+RUN apk add zip unzip build-base \
+    && wget -q -O exporter.zip https://github.com/centreon/warp10-sensision-exporter/archive/refs/heads/master.zip \
+    && unzip exporter.zip \
+    && cd warp10-sensision-exporter-master \
+    && go mod download \
+    && cd tools \
+    && go run generate_sensision_metrics.go ${WARP10_EXPORTER_VERSION} \
+    && cp sensision.go ../collector/ \
+    && cd .. \
+    && go build -a -o /usr/local/go/warp10_sensision_exporter
+
+FROM openjdk:8-jdk-alpine
 
 ENV S6_VERSION 2.0.0.1
 ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2
 
-ENV WARP10_CONF_TEMPLATES ${WARP10_HOME}/conf.templates/standalone
 ENV SENSISION_DATA_DIR /data/sensision
+ENV SENSISION_PORT 8082
+
+ENV WARP10_VOLUME=/data \
+  WARP10_HOME=/opt/warp10 \
+  WARP10_DATA_DIR=/data/warp10 \
+  SENSISION_HOME=/opt/sensision \
+  SENSISION_DATA_DIR=/data/sensision
+
+ENV WARP10_CONF_TEMPLATES ${WARP10_HOME}/conf.templates/standalone
+
+ARG WARP10_VERSION=2.8.1-95-g73e7de80
+ARG WARP10_URL=https://github.com/senx/warp10-platform/releases/download/${WARP10_VERSION}
+ENV WARP10_VERSION ${WARP10_VERSION}
+
+# Installing utils need by Warp 10 and build-dependencies
+RUN apk --no-cache add bash curl fontconfig unifont \
+  && apk --no-cache add --virtual=build-dependencies ca-certificates wget
+
+
+ARG WARPSTUDIO_VERSION=1.0.42
+ARG WARPSTUDIO_URL=https://repo1.maven.org/maven2/io/warp10/warp10-plugin-warpstudio/${WARPSTUDIO_VERSION}
+ENV WARPSTUDIO_VERSION=${WARPSTUDIO_VERSION}
+
+COPY --from=warp10 /tmp/warp10.tar.gz /opt/
+
+RUN mkdir -p /opt \
+  && cd /opt \
+  && tar xzf warp10.tar.gz \
+  && rm warp10.tar.gz \
+  && ln -s /opt/warp10-${WARP10_VERSION} ${WARP10_HOME} \
+  && adduser -D -s -H -h ${WARP10_HOME} -s /bin/bash warp10 \
+  && chown -h warp10:warp10 ${WARP10_HOME} \
+  && wget -q -P ${WARP10_HOME}/lib ${WARPSTUDIO_URL}/warp10-plugin-warpstudio-${WARPSTUDIO_VERSION}.jar
+
+ARG SENSISION_VERSION=1.0.24
+ARG SENSISION_URL=https://github.com/senx/sensision/releases/download/${SENSISION_VERSION}
+ENV SENSISION_VERSION=${SENSISION_VERSION}
+
+# Getting Sensision
+RUN cd /opt \
+  && wget -q $SENSISION_URL/sensision-service-${SENSISION_VERSION}.tar.gz \
+  && tar xzf sensision-service-${SENSISION_VERSION}.tar.gz \
+  && rm sensision-service-${SENSISION_VERSION}.tar.gz \
+  && ln -s /opt/sensision-${SENSISION_VERSION} ${SENSISION_HOME} \
+  && adduser -D -s -H -h ${SENSISION_HOME} -s /bin/bash sensision \
+  && addgroup sensision warp10 \
+  && chown -h sensision:sensision ${SENSISION_HOME}
+
+# Deleting build-dependencies
+RUN apk --no-cache del build-dependencies
+
+ENV WARP10_JAR=${WARP10_HOME}/bin/warp10-${WARP10_VERSION}.jar \
+  WARP10_CONFIG_DIR=${WARP10_HOME}/etc/conf.d \
+  WARP10_MACROS=${WARP10_VOLUME}/custom_macros
+
+ENV PATH=$PATH:${WARP10_HOME}/bin
+
+# Exposing port
+EXPOSE 8080 8081
 
 # Modify Warp 10 default config
 ENV standalone.host 0.0.0.0
@@ -23,7 +109,12 @@ RUN wget https://github.com/just-containers/s6-overlay/releases/download/v${S6_V
 # Install protobuf extestion
 ADD ./images/warp10/warp10-ext-protobuf-1.2.2-uberjar.jar /opt/warp10/lib/
 
+# Install Sensision exporter
+COPY --from=builder /usr/local/go/warp10_sensision_exporter /usr/local/bin/warp10_sensision_exporter
+
 ADD ./images/warp10/s6 /etc
 ADD ./warpscript /usr/local/share/warpscript
 ADD ./images/warp10/static.tokens /
+
 CMD /init
+

images/warp10/s6/services.d/sensision-exporter/run

@@ -0,0 +1,4 @@
+#!/usr/bin/with-contenv sh
+
+echo 'Starting sensision exporter'
+exec warp10_sensision_exporter --warp10.url=http://localhost:${SENSISION_PORT}/metrics

images/warp10/s6/services.d/sensision/run

@@ -3,7 +3,7 @@
 JAVA="/usr/bin/java"
 JAVA_OPTS=""
 
-VERSION=1.0.21
+VERSION=1.0.23
 SENSISION_CONFIG=${SENSISION_DATA_DIR}/conf/sensision.conf
 SENSISION_JAR=${SENSISION_HOME}/bin/sensision-${VERSION}.jar
 SENSISION_CP=${SENSISION_HOME}/etc:${SENSISION_JAR}
@@ -14,7 +14,7 @@ if [ -z "$SENSISION_HEAP" ]; then
     SENSISION_HEAP=64m
 fi
 
-SENSISION_CMD="${JAVA} ${JAVA_OPTS} -Xmx${SENSISION_HEAP} -Dsensision.server.port=0 ${SENSISION_OPTS} -Dsensision.config=${SENSISION_CONFIG} -cp ${SENSISION_CP} ${SENSISION_CLASS}"
+SENSISION_CMD="${JAVA} ${JAVA_OPTS} -Xmx${SENSISION_HEAP} -Dsensision.server.port=${SENSISION_PORT} ${SENSISION_OPTS} -Dsensision.config=${SENSISION_CONFIG} -cp ${SENSISION_CP} ${SENSISION_CLASS}"
 
 if [ -n "$ENABLE_SENSISION" ]; then
     echo "Starting Sensision with $SENSISION_CMD ..."

images/warp10/s6/services.d/warp10/run

@@ -28,7 +28,7 @@ if [ -n "$ENABLE_SENSISION" ]; then
     if [ -n "$SENSISION_LABELS" ]; then
         _SENSISION_LABELS="-Dsensision.default.labels=$SENSISION_LABELS"
     fi
-    SENSISION_OPTS="-Dsensision.server.port=0 ${_SENSISION_LABELS} -Dsensision.events.dir=/var/run/sensision/metrics -Dfile.encoding=UTF-8"
+    SENSISION_OPTS="${_SENSISION_LABELS} -Dsensision.events.dir=/var/run/sensision/metrics -Dfile.encoding=UTF-8"
 fi
 
 WARP10_CMD="${JAVA} -Dlog4j.configuration=file:${LOG4J_CONF} ${JAVA_OPTS} ${SENSISION_OPTS} -cp ${WARP10_CP} ${WARP10_CLASS} ${CONFIG_FILES}"

libV2/server/API/internal/prometheusMetrics.js

@@ -0,0 +1,14 @@
+const { collectDefaultMetrics, register } = require('prom-client');
+
+collectDefaultMetrics({
+    timeout: 10000,
+    gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5],
+});
+
+async function prometheusMetrics(ctx) {
+    // eslint-disable-next-line no-param-reassign
+    ctx.results.statusCode = 200;
+    ctx.results.body = await register.metrics();
+}
+
+module.exports = prometheusMetrics;

openapi.yaml

@@ -101,6 +101,12 @@ components:
                 type: string
               level:
                 type: string
+    utapi-get-prometheus-metrics:
+      description: metrics to be ingested by prometheus 
+      content:
+        text/plain:
+          schema:
+            type: string
   parameters:
     level:
       in: path
@@ -133,6 +139,16 @@ paths:
           $ref: '#/components/responses/json-error'
         200:
           description: Service is healthy
+  /_/metrics:
+    get:
+      x-router-controller: internal
+      x-iplimit: true
+      operationId: prometheusMetrics
+      responses:
+        default:
+          $ref: '#/components/responses/json-error'
+        200:
+          $ref: '#/components/responses/utapi-get-prometheus-metrics'
   /v2/ingest:
     post:
       x-router-controller: metrics

package.json

@@ -36,6 +36,7 @@
     "needle": "^2.5.0",
     "node-schedule": "^1.3.2",
     "oas-tools": "^2.1.8",
+    "prom-client": "^13.1.0",
     "uuid": "^3.3.2",
     "vaultclient": "scality/vaultclient#ff9e92f",
     "werelogs": "scality/werelogs#0a4c576"

tests/unit/v2/server/testController.js

@@ -2,6 +2,7 @@ const assert = require('assert');
 
 const APIController = require('../../../../libV2/server/controller');
 const healthcheck = require('../../../../libV2/server/API/internal/healthcheck');
+const prometheusMetrics = require('../../../../libV2/server/API/internal/prometheusMetrics');
 
 const { ResponseContainer } = require('../../../../libV2/models');
 const { ExpressResponseStub, templateRequest } = require('../../../utils/v2Data');
@@ -41,6 +42,7 @@ describe('Test APIController', () => {
         const handlers = APIController._collectHandlers('internal');
         assert.deepStrictEqual(handlers, {
             healthcheck,
+            prometheusMetrics,
         });
     });
 

yarn.lock

@@ -4633,6 +4633,13 @@ prom-client@10.2.3:
   dependencies:
     tdigest "^0.1.1"
 
+prom-client@^13.1.0:
+  version "13.1.0"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-13.1.0.tgz#1185caffd8691e28d32e373972e662964e3dba45"
+  integrity sha512-jT9VccZCWrJWXdyEtQddCDszYsiuWj5T0ekrPszi/WEegj3IZy6Mm09iOOVM86A4IKMWq8hZkT2dD9MaSe+sng==
+  dependencies:
+    tdigest "^0.1.1"
+
 promise-fs@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/promise-fs/-/promise-fs-2.1.1.tgz#0b725a592c165ff16157d1f13640ba390637e557"