diff --git a/Documentation/etcd-mixin/README.md b/Documentation/etcd-mixin/README.md index 114a4c157..224066f45 100644 --- a/Documentation/etcd-mixin/README.md +++ b/Documentation/etcd-mixin/README.md @@ -9,3 +9,17 @@ Instructions for use are the same as the [kubernetes-mixin](https://github.com/k ## Background * For more information about monitoring mixins, see this [design doc](https://docs.google.com/document/d/1A9xvzwqnFVSOZ5fD3blKODXfsat5fg6ZhnKu9LK3lB4/edit#). + +## Testing alerts + +Make sure to have [jsonnet](https://jsonnet.org/) and [gojsontoyaml](https://github.com/brancz/gojsontoyaml) installed. + +First compile the mixin to a YAML file, which the promtool will read: +``` +jsonnet -e '(import "mixin.libsonnet").prometheusAlerts' | gojsontoyaml > mixin.yaml +``` + +Then run the unit test: +``` +promtool test rules test.yaml +``` diff --git a/Documentation/etcd-mixin/mixin.libsonnet b/Documentation/etcd-mixin/mixin.libsonnet index 714ed06c9..f8c79caf7 100644 --- a/Documentation/etcd-mixin/mixin.libsonnet +++ b/Documentation/etcd-mixin/mixin.libsonnet @@ -11,7 +11,7 @@ { alert: 'EtcdInsufficientMembers', expr: ||| - count(up{%(etcd_selector)s} == 0) by (job) > (count(up{%(etcd_selector)s}) by (job) / 2 - 1) + sum(up{%(etcd_selector)s} == bool 1) by (job) < ((count(up{%(etcd_selector)s}) by (job) + 1) / 2) ||| % $._config, 'for': '3m', labels: { diff --git a/Documentation/etcd-mixin/test.yaml b/Documentation/etcd-mixin/test.yaml new file mode 100644 index 000000000..15d99c60e --- /dev/null +++ b/Documentation/etcd-mixin/test.yaml @@ -0,0 +1,35 @@ +rule_files: + - mixin.yaml + +evaluation_interval: 1m + +tests: + - interval: 1m + input_series: + - series: 'up{job="etcd",instance="10.10.10.0"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0' + - series: 'up{job="etcd",instance="10.10.10.1"}' + values: '1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0' + - series: 'up{job="etcd",instance="10.10.10.2"}' + values: '1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0' + alert_rule_test: + - eval_time: 3m + alertname: EtcdInsufficientMembers + - eval_time: 7m + alertname: EtcdInsufficientMembers + - eval_time: 11m + alertname: EtcdInsufficientMembers + exp_alerts: + - exp_labels: + job: etcd + severity: critical + exp_annotations: + message: 'Etcd cluster "etcd": insufficient members (1).' + - eval_time: 15m + alertname: EtcdInsufficientMembers + exp_alerts: + - exp_labels: + job: etcd + severity: critical + exp_annotations: + message: 'Etcd cluster "etcd": insufficient members (0).'