Documentation/op-guide: Adapt alerting rules to new gRPC metrics
parent
6f077bd74c
commit
79446ea677
|
@ -69,7 +69,7 @@ ANNOTATIONS {
|
||||||
|
|
||||||
# alert if the 99th percentile of gRPC method calls take more than 150ms
|
# alert if the 99th percentile of gRPC method calls take more than 150ms
|
||||||
ALERT GRPCRequestsSlow
|
ALERT GRPCRequestsSlow
|
||||||
IF histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m])) > 0.15
|
IF histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15
|
||||||
FOR 10m
|
FOR 10m
|
||||||
LABELS {
|
LABELS {
|
||||||
severity = "critical"
|
severity = "critical"
|
||||||
|
@ -84,8 +84,8 @@ ANNOTATIONS {
|
||||||
|
|
||||||
# alert if more than 1% of requests to an HTTP endpoint have failed within the last 5 minutes
|
# alert if more than 1% of requests to an HTTP endpoint have failed within the last 5 minutes
|
||||||
ALERT HighNumberOfFailedHTTPRequests
|
ALERT HighNumberOfFailedHTTPRequests
|
||||||
IF sum by(method) (rate(etcd_http_failed_total{job="etcd"}[5m]))
|
IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
|
||||||
/ sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.01
|
/ sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
|
||||||
FOR 10m
|
FOR 10m
|
||||||
LABELS {
|
LABELS {
|
||||||
severity = "warning"
|
severity = "warning"
|
||||||
|
@ -97,8 +97,8 @@ ANNOTATIONS {
|
||||||
|
|
||||||
# alert if more than 5% of requests to an HTTP endpoint have failed within the last 5 minutes
|
# alert if more than 5% of requests to an HTTP endpoint have failed within the last 5 minutes
|
||||||
ALERT HighNumberOfFailedHTTPRequests
|
ALERT HighNumberOfFailedHTTPRequests
|
||||||
IF sum by(method) (rate(etcd_http_failed_total{job="etcd"}[5m]))
|
IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
|
||||||
/ sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.05
|
/ sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
|
||||||
FOR 5m
|
FOR 5m
|
||||||
LABELS {
|
LABELS {
|
||||||
severity = "critical"
|
severity = "critical"
|
||||||
|
|
|
@ -26,8 +26,8 @@ groups:
|
||||||
changes within the last hour
|
changes within the last hour
|
||||||
summary: a high number of leader changes within the etcd cluster are happening
|
summary: a high number of leader changes within the etcd cluster are happening
|
||||||
- alert: HighNumberOfFailedGRPCRequests
|
- alert: HighNumberOfFailedGRPCRequests
|
||||||
expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method)
|
expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
|
||||||
/ sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.01
|
/ sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
|
@ -36,8 +36,8 @@ groups:
|
||||||
on etcd instance {{ $labels.instance }}'
|
on etcd instance {{ $labels.instance }}'
|
||||||
summary: a high number of gRPC requests are failing
|
summary: a high number of gRPC requests are failing
|
||||||
- alert: HighNumberOfFailedGRPCRequests
|
- alert: HighNumberOfFailedGRPCRequests
|
||||||
expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method)
|
expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
|
||||||
/ sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.05
|
/ sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
|
@ -46,7 +46,7 @@ groups:
|
||||||
on etcd instance {{ $labels.instance }}'
|
on etcd instance {{ $labels.instance }}'
|
||||||
summary: a high number of gRPC requests are failing
|
summary: a high number of gRPC requests are failing
|
||||||
- alert: GRPCRequestsSlow
|
- alert: GRPCRequestsSlow
|
||||||
expr: histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m]))
|
expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le))
|
||||||
> 0.15
|
> 0.15
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
|
|
Loading…
Reference in New Issue