etcd/Documentation/op-guide/etcd3_alert.rules.yml

# these rules synced manually from https://github.com/etcd-io/etcd/blob/master/Documentation/etcd-mixin/mixin.libsonnet
groups:
- name: etcd
  rules:
  - alert: etcdInsufficientMembers
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value
        }}).'
    expr: |
      sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)      
    for: 3m
    labels:
      severity: critical
  - alert: etcdNoLeader
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has
        no leader.'
    expr: |
      etcd_server_has_leader{job=~".*etcd.*"} == 0      
    for: 1m
    labels:
      severity: critical
  - alert: etcdHighNumberOfLeaderChanges
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": instance {{ $labels.instance }}
        has seen {{ $value }} leader changes within the last hour.'
    expr: |
      rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3      
    for: 15m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedGRPCRequests
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{
        $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
    expr: |
      100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
        /
      sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
        > 1      
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedGRPCRequests
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{
        $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
    expr: |
      100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
        /
      sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
        > 5      
    for: 5m
    labels:
      severity: critical
  - alert: etcdGRPCRequestsSlow
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": gRPC requests to {{ $labels.grpc_method
        }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
    expr: |
      histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
      > 0.15      
    for: 10m
    labels:
      severity: critical
  - alert: etcdMemberCommunicationSlow
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To
        }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
    expr: |
      histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
      > 0.15      
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedProposals
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within
        the last hour on etcd instance {{ $labels.instance }}.'
    expr: |
      rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5      
    for: 15m
    labels:
      severity: warning
  - alert: etcdHighFsyncDurations
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": 99th percentile fync durations are
        {{ $value }}s on etcd instance {{ $labels.instance }}.'
    expr: |
      histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
      > 0.5      
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighCommitDurations
    annotations:
      message: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations
        {{ $value }}s on etcd instance {{ $labels.instance }}.'
    expr: |
      histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
      > 0.25      
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedHTTPRequests
    annotations:
      message: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
        instance {{ $labels.instance }}'
    expr: |
      sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
      BY (method) > 0.01      
    for: 10m
    labels:
      severity: warning
  - alert: etcdHighNumberOfFailedHTTPRequests
    annotations:
      message: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
        instance {{ $labels.instance }}.'
    expr: |
      sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
      BY (method) > 0.05      
    for: 10m
    labels:
      severity: critical
  - alert: etcdHTTPRequestsSlow
    annotations:
      message: etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method
        }} are slow.
    expr: |
      histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
      > 0.15      
    for: 10m
    labels:
      severity: warning