etcdserver: add alarms metrics for server

Signed-off-by: yuzhiquanlong <yuzhiquanlong@gmail.com>

CHANGELOG/CHANGELOG-3.6.md

@@ -83,6 +83,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0).
 See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release.
 
 - Add [`etcd_disk_defrag_inflight`](https://github.com/etcd-io/etcd/pull/13371).
+- Add [`etcd_debugging_server_alarms`](https://github.com/etcd-io/etcd/pull/14276).
 
 ### Go
 - Compile with [Go 1.17+](https://golang.org/doc/devel/release.html#go1.17)

server/etcdserver/apply/apply.go

@@ -17,7 +17,6 @@ package apply
 import (
 	"context"
 
-	"github.com/coreos/go-semver/semver"
 	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
 	"go.etcd.io/etcd/api/v3/membershippb"
 	"go.etcd.io/etcd/client/pkg/v3/types"
@@ -34,9 +33,10 @@ import (
 	serverstorage "go.etcd.io/etcd/server/v3/storage"
 	"go.etcd.io/etcd/server/v3/storage/backend"
 	"go.etcd.io/etcd/server/v3/storage/mvcc"
-
-	"github.com/gogo/protobuf/proto"
 	"go.uber.org/zap"
+
+	"github.com/coreos/go-semver/semver"
+	"github.com/gogo/protobuf/proto"
 )
 
 const (
@@ -231,12 +231,14 @@ func (a *applierV3backend) Alarm(ar *pb.AlarmRequest) (*pb.AlarmResponse, error)
 			break
 		}
 		resp.Alarms = append(resp.Alarms, m)
+		alarms.WithLabelValues(types.ID(ar.MemberID).String(), m.Alarm.String()).Inc()
 	case pb.AlarmRequest_DEACTIVATE:
 		m := a.alarmStore.Deactivate(types.ID(ar.MemberID), ar.Alarm)
 		if m == nil {
 			break
 		}
 		resp.Alarms = append(resp.Alarms, m)
+		alarms.WithLabelValues(types.ID(ar.MemberID).String(), m.Alarm.String()).Dec()
 	default:
 		return nil, nil
 	}

server/etcdserver/apply/metrics.go

@@ -0,0 +1,31 @@
+// Copyright 2022 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apply
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	alarms = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "server",
+		Name:      "alarms",
+		Help:      "Alarms for every member in cluster. 1 for 'server_id' label with current ID. 2 for 'alarm_type' label with type of this alarm",
+	},
+		[]string{"server_id", "alarm_type"})
+)
+
+func init() {
+	prometheus.MustRegister(alarms)
+}