From 6a04f7fbd6487386c45115baf59689332c833713 Mon Sep 17 00:00:00 2001 From: stefanbo Date: Sat, 23 Jul 2022 12:08:26 +0800 Subject: [PATCH 1/2] With go-grpc-middleware, add grpc_zap to logging grpc request and response content in grpc-proxy mode In our test environment, it may be very useful to debug who delete etcd's key with grpc-proxy mode inspired by https://github.com/grpc-ecosystem/go-grpc-middleware Signed-off-by: stefanbo Update flag name 1. add changelog 2. update flag name to `experimental-enable-grpc-debug` Signed-off-by: stefan bo Update CHANGELOG-3.6.md Signed-off-by: stefan bo change flag name Signed-off-by: stefan bo --- CHANGELOG/CHANGELOG-3.6.md | 1 + server/etcdmain/grpc_proxy.go | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index 95960e308..5cc1c0f9a 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -56,6 +56,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). - Add [v3 discovery](https://github.com/etcd-io/etcd/pull/13635) to bootstrap a new etcd cluster. - Add [field `storage`](https://github.com/etcd-io/etcd/pull/13772) into the response body of endpoint `/version`. - Add [`etcd --max-concurrent-streams`](https://github.com/etcd-io/etcd/pull/14169) flag to configure the max concurrent streams each client can open at a time, and defaults to math.MaxUint32. +- Add [`etcd grpc-proxy --experimental-enable-grpc-debug`](https://github.com/etcd-io/etcd/pull/14266) flag to logging all grpc requests and responses. - Fix [non mutating requests pass through quotaKVServer when NOSPACE](https://github.com/etcd-io/etcd/pull/13435) - Fix [exclude the same alarm type activated by multiple peers](https://github.com/etcd-io/etcd/pull/13467). - Fix [Provide a better liveness probe for when etcd runs as a Kubernetes pod](https://github.com/etcd-io/etcd/pull/13399) diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index b13520695..aab2d6f83 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -43,6 +43,9 @@ import ( "go.etcd.io/etcd/server/v3/proxy/grpcproxy" "go.uber.org/zap/zapgrpc" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" + grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/soheilhy/cmux" "github.com/spf13/cobra" @@ -89,6 +92,7 @@ var ( grpcProxyEnablePprof bool grpcProxyEnableOrdering bool + grpcProxyEnableLogging bool grpcProxyDebug bool @@ -159,6 +163,7 @@ func newGRPCProxyStartCommand() *cobra.Command { // experimental flags cmd.Flags().BoolVar(&grpcProxyEnableOrdering, "experimental-serializable-ordering", false, "Ensure serializable reads have monotonically increasing store revisions across endpoints.") cmd.Flags().StringVar(&grpcProxyLeasing, "experimental-leasing-prefix", "", "leasing metadata prefix for disconnected linearized reads.") + cmd.Flags().BoolVar(&grpcProxyEnableLogging, "experimental-enable-grpc-logging", false, "logging all grpc requests and responses") cmd.Flags().BoolVar(&grpcProxyDebug, "debug", false, "Enable debug-level logging for grpc-proxy.") @@ -430,9 +435,32 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server { electionp := grpcproxy.NewElectionProxy(client) lockp := grpcproxy.NewLockProxy(client) + alwaysLoggingDeciderServer := func(ctx context.Context, fullMethodName string, servingObject interface{}) bool { return true } + + grpcChainStreamList := []grpc.StreamServerInterceptor{ + grpc_prometheus.StreamServerInterceptor, + } + grpcChainUnaryList := []grpc.UnaryServerInterceptor{ + grpc_prometheus.UnaryServerInterceptor, + } + if grpcProxyEnableLogging { + grpcChainStreamList = append(grpcChainStreamList, + grpc_ctxtags.StreamServerInterceptor(), + grpc_zap.PayloadStreamServerInterceptor(lg, alwaysLoggingDeciderServer), + ) + grpcChainUnaryList = append(grpcChainUnaryList, + grpc_ctxtags.UnaryServerInterceptor(), + grpc_zap.PayloadUnaryServerInterceptor(lg, alwaysLoggingDeciderServer), + ) + } + gopts := []grpc.ServerOption{ - grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), - grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), + grpc.StreamInterceptor(grpc_middleware.ChainStreamServer( + grpcChainStreamList..., + )), + grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( + grpcChainUnaryList..., + )), grpc.MaxConcurrentStreams(math.MaxUint32), } if grpcKeepAliveMinTime > time.Duration(0) { From 658d570b0b2ee3ff43a72b1b1b1c8483cec0f2a0 Mon Sep 17 00:00:00 2001 From: stefan bo Date: Wed, 10 Aug 2022 16:24:27 +0800 Subject: [PATCH 2/2] change flag name in changelog-3.6 Signed-off-by: stefan bo --- CHANGELOG/CHANGELOG-3.6.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index 5cc1c0f9a..296149a91 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -56,7 +56,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). - Add [v3 discovery](https://github.com/etcd-io/etcd/pull/13635) to bootstrap a new etcd cluster. - Add [field `storage`](https://github.com/etcd-io/etcd/pull/13772) into the response body of endpoint `/version`. - Add [`etcd --max-concurrent-streams`](https://github.com/etcd-io/etcd/pull/14169) flag to configure the max concurrent streams each client can open at a time, and defaults to math.MaxUint32. -- Add [`etcd grpc-proxy --experimental-enable-grpc-debug`](https://github.com/etcd-io/etcd/pull/14266) flag to logging all grpc requests and responses. +- Add [`etcd grpc-proxy --experimental-enable-grpc-logging`](https://github.com/etcd-io/etcd/pull/14266) flag to logging all grpc requests and responses. - Fix [non mutating requests pass through quotaKVServer when NOSPACE](https://github.com/etcd-io/etcd/pull/13435) - Fix [exclude the same alarm type activated by multiple peers](https://github.com/etcd-io/etcd/pull/13467). - Fix [Provide a better liveness probe for when etcd runs as a Kubernetes pod](https://github.com/etcd-io/etcd/pull/13399)