From 4a239070c8beaa0abcc86f7f301e9dd849f080b7 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Fri, 17 Aug 2018 16:22:00 -0700 Subject: [PATCH] etcdserver/api/rafthttp: display roundtripper name in warnings Signed-off-by: Gyuho Lee --- etcdserver/api/rafthttp/probing_status.go | 16 ++++++++++++---- etcdserver/api/rafthttp/transport.go | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/etcdserver/api/rafthttp/probing_status.go b/etcdserver/api/rafthttp/probing_status.go index a8199dfdf..d97590730 100644 --- a/etcdserver/api/rafthttp/probing_status.go +++ b/etcdserver/api/rafthttp/probing_status.go @@ -17,10 +17,16 @@ package rafthttp import ( "time" + "github.com/prometheus/client_golang/prometheus" "github.com/xiang90/probing" "go.uber.org/zap" ) +const ( + // RoundTripperNameSnapshot is the name of round-tripper that sends merged snapshot message. + RoundTripperNameSnapshot = "ROUND_TRIPPER_SNAPSHOT" +) + var ( // proberInterval must be shorter than read timeout. // Or the connection will time-out. @@ -29,7 +35,7 @@ var ( statusErrorInterval = 5 * time.Second ) -func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string) { +func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string, roundTripperName string, rttSecProm *prometheus.HistogramVec) { hus := make([]string, len(us)) for i := range us { hus[i] = us[i] + ProbingPrefix @@ -47,10 +53,10 @@ func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string) { return } - go monitorProbingStatus(lg, s, id) + go monitorProbingStatus(lg, s, id, roundTripperName, rttSecProm) } -func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) { +func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string, roundTripperName string, rttSecProm *prometheus.HistogramVec) { // set the first interval short to log error early. interval := statusErrorInterval for { @@ -60,6 +66,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) { if lg != nil { lg.Warn( "prober detected unhealthy status", + zap.String("round-tripper-name", roundTripperName), zap.String("remote-peer-id", id), zap.Duration("rtt", s.SRTT()), zap.Error(s.Err()), @@ -75,6 +82,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) { if lg != nil { lg.Warn( "prober found high clock drift", + zap.String("round-tripper-name", roundTripperName), zap.String("remote-peer-id", id), zap.Duration("clock-drift", s.SRTT()), zap.Duration("rtt", s.ClockDiff()), @@ -84,7 +92,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) { plog.Warningf("the clock difference against peer %s is too high [%v > %v]", id, s.ClockDiff(), time.Second) } } - rttSec.WithLabelValues(id).Observe(s.SRTT().Seconds()) + rttSecProm.WithLabelValues(id).Observe(s.SRTT().Seconds()) case <-s.StopNotify(): return diff --git a/etcdserver/api/rafthttp/transport.go b/etcdserver/api/rafthttp/transport.go index 78b159565..30642ced6 100644 --- a/etcdserver/api/rafthttp/transport.go +++ b/etcdserver/api/rafthttp/transport.go @@ -317,7 +317,7 @@ func (t *Transport) AddPeer(id types.ID, us []string) { } fs := t.LeaderStats.Follower(id.String()) t.peers[id] = startPeer(t, urls, id, fs) - addPeerToProber(t.Logger, t.pipelineProber, id.String(), us) + addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec) if t.Logger != nil { t.Logger.Info( @@ -389,7 +389,7 @@ func (t *Transport) UpdatePeer(id types.ID, us []string) { t.peers[id].update(urls) t.pipelineProber.Remove(id.String()) - addPeerToProber(t.Logger, t.pipelineProber, id.String(), us) + addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec) if t.Logger != nil { t.Logger.Info(