tools/functional-tester: better logging
parent
f505bcb91a
commit
69204397ee
|
@ -160,6 +160,7 @@ type stresser struct {
|
||||||
conn *grpc.ClientConn
|
conn *grpc.ClientConn
|
||||||
|
|
||||||
success int
|
success int
|
||||||
|
failure int
|
||||||
|
|
||||||
stressTable *stressTable
|
stressTable *stressTable
|
||||||
}
|
}
|
||||||
|
@ -219,7 +220,10 @@ func (s *stresser) run(ctx context.Context) {
|
||||||
scancel()
|
scancel()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
shouldContinue := false
|
s.mu.Lock()
|
||||||
|
s.failure++
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
switch grpc.ErrorDesc(err) {
|
switch grpc.ErrorDesc(err) {
|
||||||
case context.DeadlineExceeded.Error():
|
case context.DeadlineExceeded.Error():
|
||||||
// This retries when request is triggered at the same time as
|
// This retries when request is triggered at the same time as
|
||||||
|
@ -228,37 +232,45 @@ func (s *stresser) run(ctx context.Context) {
|
||||||
// to followers cannot be forwarded to the old leader, so timing out
|
// to followers cannot be forwarded to the old leader, so timing out
|
||||||
// as well. We want to keep stressing until the cluster elects a
|
// as well. We want to keep stressing until the cluster elects a
|
||||||
// new leader and start processing requests again.
|
// new leader and start processing requests again.
|
||||||
shouldContinue = true
|
continue
|
||||||
|
|
||||||
case etcdserver.ErrTimeoutDueToLeaderFail.Error(), etcdserver.ErrTimeout.Error():
|
case etcdserver.ErrTimeoutDueToLeaderFail.Error(), etcdserver.ErrTimeout.Error():
|
||||||
// This retries when request is triggered at the same time as
|
// This retries when request is triggered at the same time as
|
||||||
// leader failure and follower nodes receive time out errors
|
// leader failure and follower nodes receive time out errors
|
||||||
// from losing their leader. Followers should retry to connect
|
// from losing their leader. Followers should retry to connect
|
||||||
// to the new leader.
|
// to the new leader.
|
||||||
shouldContinue = true
|
continue
|
||||||
|
|
||||||
case etcdserver.ErrStopped.Error():
|
case etcdserver.ErrStopped.Error():
|
||||||
// one of the etcd nodes stopped from failure injection
|
// one of the etcd nodes stopped from failure injection
|
||||||
shouldContinue = true
|
continue
|
||||||
|
|
||||||
case transport.ErrConnClosing.Desc:
|
case transport.ErrConnClosing.Desc:
|
||||||
// server closed the transport (failure injected node)
|
// server closed the transport (failure injected node)
|
||||||
shouldContinue = true
|
continue
|
||||||
|
|
||||||
case rpctypes.ErrNotCapable.Error():
|
case rpctypes.ErrNotCapable.Error():
|
||||||
// capability check has not been done (in the beginning)
|
// capability check has not been done (in the beginning)
|
||||||
shouldContinue = true
|
continue
|
||||||
|
|
||||||
// default:
|
// default:
|
||||||
// errors from stresser.Cancel method:
|
// errors from stresser.Cancel method:
|
||||||
// rpc error: code = 1 desc = context canceled (type grpc.rpcError)
|
// rpc error: code = 1 desc = context canceled (type grpc.rpcError)
|
||||||
// rpc error: code = 2 desc = grpc: the client connection is closing (type grpc.rpcError)
|
// rpc error: code = 2 desc = grpc: the client connection is closing (type grpc.rpcError)
|
||||||
}
|
case rpctypes.ErrTooManyRequests.Error():
|
||||||
if shouldContinue {
|
// hitting the recovering member.
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
case context.Canceled.Error():
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
su, fa := s.Report()
|
||||||
|
plog.Warningf("stresser %v (success %d, failure %d) exited with error (%v)", s.Endpoint, su, fa, err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
s.success++
|
s.success++
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
|
@ -279,8 +291,7 @@ func (s *stresser) Cancel() {
|
||||||
func (s *stresser) Report() (int, int) {
|
func (s *stresser) Report() (int, int) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
// TODO: find a better way to report v3 tests
|
return s.success, s.failure
|
||||||
return s.success, -1
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type stresserV2 struct {
|
type stresserV2 struct {
|
||||||
|
|
Loading…
Reference in New Issue