vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

functional-tester: improve logging, remove "round" parameter from 

"Failure"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
release-3.4
Gyuho Lee 2018-04-02 09:37:50 -07:00
parent 3ced4443ba
commit 94a96e59e0
6 changed files with 41 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tools/functional-tester/tester/cluster_tester.go
View File

@ -34,9 +34,9 @@ func (clus *Cluster) StartTester() {
var preModifiedKey int64
for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
roundTotalCounter.Inc()
clus.rd = round
if err := clus.doRound(round); err != nil {
if err := clus.doRound(); err != nil {
clus.logger.Warn(
"doRound failed; returning",
zap.Int("round", clus.rd),
@ -50,6 +50,7 @@ func (clus *Cluster) StartTester() {
preModifiedKey = 0
continue
}
// -1 so that logPrefix doesn't print out 'case'
clus.cs = -1
@ -105,11 +106,16 @@ func (clus *Cluster) StartTester() {
)
}
func (clus *Cluster) doRound(round int) error {
func (clus *Cluster) doRound() error {
if clus.Tester.FailureShuffle {
clus.shuffleFailures()
}
clus.logger.Info(
"starting round",
zap.Int("round", clus.rd),
zap.Strings("failures", clus.failureStrings()),
)
for i, f := range clus.failures {
clus.cs = i
@ -126,7 +132,7 @@ func (clus *Cluster) doRound(round int) error {
zap.Int("case", clus.cs),
zap.String("desc", f.Desc()),
)
if err := f.Inject(clus, round); err != nil {
if err := f.Inject(clus); err != nil {
return fmt.Errorf("injection error: %v", err)
}
clus.logger.Info(
@ -145,7 +151,7 @@ func (clus *Cluster) doRound(round int) error {
zap.Int("case", clus.cs),
zap.String("desc", f.Desc()),
)
if err := f.Recover(clus, round); err != nil {
if err := f.Recover(clus); err != nil {
return fmt.Errorf("recovery error: %v", err)
}
clus.logger.Info(
@ -174,6 +180,12 @@ func (clus *Cluster) doRound(round int) error {
zap.String("desc", f.Desc()),
)
}
clus.logger.Info(
"finished round",
zap.Int("round", clus.rd),
zap.Strings("failures", clus.failureStrings()),
)
return nil
}

4
tools/functional-tester/tester/failure.go
View File

@ -21,10 +21,10 @@ package tester
type Failure interface {
// Inject injeccts the failure into the testing cluster at the given
// round. When calling the function, the cluster should be in health.
Inject(clus *Cluster, round int) error
Inject(clus *Cluster) error
// Recover recovers the injected failure caused by the injection of the
// given round and wait for the recovery of the testing cluster.
Recover(clus *Cluster, round int) error
Recover(clus *Cluster) error
// Desc returns a description of the failure
Desc() string
}

4
tools/functional-tester/tester/failure_case_delay.go
View File

@ -25,8 +25,8 @@ type failureDelay struct {
delayDuration time.Duration
}
func (f *failureDelay) Inject(clus *Cluster, round int) error {
if err := f.Failure.Inject(clus, round); err != nil {
func (f *failureDelay) Inject(clus *Cluster) error {
if err := f.Failure.Inject(clus); err != nil {
return err
}
if f.delayDuration > 0 {

8
tools/functional-tester/tester/failure_case_external.go
View File

@ -26,12 +26,12 @@ type failureExternal struct {
scriptPath string
}
func (f *failureExternal) Inject(clus *Cluster, round int) error {
return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", round)).Run()
func (f *failureExternal) Inject(clus *Cluster) error {
return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", clus.rd)).Run()
}
func (f *failureExternal) Recover(clus *Cluster, round int) error {
return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", round)).Run()
func (f *failureExternal) Recover(clus *Cluster) error {
return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run()
}
func (f *failureExternal) Desc() string { return f.description }

28
tools/functional-tester/tester/failure_case_kill.go
View File

@ -99,19 +99,19 @@ type failureLeader struct {
// failureUntilSnapshot injects a failure and waits for a snapshot event
type failureUntilSnapshot struct{ Failure }
func (f *failureOne) Inject(clus *Cluster, round int) error {
return f.injectMember(clus, round%len(clus.Members))
func (f *failureOne) Inject(clus *Cluster) error {
return f.injectMember(clus, clus.rd%len(clus.Members))
}
func (f *failureOne) Recover(clus *Cluster, round int) error {
if err := f.recoverMember(clus, round%len(clus.Members)); err != nil {
func (f *failureOne) Recover(clus *Cluster) error {
if err := f.recoverMember(clus, clus.rd%len(clus.Members)); err != nil {
return err
}
clus.logger.Info("wait health after recovering failureOne")
return clus.WaitHealth()
}
func (f *failureAll) Inject(clus *Cluster, round int) error {
func (f *failureAll) Inject(clus *Cluster) error {
for i := range clus.Members {
if err := f.injectMember(clus, i); err != nil {
return err
@ -120,7 +120,7 @@ func (f *failureAll) Inject(clus *Cluster, round int) error {
return nil
}
func (f *failureAll) Recover(clus *Cluster, round int) error {
func (f *failureAll) Recover(clus *Cluster) error {
for i := range clus.Members {
if err := f.recoverMember(clus, i); err != nil {
return err
@ -130,8 +130,8 @@ func (f *failureAll) Recover(clus *Cluster, round int) error {
return clus.WaitHealth()
}
func (f *failureQuorum) Inject(clus *Cluster, round int) error {
for i := range killMap(len(clus.Members), round) {
func (f *failureQuorum) Inject(clus *Cluster) error {
for i := range killMap(len(clus.Members), clus.rd) {
if err := f.injectMember(clus, i); err != nil {
return err
}
@ -139,8 +139,8 @@ func (f *failureQuorum) Inject(clus *Cluster, round int) error {
return nil
}
func (f *failureQuorum) Recover(clus *Cluster, round int) error {
for i := range killMap(len(clus.Members), round) {
func (f *failureQuorum) Recover(clus *Cluster) error {
for i := range killMap(len(clus.Members), clus.rd) {
if err := f.recoverMember(clus, i); err != nil {
return err
}
@ -148,7 +148,7 @@ func (f *failureQuorum) Recover(clus *Cluster, round int) error {
return nil
}
func (f *failureLeader) Inject(clus *Cluster, round int) error {
func (f *failureLeader) Inject(clus *Cluster) error {
idx, err := clus.GetLeader()
if err != nil {
return err
@ -157,7 +157,7 @@ func (f *failureLeader) Inject(clus *Cluster, round int) error {
return f.injectMember(clus, idx)
}
func (f *failureLeader) Recover(clus *Cluster, round int) error {
func (f *failureLeader) Recover(clus *Cluster) error {
if err := f.recoverMember(clus, f.idx); err != nil {
return err
}
@ -165,8 +165,8 @@ func (f *failureLeader) Recover(clus *Cluster, round int) error {
return clus.WaitHealth()
}
func (f *failureUntilSnapshot) Inject(clus *Cluster, round int) error {
if err := f.Failure.Inject(clus, round); err != nil {
func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
if err := f.Failure.Inject(clus); err != nil {
return err
}
if len(clus.Members) < 3 {

4
tools/functional-tester/tester/failure_case_no_op.go
View File

@ -16,8 +16,8 @@ package tester
type failureNoOp failureByFunc
func (f *failureNoOp) Inject(clus *Cluster, round int) error { return nil }
func (f *failureNoOp) Recover(clus *Cluster, round int) error { return nil }
func (f *failureNoOp) Inject(clus *Cluster) error { return nil }
func (f *failureNoOp) Recover(clus *Cluster) error { return nil }
func newFailureNoOp() Failure {
return &failureNoOp{