vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

functional/tester: fix trigger snapshot retry 

Fix negative commit entry diff:

> {"level":"info","ts":1528740474.775448,"caller":"tester/case.go:299","msg":"trigger snapshot PROGRESS","retries":53,"committed-entries":-238470,"etcd-snapshot-count":10000,"start-revision":238470,"last-revision":0,"took":77.302668629,"error":"context deadline exceeded"}

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
release-3.4
Gyuho Lee 2018-06-11 12:30:16 -07:00
parent f985e5172a
commit 19de2b2177
1 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
functional/tester/case.go
View File

@ -275,6 +275,18 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
for i := 0; i < retries; i++ {
lastRev, err = clus.maxRev()
if lastRev == 0 {
clus.lg.Info(
"trigger snapshot RETRY",
zap.Int("retries", i),
zap.Int64("etcd-snapshot-count", snapshotCount),
zap.Int64("start-revision", startRev),
zap.Error(err),
)
time.Sleep(3 * time.Second)
continue
}
// If the number of proposals committed is bigger than snapshot count,
// a new snapshot should have been created.
diff := lastRev - startRev
@ -292,12 +304,8 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
return nil
}
dur := time.Second
if diff < 0 || err != nil {
dur = 3 * time.Second
}
clus.lg.Info(
"trigger snapshot PROGRESS",
"trigger snapshot RETRY",
zap.Int("retries", i),
zap.Int64("committed-entries", diff),
zap.Int64("etcd-snapshot-count", snapshotCount),
@ -306,7 +314,10 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
zap.Duration("took", time.Since(now)),
zap.Error(err),
)
time.Sleep(dur)
time.Sleep(time.Second)
if err != nil {
time.Sleep(2 * time.Second)
}
}
return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries)