integration: retry remove in TestDecreaseClusterSize

Cluster may go through a second leader election if test machine is
overloaded. Retry remove until it passes without error.

Fixes #8225
release-3.3
Anthony Romano 2017-08-31 23:36:12 -07:00
parent 565831c21c
commit 59d232adf9
2 changed files with 12 additions and 3 deletions

View File

@ -303,10 +303,11 @@ func (c *cluster) removeMember(t *testing.T, id uint64) error {
cc := MustNewHTTPClient(t, c.URLs(), c.cfg.ClientTLS)
ma := client.NewMembersAPI(cc)
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
if err := ma.Remove(ctx, types.ID(id).String()); err != nil {
err := ma.Remove(ctx, types.ID(id).String())
cancel()
if err != nil {
return err
}
cancel()
newMembers := make([]*member, 0)
for _, m := range c.Members {
if uint64(m.s.ID()) != id {

View File

@ -151,7 +151,15 @@ func testDecreaseClusterSize(t *testing.T, size int) {
// TODO: remove the last but one member
for i := 0; i < size-1; i++ {
id := c.Members[len(c.Members)-1].s.ID()
c.RemoveMember(t, uint64(id))
// may hit second leader election on slow machines
if err := c.removeMember(t, uint64(id)); err != nil {
if strings.Contains(err.Error(), "no leader") {
t.Logf("got leader error (%v)", err)
i--
continue
}
t.Fatal(err)
}
c.waitLeader(t, c.Members)
}
clusterMustProgress(t, c.Members)