2018-04-11 11:21:09 +03:00
|
|
|
// Copyright 2018 The etcd Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package tester
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"fmt"
|
|
|
|
"sort"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
2020-04-27 00:03:37 +03:00
|
|
|
"go.etcd.io/etcd/v3/clientv3"
|
|
|
|
"go.etcd.io/etcd/v3/functional/rpcpb"
|
2018-04-11 11:42:33 +03:00
|
|
|
|
2018-04-11 11:21:09 +03:00
|
|
|
"go.uber.org/zap"
|
|
|
|
)
|
|
|
|
|
|
|
|
func inject_SIGQUIT_ETCD_AND_REMOVE_DATA(clus *Cluster, idx1 int) error {
|
|
|
|
cli1, err := clus.Members[idx1].CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer cli1.Close()
|
|
|
|
|
|
|
|
var mresp *clientv3.MemberListResponse
|
|
|
|
mresp, err = cli1.MemberList(context.Background())
|
|
|
|
mss := []string{}
|
|
|
|
if err == nil && mresp != nil {
|
|
|
|
mss = describeMembers(mresp)
|
|
|
|
}
|
|
|
|
clus.lg.Info(
|
|
|
|
"member list before disastrous machine failure",
|
|
|
|
zap.String("request-to", clus.Members[idx1].EtcdClientEndpoint),
|
|
|
|
zap.Strings("members", mss),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
sresp, serr := cli1.Status(context.Background(), clus.Members[idx1].EtcdClientEndpoint)
|
|
|
|
if serr != nil {
|
|
|
|
return serr
|
|
|
|
}
|
|
|
|
id1 := sresp.Header.MemberId
|
|
|
|
is1 := fmt.Sprintf("%016x", id1)
|
|
|
|
|
2018-04-11 12:01:08 +03:00
|
|
|
clus.lg.Info(
|
|
|
|
"disastrous machine failure START",
|
|
|
|
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
|
|
|
|
zap.String("target-member-id", is1),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
2018-04-11 11:21:09 +03:00
|
|
|
err = clus.sendOp(idx1, rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA)
|
|
|
|
clus.lg.Info(
|
2018-04-11 12:01:08 +03:00
|
|
|
"disastrous machine failure END",
|
2018-04-11 11:21:09 +03:00
|
|
|
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
|
|
|
|
zap.String("target-member-id", is1),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-04-11 11:25:07 +03:00
|
|
|
time.Sleep(2 * time.Second)
|
2018-04-11 11:21:09 +03:00
|
|
|
|
|
|
|
idx2 := (idx1 + 1) % len(clus.Members)
|
|
|
|
var cli2 *clientv3.Client
|
|
|
|
cli2, err = clus.Members[idx2].CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer cli2.Close()
|
|
|
|
|
2018-04-11 12:01:08 +03:00
|
|
|
// FIXME(bug): this may block forever during
|
|
|
|
// "SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT"
|
|
|
|
// is the new leader too busy with snapshotting?
|
|
|
|
// is raft proposal dropped?
|
|
|
|
// enable client keepalive for failover?
|
|
|
|
clus.lg.Info(
|
|
|
|
"member remove after disaster START",
|
|
|
|
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
|
|
|
|
zap.String("target-member-id", is1),
|
|
|
|
zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint),
|
|
|
|
)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
|
|
_, err = cli2.MemberRemove(ctx, id1)
|
|
|
|
cancel()
|
2018-04-11 11:21:09 +03:00
|
|
|
clus.lg.Info(
|
2018-04-11 12:01:08 +03:00
|
|
|
"member remove after disaster END",
|
2018-04-11 11:21:09 +03:00
|
|
|
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
|
|
|
|
zap.String("target-member-id", is1),
|
|
|
|
zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-04-11 11:25:07 +03:00
|
|
|
time.Sleep(2 * time.Second)
|
2018-04-11 11:21:09 +03:00
|
|
|
|
|
|
|
mresp, err = cli2.MemberList(context.Background())
|
|
|
|
mss = []string{}
|
|
|
|
if err == nil && mresp != nil {
|
|
|
|
mss = describeMembers(mresp)
|
|
|
|
}
|
|
|
|
clus.lg.Info(
|
|
|
|
"member list after member remove",
|
|
|
|
zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint),
|
|
|
|
zap.Strings("members", mss),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func recover_SIGQUIT_ETCD_AND_REMOVE_DATA(clus *Cluster, idx1 int) error {
|
|
|
|
idx2 := (idx1 + 1) % len(clus.Members)
|
|
|
|
cli2, err := clus.Members[idx2].CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer cli2.Close()
|
|
|
|
|
|
|
|
_, err = cli2.MemberAdd(context.Background(), clus.Members[idx1].Etcd.AdvertisePeerURLs)
|
|
|
|
clus.lg.Info(
|
|
|
|
"member add before fresh restart",
|
|
|
|
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
|
|
|
|
zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-04-11 11:25:07 +03:00
|
|
|
time.Sleep(2 * time.Second)
|
2018-04-11 11:21:09 +03:00
|
|
|
|
|
|
|
clus.Members[idx1].Etcd.InitialClusterState = "existing"
|
|
|
|
err = clus.sendOp(idx1, rpcpb.Operation_RESTART_ETCD)
|
|
|
|
clus.lg.Info(
|
|
|
|
"fresh restart after member add",
|
|
|
|
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-04-11 11:25:07 +03:00
|
|
|
time.Sleep(2 * time.Second)
|
2018-04-11 11:21:09 +03:00
|
|
|
|
|
|
|
var mresp *clientv3.MemberListResponse
|
|
|
|
mresp, err = cli2.MemberList(context.Background())
|
|
|
|
mss := []string{}
|
|
|
|
if err == nil && mresp != nil {
|
|
|
|
mss = describeMembers(mresp)
|
|
|
|
}
|
|
|
|
clus.lg.Info(
|
|
|
|
"member list after member add",
|
|
|
|
zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint),
|
|
|
|
zap.Strings("members", mss),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
func new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER(clus *Cluster) Case {
|
|
|
|
cc := caseByFunc{
|
|
|
|
rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER,
|
2018-04-11 11:21:09 +03:00
|
|
|
injectMember: inject_SIGQUIT_ETCD_AND_REMOVE_DATA,
|
|
|
|
recoverMember: recover_SIGQUIT_ETCD_AND_REMOVE_DATA,
|
|
|
|
}
|
2018-04-12 01:12:09 +03:00
|
|
|
c := &caseFollower{cc, -1, -1}
|
|
|
|
return &caseDelay{
|
|
|
|
Case: c,
|
|
|
|
delayDuration: clus.GetCaseDelayDuration(),
|
2018-04-11 11:21:09 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
func new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Case {
|
|
|
|
return &caseUntilSnapshot{
|
|
|
|
rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT,
|
|
|
|
Case: new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER(clus),
|
2018-04-11 11:27:39 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
func new_Case_SIGQUIT_AND_REMOVE_LEADER(clus *Cluster) Case {
|
|
|
|
cc := caseByFunc{
|
|
|
|
rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_LEADER,
|
2018-04-11 11:32:07 +03:00
|
|
|
injectMember: inject_SIGQUIT_ETCD_AND_REMOVE_DATA,
|
|
|
|
recoverMember: recover_SIGQUIT_ETCD_AND_REMOVE_DATA,
|
|
|
|
}
|
2018-04-12 01:12:09 +03:00
|
|
|
c := &caseLeader{cc, -1, -1}
|
|
|
|
return &caseDelay{
|
|
|
|
Case: c,
|
|
|
|
delayDuration: clus.GetCaseDelayDuration(),
|
2018-04-11 11:32:07 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
func new_Case_SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Case {
|
|
|
|
return &caseUntilSnapshot{
|
|
|
|
rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT,
|
|
|
|
Case: new_Case_SIGQUIT_AND_REMOVE_LEADER(clus),
|
2018-04-11 11:37:13 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-11 11:21:09 +03:00
|
|
|
func describeMembers(mresp *clientv3.MemberListResponse) (ss []string) {
|
|
|
|
ss = make([]string, len(mresp.Members))
|
|
|
|
for i, m := range mresp.Members {
|
|
|
|
ss[i] = fmt.Sprintf("Name %s / ID %016x / ClientURLs %s / PeerURLs %s",
|
|
|
|
m.Name,
|
|
|
|
m.ID,
|
|
|
|
strings.Join(m.ClientURLs, ","),
|
|
|
|
strings.Join(m.PeerURLs, ","),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
sort.Strings(ss)
|
|
|
|
return ss
|
|
|
|
}
|