2018-04-12 01:12:09 +03:00
|
|
|
// Copyright 2018 The etcd Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package tester
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"math/rand"
|
|
|
|
"time"
|
|
|
|
|
2020-04-27 00:03:37 +03:00
|
|
|
"go.etcd.io/etcd/v3/functional/rpcpb"
|
2018-04-12 01:12:09 +03:00
|
|
|
|
|
|
|
"go.uber.org/zap"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Case defines failure/test injection interface.
|
|
|
|
// To add a test case:
|
|
|
|
// 1. implement "Case" interface
|
|
|
|
// 2. define fail case name in "rpcpb.Case"
|
|
|
|
type Case interface {
|
|
|
|
// Inject injeccts the failure into the testing cluster at the given
|
|
|
|
// round. When calling the function, the cluster should be in health.
|
|
|
|
Inject(clus *Cluster) error
|
|
|
|
// Recover recovers the injected failure caused by the injection of the
|
|
|
|
// given round and wait for the recovery of the testing cluster.
|
|
|
|
Recover(clus *Cluster) error
|
|
|
|
// Desc returns a description of the failure
|
|
|
|
Desc() string
|
|
|
|
// TestCase returns "rpcpb.Case" enum type.
|
|
|
|
TestCase() rpcpb.Case
|
|
|
|
}
|
|
|
|
|
|
|
|
type injectMemberFunc func(*Cluster, int) error
|
|
|
|
type recoverMemberFunc func(*Cluster, int) error
|
|
|
|
|
|
|
|
type caseByFunc struct {
|
|
|
|
desc string
|
|
|
|
rpcpbCase rpcpb.Case
|
|
|
|
injectMember injectMemberFunc
|
|
|
|
recoverMember recoverMemberFunc
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseByFunc) Desc() string {
|
|
|
|
if c.desc != "" {
|
|
|
|
return c.desc
|
|
|
|
}
|
|
|
|
return c.rpcpbCase.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseByFunc) TestCase() rpcpb.Case {
|
|
|
|
return c.rpcpbCase
|
|
|
|
}
|
|
|
|
|
|
|
|
type caseFollower struct {
|
|
|
|
caseByFunc
|
|
|
|
last int
|
|
|
|
lead int
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseFollower) updateIndex(clus *Cluster) error {
|
|
|
|
lead, err := clus.GetLeader()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.lead = lead
|
|
|
|
|
|
|
|
n := len(clus.Members)
|
|
|
|
if c.last == -1 { // first run
|
|
|
|
c.last = clus.rd % n
|
|
|
|
if c.last == c.lead {
|
|
|
|
c.last = (c.last + 1) % n
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
c.last = (c.last + 1) % n
|
|
|
|
if c.last == c.lead {
|
|
|
|
c.last = (c.last + 1) % n
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseFollower) Inject(clus *Cluster) error {
|
|
|
|
if err := c.updateIndex(clus); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return c.injectMember(clus, c.last)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseFollower) Recover(clus *Cluster) error {
|
|
|
|
return c.recoverMember(clus, c.last)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseFollower) Desc() string {
|
|
|
|
if c.desc != "" {
|
|
|
|
return c.desc
|
|
|
|
}
|
|
|
|
return c.rpcpbCase.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseFollower) TestCase() rpcpb.Case {
|
|
|
|
return c.rpcpbCase
|
|
|
|
}
|
|
|
|
|
|
|
|
type caseLeader struct {
|
|
|
|
caseByFunc
|
|
|
|
last int
|
|
|
|
lead int
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseLeader) updateIndex(clus *Cluster) error {
|
|
|
|
lead, err := clus.GetLeader()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.lead = lead
|
|
|
|
c.last = lead
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseLeader) Inject(clus *Cluster) error {
|
|
|
|
if err := c.updateIndex(clus); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return c.injectMember(clus, c.last)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseLeader) Recover(clus *Cluster) error {
|
|
|
|
return c.recoverMember(clus, c.last)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseLeader) TestCase() rpcpb.Case {
|
|
|
|
return c.rpcpbCase
|
|
|
|
}
|
|
|
|
|
|
|
|
type caseQuorum struct {
|
|
|
|
caseByFunc
|
|
|
|
injected map[int]struct{}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseQuorum) Inject(clus *Cluster) error {
|
|
|
|
c.injected = pickQuorum(len(clus.Members))
|
|
|
|
for idx := range c.injected {
|
|
|
|
if err := c.injectMember(clus, idx); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseQuorum) Recover(clus *Cluster) error {
|
|
|
|
for idx := range c.injected {
|
|
|
|
if err := c.recoverMember(clus, idx); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseQuorum) Desc() string {
|
|
|
|
if c.desc != "" {
|
|
|
|
return c.desc
|
|
|
|
}
|
|
|
|
return c.rpcpbCase.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseQuorum) TestCase() rpcpb.Case {
|
|
|
|
return c.rpcpbCase
|
|
|
|
}
|
|
|
|
|
|
|
|
func pickQuorum(size int) (picked map[int]struct{}) {
|
|
|
|
picked = make(map[int]struct{})
|
|
|
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
|
|
quorum := size/2 + 1
|
|
|
|
for len(picked) < quorum {
|
|
|
|
idx := r.Intn(size)
|
|
|
|
picked[idx] = struct{}{}
|
|
|
|
}
|
|
|
|
return picked
|
|
|
|
}
|
|
|
|
|
|
|
|
type caseAll caseByFunc
|
|
|
|
|
|
|
|
func (c *caseAll) Inject(clus *Cluster) error {
|
|
|
|
for i := range clus.Members {
|
|
|
|
if err := c.injectMember(clus, i); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseAll) Recover(clus *Cluster) error {
|
|
|
|
for i := range clus.Members {
|
|
|
|
if err := c.recoverMember(clus, i); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseAll) Desc() string {
|
|
|
|
if c.desc != "" {
|
|
|
|
return c.desc
|
|
|
|
}
|
|
|
|
return c.rpcpbCase.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseAll) TestCase() rpcpb.Case {
|
|
|
|
return c.rpcpbCase
|
|
|
|
}
|
|
|
|
|
|
|
|
// caseUntilSnapshot injects a failure/test and waits for a snapshot event
|
|
|
|
type caseUntilSnapshot struct {
|
|
|
|
desc string
|
|
|
|
rpcpbCase rpcpb.Case
|
|
|
|
Case
|
|
|
|
}
|
|
|
|
|
|
|
|
// all delay failure cases except the ones failing with latency
|
|
|
|
// greater than election timeout (trigger leader election and
|
|
|
|
// cluster keeps operating anyways)
|
|
|
|
var slowCases = map[rpcpb.Case]bool{
|
|
|
|
rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER: true,
|
|
|
|
rpcpb.Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT: true,
|
|
|
|
rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT: true,
|
|
|
|
rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER: true,
|
|
|
|
rpcpb.Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT: true,
|
|
|
|
rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT: true,
|
|
|
|
rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM: true,
|
|
|
|
rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ALL: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
|
|
|
|
if err := c.Case.Inject(clus); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
snapshotCount := clus.Members[0].Etcd.SnapshotCount
|
|
|
|
|
|
|
|
now := time.Now()
|
|
|
|
clus.lg.Info(
|
|
|
|
"trigger snapshot START",
|
|
|
|
zap.String("desc", c.Desc()),
|
|
|
|
zap.Int64("etcd-snapshot-count", snapshotCount),
|
|
|
|
)
|
|
|
|
|
|
|
|
// maxRev may fail since failure just injected, retry if failed.
|
|
|
|
startRev, err := clus.maxRev()
|
|
|
|
for i := 0; i < 10 && startRev == 0; i++ {
|
|
|
|
startRev, err = clus.maxRev()
|
|
|
|
}
|
|
|
|
if startRev == 0 {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
lastRev := startRev
|
|
|
|
|
|
|
|
// healthy cluster could accept 1000 req/sec at least.
|
|
|
|
// 3x time to trigger snapshot.
|
|
|
|
retries := int(snapshotCount) / 1000 * 3
|
|
|
|
if v, ok := slowCases[c.TestCase()]; v && ok {
|
|
|
|
// slow network takes more retries
|
|
|
|
retries *= 5
|
|
|
|
}
|
|
|
|
|
|
|
|
for i := 0; i < retries; i++ {
|
2018-04-20 03:40:08 +03:00
|
|
|
lastRev, err = clus.maxRev()
|
functional/tester: fix trigger snapshot retry
Fix negative commit entry diff:
> {"level":"info","ts":1528740474.775448,"caller":"tester/case.go:299","msg":"trigger snapshot PROGRESS","retries":53,"committed-entries":-238470,"etcd-snapshot-count":10000,"start-revision":238470,"last-revision":0,"took":77.302668629,"error":"context deadline exceeded"}
Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
2018-06-11 22:30:16 +03:00
|
|
|
if lastRev == 0 {
|
|
|
|
clus.lg.Info(
|
|
|
|
"trigger snapshot RETRY",
|
|
|
|
zap.Int("retries", i),
|
|
|
|
zap.Int64("etcd-snapshot-count", snapshotCount),
|
|
|
|
zap.Int64("start-revision", startRev),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
time.Sleep(3 * time.Second)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
// If the number of proposals committed is bigger than snapshot count,
|
|
|
|
// a new snapshot should have been created.
|
2018-04-20 03:40:08 +03:00
|
|
|
diff := lastRev - startRev
|
|
|
|
if diff > snapshotCount {
|
2018-04-12 01:12:09 +03:00
|
|
|
clus.lg.Info(
|
|
|
|
"trigger snapshot PASS",
|
|
|
|
zap.Int("retries", i),
|
|
|
|
zap.String("desc", c.Desc()),
|
2018-04-20 03:40:08 +03:00
|
|
|
zap.Int64("committed-entries", diff),
|
2018-04-12 01:12:09 +03:00
|
|
|
zap.Int64("etcd-snapshot-count", snapshotCount),
|
2018-04-20 03:40:08 +03:00
|
|
|
zap.Int64("start-revision", startRev),
|
2018-04-12 01:12:09 +03:00
|
|
|
zap.Int64("last-revision", lastRev),
|
|
|
|
zap.Duration("took", time.Since(now)),
|
|
|
|
)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
clus.lg.Info(
|
functional/tester: fix trigger snapshot retry
Fix negative commit entry diff:
> {"level":"info","ts":1528740474.775448,"caller":"tester/case.go:299","msg":"trigger snapshot PROGRESS","retries":53,"committed-entries":-238470,"etcd-snapshot-count":10000,"start-revision":238470,"last-revision":0,"took":77.302668629,"error":"context deadline exceeded"}
Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
2018-06-11 22:30:16 +03:00
|
|
|
"trigger snapshot RETRY",
|
2018-04-12 01:12:09 +03:00
|
|
|
zap.Int("retries", i),
|
2018-04-20 03:40:08 +03:00
|
|
|
zap.Int64("committed-entries", diff),
|
2018-04-12 01:12:09 +03:00
|
|
|
zap.Int64("etcd-snapshot-count", snapshotCount),
|
2018-04-20 03:40:08 +03:00
|
|
|
zap.Int64("start-revision", startRev),
|
2018-04-12 01:12:09 +03:00
|
|
|
zap.Int64("last-revision", lastRev),
|
|
|
|
zap.Duration("took", time.Since(now)),
|
2018-04-20 03:40:08 +03:00
|
|
|
zap.Error(err),
|
2018-04-12 01:12:09 +03:00
|
|
|
)
|
functional/tester: fix trigger snapshot retry
Fix negative commit entry diff:
> {"level":"info","ts":1528740474.775448,"caller":"tester/case.go:299","msg":"trigger snapshot PROGRESS","retries":53,"committed-entries":-238470,"etcd-snapshot-count":10000,"start-revision":238470,"last-revision":0,"took":77.302668629,"error":"context deadline exceeded"}
Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
2018-06-11 22:30:16 +03:00
|
|
|
time.Sleep(time.Second)
|
|
|
|
if err != nil {
|
|
|
|
time.Sleep(2 * time.Second)
|
|
|
|
}
|
2018-04-12 01:12:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseUntilSnapshot) Desc() string {
|
|
|
|
if c.desc != "" {
|
|
|
|
return c.desc
|
|
|
|
}
|
|
|
|
if c.rpcpbCase.String() != "" {
|
|
|
|
return c.rpcpbCase.String()
|
|
|
|
}
|
|
|
|
return c.Case.Desc()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *caseUntilSnapshot) TestCase() rpcpb.Case {
|
|
|
|
return c.rpcpbCase
|
|
|
|
}
|