vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

functional-tester/tester: add "failure-delay-ms" for configurable triggerElectionDur 

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
release-3.4
Gyuho Lee 2018-04-04 13:15:31 -07:00
parent ca310c5918
commit 33753d4ae6
6 changed files with 20 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/functional-tester/tester/cluster.go
View File

@ -276,11 +276,11 @@ func (clus *Cluster) updateFailures() {
case "KILL_ALL":
clus.failures = append(clus.failures, newFailureKillAll())
case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER":
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower())
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus))
case "BLACKHOLE_PEER_PORT_TX_RX_LEADER":
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader())
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus))
case "BLACKHOLE_PEER_PORT_TX_RX_ALL":
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll())
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus))
case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER":
clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus))
case "DELAY_PEER_PORT_TX_RX_LEADER":

1
tools/functional-tester/tester/cluster_test.go
View File

@ -134,6 +134,7 @@ func Test_newCluster(t *testing.T) {
"NO_FAIL_WITH_STRESS",
"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
},
FailureDelayMs: 7000,
FailureShuffle: true,
FailpointCommands: []string{`panic("etcd-tester")`},
RunnerExecPath: "/etcd-runner",

2
tools/functional-tester/tester/failure_case_failpoints.go
View File

@ -59,7 +59,7 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) {
} else {
fpFails[i] = &failureDelay{
Failure: fpf,
delayDuration: 3 * time.Second,
delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
}
}
}

18
tools/functional-tester/tester/failure_case_network_blackhole.go
View File

@ -14,7 +14,11 @@
package tester
import "github.com/coreos/etcd/tools/functional-tester/rpcpb"
import (
"time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
)
func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error {
return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx)
@ -24,7 +28,7 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error {
return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx)
}
func newFailureBlackholePeerPortTxRxOneFollower() Failure {
func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure {
ff := failureByFunc{
failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER,
injectMember: injectBlackholePeerPortTxRx,
@ -33,11 +37,11 @@ func newFailureBlackholePeerPortTxRxOneFollower() Failure {
f := &failureFollower{ff, -1, -1}
return &failureDelay{
Failure: f,
delayDuration: triggerElectionDur,
delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
}
}
func newFailureBlackholePeerPortTxRxLeader() Failure {
func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure {
ff := failureByFunc{
failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER,
injectMember: injectBlackholePeerPortTxRx,
@ -46,11 +50,11 @@ func newFailureBlackholePeerPortTxRxLeader() Failure {
f := &failureLeader{ff, -1, -1}
return &failureDelay{
Failure: f,
delayDuration: triggerElectionDur,
delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
}
}
func newFailureBlackholePeerPortTxRxAll() Failure {
func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure {
f := &failureAll{
failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL,
injectMember: injectBlackholePeerPortTxRx,
@ -58,6 +62,6 @@ func newFailureBlackholePeerPortTxRxAll() Failure {
}
return &failureDelay{
Failure: f,
delayDuration: triggerElectionDur,
delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
}
}

9
tools/functional-tester/tester/failure_case_network_slow.go
View File

@ -21,9 +21,6 @@ import (
)
const (
// delay duration to trigger leader election (default election timeout 1s)
triggerElectionDur = 5 * time.Second
// Wait more when it recovers from slow network, because network layer
// needs extra time to propagate traffic control (tc command) change.
// Otherwise, we get different hash values from the previous revision.
@ -50,7 +47,7 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure {
f := &failureFollower{ff, -1, -1}
return &failureDelay{
Failure: f,
delayDuration: triggerElectionDur,
delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
}
}
@ -63,7 +60,7 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure {
f := &failureLeader{ff, -1, -1}
return &failureDelay{
Failure: f,
delayDuration: triggerElectionDur,
delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
}
}
@ -75,6 +72,6 @@ func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure {
}
return &failureDelay{
Failure: f,
delayDuration: triggerElectionDur,
delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
}
}

1
tools/functional-tester/tester/local-test.yaml
View File

@ -100,6 +100,7 @@ tester-config:
- NO_FAIL_WITH_STRESS
- NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS
failure-delay-ms: 7000
failure-shuffle: true
failpoint-commands:
- panic("etcd-tester")