2018-03-28 04:27:28 +03:00
|
|
|
// Copyright 2018 The etcd Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package tester
|
|
|
|
|
|
|
|
import (
|
|
|
|
"reflect"
|
2018-04-01 04:43:24 +03:00
|
|
|
"sort"
|
2018-03-28 04:27:28 +03:00
|
|
|
"testing"
|
|
|
|
|
2019-05-29 00:59:02 +03:00
|
|
|
"go.etcd.io/etcd/functional/rpcpb"
|
2018-03-28 04:27:28 +03:00
|
|
|
|
|
|
|
"go.uber.org/zap"
|
|
|
|
)
|
|
|
|
|
2018-04-09 17:09:48 +03:00
|
|
|
func Test_read(t *testing.T) {
|
2018-03-28 04:27:28 +03:00
|
|
|
exp := &Cluster{
|
|
|
|
Members: []*rpcpb.Member{
|
|
|
|
{
|
2018-04-17 02:07:19 +03:00
|
|
|
EtcdExec: "./bin/etcd",
|
2018-03-28 04:27:28 +03:00
|
|
|
AgentAddr: "127.0.0.1:19027",
|
|
|
|
FailpointHTTPAddr: "http://127.0.0.1:7381",
|
2018-04-11 22:51:54 +03:00
|
|
|
BaseDir: "/tmp/etcd-functional-1",
|
2018-03-28 04:27:28 +03:00
|
|
|
EtcdClientProxy: false,
|
|
|
|
EtcdPeerProxy: true,
|
|
|
|
EtcdClientEndpoint: "127.0.0.1:1379",
|
|
|
|
Etcd: &rpcpb.Etcd{
|
2018-04-06 00:05:04 +03:00
|
|
|
Name: "s1",
|
2018-04-11 22:51:54 +03:00
|
|
|
DataDir: "/tmp/etcd-functional-1/etcd.data",
|
|
|
|
WALDir: "/tmp/etcd-functional-1/etcd.data/member/wal",
|
2018-04-06 00:05:04 +03:00
|
|
|
HeartbeatIntervalMs: 100,
|
|
|
|
ElectionTimeoutMs: 1000,
|
|
|
|
ListenClientURLs: []string{"https://127.0.0.1:1379"},
|
|
|
|
AdvertiseClientURLs: []string{"https://127.0.0.1:1379"},
|
|
|
|
ClientAutoTLS: true,
|
|
|
|
ClientCertAuth: false,
|
|
|
|
ClientCertFile: "",
|
|
|
|
ClientKeyFile: "",
|
|
|
|
ClientTrustedCAFile: "",
|
|
|
|
ListenPeerURLs: []string{"https://127.0.0.1:1380"},
|
2018-04-20 03:44:16 +03:00
|
|
|
AdvertisePeerURLs: []string{"https://127.0.0.1:1381"},
|
2018-04-06 00:05:04 +03:00
|
|
|
PeerAutoTLS: true,
|
|
|
|
PeerClientCertAuth: false,
|
|
|
|
PeerCertFile: "",
|
|
|
|
PeerKeyFile: "",
|
|
|
|
PeerTrustedCAFile: "",
|
2018-04-20 03:44:16 +03:00
|
|
|
InitialCluster: "s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381",
|
2018-04-06 00:05:04 +03:00
|
|
|
InitialClusterState: "new",
|
|
|
|
InitialClusterToken: "tkn",
|
|
|
|
SnapshotCount: 10000,
|
|
|
|
QuotaBackendBytes: 10740000000,
|
|
|
|
PreVote: true,
|
|
|
|
InitialCorruptCheck: true,
|
2018-04-15 06:00:20 +03:00
|
|
|
Logger: "zap",
|
2018-04-25 21:42:05 +03:00
|
|
|
LogOutputs: []string{"/tmp/etcd-functional-1/etcd.log"},
|
2018-04-15 06:00:20 +03:00
|
|
|
Debug: true,
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
2018-04-11 22:51:54 +03:00
|
|
|
ClientCertData: "",
|
|
|
|
ClientCertPath: "",
|
|
|
|
ClientKeyData: "",
|
|
|
|
ClientKeyPath: "",
|
|
|
|
ClientTrustedCAData: "",
|
|
|
|
ClientTrustedCAPath: "",
|
|
|
|
PeerCertData: "",
|
|
|
|
PeerCertPath: "",
|
|
|
|
PeerKeyData: "",
|
|
|
|
PeerKeyPath: "",
|
|
|
|
PeerTrustedCAData: "",
|
|
|
|
PeerTrustedCAPath: "",
|
|
|
|
SnapshotPath: "/tmp/etcd-functional-1.snapshot.db",
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
|
|
|
{
|
2018-04-17 02:07:19 +03:00
|
|
|
EtcdExec: "./bin/etcd",
|
2018-03-28 04:27:28 +03:00
|
|
|
AgentAddr: "127.0.0.1:29027",
|
|
|
|
FailpointHTTPAddr: "http://127.0.0.1:7382",
|
2018-04-11 22:51:54 +03:00
|
|
|
BaseDir: "/tmp/etcd-functional-2",
|
2018-03-28 04:27:28 +03:00
|
|
|
EtcdClientProxy: false,
|
|
|
|
EtcdPeerProxy: true,
|
|
|
|
EtcdClientEndpoint: "127.0.0.1:2379",
|
|
|
|
Etcd: &rpcpb.Etcd{
|
2018-04-06 00:05:04 +03:00
|
|
|
Name: "s2",
|
2018-04-11 22:51:54 +03:00
|
|
|
DataDir: "/tmp/etcd-functional-2/etcd.data",
|
|
|
|
WALDir: "/tmp/etcd-functional-2/etcd.data/member/wal",
|
2018-04-06 00:05:04 +03:00
|
|
|
HeartbeatIntervalMs: 100,
|
|
|
|
ElectionTimeoutMs: 1000,
|
|
|
|
ListenClientURLs: []string{"https://127.0.0.1:2379"},
|
|
|
|
AdvertiseClientURLs: []string{"https://127.0.0.1:2379"},
|
|
|
|
ClientAutoTLS: true,
|
|
|
|
ClientCertAuth: false,
|
|
|
|
ClientCertFile: "",
|
|
|
|
ClientKeyFile: "",
|
|
|
|
ClientTrustedCAFile: "",
|
|
|
|
ListenPeerURLs: []string{"https://127.0.0.1:2380"},
|
2018-04-20 03:44:16 +03:00
|
|
|
AdvertisePeerURLs: []string{"https://127.0.0.1:2381"},
|
2018-04-06 00:05:04 +03:00
|
|
|
PeerAutoTLS: true,
|
|
|
|
PeerClientCertAuth: false,
|
|
|
|
PeerCertFile: "",
|
|
|
|
PeerKeyFile: "",
|
|
|
|
PeerTrustedCAFile: "",
|
2018-04-20 03:44:16 +03:00
|
|
|
InitialCluster: "s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381",
|
2018-04-06 00:05:04 +03:00
|
|
|
InitialClusterState: "new",
|
|
|
|
InitialClusterToken: "tkn",
|
|
|
|
SnapshotCount: 10000,
|
|
|
|
QuotaBackendBytes: 10740000000,
|
|
|
|
PreVote: true,
|
|
|
|
InitialCorruptCheck: true,
|
2018-04-15 06:00:20 +03:00
|
|
|
Logger: "zap",
|
2018-04-25 21:42:05 +03:00
|
|
|
LogOutputs: []string{"/tmp/etcd-functional-2/etcd.log"},
|
2018-04-15 06:00:20 +03:00
|
|
|
Debug: true,
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
2018-04-11 22:51:54 +03:00
|
|
|
ClientCertData: "",
|
|
|
|
ClientCertPath: "",
|
|
|
|
ClientKeyData: "",
|
|
|
|
ClientKeyPath: "",
|
|
|
|
ClientTrustedCAData: "",
|
|
|
|
ClientTrustedCAPath: "",
|
|
|
|
PeerCertData: "",
|
|
|
|
PeerCertPath: "",
|
|
|
|
PeerKeyData: "",
|
|
|
|
PeerKeyPath: "",
|
|
|
|
PeerTrustedCAData: "",
|
|
|
|
PeerTrustedCAPath: "",
|
|
|
|
SnapshotPath: "/tmp/etcd-functional-2.snapshot.db",
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
|
|
|
{
|
2018-04-17 02:07:19 +03:00
|
|
|
EtcdExec: "./bin/etcd",
|
2018-03-28 04:27:28 +03:00
|
|
|
AgentAddr: "127.0.0.1:39027",
|
|
|
|
FailpointHTTPAddr: "http://127.0.0.1:7383",
|
2018-04-11 22:51:54 +03:00
|
|
|
BaseDir: "/tmp/etcd-functional-3",
|
2018-03-28 04:27:28 +03:00
|
|
|
EtcdClientProxy: false,
|
|
|
|
EtcdPeerProxy: true,
|
|
|
|
EtcdClientEndpoint: "127.0.0.1:3379",
|
|
|
|
Etcd: &rpcpb.Etcd{
|
2018-04-06 00:05:04 +03:00
|
|
|
Name: "s3",
|
2018-04-11 22:51:54 +03:00
|
|
|
DataDir: "/tmp/etcd-functional-3/etcd.data",
|
|
|
|
WALDir: "/tmp/etcd-functional-3/etcd.data/member/wal",
|
2018-04-06 00:05:04 +03:00
|
|
|
HeartbeatIntervalMs: 100,
|
|
|
|
ElectionTimeoutMs: 1000,
|
|
|
|
ListenClientURLs: []string{"https://127.0.0.1:3379"},
|
|
|
|
AdvertiseClientURLs: []string{"https://127.0.0.1:3379"},
|
|
|
|
ClientAutoTLS: true,
|
|
|
|
ClientCertAuth: false,
|
|
|
|
ClientCertFile: "",
|
|
|
|
ClientKeyFile: "",
|
|
|
|
ClientTrustedCAFile: "",
|
|
|
|
ListenPeerURLs: []string{"https://127.0.0.1:3380"},
|
2018-04-20 03:44:16 +03:00
|
|
|
AdvertisePeerURLs: []string{"https://127.0.0.1:3381"},
|
2018-04-06 00:05:04 +03:00
|
|
|
PeerAutoTLS: true,
|
|
|
|
PeerClientCertAuth: false,
|
|
|
|
PeerCertFile: "",
|
|
|
|
PeerKeyFile: "",
|
|
|
|
PeerTrustedCAFile: "",
|
2018-04-20 03:44:16 +03:00
|
|
|
InitialCluster: "s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381",
|
2018-04-06 00:05:04 +03:00
|
|
|
InitialClusterState: "new",
|
|
|
|
InitialClusterToken: "tkn",
|
|
|
|
SnapshotCount: 10000,
|
|
|
|
QuotaBackendBytes: 10740000000,
|
|
|
|
PreVote: true,
|
|
|
|
InitialCorruptCheck: true,
|
2018-04-15 06:00:20 +03:00
|
|
|
Logger: "zap",
|
2018-04-25 21:42:05 +03:00
|
|
|
LogOutputs: []string{"/tmp/etcd-functional-3/etcd.log"},
|
2018-04-15 06:00:20 +03:00
|
|
|
Debug: true,
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
2018-04-11 22:51:54 +03:00
|
|
|
ClientCertData: "",
|
|
|
|
ClientCertPath: "",
|
|
|
|
ClientKeyData: "",
|
|
|
|
ClientKeyPath: "",
|
|
|
|
ClientTrustedCAData: "",
|
|
|
|
ClientTrustedCAPath: "",
|
|
|
|
PeerCertData: "",
|
|
|
|
PeerCertPath: "",
|
|
|
|
PeerKeyData: "",
|
|
|
|
PeerKeyPath: "",
|
|
|
|
PeerTrustedCAData: "",
|
|
|
|
PeerTrustedCAPath: "",
|
|
|
|
SnapshotPath: "/tmp/etcd-functional-3.snapshot.db",
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
|
|
|
},
|
|
|
|
Tester: &rpcpb.Tester{
|
2018-04-09 17:09:48 +03:00
|
|
|
DataDir: "/tmp/etcd-tester-data",
|
|
|
|
Network: "tcp",
|
|
|
|
Addr: "127.0.0.1:9028",
|
2018-04-05 21:45:44 +03:00
|
|
|
DelayLatencyMs: 5000,
|
|
|
|
DelayLatencyMsRv: 500,
|
|
|
|
UpdatedDelayLatencyMs: 5000,
|
|
|
|
RoundLimit: 1,
|
2018-04-12 01:12:09 +03:00
|
|
|
ExitOnCaseFail: true,
|
2018-04-05 21:45:44 +03:00
|
|
|
EnablePprof: true,
|
2018-04-12 01:12:09 +03:00
|
|
|
CaseDelayMs: 7000,
|
|
|
|
CaseShuffle: true,
|
|
|
|
Cases: []string{
|
2018-04-10 02:18:27 +03:00
|
|
|
"SIGTERM_ONE_FOLLOWER",
|
|
|
|
"SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
|
|
|
|
"SIGTERM_LEADER",
|
|
|
|
"SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT",
|
|
|
|
"SIGTERM_QUORUM",
|
|
|
|
"SIGTERM_ALL",
|
2018-04-11 11:21:09 +03:00
|
|
|
"SIGQUIT_AND_REMOVE_ONE_FOLLOWER",
|
2018-04-11 11:27:39 +03:00
|
|
|
"SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
|
2018-04-11 20:24:42 +03:00
|
|
|
// "SIGQUIT_AND_REMOVE_LEADER",
|
|
|
|
// "SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT",
|
2018-04-12 20:39:45 +03:00
|
|
|
// "SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH",
|
2018-04-20 03:44:16 +03:00
|
|
|
// "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER",
|
|
|
|
// "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
|
2018-04-02 21:00:24 +03:00
|
|
|
"BLACKHOLE_PEER_PORT_TX_RX_LEADER",
|
2018-04-05 05:13:06 +03:00
|
|
|
"BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT",
|
|
|
|
"BLACKHOLE_PEER_PORT_TX_RX_QUORUM",
|
2018-03-28 04:27:28 +03:00
|
|
|
"BLACKHOLE_PEER_PORT_TX_RX_ALL",
|
2018-04-20 03:44:16 +03:00
|
|
|
// "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
|
|
|
|
// "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
|
|
|
|
// "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
|
|
|
|
// "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
|
2018-03-28 04:27:28 +03:00
|
|
|
"DELAY_PEER_PORT_TX_RX_LEADER",
|
2018-04-05 21:17:46 +03:00
|
|
|
"RANDOM_DELAY_PEER_PORT_TX_RX_LEADER",
|
2018-04-05 05:13:06 +03:00
|
|
|
"DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT",
|
2018-04-05 21:17:46 +03:00
|
|
|
"RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT",
|
2018-04-05 05:13:06 +03:00
|
|
|
"DELAY_PEER_PORT_TX_RX_QUORUM",
|
2018-04-05 21:17:46 +03:00
|
|
|
"RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM",
|
2018-03-28 04:27:28 +03:00
|
|
|
"DELAY_PEER_PORT_TX_RX_ALL",
|
2018-04-05 21:17:46 +03:00
|
|
|
"RANDOM_DELAY_PEER_PORT_TX_RX_ALL",
|
2018-04-04 23:04:21 +03:00
|
|
|
"NO_FAIL_WITH_STRESS",
|
|
|
|
"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
2018-05-09 02:18:45 +03:00
|
|
|
FailpointCommands: []string{`panic("etcd-tester")`},
|
|
|
|
RunnerExecPath: "./bin/etcd-runner",
|
|
|
|
ExternalExecPath: "",
|
|
|
|
Stressers: []*rpcpb.Stresser{
|
|
|
|
{Type: "KV_WRITE_SMALL", Weight: 0.35},
|
|
|
|
{Type: "KV_WRITE_LARGE", Weight: 0.002},
|
|
|
|
{Type: "KV_READ_ONE_KEY", Weight: 0.07},
|
|
|
|
{Type: "KV_READ_RANGE", Weight: 0.07},
|
|
|
|
{Type: "KV_DELETE_ONE_KEY", Weight: 0.07},
|
|
|
|
{Type: "KV_DELETE_RANGE", Weight: 0.07},
|
|
|
|
{Type: "KV_TXN_WRITE_DELETE", Weight: 0.35},
|
|
|
|
{Type: "LEASE", Weight: 0.0},
|
|
|
|
},
|
2018-04-12 05:10:01 +03:00
|
|
|
Checkers: []string{"KV_HASH", "LEASE_EXPIRE"},
|
2018-03-28 04:27:28 +03:00
|
|
|
StressKeySize: 100,
|
|
|
|
StressKeySizeLarge: 32769,
|
|
|
|
StressKeySuffixRange: 250000,
|
|
|
|
StressKeySuffixRangeTxn: 100,
|
|
|
|
StressKeyTxnOps: 10,
|
2018-04-04 23:04:21 +03:00
|
|
|
StressClients: 100,
|
2018-04-05 05:13:06 +03:00
|
|
|
StressQPS: 2000,
|
2018-03-28 04:27:28 +03:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
logger, err := zap.NewProduction()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
defer logger.Sync()
|
|
|
|
|
2018-04-09 17:09:48 +03:00
|
|
|
cfg, err := read(logger, "../../functional.yaml")
|
2018-03-28 04:27:28 +03:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2018-04-03 21:12:12 +03:00
|
|
|
cfg.lg = nil
|
2018-03-28 04:27:28 +03:00
|
|
|
|
|
|
|
if !reflect.DeepEqual(exp, cfg) {
|
|
|
|
t.Fatalf("expected %+v, got %+v", exp, cfg)
|
|
|
|
}
|
2018-04-01 04:35:06 +03:00
|
|
|
|
2018-04-03 21:12:12 +03:00
|
|
|
cfg.lg = logger
|
2018-04-01 04:35:06 +03:00
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
cfg.updateCases()
|
2018-04-12 05:35:17 +03:00
|
|
|
fs1 := cfg.listCases()
|
2018-04-01 04:35:06 +03:00
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
cfg.shuffleCases()
|
2018-04-12 05:35:17 +03:00
|
|
|
fs2 := cfg.listCases()
|
2018-04-01 04:35:06 +03:00
|
|
|
if reflect.DeepEqual(fs1, fs2) {
|
|
|
|
t.Fatalf("expected shuffled failure cases, got %q", fs2)
|
|
|
|
}
|
|
|
|
|
2018-04-12 01:12:09 +03:00
|
|
|
cfg.shuffleCases()
|
2018-04-12 05:35:17 +03:00
|
|
|
fs3 := cfg.listCases()
|
2018-04-01 04:35:06 +03:00
|
|
|
if reflect.DeepEqual(fs2, fs3) {
|
|
|
|
t.Fatalf("expected reshuffled failure cases from %q, got %q", fs2, fs3)
|
|
|
|
}
|
2018-04-01 04:43:24 +03:00
|
|
|
|
|
|
|
// shuffle ensures visit all exactly once
|
|
|
|
// so when sorted, failure cases must be equal
|
|
|
|
sort.Strings(fs1)
|
|
|
|
sort.Strings(fs2)
|
|
|
|
sort.Strings(fs3)
|
2018-04-09 17:09:48 +03:00
|
|
|
|
2018-04-01 04:43:24 +03:00
|
|
|
if !reflect.DeepEqual(fs1, fs2) {
|
|
|
|
t.Fatalf("expected %q, got %q", fs1, fs2)
|
|
|
|
}
|
|
|
|
if !reflect.DeepEqual(fs2, fs3) {
|
|
|
|
t.Fatalf("expected %q, got %q", fs2, fs3)
|
|
|
|
}
|
2018-03-28 04:27:28 +03:00
|
|
|
}
|