diff --git a/test b/test index db2ce994a..2bee76399 100755 --- a/test +++ b/test @@ -234,7 +234,7 @@ function cov_pass { mkdir -p "$COVERDIR" # run code coverage for unit and integration tests - GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 15m" + GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 20m" # shellcheck disable=SC2206 GOCOVFLAGS=($GOCOVFLAGS) failed="" @@ -292,7 +292,7 @@ function e2e_pass { # expectation could be different USERTIMEOUT="" if [ -z "${TIMEOUT}" ]; then - USERTIMEOUT="15m" + USERTIMEOUT="20m" else USERTIMEOUT="${TIMEOUT}" fi @@ -303,9 +303,9 @@ function e2e_pass { function integration_e2e_pass { echo "Running integration and e2e tests..." - go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" & + go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" & e2epid="$!" - go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" & + go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" & intpid="$!" wait $e2epid wait $intpid @@ -315,7 +315,7 @@ function integration_e2e_pass { function grpcproxy_pass { go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration" go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration" - go test -timeout 15m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e" + go test -timeout 20m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e" } function release_pass { diff --git a/tools/functional-tester/rpcpb/rpc.pb.go b/tools/functional-tester/rpcpb/rpc.pb.go index 11e9924e4..2ff92826c 100644 --- a/tools/functional-tester/rpcpb/rpc.pb.go +++ b/tools/functional-tester/rpcpb/rpc.pb.go @@ -104,9 +104,14 @@ const ( FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 9 FailureCase_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 10 FailureCase_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 11 - FailureCase_FAILPOINTS FailureCase = 100 - FailureCase_NO_FAIL FailureCase = 200 - FailureCase_EXTERNAL FailureCase = 300 + // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period + // while stressers are still sending requests. + FailureCase_NO_FAIL_WITH_STRESS FailureCase = 100 + // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection + // with all stressers stopped. + FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS FailureCase = 101 + FailureCase_FAILPOINTS FailureCase = 200 + FailureCase_EXTERNAL FailureCase = 300 ) var FailureCase_name = map[int32]string{ @@ -122,8 +127,9 @@ var FailureCase_name = map[int32]string{ 9: "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", 10: "DELAY_PEER_PORT_TX_RX_LEADER", 11: "DELAY_PEER_PORT_TX_RX_ALL", - 100: "FAILPOINTS", - 200: "NO_FAIL", + 100: "NO_FAIL_WITH_STRESS", + 101: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", + 200: "FAILPOINTS", 300: "EXTERNAL", } var FailureCase_value = map[string]int32{ @@ -139,8 +145,9 @@ var FailureCase_value = map[string]int32{ "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": 9, "DELAY_PEER_PORT_TX_RX_LEADER": 10, "DELAY_PEER_PORT_TX_RX_ALL": 11, - "FAILPOINTS": 100, - "NO_FAIL": 200, + "NO_FAIL_WITH_STRESS": 100, + "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": 101, + "FAILPOINTS": 200, "EXTERNAL": 300, } @@ -154,30 +161,27 @@ type StressType int32 const ( StressType_KV StressType = 0 StressType_LEASE StressType = 1 - StressType_NO_STRESS StressType = 2 - StressType_ELECTION_RUNNER StressType = 3 - StressType_WATCH_RUNNER StressType = 4 - StressType_LOCK_RACER_RUNNER StressType = 5 - StressType_LEASE_RUNNER StressType = 6 + StressType_ELECTION_RUNNER StressType = 2 + StressType_WATCH_RUNNER StressType = 3 + StressType_LOCK_RACER_RUNNER StressType = 4 + StressType_LEASE_RUNNER StressType = 5 ) var StressType_name = map[int32]string{ 0: "KV", 1: "LEASE", - 2: "NO_STRESS", - 3: "ELECTION_RUNNER", - 4: "WATCH_RUNNER", - 5: "LOCK_RACER_RUNNER", - 6: "LEASE_RUNNER", + 2: "ELECTION_RUNNER", + 3: "WATCH_RUNNER", + 4: "LOCK_RACER_RUNNER", + 5: "LEASE_RUNNER", } var StressType_value = map[string]int32{ "KV": 0, "LEASE": 1, - "NO_STRESS": 2, - "ELECTION_RUNNER": 3, - "WATCH_RUNNER": 4, - "LOCK_RACER_RUNNER": 5, - "LEASE_RUNNER": 6, + "ELECTION_RUNNER": 2, + "WATCH_RUNNER": 3, + "LOCK_RACER_RUNNER": 4, + "LEASE_RUNNER": 5, } func (x StressType) String() string { @@ -255,12 +259,14 @@ type Tester struct { EnablePprof bool `protobuf:"varint,24,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"` // FailureCases is the selected test cases to schedule. // If empty, run all failure cases. - // TODO: support no-op FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"` + // FailureDelayMs is the delay duration after failure is injected. + // Useful when triggering snapshot or no-op failure cases. + FailureDelayMs uint32 `protobuf:"varint,32,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"` // FailureShuffle is true to randomize failure injecting order. - FailureShuffle bool `protobuf:"varint,32,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"` + FailureShuffle bool `protobuf:"varint,33,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"` // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). - FailpointCommands []string `protobuf:"bytes,33,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"` + FailpointCommands []string `protobuf:"bytes,34,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"` // RunnerExecPath is a path of etcd-runner binary. RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"` // ExternalExecPath is a path of script for enabling/disabling an external fault injector. @@ -280,8 +286,11 @@ type Tester struct { StressKeySuffixRangeTxn int32 `protobuf:"varint,105,opt,name=StressKeySuffixRangeTxn,proto3" json:"StressKeySuffixRangeTxn,omitempty" yaml:"stress-key-suffix-range-txn"` // StressKeyTxnOps is the number of operations per a transaction (max 64). StressKeyTxnOps int32 `protobuf:"varint,106,opt,name=StressKeyTxnOps,proto3" json:"StressKeyTxnOps,omitempty" yaml:"stress-key-txn-ops"` + // StressClients is the number of concurrent stressing clients + // with "one" shared TCP connection. + StressClients int32 `protobuf:"varint,201,opt,name=StressClients,proto3" json:"StressClients,omitempty" yaml:"stress-clients"` // StressQPS is the maximum number of stresser requests per second. - StressQPS int32 `protobuf:"varint,107,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"` + StressQPS int32 `protobuf:"varint,202,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"` } func (m *Tester) Reset() { *m = Tester{} } @@ -774,11 +783,18 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.FailureShuffle { + if m.FailureDelayMs != 0 { dAtA[i] = 0x80 i++ dAtA[i] = 0x2 i++ + i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs)) + } + if m.FailureShuffle { + dAtA[i] = 0x88 + i++ + dAtA[i] = 0x2 + i++ if m.FailureShuffle { dAtA[i] = 1 } else { @@ -788,7 +804,7 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { } if len(m.FailpointCommands) > 0 { for _, s := range m.FailpointCommands { - dAtA[i] = 0x8a + dAtA[i] = 0x92 i++ dAtA[i] = 0x2 i++ @@ -871,10 +887,17 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintRpc(dAtA, i, uint64(m.StressKeyTxnOps)) } - if m.StressQPS != 0 { - dAtA[i] = 0xd8 + if m.StressClients != 0 { + dAtA[i] = 0xc8 i++ - dAtA[i] = 0x6 + dAtA[i] = 0xc + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressClients)) + } + if m.StressQPS != 0 { + dAtA[i] = 0xd0 + i++ + dAtA[i] = 0xc i++ i = encodeVarintRpc(dAtA, i, uint64(m.StressQPS)) } @@ -1111,6 +1134,9 @@ func (m *Tester) Size() (n int) { n += 2 + l + sovRpc(uint64(l)) } } + if m.FailureDelayMs != 0 { + n += 2 + sovRpc(uint64(m.FailureDelayMs)) + } if m.FailureShuffle { n += 3 } @@ -1149,6 +1175,9 @@ func (m *Tester) Size() (n int) { if m.StressKeyTxnOps != 0 { n += 2 + sovRpc(uint64(m.StressKeyTxnOps)) } + if m.StressClients != 0 { + n += 2 + sovRpc(uint64(m.StressClients)) + } if m.StressQPS != 0 { n += 2 + sovRpc(uint64(m.StressQPS)) } @@ -2167,6 +2196,25 @@ func (m *Tester) Unmarshal(dAtA []byte) error { m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 32: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType) + } + m.FailureDelayMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FailureDelayMs |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 33: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType) } @@ -2186,7 +2234,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error { } } m.FailureShuffle = bool(v != 0) - case 33: + case 34: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType) } @@ -2397,7 +2445,26 @@ func (m *Tester) Unmarshal(dAtA []byte) error { break } } - case 107: + case 201: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressClients", wireType) + } + m.StressClients = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressClients |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 202: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field StressQPS", wireType) } @@ -2779,119 +2846,122 @@ var ( func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 1813 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x57, 0x51, 0x73, 0xda, 0xca, - 0x15, 0x36, 0xb6, 0xc1, 0x66, 0x6d, 0x6c, 0xdd, 0xb5, 0x1d, 0x2b, 0x4e, 0x62, 0x71, 0xd5, 0xde, - 0x8c, 0xeb, 0x19, 0x39, 0x6d, 0xee, 0x4c, 0x67, 0x7a, 0xe7, 0x76, 0x52, 0xc0, 0xca, 0x8d, 0x6b, - 0x05, 0xc8, 0x82, 0x93, 0xf4, 0x89, 0x0a, 0x69, 0x01, 0xd5, 0x42, 0x52, 0x56, 0x8b, 0x2f, 0xdc, - 0x87, 0xbe, 0x76, 0xfa, 0x0f, 0xda, 0xff, 0xd0, 0xfe, 0x8f, 0xb4, 0xbd, 0x0f, 0xfd, 0x05, 0xb4, - 0x4d, 0xff, 0x01, 0x2f, 0x7d, 0xed, 0xec, 0xae, 0x80, 0x15, 0xe0, 0xf6, 0x8d, 0x3d, 0xe7, 0xfb, - 0xbe, 0x3d, 0x7b, 0x74, 0xce, 0xd9, 0x05, 0xec, 0x93, 0xc8, 0x89, 0xda, 0xcf, 0x48, 0xe4, 0x5c, - 0x44, 0x24, 0xa4, 0x21, 0xcc, 0x72, 0xc3, 0x89, 0xd1, 0xf5, 0x68, 0x6f, 0xd0, 0xbe, 0x70, 0xc2, - 0xfe, 0xb3, 0x6e, 0xd8, 0x0d, 0x9f, 0x71, 0x6f, 0x7b, 0xd0, 0xe1, 0x2b, 0xbe, 0xe0, 0xbf, 0x04, - 0x4b, 0xff, 0x7e, 0x0b, 0x6c, 0x9a, 0xd4, 0x71, 0xe1, 0x0f, 0xc0, 0x66, 0xd5, 0xee, 0x63, 0x35, - 0x53, 0xcc, 0x9c, 0xe5, 0xcb, 0xfb, 0x93, 0xb1, 0xb6, 0x33, 0xb2, 0xfb, 0xfe, 0x57, 0x7a, 0x60, - 0xf7, 0xb1, 0x8e, 0xb8, 0x13, 0x1a, 0x60, 0xeb, 0xd2, 0xa6, 0xf6, 0xa5, 0x47, 0xd4, 0x75, 0x8e, - 0x3b, 0x98, 0x8c, 0xb5, 0x7d, 0x81, 0x73, 0x6d, 0x6a, 0x1b, 0xae, 0x47, 0x74, 0x34, 0xc5, 0xc0, - 0x73, 0x90, 0x7b, 0x57, 0xb2, 0x18, 0x7a, 0x83, 0xa3, 0xe1, 0x64, 0xac, 0xed, 0x09, 0xf4, 0xb7, - 0xb6, 0x2f, 0xc0, 0x09, 0x02, 0x5e, 0x01, 0xc5, 0xf2, 0x62, 0x8a, 0x83, 0x8a, 0xef, 0xe1, 0x80, - 0xde, 0x20, 0x2b, 0x56, 0x37, 0x8b, 0x1b, 0x67, 0xf9, 0xf2, 0x93, 0xc9, 0x58, 0x7b, 0x28, 0x58, - 0x3e, 0x47, 0x18, 0x0e, 0x87, 0x18, 0x03, 0xe2, 0xc7, 0x3a, 0x5a, 0xa2, 0x41, 0x04, 0x0e, 0x4a, - 0xee, 0x1d, 0x26, 0xd4, 0x8b, 0xb1, 0xa4, 0x96, 0xe5, 0x6a, 0xc5, 0xc9, 0x58, 0x7b, 0x2c, 0xd4, - 0xec, 0x29, 0x28, 0x2d, 0xb8, 0x8a, 0x0c, 0x2b, 0x60, 0x4f, 0xec, 0x53, 0xc7, 0x98, 0x70, 0xb9, - 0x1c, 0x97, 0x7b, 0x34, 0x19, 0x6b, 0xc7, 0xa9, 0xe0, 0x22, 0x8c, 0x49, 0xa2, 0xb4, 0x40, 0x81, - 0x6d, 0xa0, 0x5e, 0x05, 0x1e, 0xf5, 0x6c, 0x7f, 0xb6, 0xc5, 0x4c, 0x6e, 0x8b, 0xcb, 0x3d, 0x9d, - 0x8c, 0x35, 0x5d, 0xc8, 0x79, 0x02, 0x69, 0xcc, 0xa3, 0x94, 0x94, 0xef, 0xd5, 0x81, 0x65, 0xb0, - 0x97, 0xf8, 0x2a, 0xfe, 0x20, 0xa6, 0x98, 0xa8, 0xdb, 0x3c, 0xf7, 0x27, 0x93, 0xb1, 0xf6, 0x20, - 0xad, 0xec, 0x08, 0x80, 0x8e, 0x16, 0x18, 0x2c, 0x81, 0x69, 0x4b, 0x83, 0xda, 0x14, 0xab, 0x79, - 0x2e, 0x24, 0x25, 0x70, 0x41, 0xc8, 0x88, 0x19, 0x4c, 0x47, 0xab, 0xc8, 0xcb, 0x9a, 0xcd, 0xf0, - 0x16, 0x07, 0x2a, 0xf8, 0x7f, 0x9a, 0x94, 0xc1, 0x96, 0x34, 0x39, 0x19, 0xbe, 0x00, 0x85, 0x46, - 0x60, 0x47, 0x71, 0x2f, 0xa4, 0x95, 0x70, 0x10, 0x50, 0x75, 0xa7, 0x98, 0x39, 0xdb, 0x28, 0x3f, - 0x9c, 0x8c, 0xb5, 0x23, 0xa1, 0x16, 0x27, 0x6e, 0xc3, 0x61, 0x7e, 0x1d, 0xa5, 0xf1, 0xd0, 0x02, - 0x9f, 0xbd, 0x19, 0x84, 0xd4, 0x2e, 0xdb, 0xce, 0x2d, 0x0e, 0xdc, 0xf2, 0x88, 0xe2, 0x58, 0xdd, - 0xe5, 0x22, 0xa7, 0x93, 0xb1, 0x76, 0x22, 0x44, 0x3e, 0x30, 0x88, 0xd1, 0x16, 0x18, 0xa3, 0xcd, - 0x40, 0x3a, 0x5a, 0x26, 0xb2, 0xee, 0xa8, 0x13, 0xfc, 0x36, 0xa4, 0x58, 0x2d, 0x14, 0x33, 0x67, - 0xdb, 0x72, 0x77, 0x44, 0x04, 0x1b, 0x77, 0x21, 0xcb, 0xce, 0x14, 0x23, 0x67, 0x24, 0x24, 0x64, - 0x10, 0xd1, 0x4a, 0x0f, 0x3b, 0xb7, 0xea, 0x1e, 0xa7, 0xae, 0xca, 0x88, 0x40, 0x19, 0x0e, 0x83, - 0x49, 0x19, 0x91, 0xc8, 0xfa, 0xef, 0xb3, 0x20, 0xf7, 0x1a, 0xf7, 0xdb, 0x98, 0xc0, 0x9f, 0x83, - 0x5d, 0xd6, 0xd8, 0xe6, 0x10, 0x3b, 0x75, 0x9b, 0xf6, 0x92, 0xc6, 0x96, 0x72, 0x83, 0xa9, 0xe3, - 0x1a, 0x78, 0x88, 0x1d, 0x23, 0xb2, 0x69, 0x4f, 0x47, 0x29, 0x38, 0xfc, 0x12, 0xe4, 0x4b, 0x5d, - 0x1c, 0xd0, 0x92, 0xeb, 0x12, 0x9e, 0xd7, 0x7c, 0xf9, 0x68, 0x32, 0xd6, 0x3e, 0x4b, 0x5a, 0x87, - 0xb9, 0x0c, 0xdb, 0x75, 0x89, 0x8e, 0xe6, 0x38, 0x96, 0xcf, 0x97, 0xb6, 0xe7, 0x47, 0xa1, 0x17, - 0xd0, 0x57, 0xcd, 0x66, 0x9d, 0x93, 0x77, 0x39, 0x59, 0xca, 0x67, 0x67, 0x0a, 0x31, 0x7a, 0x94, - 0x46, 0x89, 0xca, 0x32, 0x91, 0xe5, 0xb3, 0x6c, 0xc7, 0x98, 0xcd, 0x0f, 0xbc, 0x38, 0x6d, 0xda, - 0x76, 0x8c, 0x93, 0x69, 0x93, 0x60, 0xe0, 0x57, 0x60, 0x87, 0x9d, 0xc0, 0x0a, 0xbb, 0xfc, 0xbc, - 0x1d, 0x4e, 0x51, 0x27, 0x63, 0xed, 0x50, 0x3a, 0xaf, 0x1f, 0x76, 0x93, 0xe3, 0xca, 0x60, 0x58, - 0x02, 0x05, 0xb6, 0x14, 0x0d, 0xdf, 0xb4, 0x1a, 0xea, 0x5f, 0x32, 0xfc, 0x33, 0x48, 0x5d, 0xc3, - 0xe9, 0xc9, 0xa0, 0xa0, 0xac, 0x07, 0xd3, 0x0c, 0xf8, 0x0d, 0xd8, 0x9f, 0x1b, 0xea, 0x24, 0x1c, - 0x8e, 0xd4, 0xbf, 0x0a, 0x91, 0xc7, 0x93, 0xb1, 0xa6, 0x2e, 0x8b, 0x44, 0x0c, 0xa3, 0xa3, 0x45, - 0xd6, 0x34, 0x16, 0xd6, 0xd1, 0x42, 0xe6, 0x6f, 0xab, 0x63, 0xe1, 0xe3, 0x20, 0x11, 0x49, 0x33, - 0x60, 0x1d, 0xc0, 0xb9, 0xaa, 0x19, 0xb8, 0x3c, 0xaf, 0xea, 0xf7, 0xa2, 0x04, 0xb4, 0xc9, 0x58, - 0x7b, 0xb4, 0x1c, 0x0e, 0x4e, 0x60, 0x3a, 0x5a, 0xc1, 0x85, 0x3f, 0x11, 0xd7, 0x84, 0xfa, 0x67, - 0x36, 0xf7, 0x77, 0x9e, 0xef, 0x5c, 0xf0, 0xdb, 0xe6, 0x82, 0xd9, 0xe4, 0xcb, 0x82, 0x09, 0xea, - 0x88, 0x43, 0xf5, 0xff, 0x00, 0x90, 0x6b, 0x62, 0x3e, 0x50, 0x5e, 0x80, 0x82, 0xf8, 0x55, 0xc5, - 0xf4, 0xdb, 0x90, 0xdc, 0x2e, 0x17, 0x23, 0xe5, 0x6e, 0x23, 0x10, 0x7e, 0x1d, 0xa5, 0xf1, 0xf0, - 0xa7, 0x00, 0x08, 0x03, 0xaf, 0x28, 0x71, 0xf7, 0x3c, 0x98, 0x8c, 0x35, 0x98, 0x62, 0x8b, 0x4a, - 0x92, 0x90, 0x6c, 0x6c, 0x5f, 0x62, 0xdf, 0x1e, 0x59, 0x36, 0xc5, 0x81, 0x33, 0x7a, 0x1d, 0xf3, - 0x52, 0x2e, 0xc8, 0x63, 0xdb, 0x65, 0x7e, 0xc3, 0x17, 0x00, 0xa3, 0xcf, 0xc6, 0x76, 0x9a, 0x02, - 0x7f, 0x09, 0x94, 0xb4, 0x05, 0xdd, 0xf1, 0xa2, 0x2e, 0xc8, 0x45, 0xbd, 0x28, 0x63, 0x90, 0x3b, - 0x1d, 0x2d, 0xf1, 0xd8, 0x41, 0x50, 0x38, 0x08, 0x5c, 0xcb, 0xeb, 0x7b, 0x54, 0x3d, 0x2a, 0x66, - 0xce, 0xb2, 0xf2, 0x41, 0x08, 0xf3, 0x19, 0x3e, 0x73, 0xea, 0x48, 0x42, 0xc2, 0x5f, 0x80, 0x82, - 0x39, 0xf4, 0x68, 0x2d, 0x60, 0x6d, 0x32, 0x20, 0x58, 0x7d, 0xb0, 0x54, 0x13, 0x43, 0x8f, 0x1a, - 0x61, 0x60, 0x74, 0x04, 0x80, 0xd5, 0x84, 0x4c, 0x80, 0xaf, 0x80, 0x52, 0x09, 0x83, 0x98, 0xdf, - 0x48, 0xce, 0x48, 0xcc, 0x9a, 0xe3, 0xc5, 0xfa, 0x74, 0xe6, 0x88, 0xe9, 0x9c, 0x59, 0x62, 0xc1, - 0x9f, 0x81, 0x1d, 0x33, 0xb0, 0xdb, 0x3e, 0xae, 0x47, 0x24, 0xec, 0xa8, 0x2a, 0x17, 0x39, 0x9e, - 0x8c, 0xb5, 0x83, 0x24, 0x12, 0xee, 0x34, 0x22, 0xe6, 0x65, 0x7d, 0x36, 0xc7, 0xc2, 0xaf, 0xc1, - 0x6e, 0x12, 0x4f, 0xc5, 0x8e, 0x71, 0xac, 0x6a, 0xfc, 0xd6, 0x93, 0x9a, 0x34, 0x89, 0xde, 0x70, - 0x98, 0x5b, 0x47, 0x29, 0x34, 0xbb, 0xdb, 0x92, 0x75, 0xa3, 0x37, 0xe8, 0x74, 0x7c, 0xac, 0x16, - 0x17, 0xb3, 0x30, 0xe5, 0xc7, 0x02, 0xa0, 0xa3, 0x05, 0x06, 0xbc, 0x96, 0x46, 0x54, 0x25, 0xec, - 0xf7, 0xed, 0xc0, 0x8d, 0xd5, 0xcf, 0x17, 0x1f, 0x1a, 0xf3, 0x11, 0xe5, 0x24, 0x18, 0x79, 0x42, - 0x4d, 0x79, 0xac, 0xbc, 0xd0, 0x20, 0x08, 0x30, 0x99, 0x4d, 0xd9, 0x1f, 0xf1, 0xd2, 0x94, 0xca, - 0x8b, 0x70, 0xbf, 0x3c, 0x67, 0x17, 0x28, 0xec, 0xe5, 0x63, 0x0e, 0x29, 0x26, 0x81, 0xed, 0xcf, - 0x64, 0xce, 0xb9, 0x8c, 0x14, 0x10, 0x4e, 0x10, 0xb2, 0xd0, 0x12, 0x8d, 0x7d, 0x99, 0x06, 0x25, - 0x38, 0x8e, 0x9b, 0xa3, 0x08, 0xc7, 0x2a, 0xe6, 0xc7, 0x92, 0xbe, 0x4c, 0xcc, 0x9d, 0x06, 0x65, - 0x5e, 0x1d, 0xc9, 0x58, 0x56, 0x60, 0x62, 0x79, 0x8d, 0x47, 0x0d, 0xef, 0x3b, 0xcc, 0xe7, 0x67, - 0x56, 0x4e, 0x6d, 0x42, 0xbe, 0xc5, 0x23, 0x23, 0xf6, 0xbe, 0x63, 0x05, 0x96, 0x22, 0xb0, 0xa1, - 0x93, 0x32, 0x58, 0x36, 0xe9, 0x62, 0xb5, 0xcb, 0x65, 0xa4, 0xeb, 0x6c, 0x41, 0xc6, 0xf0, 0x19, - 0x4c, 0x47, 0x2b, 0xb8, 0xf0, 0x2d, 0x38, 0x9c, 0x5b, 0x07, 0x9d, 0x8e, 0x37, 0x44, 0x76, 0xd0, - 0xc5, 0x6a, 0x8f, 0x6b, 0xea, 0x93, 0xb1, 0x76, 0xba, 0xac, 0xc9, 0x71, 0x06, 0x61, 0x40, 0x1d, - 0xad, 0xe4, 0xc3, 0x5f, 0x83, 0xe3, 0x55, 0xf6, 0xe6, 0x30, 0x50, 0x3d, 0x2e, 0x2d, 0x3d, 0xc3, - 0xee, 0x91, 0x36, 0xe8, 0x30, 0xd0, 0xd1, 0x7d, 0x32, 0xec, 0x32, 0x98, 0xb9, 0x9a, 0xc3, 0xa0, - 0x16, 0xc5, 0xea, 0x6f, 0xb8, 0xb2, 0xf4, 0x49, 0x25, 0x65, 0x3a, 0x0c, 0x8c, 0x30, 0x8a, 0x75, - 0xb4, 0xc8, 0x62, 0xd7, 0xb0, 0x30, 0xbd, 0xa9, 0x37, 0xd4, 0x5b, 0x2e, 0x21, 0x5d, 0xc3, 0x89, - 0xc4, 0x07, 0x46, 0x9d, 0xe3, 0xf4, 0xdf, 0x65, 0xc0, 0x16, 0xc2, 0x1f, 0x06, 0x38, 0xa6, 0xf0, - 0x02, 0xe4, 0x6b, 0x11, 0x26, 0x36, 0xf5, 0xc2, 0x80, 0x8f, 0xdd, 0xbd, 0xe7, 0x4a, 0x32, 0xbc, - 0x67, 0x76, 0x34, 0x87, 0xc0, 0x2f, 0xa6, 0x0f, 0x08, 0x55, 0x4c, 0xfa, 0x42, 0x02, 0x16, 0x46, - 0x34, 0x7d, 0x5d, 0x7c, 0x31, 0x9d, 0xed, 0xfc, 0x69, 0x3f, 0x87, 0x09, 0x23, 0x4a, 0x9c, 0xfa, - 0xd7, 0x60, 0x1b, 0xe1, 0x38, 0x0a, 0x83, 0x18, 0x43, 0x15, 0x6c, 0x35, 0x06, 0x8e, 0x83, 0xe3, - 0x98, 0xc7, 0xb1, 0x8d, 0xa6, 0x4b, 0xf8, 0x00, 0xe4, 0xd8, 0x23, 0x71, 0x10, 0x8b, 0xc9, 0x8e, - 0x92, 0xd5, 0xf9, 0x3f, 0x32, 0x52, 0xf0, 0x70, 0x0f, 0x80, 0x6a, 0x48, 0x1b, 0xd4, 0x26, 0x14, - 0xbb, 0xca, 0x1a, 0x3c, 0x04, 0x4a, 0xf2, 0x04, 0xe2, 0x36, 0x76, 0xe7, 0x28, 0x19, 0xb8, 0x0f, - 0x76, 0x10, 0x8e, 0x67, 0x86, 0x75, 0xb8, 0x0b, 0xb6, 0xaf, 0x3d, 0xdf, 0xe7, 0xab, 0x0d, 0xe6, - 0x66, 0x6d, 0x5c, 0x22, 0x4e, 0xcf, 0xbb, 0xc3, 0xca, 0x26, 0x53, 0xb9, 0xc4, 0x31, 0x25, 0xe1, - 0x88, 0x21, 0xf8, 0x53, 0x46, 0xc9, 0xc2, 0x87, 0xe0, 0xa8, 0xec, 0xdb, 0xce, 0x6d, 0x2f, 0xf4, - 0xf9, 0xd3, 0xba, 0x1e, 0x12, 0xda, 0x1c, 0xa2, 0xa1, 0xe2, 0xc2, 0x47, 0xe0, 0xf8, 0x26, 0x68, - 0xaf, 0x74, 0x62, 0x78, 0x04, 0x3e, 0xe3, 0x23, 0x3f, 0x65, 0xee, 0xc0, 0x63, 0x70, 0x70, 0x13, - 0xb8, 0x4b, 0x8e, 0xee, 0xf9, 0x1f, 0x37, 0x44, 0x3c, 0xc9, 0x88, 0x63, 0xfc, 0xeb, 0x2b, 0xcb, - 0x6a, 0xd5, 0xaa, 0x66, 0xeb, 0x65, 0xcd, 0xb2, 0x6a, 0xef, 0x4c, 0xa4, 0xac, 0xb1, 0xa8, 0xb9, - 0xd9, 0x32, 0x4b, 0x97, 0x26, 0x52, 0x32, 0xf0, 0x14, 0x9c, 0x2c, 0xe1, 0x5a, 0x2f, 0x6b, 0xa8, - 0x65, 0xd5, 0xaa, 0xdf, 0x28, 0xeb, 0x50, 0x05, 0x87, 0x12, 0x61, 0xee, 0xd9, 0x98, 0x49, 0xbd, - 0xb9, 0xa9, 0xa1, 0x9b, 0xd7, 0xca, 0x26, 0xcf, 0x0f, 0x33, 0x94, 0x2c, 0x4b, 0xc9, 0xc2, 0x73, - 0xf0, 0xb4, 0x6c, 0x95, 0x2a, 0xd7, 0xaf, 0x6a, 0x96, 0xd9, 0xaa, 0x9b, 0x26, 0x6a, 0xd5, 0x6b, - 0xa8, 0xd9, 0x6a, 0xbe, 0x6f, 0xa1, 0xf7, 0xe9, 0xa8, 0x72, 0xf0, 0x87, 0xa0, 0x78, 0x3f, 0x36, - 0x09, 0x75, 0x0b, 0x7e, 0x0e, 0x9e, 0xdc, 0x8f, 0x62, 0x9b, 0x6e, 0xc3, 0xa7, 0x40, 0xbf, 0x34, - 0xad, 0xd2, 0xaf, 0xfe, 0xf7, 0x86, 0x79, 0x58, 0x04, 0x8f, 0x57, 0xe3, 0x92, 0xcd, 0x00, 0x7c, - 0x02, 0x1e, 0xae, 0x46, 0xb0, 0x8d, 0x76, 0x58, 0x09, 0xbd, 0x2c, 0x5d, 0x59, 0xf5, 0xda, 0x55, - 0xb5, 0xd9, 0x50, 0x5c, 0xb8, 0x0b, 0xb6, 0xaa, 0xb5, 0x16, 0x33, 0x29, 0x1f, 0x33, 0xb0, 0x00, - 0xb6, 0xcd, 0xf7, 0x4d, 0x13, 0x55, 0x4b, 0x96, 0xf2, 0xa7, 0xf5, 0xf3, 0xdf, 0x02, 0x30, 0x1f, - 0x90, 0x30, 0x07, 0xd6, 0xaf, 0xdf, 0x2a, 0x6b, 0x30, 0x0f, 0xb2, 0x96, 0x59, 0x6a, 0x98, 0x0a, - 0xc3, 0xe7, 0xab, 0xb5, 0x56, 0xa3, 0x89, 0xcc, 0x46, 0x43, 0x59, 0x87, 0x07, 0x60, 0xdf, 0xb4, - 0xcc, 0x4a, 0xf3, 0xaa, 0x56, 0x6d, 0xa1, 0x9b, 0x6a, 0xd5, 0x44, 0xca, 0x06, 0x54, 0xc0, 0xee, - 0xbb, 0x52, 0xb3, 0xf2, 0x6a, 0x6a, 0xd9, 0x64, 0x9f, 0xd8, 0xaa, 0x55, 0xae, 0x5b, 0xa8, 0x54, - 0x31, 0xd1, 0xd4, 0x9c, 0x65, 0x40, 0xae, 0x3b, 0xb5, 0xe4, 0x9e, 0xbf, 0x00, 0xf9, 0x26, 0xb1, - 0x83, 0x38, 0x0a, 0x09, 0x85, 0xcf, 0xe5, 0xc5, 0x5e, 0xd2, 0x6c, 0x49, 0x8f, 0x9f, 0xec, 0xcf, - 0xd6, 0xa2, 0xd5, 0xf4, 0xb5, 0xb3, 0xcc, 0x8f, 0x33, 0xe5, 0xc3, 0x8f, 0xff, 0x3a, 0x5d, 0xfb, - 0xf8, 0xe9, 0x34, 0xf3, 0xf7, 0x4f, 0xa7, 0x99, 0x7f, 0x7e, 0x3a, 0xcd, 0xfc, 0xe1, 0xdf, 0xa7, - 0x6b, 0xed, 0x1c, 0xff, 0xe3, 0xff, 0xe5, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc2, 0x76, 0x9b, - 0x6a, 0x41, 0x10, 0x00, 0x00, + // 1870 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x58, 0x7f, 0x72, 0xdb, 0xb8, + 0x15, 0xb6, 0x6c, 0xcb, 0xb6, 0xe0, 0x5f, 0x0c, 0x6c, 0xc7, 0x8c, 0x93, 0x98, 0x5e, 0x6e, 0x37, + 0x75, 0x3d, 0x43, 0xa7, 0xcd, 0x76, 0x3a, 0xd3, 0x9d, 0xed, 0xa4, 0x92, 0xcc, 0xac, 0x5d, 0x33, + 0x92, 0x02, 0xc9, 0x49, 0xfa, 0x97, 0x4a, 0x51, 0x90, 0xc4, 0x9a, 0x22, 0x19, 0x00, 0xf2, 0x4a, + 0x7b, 0x81, 0x4e, 0x6f, 0xd0, 0x43, 0xb4, 0xf7, 0x48, 0xdb, 0x9d, 0x4e, 0x4f, 0xa0, 0xb6, 0xe9, + 0x0d, 0x74, 0x81, 0x76, 0x00, 0x50, 0x12, 0x28, 0xc9, 0xed, 0x7f, 0xc2, 0x7b, 0xdf, 0xf7, 0xf1, + 0xe1, 0x01, 0xef, 0x3d, 0xd8, 0x60, 0x97, 0xc4, 0x5e, 0xdc, 0x78, 0x4e, 0x62, 0xef, 0x3c, 0x26, + 0x11, 0x8b, 0x60, 0x56, 0x18, 0x8e, 0xac, 0xb6, 0xcf, 0x3a, 0xbd, 0xc6, 0xb9, 0x17, 0x75, 0x9f, + 0xb7, 0xa3, 0x76, 0xf4, 0x5c, 0x78, 0x1b, 0xbd, 0x96, 0x58, 0x89, 0x85, 0xf8, 0x25, 0x59, 0xe6, + 0xf7, 0xeb, 0x60, 0xd5, 0x66, 0x5e, 0x13, 0x7e, 0x0e, 0x56, 0x4b, 0x6e, 0x17, 0xeb, 0x99, 0x93, + 0xcc, 0x69, 0xae, 0xb0, 0x3b, 0x1a, 0x1a, 0x9b, 0x03, 0xb7, 0x1b, 0x7c, 0x65, 0x86, 0x6e, 0x17, + 0x9b, 0x48, 0x38, 0xa1, 0x05, 0xd6, 0x2f, 0x5c, 0xe6, 0x5e, 0xf8, 0x44, 0x5f, 0x16, 0xb8, 0xbd, + 0xd1, 0xd0, 0xd8, 0x95, 0xb8, 0xa6, 0xcb, 0x5c, 0xab, 0xe9, 0x13, 0x13, 0x8d, 0x31, 0xf0, 0x0c, + 0xac, 0xbd, 0xcb, 0x3b, 0x1c, 0xbd, 0x22, 0xd0, 0x70, 0x34, 0x34, 0x76, 0x24, 0xfa, 0x5b, 0x37, + 0x90, 0xe0, 0x04, 0x01, 0xaf, 0x80, 0xe6, 0xf8, 0x94, 0xe1, 0xb0, 0x18, 0xf8, 0x38, 0x64, 0x37, + 0xc8, 0xa1, 0xfa, 0xea, 0xc9, 0xca, 0x69, 0xae, 0xf0, 0x74, 0x34, 0x34, 0x1e, 0x49, 0x56, 0x20, + 0x10, 0x96, 0x27, 0x20, 0x56, 0x8f, 0x04, 0xd4, 0x44, 0x73, 0x34, 0x88, 0xc0, 0x5e, 0xbe, 0x79, + 0x87, 0x09, 0xf3, 0x29, 0x56, 0xd4, 0xb2, 0x42, 0xed, 0x64, 0x34, 0x34, 0x9e, 0x48, 0x35, 0x77, + 0x0c, 0x4a, 0x0b, 0x2e, 0x22, 0xc3, 0x22, 0xd8, 0x91, 0xdf, 0xa9, 0x60, 0x4c, 0x84, 0xdc, 0x9a, + 0x90, 0x7b, 0x3c, 0x1a, 0x1a, 0x87, 0xa9, 0xe0, 0x62, 0x8c, 0x49, 0xa2, 0x34, 0x43, 0x81, 0x0d, + 0xa0, 0x5f, 0x85, 0x3e, 0xf3, 0xdd, 0x60, 0xf2, 0x89, 0x89, 0xdc, 0xba, 0x90, 0x7b, 0x36, 0x1a, + 0x1a, 0xa6, 0x94, 0xf3, 0x25, 0xd2, 0x9a, 0x46, 0xa9, 0x28, 0xdf, 0xab, 0x03, 0x0b, 0x60, 0x27, + 0xf1, 0x15, 0x83, 0x1e, 0x65, 0x98, 0xe8, 0x1b, 0x22, 0xf7, 0x47, 0xa3, 0xa1, 0xf1, 0x30, 0xad, + 0xec, 0x49, 0x80, 0x89, 0x66, 0x18, 0x3c, 0x81, 0x69, 0x4b, 0x95, 0xb9, 0x0c, 0xeb, 0x39, 0x21, + 0xa4, 0x24, 0x70, 0x46, 0xc8, 0xa2, 0x1c, 0x66, 0xa2, 0x45, 0xe4, 0x79, 0xcd, 0x5a, 0x74, 0x8b, + 0x43, 0x1d, 0xfc, 0x3f, 0x4d, 0xc6, 0x61, 0x73, 0x9a, 0x82, 0x0c, 0x5f, 0x82, 0xed, 0x6a, 0xe8, + 0xc6, 0xb4, 0x13, 0xb1, 0x62, 0xd4, 0x0b, 0x99, 0xbe, 0x79, 0x92, 0x39, 0x5d, 0x29, 0x3c, 0x1a, + 0x0d, 0x8d, 0x03, 0xa9, 0x46, 0x13, 0xb7, 0xe5, 0x71, 0xbf, 0x89, 0xd2, 0x78, 0xe8, 0x80, 0x07, + 0x6f, 0x7a, 0x11, 0x73, 0x0b, 0xae, 0x77, 0x8b, 0xc3, 0x66, 0x61, 0xc0, 0x30, 0xd5, 0xb7, 0x84, + 0xc8, 0xf1, 0x68, 0x68, 0x1c, 0x49, 0x91, 0x0f, 0x1c, 0x62, 0x35, 0x24, 0xc6, 0x6a, 0x70, 0x90, + 0x89, 0xe6, 0x89, 0xbc, 0x3a, 0x2a, 0x04, 0xbf, 0x8d, 0x18, 0xd6, 0xb7, 0x4f, 0x32, 0xa7, 0x1b, + 0x6a, 0x75, 0xc4, 0x04, 0x5b, 0x77, 0x11, 0xcf, 0xce, 0x18, 0xa3, 0x66, 0x24, 0x22, 0xa4, 0x17, + 0xb3, 0x62, 0x07, 0x7b, 0xb7, 0xfa, 0x8e, 0xa0, 0x2e, 0xca, 0x88, 0x44, 0x59, 0x1e, 0x87, 0x29, + 0x19, 0x51, 0xc8, 0xe6, 0xef, 0xb3, 0x60, 0xed, 0x35, 0xee, 0x36, 0x30, 0x81, 0xbf, 0x00, 0x5b, + 0xbc, 0xb0, 0xed, 0x3e, 0xf6, 0x2a, 0x2e, 0xeb, 0x24, 0x85, 0xad, 0xe4, 0x06, 0x33, 0xaf, 0x69, + 0xe1, 0x3e, 0xf6, 0xac, 0xd8, 0x65, 0x1d, 0x13, 0xa5, 0xe0, 0xf0, 0x4b, 0x90, 0xcb, 0xb7, 0x71, + 0xc8, 0xf2, 0xcd, 0x26, 0x11, 0x79, 0xcd, 0x15, 0x0e, 0x46, 0x43, 0xe3, 0x41, 0x52, 0x3a, 0xdc, + 0x65, 0xb9, 0xcd, 0x26, 0x31, 0xd1, 0x14, 0xc7, 0xf3, 0xf9, 0xca, 0xf5, 0x83, 0x38, 0xf2, 0x43, + 0x76, 0x59, 0xab, 0x55, 0x04, 0x79, 0x4b, 0x90, 0x95, 0x7c, 0xb6, 0xc6, 0x10, 0xab, 0xc3, 0x58, + 0x9c, 0xa8, 0xcc, 0x13, 0x79, 0x3e, 0x0b, 0x2e, 0xc5, 0xbc, 0x7f, 0xe0, 0xd9, 0x6e, 0xd3, 0x70, + 0x29, 0x4e, 0xba, 0x4d, 0x82, 0x81, 0x5f, 0x81, 0x4d, 0xbe, 0x03, 0x27, 0x6a, 0x8b, 0xfd, 0xb6, + 0x04, 0x45, 0x1f, 0x0d, 0x8d, 0x7d, 0x65, 0xbf, 0x41, 0xd4, 0x4e, 0xb6, 0xab, 0x82, 0x61, 0x1e, + 0x6c, 0xf3, 0xa5, 0x2c, 0xf8, 0x9a, 0x53, 0xd5, 0xff, 0x9c, 0x11, 0xc7, 0xa0, 0x54, 0x8d, 0xa0, + 0x27, 0x8d, 0x82, 0xf1, 0x1a, 0x4c, 0x33, 0xe0, 0x37, 0x60, 0x77, 0x6a, 0xa8, 0x90, 0xa8, 0x3f, + 0xd0, 0xff, 0x22, 0x45, 0x9e, 0x8c, 0x86, 0x86, 0x3e, 0x2f, 0x12, 0x73, 0x8c, 0x89, 0x66, 0x59, + 0xe3, 0x58, 0x78, 0x45, 0x4b, 0x99, 0xbf, 0x2e, 0x8e, 0x45, 0xb4, 0x83, 0x44, 0x24, 0xcd, 0x80, + 0x15, 0x00, 0xa7, 0xaa, 0x76, 0xd8, 0x14, 0x79, 0xd5, 0xbf, 0x97, 0x57, 0xc0, 0x18, 0x0d, 0x8d, + 0xc7, 0xf3, 0xe1, 0xe0, 0x04, 0x66, 0xa2, 0x05, 0x5c, 0xf8, 0x13, 0x39, 0x26, 0xf4, 0x3f, 0xf1, + 0xbe, 0xbf, 0xf9, 0x62, 0xf3, 0x5c, 0x4c, 0x9b, 0x73, 0x6e, 0x53, 0x87, 0x05, 0x17, 0x34, 0x91, + 0x80, 0x9a, 0xff, 0xd9, 0x04, 0x6b, 0x35, 0x2c, 0x1a, 0xca, 0x4b, 0xb0, 0x2d, 0x7f, 0x95, 0x30, + 0xfb, 0x36, 0x22, 0xb7, 0xf3, 0x97, 0x91, 0x09, 0xb7, 0x15, 0x4a, 0xbf, 0x89, 0xd2, 0x78, 0xf8, + 0x33, 0x00, 0xa4, 0x41, 0xdc, 0x28, 0x39, 0x7b, 0x1e, 0x8e, 0x86, 0x06, 0x4c, 0xb1, 0xe5, 0x4d, + 0x52, 0x90, 0xbc, 0x6d, 0x5f, 0xe0, 0xc0, 0x1d, 0x38, 0x2e, 0xc3, 0xa1, 0x37, 0x78, 0x4d, 0xc5, + 0x55, 0xde, 0x56, 0xdb, 0x76, 0x93, 0xfb, 0xad, 0x40, 0x02, 0xac, 0x2e, 0x6f, 0xdb, 0x69, 0x0a, + 0xfc, 0x15, 0xd0, 0xd2, 0x16, 0x74, 0x27, 0x2e, 0xf5, 0xb6, 0x7a, 0xa9, 0x67, 0x65, 0x2c, 0x72, + 0x67, 0xa2, 0x39, 0x1e, 0xdf, 0x08, 0x8a, 0x7a, 0x61, 0xd3, 0xf1, 0xbb, 0x3e, 0xd3, 0x0f, 0x4e, + 0x32, 0xa7, 0x59, 0x75, 0x23, 0x84, 0xfb, 0xac, 0x80, 0x3b, 0x4d, 0xa4, 0x20, 0xe1, 0x2f, 0xc1, + 0xb6, 0xdd, 0xf7, 0x59, 0x39, 0xe4, 0x65, 0xd2, 0x23, 0x58, 0x7f, 0x38, 0x77, 0x27, 0xfa, 0x3e, + 0xb3, 0xa2, 0xd0, 0x6a, 0x49, 0x00, 0xbf, 0x13, 0x2a, 0x01, 0x5e, 0x02, 0xad, 0x18, 0x85, 0x54, + 0x4c, 0x24, 0x6f, 0x20, 0x7b, 0xcd, 0xe1, 0xec, 0xfd, 0xf4, 0xa6, 0x88, 0x71, 0x9f, 0x99, 0x63, + 0xc1, 0x9f, 0x83, 0x4d, 0x3b, 0x74, 0x1b, 0x01, 0xae, 0xc4, 0x24, 0x6a, 0xe9, 0xba, 0x10, 0x39, + 0x1c, 0x0d, 0x8d, 0xbd, 0x24, 0x12, 0xe1, 0xb4, 0x62, 0xee, 0xe5, 0x75, 0x36, 0xc5, 0xc2, 0xaf, + 0xc1, 0x56, 0x12, 0x4f, 0xd1, 0xa5, 0x98, 0xea, 0x86, 0x98, 0x7a, 0x4a, 0x91, 0x26, 0xd1, 0x5b, + 0x1e, 0x77, 0x9b, 0x28, 0x85, 0xe6, 0xa7, 0x99, 0xac, 0x45, 0x5e, 0x5f, 0x53, 0xfd, 0x64, 0xf6, + 0x34, 0xc7, 0x7c, 0x79, 0x1c, 0xe2, 0x34, 0xd3, 0x14, 0x3e, 0x20, 0x13, 0x4b, 0xb5, 0xd3, 0x6b, + 0xb5, 0x02, 0xac, 0x7f, 0x36, 0x9b, 0xca, 0xb1, 0x08, 0x95, 0x80, 0xa9, 0x46, 0xc2, 0x80, 0xd7, + 0x4a, 0x9f, 0x2b, 0x46, 0xdd, 0xae, 0x1b, 0x36, 0xa9, 0x6e, 0xce, 0xbe, 0x56, 0xa6, 0x7d, 0xce, + 0x4b, 0x30, 0x6a, 0x9b, 0x1b, 0xf3, 0xf8, 0xae, 0x50, 0x2f, 0x0c, 0x31, 0x99, 0xb4, 0xea, 0x1f, + 0x89, 0xfb, 0xad, 0xec, 0x8a, 0x08, 0xbf, 0xda, 0xac, 0x67, 0x28, 0xfc, 0xf9, 0x64, 0xf7, 0x19, + 0x26, 0xa1, 0x1b, 0x4c, 0x64, 0xce, 0x84, 0x8c, 0x12, 0x10, 0x4e, 0x10, 0xaa, 0xd0, 0x1c, 0x8d, + 0x1f, 0x6f, 0x95, 0x11, 0x4c, 0x69, 0x6d, 0x10, 0x63, 0xaa, 0x63, 0xb1, 0x2d, 0xe5, 0x78, 0xa9, + 0x70, 0x5a, 0x8c, 0x7b, 0x4d, 0xa4, 0x62, 0xf9, 0x2d, 0x95, 0xcb, 0x6b, 0x3c, 0xa8, 0xfa, 0xdf, + 0x61, 0xd1, 0x84, 0xb3, 0x6a, 0x6a, 0x13, 0xf2, 0x2d, 0x1e, 0x58, 0xd4, 0xff, 0x8e, 0xdf, 0xd2, + 0x14, 0x81, 0x77, 0xae, 0x94, 0xc1, 0x71, 0x49, 0x1b, 0xeb, 0x6d, 0x21, 0xa3, 0xcc, 0xc4, 0x19, + 0x19, 0x2b, 0xe0, 0x30, 0x13, 0x2d, 0xe0, 0xc2, 0xb7, 0x60, 0x7f, 0x6a, 0xed, 0xb5, 0x5a, 0x7e, + 0x1f, 0xb9, 0x61, 0x1b, 0xeb, 0x1d, 0xa1, 0x69, 0x8e, 0x86, 0xc6, 0xf1, 0xbc, 0xa6, 0xc0, 0x59, + 0x84, 0x03, 0x4d, 0xb4, 0x90, 0x0f, 0x7f, 0x03, 0x0e, 0x17, 0xd9, 0x6b, 0xfd, 0x50, 0xf7, 0x85, + 0xb4, 0xf2, 0x96, 0xbb, 0x47, 0xda, 0x62, 0xfd, 0xd0, 0x44, 0xf7, 0xc9, 0xf0, 0x89, 0x32, 0x71, + 0xd5, 0xfa, 0x61, 0x39, 0xa6, 0xfa, 0x6f, 0x85, 0xb2, 0x72, 0xa4, 0x8a, 0x32, 0xeb, 0x87, 0x56, + 0x14, 0x53, 0x13, 0xcd, 0xb2, 0xa6, 0xc7, 0x22, 0x9b, 0x3a, 0x95, 0xd3, 0x2d, 0x9b, 0x7a, 0x28, + 0x49, 0x1d, 0x39, 0x0b, 0xe8, 0xe4, 0x58, 0x12, 0x02, 0xfc, 0x29, 0xc8, 0x49, 0xc3, 0x9b, 0x4a, + 0x55, 0x8e, 0xb5, 0xac, 0xfa, 0x1c, 0x48, 0xd8, 0x1f, 0xf8, 0xd7, 0xa7, 0x40, 0xf3, 0x77, 0x19, + 0xb0, 0x8e, 0xf0, 0x87, 0x1e, 0xa6, 0x0c, 0x9e, 0x83, 0x5c, 0x39, 0xc6, 0xc4, 0x65, 0x7e, 0x14, + 0x8a, 0xf6, 0xbf, 0xf3, 0x42, 0x4b, 0x86, 0xc8, 0xc4, 0x8e, 0xa6, 0x10, 0xf8, 0xc5, 0xf8, 0x21, + 0xa3, 0xcb, 0x89, 0xb3, 0x9d, 0x80, 0xa5, 0x11, 0x8d, 0x5f, 0x39, 0x5f, 0x8c, 0x67, 0x8c, 0xf8, + 0x13, 0x63, 0x0a, 0x93, 0x46, 0x94, 0x38, 0xcd, 0xaf, 0xc1, 0x06, 0xc2, 0x34, 0x8e, 0x42, 0x8a, + 0xa1, 0x0e, 0xd6, 0xab, 0x3d, 0xcf, 0xc3, 0x94, 0x8a, 0x38, 0x36, 0xd0, 0x78, 0x09, 0x1f, 0x82, + 0x35, 0xfe, 0x58, 0xed, 0x51, 0x39, 0x61, 0x50, 0xb2, 0x3a, 0xfb, 0x47, 0x46, 0x09, 0x1e, 0xee, + 0x00, 0x50, 0x8a, 0x58, 0x95, 0xb9, 0x84, 0xe1, 0xa6, 0xb6, 0x04, 0xf7, 0x81, 0x96, 0x3c, 0xc5, + 0x84, 0x8d, 0xcf, 0x3e, 0x2d, 0x03, 0x77, 0xc1, 0x26, 0xc2, 0x74, 0x62, 0x58, 0x86, 0x5b, 0x60, + 0xe3, 0xda, 0x0f, 0x02, 0xb1, 0x5a, 0xe1, 0x6e, 0xde, 0x09, 0xf2, 0xc4, 0xeb, 0xf8, 0x77, 0x58, + 0x5b, 0xe5, 0x2a, 0x17, 0x98, 0x32, 0x12, 0x0d, 0x38, 0x42, 0x3c, 0xa9, 0xb4, 0x2c, 0x7c, 0x04, + 0x0e, 0x0a, 0x81, 0xeb, 0xdd, 0x76, 0xa2, 0x40, 0x3c, 0xf1, 0x2b, 0x11, 0x61, 0xb5, 0x3e, 0xea, + 0x6b, 0x4d, 0xf8, 0x18, 0x1c, 0xde, 0x84, 0x8d, 0x85, 0x4e, 0x0c, 0x0f, 0xc0, 0x03, 0xd1, 0xef, + 0x52, 0xe6, 0x16, 0x3c, 0x04, 0x7b, 0x37, 0x61, 0x73, 0xce, 0xd1, 0x3e, 0xfb, 0xdb, 0x8a, 0x8c, + 0x27, 0x69, 0xb5, 0x9c, 0x7f, 0x7d, 0xe5, 0x38, 0xf5, 0x72, 0xc9, 0xae, 0xbf, 0x2a, 0x3b, 0x4e, + 0xf9, 0x9d, 0x8d, 0xb4, 0x25, 0x1e, 0xb5, 0x30, 0x3b, 0x76, 0xfe, 0xc2, 0x46, 0x5a, 0x06, 0x1e, + 0x83, 0xa3, 0x39, 0x5c, 0xfd, 0x55, 0x19, 0xd5, 0x9d, 0x72, 0xe9, 0x1b, 0x6d, 0x19, 0xea, 0x60, + 0x5f, 0x21, 0x4c, 0x3d, 0x2b, 0x13, 0xa9, 0x37, 0x37, 0x65, 0x74, 0xf3, 0x5a, 0x5b, 0x15, 0xf9, + 0xe1, 0x86, 0xbc, 0xe3, 0x68, 0x59, 0x78, 0x06, 0x9e, 0x15, 0x9c, 0x7c, 0xf1, 0xfa, 0xb2, 0xec, + 0xd8, 0xf5, 0x8a, 0x6d, 0xa3, 0x7a, 0xa5, 0x8c, 0x6a, 0xf5, 0xda, 0xfb, 0x3a, 0x7a, 0x9f, 0x8e, + 0x6a, 0x0d, 0xfe, 0x00, 0x9c, 0xdc, 0x8f, 0x4d, 0x42, 0x5d, 0x87, 0x9f, 0x81, 0xa7, 0xf7, 0xa3, + 0xf8, 0x47, 0x37, 0xe0, 0x33, 0x60, 0x5e, 0xd8, 0x4e, 0xfe, 0xd7, 0xff, 0xfb, 0x83, 0x39, 0x78, + 0x02, 0x9e, 0x2c, 0xc6, 0x25, 0x1f, 0x03, 0xf0, 0x29, 0x78, 0xb4, 0x18, 0xc1, 0x3f, 0xb4, 0xc9, + 0xcf, 0xa1, 0x54, 0xae, 0xbf, 0xca, 0x5f, 0x39, 0xf5, 0x77, 0x57, 0xb5, 0xcb, 0x7a, 0xb5, 0x86, + 0xec, 0x6a, 0x55, 0x6b, 0xc2, 0x1f, 0x82, 0xcf, 0x53, 0x8e, 0x52, 0x39, 0xf1, 0xc9, 0xd4, 0x5d, + 0xbd, 0xb5, 0x4b, 0x1c, 0x88, 0xe1, 0x2e, 0x00, 0x1c, 0x55, 0x29, 0x5f, 0x95, 0x6a, 0x55, 0xed, + 0x63, 0x06, 0x6e, 0x83, 0x0d, 0xfb, 0x7d, 0xcd, 0x46, 0xa5, 0xbc, 0xa3, 0xfd, 0x71, 0xf9, 0x2c, + 0x02, 0x60, 0xda, 0x98, 0xe1, 0x1a, 0x58, 0xbe, 0x7e, 0xab, 0x2d, 0xc1, 0x1c, 0xc8, 0x3a, 0x76, + 0xbe, 0x6a, 0x6b, 0x19, 0xb8, 0x07, 0x76, 0x6d, 0xc7, 0x2e, 0xd6, 0xae, 0xca, 0xa5, 0x3a, 0xba, + 0x29, 0x95, 0x6c, 0xa4, 0x2d, 0x43, 0x0d, 0x6c, 0xbd, 0xcb, 0xd7, 0x8a, 0x97, 0x63, 0xcb, 0x0a, + 0xbf, 0x08, 0x4e, 0xb9, 0x78, 0x5d, 0x47, 0xf9, 0xa2, 0x8d, 0xc6, 0xe6, 0x55, 0x0e, 0x14, 0x42, + 0x63, 0x4b, 0xf6, 0xc5, 0x4b, 0x90, 0xab, 0x11, 0x37, 0xa4, 0x71, 0x44, 0x18, 0x7c, 0xa1, 0x2e, + 0x76, 0x92, 0x92, 0x4c, 0x3a, 0xc1, 0xd1, 0xee, 0x64, 0x2d, 0x0b, 0xd2, 0x5c, 0x3a, 0xcd, 0xfc, + 0x38, 0x53, 0xd8, 0xff, 0xf8, 0xaf, 0xe3, 0xa5, 0x8f, 0x9f, 0x8e, 0x33, 0x7f, 0xff, 0x74, 0x9c, + 0xf9, 0xe7, 0xa7, 0xe3, 0xcc, 0x1f, 0xfe, 0x7d, 0xbc, 0xd4, 0x58, 0x13, 0xff, 0xa6, 0xf8, 0xf2, + 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x33, 0x62, 0x54, 0x24, 0xef, 0x10, 0x00, 0x00, } diff --git a/tools/functional-tester/rpcpb/rpc.proto b/tools/functional-tester/rpcpb/rpc.proto index dcb9aef37..cbe206d56 100644 --- a/tools/functional-tester/rpcpb/rpc.proto +++ b/tools/functional-tester/rpcpb/rpc.proto @@ -100,23 +100,24 @@ enum FailureCase { DELAY_PEER_PORT_TX_RX_LEADER = 10; DELAY_PEER_PORT_TX_RX_ALL = 11; - FAILPOINTS = 100; - - NO_FAIL = 200; - // TODO: support no-op of liveness duration - // NO_FAIL_LIVENESS = 201; + // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period + // while stressers are still sending requests. + NO_FAIL_WITH_STRESS = 100; + // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection + // with all stressers stopped. + NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS = 101; + FAILPOINTS = 200; EXTERNAL = 300; } enum StressType { KV = 0; LEASE = 1; - NO_STRESS = 2; - ELECTION_RUNNER = 3; - WATCH_RUNNER = 4; - LOCK_RACER_RUNNER = 5; - LEASE_RUNNER = 6; + ELECTION_RUNNER = 2; + WATCH_RUNNER = 3; + LOCK_RACER_RUNNER = 4; + LEASE_RUNNER = 5; } message Tester { @@ -140,12 +141,14 @@ message Tester { // FailureCases is the selected test cases to schedule. // If empty, run all failure cases. - // TODO: support no-op repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""]; + // FailureDelayMs is the delay duration after failure is injected. + // Useful when triggering snapshot or no-op failure cases. + uint32 FailureDelayMs = 32 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""]; // FailureShuffle is true to randomize failure injecting order. - bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; + bool FailureShuffle = 33 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). - repeated string FailpointCommands = 33 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; + repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; // RunnerExecPath is a path of etcd-runner binary. string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""]; @@ -167,8 +170,12 @@ message Tester { int32 StressKeySuffixRangeTxn = 105 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""]; // StressKeyTxnOps is the number of operations per a transaction (max 64). int32 StressKeyTxnOps = 106 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""]; + + // StressClients is the number of concurrent stressing clients + // with "one" shared TCP connection. + int32 StressClients = 201 [(gogoproto.moretags) = "yaml:\"stress-clients\""]; // StressQPS is the maximum number of stresser requests per second. - int32 StressQPS = 107 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; + int32 StressQPS = 202 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; } message Request { diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 3113e8465..cc747a800 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -276,27 +276,29 @@ func (clus *Cluster) updateFailures() { case "KILL_ALL": clus.failures = append(clus.failures, newFailureKillAll()) case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_LEADER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ALL": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus)) case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus)) case "DELAY_PEER_PORT_TX_RX_LEADER": clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeader(clus)) case "DELAY_PEER_PORT_TX_RX_ALL": clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxAll(clus)) + case "NO_FAIL_WITH_STRESS": + clus.failures = append(clus.failures, newFailureNoFailWithStress(clus)) + case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": + clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness(clus)) + case "EXTERNAL": + clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath)) case "FAILPOINTS": fpFailures, fperr := failpointFailures(clus) if len(fpFailures) == 0 { clus.lg.Info("no failpoints found!", zap.Error(fperr)) } clus.failures = append(clus.failures, fpFailures...) - case "NO_FAIL": - clus.failures = append(clus.failures, newFailureNoOp()) - case "EXTERNAL": - clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath)) } } } @@ -360,8 +362,8 @@ func (clus *Cluster) updateStresserChecker() { ) cs := &compositeStresser{} - for idx := range clus.Members { - cs.stressers = append(cs.stressers, newStresser(clus, idx)) + for _, m := range clus.Members { + cs.stressers = append(cs.stressers, newStresser(clus, m)) } clus.stresser = cs @@ -381,49 +383,6 @@ func (clus *Cluster) updateStresserChecker() { ) } -func (clus *Cluster) startStresser() (err error) { - clus.lg.Info( - "starting stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - err = clus.stresser.Stress() - clus.lg.Info( - "started stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - return err -} - -func (clus *Cluster) closeStresser() { - clus.lg.Info( - "closing stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - clus.stresser.Close() - clus.lg.Info( - "closed stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) -} - -func (clus *Cluster) pauseStresser() { - clus.lg.Info( - "pausing stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - clus.stresser.Pause() - clus.lg.Info( - "paused stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) -} - func (clus *Cluster) checkConsistency() (err error) { defer func() { if err != nil { @@ -436,7 +395,6 @@ func (clus *Cluster) checkConsistency() (err error) { ) return } - err = clus.startStresser() }() clus.lg.Info( @@ -759,4 +717,12 @@ func (clus *Cluster) defrag() error { return nil } -func (clus *Cluster) Report() int64 { return clus.stresser.ModifiedKeys() } +// GetFailureDelayDuration computes failure delay duration. +func (clus *Cluster) GetFailureDelayDuration() time.Duration { + return time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond +} + +// Report reports the number of modified keys. +func (clus *Cluster) Report() int64 { + return clus.stresser.ModifiedKeys() +} diff --git a/tools/functional-tester/tester/cluster_test.go b/tools/functional-tester/tester/cluster_test.go index a6ac44c21..e0627453e 100644 --- a/tools/functional-tester/tester/cluster_test.go +++ b/tools/functional-tester/tester/cluster_test.go @@ -131,7 +131,10 @@ func Test_newCluster(t *testing.T) { "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", "DELAY_PEER_PORT_TX_RX_LEADER", "DELAY_PEER_PORT_TX_RX_ALL", + "NO_FAIL_WITH_STRESS", + "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", }, + FailureDelayMs: 7000, FailureShuffle: true, FailpointCommands: []string{`panic("etcd-tester")`}, RunnerExecPath: "/etcd-runner", @@ -142,6 +145,7 @@ func Test_newCluster(t *testing.T) { StressKeySuffixRange: 250000, StressKeySuffixRangeTxn: 100, StressKeyTxnOps: 10, + StressClients: 100, StressQPS: 1000, }, } diff --git a/tools/functional-tester/tester/cluster_tester.go b/tools/functional-tester/tester/cluster_tester.go index 9d6928ce8..453556462 100644 --- a/tools/functional-tester/tester/cluster_tester.go +++ b/tools/functional-tester/tester/cluster_tester.go @@ -19,6 +19,8 @@ import ( "os" "time" + "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "go.uber.org/zap" ) @@ -29,7 +31,6 @@ const compactQPS = 50000 // StartTester starts tester. func (clus *Cluster) StartTester() { // TODO: upate status - clus.startStresser() var preModifiedKey int64 for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ { @@ -116,68 +117,80 @@ func (clus *Cluster) doRound() error { zap.Int("round", clus.rd), zap.Strings("failures", clus.failureStrings()), ) - for i, f := range clus.failures { + + for i, fa := range clus.failures { clus.cs = i - caseTotalCounter.WithLabelValues(f.Desc()).Inc() + caseTotalCounter.WithLabelValues(fa.Desc()).Inc() + clus.lg.Info( + "failure case START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", fa.Desc()), + ) clus.lg.Info("wait health before injecting failures") if err := clus.WaitHealth(); err != nil { return fmt.Errorf("wait full health error: %v", err) } + stressStarted := false + if fa.FailureCase() != rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { + clus.lg.Info( + "starting stressers before injecting failures", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", fa.Desc()), + ) + if err := clus.stresser.Stress(); err != nil { + return fmt.Errorf("start stresser error: %v", err) + } + stressStarted = true + } + clus.lg.Info( - "injecting failure", + "injecting", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) - if err := f.Inject(clus); err != nil { + if err := fa.Inject(clus); err != nil { return fmt.Errorf("injection error: %v", err) } - clus.lg.Info( - "injected failure", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), - ) // if run local, recovering server may conflict // with stressing client ports // TODO: use unix for local tests clus.lg.Info( - "recovering failure", + "recovering", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) - if err := f.Recover(clus); err != nil { + if err := fa.Recover(clus); err != nil { return fmt.Errorf("recovery error: %v", err) } - clus.lg.Info( - "recovered failure", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), - ) - clus.lg.Info("pausing stresser after failure recovery, before wait health") - clus.pauseStresser() + if stressStarted { + clus.lg.Info("pausing stresser after failure recovery, before wait health") + clus.stresser.Pause() + } - clus.lg.Info("wait health after recovering failures") + clus.lg.Info("wait health after recover") if err := clus.WaitHealth(); err != nil { return fmt.Errorf("wait full health error: %v", err) } - clus.lg.Info("check consistency after recovering failures") + + clus.lg.Info("check consistency after recover") if err := clus.checkConsistency(); err != nil { return fmt.Errorf("tt.checkConsistency error (%v)", err) } clus.lg.Info( - "failure case passed", + "failure case PASS", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) } @@ -186,6 +199,7 @@ func (clus *Cluster) doRound() error { zap.Int("round", clus.rd), zap.Strings("failures", clus.failureStrings()), ) + return nil } @@ -204,14 +218,6 @@ func (clus *Cluster) updateRevision() error { } func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) { - clus.lg.Info("pausing stresser before compact") - clus.pauseStresser() - defer func() { - if err == nil { - err = clus.startStresser() - } - }() - clus.lg.Info( "compacting storage", zap.Int64("current-revision", clus.currentRevision), @@ -273,7 +279,13 @@ func (clus *Cluster) cleanup() error { } caseFailedTotalCounter.WithLabelValues(desc).Inc() - clus.closeStresser() + clus.lg.Info( + "closing stressers before archiving failure data", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + ) + clus.stresser.Close() + if err := clus.FailArchive(); err != nil { clus.lg.Warn( "cleanup failed", diff --git a/tools/functional-tester/tester/failure.go b/tools/functional-tester/tester/failure.go index b0fbb0b86..f9c205512 100644 --- a/tools/functional-tester/tester/failure.go +++ b/tools/functional-tester/tester/failure.go @@ -18,6 +18,8 @@ import ( "fmt" "math/rand" "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) // Failure defines failure injection interface. @@ -33,28 +35,32 @@ type Failure interface { Recover(clus *Cluster) error // Desc returns a description of the failure Desc() string + // FailureCase returns "rpcpb.FailureCase" enum type. + FailureCase() rpcpb.FailureCase } -type description string - -func (d description) Desc() string { return string(d) } - type injectMemberFunc func(*Cluster, int) error type recoverMemberFunc func(*Cluster, int) error type failureByFunc struct { - description + desc + failureCase rpcpb.FailureCase injectMember injectMemberFunc recoverMember recoverMemberFunc } -type failureFollower struct { - failureByFunc - last int - lead int +func (f *failureByFunc) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() } -type failureLeader struct { +func (f *failureByFunc) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +type failureFollower struct { failureByFunc last int lead int @@ -82,22 +88,6 @@ func (f *failureFollower) updateIndex(clus *Cluster) error { return nil } -func (f *failureLeader) updateIndex(clus *Cluster) error { - idx, err := clus.GetLeader() - if err != nil { - return err - } - f.lead = idx - f.last = idx - return nil -} - -type failureQuorum failureByFunc -type failureAll failureByFunc - -// failureUntilSnapshot injects a failure and waits for a snapshot event -type failureUntilSnapshot struct{ Failure } - func (f *failureFollower) Inject(clus *Cluster) error { if err := f.updateIndex(clus); err != nil { return err @@ -109,6 +99,33 @@ func (f *failureFollower) Recover(clus *Cluster) error { return f.recoverMember(clus, f.last) } +func (f *failureFollower) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() +} + +func (f *failureFollower) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +type failureLeader struct { + failureByFunc + last int + lead int +} + +func (f *failureLeader) updateIndex(clus *Cluster) error { + idx, err := clus.GetLeader() + if err != nil { + return err + } + f.lead = idx + f.last = idx + return nil +} + func (f *failureLeader) Inject(clus *Cluster) error { if err := f.updateIndex(clus); err != nil { return err @@ -120,6 +137,12 @@ func (f *failureLeader) Recover(clus *Cluster) error { return f.recoverMember(clus, f.last) } +func (f *failureLeader) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +type failureQuorum failureByFunc + func (f *failureQuorum) Inject(clus *Cluster) error { for i := range killMap(len(clus.Members), clus.rd) { if err := f.injectMember(clus, i); err != nil { @@ -138,6 +161,19 @@ func (f *failureQuorum) Recover(clus *Cluster) error { return nil } +func (f *failureQuorum) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() +} + +func (f *failureQuorum) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +type failureAll failureByFunc + func (f *failureAll) Inject(clus *Cluster) error { for i := range clus.Members { if err := f.injectMember(clus, i); err != nil { @@ -156,6 +192,25 @@ func (f *failureAll) Recover(clus *Cluster) error { return nil } +func (f *failureAll) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() +} + +func (f *failureAll) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +// failureUntilSnapshot injects a failure and waits for a snapshot event +type failureUntilSnapshot struct { + desc desc + failureCase rpcpb.FailureCase + + Failure +} + const snapshotCount = 10000 func (f *failureUntilSnapshot) Inject(clus *Cluster) error { @@ -190,7 +245,14 @@ func (f *failureUntilSnapshot) Inject(clus *Cluster) error { } func (f *failureUntilSnapshot) Desc() string { - return f.Failure.Desc() + " for a long time and expect it to recover from an incoming snapshot" + if f.desc.Desc() != "" { + return f.desc.Desc() + } + return f.failureCase.String() +} + +func (f *failureUntilSnapshot) FailureCase() rpcpb.FailureCase { + return f.failureCase } func killMap(size int, seed int) map[int]bool { @@ -204,3 +266,7 @@ func killMap(size int, seed int) map[int]bool { } } } + +type desc string + +func (d desc) Desc() string { return string(d) } diff --git a/tools/functional-tester/tester/failure_case_external.go b/tools/functional-tester/tester/failure_case_external.go index e5b3b6a18..0d73422b1 100644 --- a/tools/functional-tester/tester/failure_case_external.go +++ b/tools/functional-tester/tester/failure_case_external.go @@ -17,13 +17,17 @@ package tester import ( "fmt" "os/exec" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) type failureExternal struct { Failure - description string - scriptPath string + desc string + failureCase rpcpb.FailureCase + + scriptPath string } func (f *failureExternal) Inject(clus *Cluster) error { @@ -34,11 +38,18 @@ func (f *failureExternal) Recover(clus *Cluster) error { return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run() } -func (f *failureExternal) Desc() string { return f.description } +func (f *failureExternal) Desc() string { + return f.desc +} + +func (f *failureExternal) FailureCase() rpcpb.FailureCase { + return f.failureCase +} func newFailureExternal(scriptPath string) Failure { return &failureExternal{ - description: fmt.Sprintf("external fault injector (script: %q)", scriptPath), + desc: fmt.Sprintf("external fault injector (script: %q)", scriptPath), + failureCase: rpcpb.FailureCase_EXTERNAL, scriptPath: scriptPath, } } diff --git a/tools/functional-tester/tester/failure_case_failpoints.go b/tools/functional-tester/tester/failure_case_failpoints.go index ede60916a..c55e34577 100644 --- a/tools/functional-tester/tester/failure_case_failpoints.go +++ b/tools/functional-tester/tester/failure_case_failpoints.go @@ -20,7 +20,8 @@ import ( "net/http" "strings" "sync" - "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) type failpointStats struct { @@ -42,14 +43,23 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) { if len(fp) == 0 { continue } + fpFails := failuresFromFailpoint(fp, clus.Tester.FailpointCommands) + // wrap in delays so failpoint has time to trigger for i, fpf := range fpFails { if strings.Contains(fp, "Snap") { // hack to trigger snapshot failpoints - fpFails[i] = &failureUntilSnapshot{fpf} + fpFails[i] = &failureUntilSnapshot{ + desc: desc(fpf.Desc()), + failureCase: rpcpb.FailureCase_FAILPOINTS, + Failure: fpf, + } } else { - fpFails[i] = &failureDelay{fpf, 3 * time.Second} + fpFails[i] = &failureDelay{ + Failure: fpf, + delayDuration: clus.GetFailureDelayDuration(), + } } } ret = append(ret, fpFails...) @@ -85,7 +95,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure) fs = append(fs, []Failure{ &failureFollower{ failureByFunc: failureByFunc{ - description: description(fmt.Sprintf("failpoint %s (one: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (one: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, @@ -94,7 +105,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure) }, &failureLeader{ failureByFunc: failureByFunc{ - description: description(fmt.Sprintf("failpoint %s (leader: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (leader: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, @@ -102,12 +114,14 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure) lead: -1, }, &failureQuorum{ - description: description(fmt.Sprintf("failpoint %s (quorum: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, &failureAll{ - description: description(fmt.Sprintf("failpoint %s (all: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (all: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, diff --git a/tools/functional-tester/tester/failure_case_kill.go b/tools/functional-tester/tester/failure_case_kill.go index d59fddf8f..9e88efe03 100644 --- a/tools/functional-tester/tester/failure_case_kill.go +++ b/tools/functional-tester/tester/failure_case_kill.go @@ -26,7 +26,7 @@ func recoverKill(clus *Cluster, idx int) error { func newFailureKillOneFollower() Failure { ff := failureByFunc{ - description: "kill one follower", + failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER, injectMember: injectKill, recoverMember: recoverKill, } @@ -35,7 +35,7 @@ func newFailureKillOneFollower() Failure { func newFailureKillLeader() Failure { ff := failureByFunc{ - description: "kill leader", + failureCase: rpcpb.FailureCase_KILL_LEADER, injectMember: injectKill, recoverMember: recoverKill, } @@ -44,7 +44,7 @@ func newFailureKillLeader() Failure { func newFailureKillQuorum() Failure { return &failureQuorum{ - description: "kill quorum", + failureCase: rpcpb.FailureCase_KILL_QUORUM, injectMember: injectKill, recoverMember: recoverKill, } @@ -52,16 +52,22 @@ func newFailureKillQuorum() Failure { func newFailureKillAll() Failure { return &failureAll{ - description: "kill all", + failureCase: rpcpb.FailureCase_KILL_ALL, injectMember: injectKill, recoverMember: recoverKill, } } func newFailureKillOneFollowerForLongTime() Failure { - return &failureUntilSnapshot{newFailureKillOneFollower()} + return &failureUntilSnapshot{ + failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER_FOR_LONG, + Failure: newFailureKillOneFollower(), + } } func newFailureKillLeaderForLongTime() Failure { - return &failureUntilSnapshot{newFailureKillLeader()} + return &failureUntilSnapshot{ + failureCase: rpcpb.FailureCase_KILL_LEADER_FOR_LONG, + Failure: newFailureKillLeader(), + } } diff --git a/tools/functional-tester/tester/failure_case_network_blackhole.go b/tools/functional-tester/tester/failure_case_network_blackhole.go index 6951c892f..0f7581280 100644 --- a/tools/functional-tester/tester/failure_case_network_blackhole.go +++ b/tools/functional-tester/tester/failure_case_network_blackhole.go @@ -14,7 +14,9 @@ package tester -import "github.com/coreos/etcd/tools/functional-tester/rpcpb" +import ( + "github.com/coreos/etcd/tools/functional-tester/rpcpb" +) func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx) @@ -24,40 +26,40 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx) } -func newFailureBlackholePeerPortTxRxOneFollower() Failure { +func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure { ff := failureByFunc{ - description: "blackhole peer port on one follower", + failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER, injectMember: injectBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx, } f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: clus.GetFailureDelayDuration(), } } -func newFailureBlackholePeerPortTxRxLeader() Failure { +func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure { ff := failureByFunc{ - description: "blackhole peer port on leader", + failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER, injectMember: injectBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx, } f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: clus.GetFailureDelayDuration(), } } -func newFailureBlackholePeerPortTxRxAll() Failure { +func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure { f := &failureAll{ - description: "blackhole peer port on all", + failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL, injectMember: injectBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx, } return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: clus.GetFailureDelayDuration(), } } diff --git a/tools/functional-tester/tester/failure_case_network_slow.go b/tools/functional-tester/tester/failure_case_network_slow.go index 840d2101d..274ba6383 100644 --- a/tools/functional-tester/tester/failure_case_network_slow.go +++ b/tools/functional-tester/tester/failure_case_network_slow.go @@ -15,16 +15,12 @@ package tester import ( - "fmt" "time" "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) const ( - // delay duration to trigger leader election (default election timeout 1s) - triggerElectionDur = 5 * time.Second - // Wait more when it recovers from slow network, because network layer // needs extra time to propagate traffic control (tc command) change. // Otherwise, we get different hash values from the previous revision. @@ -43,42 +39,39 @@ func recoverDelayPeerPortTxRx(clus *Cluster, idx int) error { } func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure { - desc := fmt.Sprintf("delay follower peer port by %d ms", clus.Tester.DelayLatencyMs) ff := failureByFunc{ - description: description(desc), + failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER, injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: clus.GetFailureDelayDuration(), } } func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure { - desc := fmt.Sprintf("delay leader peer port by %d ms", clus.Tester.DelayLatencyMs) ff := failureByFunc{ - description: description(desc), + failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_LEADER, injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: clus.GetFailureDelayDuration(), } } func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure { - desc := fmt.Sprintf("delay all peer port by %d ms", clus.Tester.DelayLatencyMs) f := &failureAll{ - description: description(desc), + failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ALL, injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: clus.GetFailureDelayDuration(), } } diff --git a/tools/functional-tester/tester/failure_case_no_fail.go b/tools/functional-tester/tester/failure_case_no_fail.go new file mode 100644 index 000000000..221929960 --- /dev/null +++ b/tools/functional-tester/tester/failure_case_no_fail.go @@ -0,0 +1,99 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" + + "go.uber.org/zap" +) + +type failureNoFailWithStress failureByFunc + +func (f *failureNoFailWithStress) Inject(clus *Cluster) error { + return nil +} + +func (f *failureNoFailWithStress) Recover(clus *Cluster) error { + return nil +} + +func (f *failureNoFailWithStress) Desc() string { + if f.desc.Desc() != "" { + return f.desc.Desc() + } + return f.failureCase.String() +} + +func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +func newFailureNoFailWithStress(clus *Cluster) Failure { + f := &failureNoFailWithStress{ + failureCase: rpcpb.FailureCase_NO_FAIL_WITH_STRESS, + } + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } +} + +type failureNoFailWithNoStressForLiveness failureByFunc + +func (f *failureNoFailWithNoStressForLiveness) Inject(clus *Cluster) error { + clus.lg.Info( + "extra delay for liveness mode with no stresser", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", f.Desc()), + ) + time.Sleep(clus.GetFailureDelayDuration()) + + clus.lg.Info( + "wait health in liveness mode", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", f.Desc()), + ) + return clus.WaitHealth() +} + +func (f *failureNoFailWithNoStressForLiveness) Recover(clus *Cluster) error { + return nil +} + +func (f *failureNoFailWithNoStressForLiveness) Desc() string { + if f.desc.Desc() != "" { + return f.desc.Desc() + } + return f.failureCase.String() +} + +func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +func newFailureNoFailWithNoStressForLiveness(clus *Cluster) Failure { + f := &failureNoFailWithNoStressForLiveness{ + failureCase: rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS, + } + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } +} diff --git a/tools/functional-tester/tester/failure_case_no_op.go b/tools/functional-tester/tester/failure_case_no_op.go deleted file mode 100644 index fdcddd306..000000000 --- a/tools/functional-tester/tester/failure_case_no_op.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tester - -type failureNoOp failureByFunc - -func (f *failureNoOp) Inject(clus *Cluster) error { return nil } -func (f *failureNoOp) Recover(clus *Cluster) error { return nil } - -func newFailureNoOp() Failure { - return &failureNoOp{ - description: "no failure", - } -} diff --git a/tools/functional-tester/tester/local-test.yaml b/tools/functional-tester/tester/local-test.yaml index 4104691d4..f5068c236 100644 --- a/tools/functional-tester/tester/local-test.yaml +++ b/tools/functional-tester/tester/local-test.yaml @@ -97,7 +97,10 @@ tester-config: - DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER - DELAY_PEER_PORT_TX_RX_LEADER - DELAY_PEER_PORT_TX_RX_ALL + - NO_FAIL_WITH_STRESS + - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS + failure-delay-ms: 7000 failure-shuffle: true failpoint-commands: - panic("etcd-tester") @@ -110,7 +113,6 @@ tester-config: stress-types: - KV - LEASE - # - NO_STRESS # - ELECTION_RUNNER # - WATCH_RUNNER # - LOCK_RACER_RUNNER @@ -121,4 +123,6 @@ tester-config: stress-key-suffix-range: 250000 stress-key-suffix-range-txn: 100 stress-key-txn-ops: 10 + + stress-clients: 100 stress-qps: 1000 diff --git a/tools/functional-tester/tester/stress.go b/tools/functional-tester/tester/stress.go index 421a6d93f..816189a10 100644 --- a/tools/functional-tester/tester/stress.go +++ b/tools/functional-tester/tester/stress.go @@ -18,6 +18,8 @@ import ( "fmt" "time" + "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "go.uber.org/zap" ) @@ -36,34 +38,31 @@ type Stresser interface { } // newStresser creates stresser from a comma separated list of stresser types. -func newStresser(clus *Cluster, idx int) Stresser { +func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { stressers := make([]Stresser, len(clus.Tester.StressTypes)) for i, stype := range clus.Tester.StressTypes { clus.lg.Info("creating stresser", zap.String("type", stype)) switch stype { - case "NO_STRESS": - stressers[i] = &nopStresser{start: time.Now(), qps: int(clus.rateLimiter.Limit())} - case "KV": // TODO: Too intensive stressing clients can panic etcd member with // 'out of memory' error. Put rate limits in server side. stressers[i] = &keyStresser{ lg: clus.lg, - m: clus.Members[idx], + m: m, keySize: int(clus.Tester.StressKeySize), keyLargeSize: int(clus.Tester.StressKeySizeLarge), keySuffixRange: int(clus.Tester.StressKeySuffixRange), keyTxnSuffixRange: int(clus.Tester.StressKeySuffixRangeTxn), keyTxnOps: int(clus.Tester.StressKeyTxnOps), - N: 100, + clientsN: int(clus.Tester.StressClients), rateLimiter: clus.rateLimiter, } case "LEASE": stressers[i] = &leaseStresser{ lg: clus.lg, - m: clus.Members[idx], + m: m, numLeases: 10, // TODO: configurable keysPerLease: 10, // TODO: configurable rateLimiter: clus.rateLimiter, @@ -75,7 +74,7 @@ func newStresser(clus *Cluster, idx int) Stresser { "election", fmt.Sprintf("%v", time.Now().UnixNano()), // election name as current nano time "--dial-timeout=10s", - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, "--total-client-connections=10", "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), @@ -95,7 +94,7 @@ func newStresser(clus *Cluster, idx int) Stresser { "--total-keys=1", "--total-prefixes=1", "--watch-per-prefix=1", - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), } @@ -106,7 +105,7 @@ func newStresser(clus *Cluster, idx int) Stresser { args := []string{ "lock-racer", fmt.Sprintf("%v", time.Now().UnixNano()), // locker name as current nano time - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, "--total-client-connections=10", "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), @@ -117,7 +116,7 @@ func newStresser(clus *Cluster, idx int) Stresser { args := []string{ "lease-renewer", "--ttl=30", - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, } stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, 0) } diff --git a/tools/functional-tester/tester/stress_key.go b/tools/functional-tester/tester/stress_key.go index e3868c2ae..2359e87e1 100644 --- a/tools/functional-tester/tester/stress_key.go +++ b/tools/functional-tester/tester/stress_key.go @@ -44,14 +44,15 @@ type keyStresser struct { keyTxnSuffixRange int keyTxnOps int - N int - rateLimiter *rate.Limiter - wg sync.WaitGroup + wg sync.WaitGroup + clientsN int + ctx context.Context cancel func() cli *clientv3.Client + // atomicModifiedKeys records the number of keys created and deleted by the stresser. atomicModifiedKeys int64 @@ -59,40 +60,37 @@ type keyStresser struct { } func (s *keyStresser) Stress() error { - // TODO: add backoff option - cli, err := s.m.CreateEtcdClient() + var err error + s.cli, err = s.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(1 * time.Second)) if err != nil { return fmt.Errorf("%v (%q)", err, s.m.EtcdClientEndpoint) } - ctx, cancel := context.WithCancel(context.Background()) - - s.wg.Add(s.N) - s.cli = cli - s.cancel = cancel + s.ctx, s.cancel = context.WithCancel(context.Background()) + s.wg.Add(s.clientsN) var stressEntries = []stressEntry{ - {weight: 0.7, f: newStressPut(cli, s.keySuffixRange, s.keySize)}, + {weight: 0.7, f: newStressPut(s.cli, s.keySuffixRange, s.keySize)}, { weight: 0.7 * float32(s.keySize) / float32(s.keyLargeSize), - f: newStressPut(cli, s.keySuffixRange, s.keyLargeSize), + f: newStressPut(s.cli, s.keySuffixRange, s.keyLargeSize), }, - {weight: 0.07, f: newStressRange(cli, s.keySuffixRange)}, - {weight: 0.07, f: newStressRangeInterval(cli, s.keySuffixRange)}, - {weight: 0.07, f: newStressDelete(cli, s.keySuffixRange)}, - {weight: 0.07, f: newStressDeleteInterval(cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressRange(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressRangeInterval(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressDelete(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressDeleteInterval(s.cli, s.keySuffixRange)}, } if s.keyTxnSuffixRange > 0 { // adjust to make up ±70% of workloads with writes stressEntries[0].weight = 0.35 stressEntries = append(stressEntries, stressEntry{ weight: 0.35, - f: newStressTxn(cli, s.keyTxnSuffixRange, s.keyTxnOps), + f: newStressTxn(s.cli, s.keyTxnSuffixRange, s.keyTxnOps), }) } s.stressTable = createStressTable(stressEntries) - for i := 0; i < s.N; i++ { - go s.run(ctx) + for i := 0; i < s.clientsN; i++ { + go s.run() } s.lg.Info( @@ -102,18 +100,18 @@ func (s *keyStresser) Stress() error { return nil } -func (s *keyStresser) run(ctx context.Context) { +func (s *keyStresser) run() { defer s.wg.Done() for { - if err := s.rateLimiter.Wait(ctx); err == context.Canceled { + if err := s.rateLimiter.Wait(s.ctx); err == context.Canceled { return } // TODO: 10-second is enough timeout to cover leader failure // and immediate leader election. Find out what other cases this // could be timed out. - sctx, scancel := context.WithTimeout(ctx, 10*time.Second) + sctx, scancel := context.WithTimeout(s.ctx, 10*time.Second) err, modifiedKeys := s.stressTable.choose()(sctx) scancel() if err == nil { diff --git a/tools/functional-tester/tester/stress_lease.go b/tools/functional-tester/tester/stress_lease.go index 264f3ee99..ed65a9ec7 100644 --- a/tools/functional-tester/tester/stress_lease.go +++ b/tools/functional-tester/tester/stress_lease.go @@ -33,8 +33,8 @@ import ( const ( // time to live for lease - TTL = 120 - TTLShort = 2 + defaultTTL = 120 + defaultTTLShort = 2 ) type leaseStresser struct { @@ -201,7 +201,7 @@ func (ls *leaseStresser) createAliveLeases() { wg.Add(1) go func() { defer wg.Done() - leaseID, err := ls.createLeaseWithKeys(TTL) + leaseID, err := ls.createLeaseWithKeys(defaultTTL) if err != nil { ls.lg.Debug( "createLeaseWithKeys failed", @@ -228,7 +228,7 @@ func (ls *leaseStresser) createShortLivedLeases() { wg.Add(1) go func() { defer wg.Done() - leaseID, err := ls.createLeaseWithKeys(TTLShort) + leaseID, err := ls.createLeaseWithKeys(defaultTTLShort) if err != nil { return } @@ -323,7 +323,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { // if it is renewed, this means that invariant checking have at least ttl/2 time before lease exipres which is long enough for the checking to finish. // if it is not renewed, we remove the lease from the alive map so that the lease doesn't exipre during invariant checking renewTime, ok := ls.aliveLeases.read(leaseID) - if ok && renewTime.Add(TTL/2*time.Second).Before(time.Now()) { + if ok && renewTime.Add(defaultTTL/2*time.Second).Before(time.Now()) { ls.aliveLeases.remove(leaseID) ls.lg.Debug( "keepLeaseAlive lease has not been renewed, dropped it", diff --git a/tools/functional-tester/tester/stress_nop.go b/tools/functional-tester/tester/stress_nop.go deleted file mode 100644 index 8254a12d7..000000000 --- a/tools/functional-tester/tester/stress_nop.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tester - -import "time" - -// nopStresser implements Stresser that does nothing -type nopStresser struct { - start time.Time - qps int -} - -func (s *nopStresser) Stress() error { return nil } -func (s *nopStresser) Pause() {} -func (s *nopStresser) Close() {} -func (s *nopStresser) ModifiedKeys() int64 { - return 0 -} -func (s *nopStresser) Checker() Checker { return nil }