From 88a0f4c6b24dbe7e4ff285682ff7691edcd077a3 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 12:37:51 -0700 Subject: [PATCH 01/12] functional-tester/rpcpb: add liveness mode for failure case Signed-off-by: Gyuho Lee --- tools/functional-tester/rpcpb/rpc.pb.go | 322 +++++++++++++----------- tools/functional-tester/rpcpb/rpc.proto | 27 +- 2 files changed, 196 insertions(+), 153 deletions(-) diff --git a/tools/functional-tester/rpcpb/rpc.pb.go b/tools/functional-tester/rpcpb/rpc.pb.go index 11e9924e4..f28d0b8d3 100644 --- a/tools/functional-tester/rpcpb/rpc.pb.go +++ b/tools/functional-tester/rpcpb/rpc.pb.go @@ -104,9 +104,14 @@ const ( FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 9 FailureCase_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 10 FailureCase_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 11 - FailureCase_FAILPOINTS FailureCase = 100 - FailureCase_NO_FAIL FailureCase = 200 - FailureCase_EXTERNAL FailureCase = 300 + // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period + // while stressers are still sending requests. + FailureCase_NO_FAIL_WITH_STRESS FailureCase = 100 + // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection + // with all stressers stopped. + FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS FailureCase = 101 + FailureCase_FAILPOINTS FailureCase = 200 + FailureCase_EXTERNAL FailureCase = 300 ) var FailureCase_name = map[int32]string{ @@ -122,8 +127,9 @@ var FailureCase_name = map[int32]string{ 9: "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", 10: "DELAY_PEER_PORT_TX_RX_LEADER", 11: "DELAY_PEER_PORT_TX_RX_ALL", - 100: "FAILPOINTS", - 200: "NO_FAIL", + 100: "NO_FAIL_WITH_STRESS", + 101: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", + 200: "FAILPOINTS", 300: "EXTERNAL", } var FailureCase_value = map[string]int32{ @@ -139,8 +145,9 @@ var FailureCase_value = map[string]int32{ "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": 9, "DELAY_PEER_PORT_TX_RX_LEADER": 10, "DELAY_PEER_PORT_TX_RX_ALL": 11, - "FAILPOINTS": 100, - "NO_FAIL": 200, + "NO_FAIL_WITH_STRESS": 100, + "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": 101, + "FAILPOINTS": 200, "EXTERNAL": 300, } @@ -154,30 +161,27 @@ type StressType int32 const ( StressType_KV StressType = 0 StressType_LEASE StressType = 1 - StressType_NO_STRESS StressType = 2 - StressType_ELECTION_RUNNER StressType = 3 - StressType_WATCH_RUNNER StressType = 4 - StressType_LOCK_RACER_RUNNER StressType = 5 - StressType_LEASE_RUNNER StressType = 6 + StressType_ELECTION_RUNNER StressType = 2 + StressType_WATCH_RUNNER StressType = 3 + StressType_LOCK_RACER_RUNNER StressType = 4 + StressType_LEASE_RUNNER StressType = 5 ) var StressType_name = map[int32]string{ 0: "KV", 1: "LEASE", - 2: "NO_STRESS", - 3: "ELECTION_RUNNER", - 4: "WATCH_RUNNER", - 5: "LOCK_RACER_RUNNER", - 6: "LEASE_RUNNER", + 2: "ELECTION_RUNNER", + 3: "WATCH_RUNNER", + 4: "LOCK_RACER_RUNNER", + 5: "LEASE_RUNNER", } var StressType_value = map[string]int32{ "KV": 0, "LEASE": 1, - "NO_STRESS": 2, - "ELECTION_RUNNER": 3, - "WATCH_RUNNER": 4, - "LOCK_RACER_RUNNER": 5, - "LEASE_RUNNER": 6, + "ELECTION_RUNNER": 2, + "WATCH_RUNNER": 3, + "LOCK_RACER_RUNNER": 4, + "LEASE_RUNNER": 5, } func (x StressType) String() string { @@ -280,8 +284,11 @@ type Tester struct { StressKeySuffixRangeTxn int32 `protobuf:"varint,105,opt,name=StressKeySuffixRangeTxn,proto3" json:"StressKeySuffixRangeTxn,omitempty" yaml:"stress-key-suffix-range-txn"` // StressKeyTxnOps is the number of operations per a transaction (max 64). StressKeyTxnOps int32 `protobuf:"varint,106,opt,name=StressKeyTxnOps,proto3" json:"StressKeyTxnOps,omitempty" yaml:"stress-key-txn-ops"` + // StressClients is the number of concurrent stressing clients + // with "one" shared TCP connection. + StressClients int32 `protobuf:"varint,201,opt,name=StressClients,proto3" json:"StressClients,omitempty" yaml:"stress-clients"` // StressQPS is the maximum number of stresser requests per second. - StressQPS int32 `protobuf:"varint,107,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"` + StressQPS int32 `protobuf:"varint,202,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"` } func (m *Tester) Reset() { *m = Tester{} } @@ -871,10 +878,17 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintRpc(dAtA, i, uint64(m.StressKeyTxnOps)) } - if m.StressQPS != 0 { - dAtA[i] = 0xd8 + if m.StressClients != 0 { + dAtA[i] = 0xc8 i++ - dAtA[i] = 0x6 + dAtA[i] = 0xc + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressClients)) + } + if m.StressQPS != 0 { + dAtA[i] = 0xd0 + i++ + dAtA[i] = 0xc i++ i = encodeVarintRpc(dAtA, i, uint64(m.StressQPS)) } @@ -1149,6 +1163,9 @@ func (m *Tester) Size() (n int) { if m.StressKeyTxnOps != 0 { n += 2 + sovRpc(uint64(m.StressKeyTxnOps)) } + if m.StressClients != 0 { + n += 2 + sovRpc(uint64(m.StressClients)) + } if m.StressQPS != 0 { n += 2 + sovRpc(uint64(m.StressQPS)) } @@ -2397,7 +2414,26 @@ func (m *Tester) Unmarshal(dAtA []byte) error { break } } - case 107: + case 201: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressClients", wireType) + } + m.StressClients = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressClients |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 202: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field StressQPS", wireType) } @@ -2779,119 +2815,121 @@ var ( func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 1813 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x57, 0x51, 0x73, 0xda, 0xca, - 0x15, 0x36, 0xb6, 0xc1, 0x66, 0x6d, 0x6c, 0xdd, 0xb5, 0x1d, 0x2b, 0x4e, 0x62, 0x71, 0xd5, 0xde, - 0x8c, 0xeb, 0x19, 0x39, 0x6d, 0xee, 0x4c, 0x67, 0x7a, 0xe7, 0x76, 0x52, 0xc0, 0xca, 0x8d, 0x6b, - 0x05, 0xc8, 0x82, 0x93, 0xf4, 0x89, 0x0a, 0x69, 0x01, 0xd5, 0x42, 0x52, 0x56, 0x8b, 0x2f, 0xdc, - 0x87, 0xbe, 0x76, 0xfa, 0x0f, 0xda, 0xff, 0xd0, 0xfe, 0x8f, 0xb4, 0xbd, 0x0f, 0xfd, 0x05, 0xb4, - 0x4d, 0xff, 0x01, 0x2f, 0x7d, 0xed, 0xec, 0xae, 0x80, 0x15, 0xe0, 0xf6, 0x8d, 0x3d, 0xe7, 0xfb, - 0xbe, 0x3d, 0x7b, 0x74, 0xce, 0xd9, 0x05, 0xec, 0x93, 0xc8, 0x89, 0xda, 0xcf, 0x48, 0xe4, 0x5c, - 0x44, 0x24, 0xa4, 0x21, 0xcc, 0x72, 0xc3, 0x89, 0xd1, 0xf5, 0x68, 0x6f, 0xd0, 0xbe, 0x70, 0xc2, - 0xfe, 0xb3, 0x6e, 0xd8, 0x0d, 0x9f, 0x71, 0x6f, 0x7b, 0xd0, 0xe1, 0x2b, 0xbe, 0xe0, 0xbf, 0x04, - 0x4b, 0xff, 0x7e, 0x0b, 0x6c, 0x9a, 0xd4, 0x71, 0xe1, 0x0f, 0xc0, 0x66, 0xd5, 0xee, 0x63, 0x35, - 0x53, 0xcc, 0x9c, 0xe5, 0xcb, 0xfb, 0x93, 0xb1, 0xb6, 0x33, 0xb2, 0xfb, 0xfe, 0x57, 0x7a, 0x60, - 0xf7, 0xb1, 0x8e, 0xb8, 0x13, 0x1a, 0x60, 0xeb, 0xd2, 0xa6, 0xf6, 0xa5, 0x47, 0xd4, 0x75, 0x8e, - 0x3b, 0x98, 0x8c, 0xb5, 0x7d, 0x81, 0x73, 0x6d, 0x6a, 0x1b, 0xae, 0x47, 0x74, 0x34, 0xc5, 0xc0, - 0x73, 0x90, 0x7b, 0x57, 0xb2, 0x18, 0x7a, 0x83, 0xa3, 0xe1, 0x64, 0xac, 0xed, 0x09, 0xf4, 0xb7, - 0xb6, 0x2f, 0xc0, 0x09, 0x02, 0x5e, 0x01, 0xc5, 0xf2, 0x62, 0x8a, 0x83, 0x8a, 0xef, 0xe1, 0x80, - 0xde, 0x20, 0x2b, 0x56, 0x37, 0x8b, 0x1b, 0x67, 0xf9, 0xf2, 0x93, 0xc9, 0x58, 0x7b, 0x28, 0x58, - 0x3e, 0x47, 0x18, 0x0e, 0x87, 0x18, 0x03, 0xe2, 0xc7, 0x3a, 0x5a, 0xa2, 0x41, 0x04, 0x0e, 0x4a, - 0xee, 0x1d, 0x26, 0xd4, 0x8b, 0xb1, 0xa4, 0x96, 0xe5, 0x6a, 0xc5, 0xc9, 0x58, 0x7b, 0x2c, 0xd4, - 0xec, 0x29, 0x28, 0x2d, 0xb8, 0x8a, 0x0c, 0x2b, 0x60, 0x4f, 0xec, 0x53, 0xc7, 0x98, 0x70, 0xb9, - 0x1c, 0x97, 0x7b, 0x34, 0x19, 0x6b, 0xc7, 0xa9, 0xe0, 0x22, 0x8c, 0x49, 0xa2, 0xb4, 0x40, 0x81, - 0x6d, 0xa0, 0x5e, 0x05, 0x1e, 0xf5, 0x6c, 0x7f, 0xb6, 0xc5, 0x4c, 0x6e, 0x8b, 0xcb, 0x3d, 0x9d, - 0x8c, 0x35, 0x5d, 0xc8, 0x79, 0x02, 0x69, 0xcc, 0xa3, 0x94, 0x94, 0xef, 0xd5, 0x81, 0x65, 0xb0, - 0x97, 0xf8, 0x2a, 0xfe, 0x20, 0xa6, 0x98, 0xa8, 0xdb, 0x3c, 0xf7, 0x27, 0x93, 0xb1, 0xf6, 0x20, - 0xad, 0xec, 0x08, 0x80, 0x8e, 0x16, 0x18, 0x2c, 0x81, 0x69, 0x4b, 0x83, 0xda, 0x14, 0xab, 0x79, - 0x2e, 0x24, 0x25, 0x70, 0x41, 0xc8, 0x88, 0x19, 0x4c, 0x47, 0xab, 0xc8, 0xcb, 0x9a, 0xcd, 0xf0, - 0x16, 0x07, 0x2a, 0xf8, 0x7f, 0x9a, 0x94, 0xc1, 0x96, 0x34, 0x39, 0x19, 0xbe, 0x00, 0x85, 0x46, - 0x60, 0x47, 0x71, 0x2f, 0xa4, 0x95, 0x70, 0x10, 0x50, 0x75, 0xa7, 0x98, 0x39, 0xdb, 0x28, 0x3f, - 0x9c, 0x8c, 0xb5, 0x23, 0xa1, 0x16, 0x27, 0x6e, 0xc3, 0x61, 0x7e, 0x1d, 0xa5, 0xf1, 0xd0, 0x02, - 0x9f, 0xbd, 0x19, 0x84, 0xd4, 0x2e, 0xdb, 0xce, 0x2d, 0x0e, 0xdc, 0xf2, 0x88, 0xe2, 0x58, 0xdd, - 0xe5, 0x22, 0xa7, 0x93, 0xb1, 0x76, 0x22, 0x44, 0x3e, 0x30, 0x88, 0xd1, 0x16, 0x18, 0xa3, 0xcd, - 0x40, 0x3a, 0x5a, 0x26, 0xb2, 0xee, 0xa8, 0x13, 0xfc, 0x36, 0xa4, 0x58, 0x2d, 0x14, 0x33, 0x67, - 0xdb, 0x72, 0x77, 0x44, 0x04, 0x1b, 0x77, 0x21, 0xcb, 0xce, 0x14, 0x23, 0x67, 0x24, 0x24, 0x64, - 0x10, 0xd1, 0x4a, 0x0f, 0x3b, 0xb7, 0xea, 0x1e, 0xa7, 0xae, 0xca, 0x88, 0x40, 0x19, 0x0e, 0x83, - 0x49, 0x19, 0x91, 0xc8, 0xfa, 0xef, 0xb3, 0x20, 0xf7, 0x1a, 0xf7, 0xdb, 0x98, 0xc0, 0x9f, 0x83, - 0x5d, 0xd6, 0xd8, 0xe6, 0x10, 0x3b, 0x75, 0x9b, 0xf6, 0x92, 0xc6, 0x96, 0x72, 0x83, 0xa9, 0xe3, - 0x1a, 0x78, 0x88, 0x1d, 0x23, 0xb2, 0x69, 0x4f, 0x47, 0x29, 0x38, 0xfc, 0x12, 0xe4, 0x4b, 0x5d, - 0x1c, 0xd0, 0x92, 0xeb, 0x12, 0x9e, 0xd7, 0x7c, 0xf9, 0x68, 0x32, 0xd6, 0x3e, 0x4b, 0x5a, 0x87, - 0xb9, 0x0c, 0xdb, 0x75, 0x89, 0x8e, 0xe6, 0x38, 0x96, 0xcf, 0x97, 0xb6, 0xe7, 0x47, 0xa1, 0x17, - 0xd0, 0x57, 0xcd, 0x66, 0x9d, 0x93, 0x77, 0x39, 0x59, 0xca, 0x67, 0x67, 0x0a, 0x31, 0x7a, 0x94, - 0x46, 0x89, 0xca, 0x32, 0x91, 0xe5, 0xb3, 0x6c, 0xc7, 0x98, 0xcd, 0x0f, 0xbc, 0x38, 0x6d, 0xda, - 0x76, 0x8c, 0x93, 0x69, 0x93, 0x60, 0xe0, 0x57, 0x60, 0x87, 0x9d, 0xc0, 0x0a, 0xbb, 0xfc, 0xbc, - 0x1d, 0x4e, 0x51, 0x27, 0x63, 0xed, 0x50, 0x3a, 0xaf, 0x1f, 0x76, 0x93, 0xe3, 0xca, 0x60, 0x58, - 0x02, 0x05, 0xb6, 0x14, 0x0d, 0xdf, 0xb4, 0x1a, 0xea, 0x5f, 0x32, 0xfc, 0x33, 0x48, 0x5d, 0xc3, - 0xe9, 0xc9, 0xa0, 0xa0, 0xac, 0x07, 0xd3, 0x0c, 0xf8, 0x0d, 0xd8, 0x9f, 0x1b, 0xea, 0x24, 0x1c, - 0x8e, 0xd4, 0xbf, 0x0a, 0x91, 0xc7, 0x93, 0xb1, 0xa6, 0x2e, 0x8b, 0x44, 0x0c, 0xa3, 0xa3, 0x45, - 0xd6, 0x34, 0x16, 0xd6, 0xd1, 0x42, 0xe6, 0x6f, 0xab, 0x63, 0xe1, 0xe3, 0x20, 0x11, 0x49, 0x33, - 0x60, 0x1d, 0xc0, 0xb9, 0xaa, 0x19, 0xb8, 0x3c, 0xaf, 0xea, 0xf7, 0xa2, 0x04, 0xb4, 0xc9, 0x58, - 0x7b, 0xb4, 0x1c, 0x0e, 0x4e, 0x60, 0x3a, 0x5a, 0xc1, 0x85, 0x3f, 0x11, 0xd7, 0x84, 0xfa, 0x67, - 0x36, 0xf7, 0x77, 0x9e, 0xef, 0x5c, 0xf0, 0xdb, 0xe6, 0x82, 0xd9, 0xe4, 0xcb, 0x82, 0x09, 0xea, - 0x88, 0x43, 0xf5, 0xff, 0x00, 0x90, 0x6b, 0x62, 0x3e, 0x50, 0x5e, 0x80, 0x82, 0xf8, 0x55, 0xc5, - 0xf4, 0xdb, 0x90, 0xdc, 0x2e, 0x17, 0x23, 0xe5, 0x6e, 0x23, 0x10, 0x7e, 0x1d, 0xa5, 0xf1, 0xf0, - 0xa7, 0x00, 0x08, 0x03, 0xaf, 0x28, 0x71, 0xf7, 0x3c, 0x98, 0x8c, 0x35, 0x98, 0x62, 0x8b, 0x4a, - 0x92, 0x90, 0x6c, 0x6c, 0x5f, 0x62, 0xdf, 0x1e, 0x59, 0x36, 0xc5, 0x81, 0x33, 0x7a, 0x1d, 0xf3, - 0x52, 0x2e, 0xc8, 0x63, 0xdb, 0x65, 0x7e, 0xc3, 0x17, 0x00, 0xa3, 0xcf, 0xc6, 0x76, 0x9a, 0x02, - 0x7f, 0x09, 0x94, 0xb4, 0x05, 0xdd, 0xf1, 0xa2, 0x2e, 0xc8, 0x45, 0xbd, 0x28, 0x63, 0x90, 0x3b, - 0x1d, 0x2d, 0xf1, 0xd8, 0x41, 0x50, 0x38, 0x08, 0x5c, 0xcb, 0xeb, 0x7b, 0x54, 0x3d, 0x2a, 0x66, - 0xce, 0xb2, 0xf2, 0x41, 0x08, 0xf3, 0x19, 0x3e, 0x73, 0xea, 0x48, 0x42, 0xc2, 0x5f, 0x80, 0x82, - 0x39, 0xf4, 0x68, 0x2d, 0x60, 0x6d, 0x32, 0x20, 0x58, 0x7d, 0xb0, 0x54, 0x13, 0x43, 0x8f, 0x1a, - 0x61, 0x60, 0x74, 0x04, 0x80, 0xd5, 0x84, 0x4c, 0x80, 0xaf, 0x80, 0x52, 0x09, 0x83, 0x98, 0xdf, - 0x48, 0xce, 0x48, 0xcc, 0x9a, 0xe3, 0xc5, 0xfa, 0x74, 0xe6, 0x88, 0xe9, 0x9c, 0x59, 0x62, 0xc1, - 0x9f, 0x81, 0x1d, 0x33, 0xb0, 0xdb, 0x3e, 0xae, 0x47, 0x24, 0xec, 0xa8, 0x2a, 0x17, 0x39, 0x9e, - 0x8c, 0xb5, 0x83, 0x24, 0x12, 0xee, 0x34, 0x22, 0xe6, 0x65, 0x7d, 0x36, 0xc7, 0xc2, 0xaf, 0xc1, - 0x6e, 0x12, 0x4f, 0xc5, 0x8e, 0x71, 0xac, 0x6a, 0xfc, 0xd6, 0x93, 0x9a, 0x34, 0x89, 0xde, 0x70, - 0x98, 0x5b, 0x47, 0x29, 0x34, 0xbb, 0xdb, 0x92, 0x75, 0xa3, 0x37, 0xe8, 0x74, 0x7c, 0xac, 0x16, - 0x17, 0xb3, 0x30, 0xe5, 0xc7, 0x02, 0xa0, 0xa3, 0x05, 0x06, 0xbc, 0x96, 0x46, 0x54, 0x25, 0xec, - 0xf7, 0xed, 0xc0, 0x8d, 0xd5, 0xcf, 0x17, 0x1f, 0x1a, 0xf3, 0x11, 0xe5, 0x24, 0x18, 0x79, 0x42, - 0x4d, 0x79, 0xac, 0xbc, 0xd0, 0x20, 0x08, 0x30, 0x99, 0x4d, 0xd9, 0x1f, 0xf1, 0xd2, 0x94, 0xca, - 0x8b, 0x70, 0xbf, 0x3c, 0x67, 0x17, 0x28, 0xec, 0xe5, 0x63, 0x0e, 0x29, 0x26, 0x81, 0xed, 0xcf, - 0x64, 0xce, 0xb9, 0x8c, 0x14, 0x10, 0x4e, 0x10, 0xb2, 0xd0, 0x12, 0x8d, 0x7d, 0x99, 0x06, 0x25, - 0x38, 0x8e, 0x9b, 0xa3, 0x08, 0xc7, 0x2a, 0xe6, 0xc7, 0x92, 0xbe, 0x4c, 0xcc, 0x9d, 0x06, 0x65, - 0x5e, 0x1d, 0xc9, 0x58, 0x56, 0x60, 0x62, 0x79, 0x8d, 0x47, 0x0d, 0xef, 0x3b, 0xcc, 0xe7, 0x67, - 0x56, 0x4e, 0x6d, 0x42, 0xbe, 0xc5, 0x23, 0x23, 0xf6, 0xbe, 0x63, 0x05, 0x96, 0x22, 0xb0, 0xa1, - 0x93, 0x32, 0x58, 0x36, 0xe9, 0x62, 0xb5, 0xcb, 0x65, 0xa4, 0xeb, 0x6c, 0x41, 0xc6, 0xf0, 0x19, - 0x4c, 0x47, 0x2b, 0xb8, 0xf0, 0x2d, 0x38, 0x9c, 0x5b, 0x07, 0x9d, 0x8e, 0x37, 0x44, 0x76, 0xd0, - 0xc5, 0x6a, 0x8f, 0x6b, 0xea, 0x93, 0xb1, 0x76, 0xba, 0xac, 0xc9, 0x71, 0x06, 0x61, 0x40, 0x1d, - 0xad, 0xe4, 0xc3, 0x5f, 0x83, 0xe3, 0x55, 0xf6, 0xe6, 0x30, 0x50, 0x3d, 0x2e, 0x2d, 0x3d, 0xc3, - 0xee, 0x91, 0x36, 0xe8, 0x30, 0xd0, 0xd1, 0x7d, 0x32, 0xec, 0x32, 0x98, 0xb9, 0x9a, 0xc3, 0xa0, - 0x16, 0xc5, 0xea, 0x6f, 0xb8, 0xb2, 0xf4, 0x49, 0x25, 0x65, 0x3a, 0x0c, 0x8c, 0x30, 0x8a, 0x75, - 0xb4, 0xc8, 0x62, 0xd7, 0xb0, 0x30, 0xbd, 0xa9, 0x37, 0xd4, 0x5b, 0x2e, 0x21, 0x5d, 0xc3, 0x89, - 0xc4, 0x07, 0x46, 0x9d, 0xe3, 0xf4, 0xdf, 0x65, 0xc0, 0x16, 0xc2, 0x1f, 0x06, 0x38, 0xa6, 0xf0, - 0x02, 0xe4, 0x6b, 0x11, 0x26, 0x36, 0xf5, 0xc2, 0x80, 0x8f, 0xdd, 0xbd, 0xe7, 0x4a, 0x32, 0xbc, - 0x67, 0x76, 0x34, 0x87, 0xc0, 0x2f, 0xa6, 0x0f, 0x08, 0x55, 0x4c, 0xfa, 0x42, 0x02, 0x16, 0x46, - 0x34, 0x7d, 0x5d, 0x7c, 0x31, 0x9d, 0xed, 0xfc, 0x69, 0x3f, 0x87, 0x09, 0x23, 0x4a, 0x9c, 0xfa, - 0xd7, 0x60, 0x1b, 0xe1, 0x38, 0x0a, 0x83, 0x18, 0x43, 0x15, 0x6c, 0x35, 0x06, 0x8e, 0x83, 0xe3, - 0x98, 0xc7, 0xb1, 0x8d, 0xa6, 0x4b, 0xf8, 0x00, 0xe4, 0xd8, 0x23, 0x71, 0x10, 0x8b, 0xc9, 0x8e, - 0x92, 0xd5, 0xf9, 0x3f, 0x32, 0x52, 0xf0, 0x70, 0x0f, 0x80, 0x6a, 0x48, 0x1b, 0xd4, 0x26, 0x14, - 0xbb, 0xca, 0x1a, 0x3c, 0x04, 0x4a, 0xf2, 0x04, 0xe2, 0x36, 0x76, 0xe7, 0x28, 0x19, 0xb8, 0x0f, - 0x76, 0x10, 0x8e, 0x67, 0x86, 0x75, 0xb8, 0x0b, 0xb6, 0xaf, 0x3d, 0xdf, 0xe7, 0xab, 0x0d, 0xe6, - 0x66, 0x6d, 0x5c, 0x22, 0x4e, 0xcf, 0xbb, 0xc3, 0xca, 0x26, 0x53, 0xb9, 0xc4, 0x31, 0x25, 0xe1, - 0x88, 0x21, 0xf8, 0x53, 0x46, 0xc9, 0xc2, 0x87, 0xe0, 0xa8, 0xec, 0xdb, 0xce, 0x6d, 0x2f, 0xf4, - 0xf9, 0xd3, 0xba, 0x1e, 0x12, 0xda, 0x1c, 0xa2, 0xa1, 0xe2, 0xc2, 0x47, 0xe0, 0xf8, 0x26, 0x68, - 0xaf, 0x74, 0x62, 0x78, 0x04, 0x3e, 0xe3, 0x23, 0x3f, 0x65, 0xee, 0xc0, 0x63, 0x70, 0x70, 0x13, - 0xb8, 0x4b, 0x8e, 0xee, 0xf9, 0x1f, 0x37, 0x44, 0x3c, 0xc9, 0x88, 0x63, 0xfc, 0xeb, 0x2b, 0xcb, - 0x6a, 0xd5, 0xaa, 0x66, 0xeb, 0x65, 0xcd, 0xb2, 0x6a, 0xef, 0x4c, 0xa4, 0xac, 0xb1, 0xa8, 0xb9, - 0xd9, 0x32, 0x4b, 0x97, 0x26, 0x52, 0x32, 0xf0, 0x14, 0x9c, 0x2c, 0xe1, 0x5a, 0x2f, 0x6b, 0xa8, - 0x65, 0xd5, 0xaa, 0xdf, 0x28, 0xeb, 0x50, 0x05, 0x87, 0x12, 0x61, 0xee, 0xd9, 0x98, 0x49, 0xbd, - 0xb9, 0xa9, 0xa1, 0x9b, 0xd7, 0xca, 0x26, 0xcf, 0x0f, 0x33, 0x94, 0x2c, 0x4b, 0xc9, 0xc2, 0x73, - 0xf0, 0xb4, 0x6c, 0x95, 0x2a, 0xd7, 0xaf, 0x6a, 0x96, 0xd9, 0xaa, 0x9b, 0x26, 0x6a, 0xd5, 0x6b, - 0xa8, 0xd9, 0x6a, 0xbe, 0x6f, 0xa1, 0xf7, 0xe9, 0xa8, 0x72, 0xf0, 0x87, 0xa0, 0x78, 0x3f, 0x36, - 0x09, 0x75, 0x0b, 0x7e, 0x0e, 0x9e, 0xdc, 0x8f, 0x62, 0x9b, 0x6e, 0xc3, 0xa7, 0x40, 0xbf, 0x34, - 0xad, 0xd2, 0xaf, 0xfe, 0xf7, 0x86, 0x79, 0x58, 0x04, 0x8f, 0x57, 0xe3, 0x92, 0xcd, 0x00, 0x7c, - 0x02, 0x1e, 0xae, 0x46, 0xb0, 0x8d, 0x76, 0x58, 0x09, 0xbd, 0x2c, 0x5d, 0x59, 0xf5, 0xda, 0x55, - 0xb5, 0xd9, 0x50, 0x5c, 0xb8, 0x0b, 0xb6, 0xaa, 0xb5, 0x16, 0x33, 0x29, 0x1f, 0x33, 0xb0, 0x00, - 0xb6, 0xcd, 0xf7, 0x4d, 0x13, 0x55, 0x4b, 0x96, 0xf2, 0xa7, 0xf5, 0xf3, 0xdf, 0x02, 0x30, 0x1f, - 0x90, 0x30, 0x07, 0xd6, 0xaf, 0xdf, 0x2a, 0x6b, 0x30, 0x0f, 0xb2, 0x96, 0x59, 0x6a, 0x98, 0x0a, - 0xc3, 0xe7, 0xab, 0xb5, 0x56, 0xa3, 0x89, 0xcc, 0x46, 0x43, 0x59, 0x87, 0x07, 0x60, 0xdf, 0xb4, - 0xcc, 0x4a, 0xf3, 0xaa, 0x56, 0x6d, 0xa1, 0x9b, 0x6a, 0xd5, 0x44, 0xca, 0x06, 0x54, 0xc0, 0xee, - 0xbb, 0x52, 0xb3, 0xf2, 0x6a, 0x6a, 0xd9, 0x64, 0x9f, 0xd8, 0xaa, 0x55, 0xae, 0x5b, 0xa8, 0x54, - 0x31, 0xd1, 0xd4, 0x9c, 0x65, 0x40, 0xae, 0x3b, 0xb5, 0xe4, 0x9e, 0xbf, 0x00, 0xf9, 0x26, 0xb1, - 0x83, 0x38, 0x0a, 0x09, 0x85, 0xcf, 0xe5, 0xc5, 0x5e, 0xd2, 0x6c, 0x49, 0x8f, 0x9f, 0xec, 0xcf, - 0xd6, 0xa2, 0xd5, 0xf4, 0xb5, 0xb3, 0xcc, 0x8f, 0x33, 0xe5, 0xc3, 0x8f, 0xff, 0x3a, 0x5d, 0xfb, - 0xf8, 0xe9, 0x34, 0xf3, 0xf7, 0x4f, 0xa7, 0x99, 0x7f, 0x7e, 0x3a, 0xcd, 0xfc, 0xe1, 0xdf, 0xa7, - 0x6b, 0xed, 0x1c, 0xff, 0xe3, 0xff, 0xe5, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc2, 0x76, 0x9b, - 0x6a, 0x41, 0x10, 0x00, 0x00, + // 1851 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x58, 0xff, 0x72, 0xdb, 0x4a, + 0x15, 0x8e, 0x93, 0x38, 0x89, 0x37, 0xbf, 0xd4, 0x4d, 0xd2, 0xa8, 0x69, 0x1b, 0xe5, 0xea, 0x72, + 0x4b, 0xc8, 0x8c, 0x52, 0xe8, 0x65, 0x98, 0xe1, 0xce, 0x65, 0x8a, 0xed, 0xa8, 0x37, 0x21, 0xaa, + 0xed, 0xae, 0x95, 0xb6, 0xfc, 0x65, 0x64, 0x69, 0x6d, 0x8b, 0xc8, 0x92, 0xba, 0x5a, 0xe7, 0xda, + 0xf7, 0x05, 0x18, 0xde, 0x80, 0x87, 0x80, 0x17, 0xe0, 0x09, 0x0a, 0xdc, 0x61, 0x78, 0x02, 0x03, + 0xe5, 0x0d, 0xfc, 0x04, 0xcc, 0xee, 0xca, 0xf6, 0xca, 0x76, 0xe0, 0x3f, 0xef, 0x39, 0xdf, 0xf7, + 0xe9, 0xe8, 0xe8, 0xfc, 0xd8, 0x04, 0xec, 0x92, 0xd8, 0x8d, 0x9b, 0xcf, 0x49, 0xec, 0x9e, 0xc7, + 0x24, 0xa2, 0x11, 0xcc, 0x73, 0xc3, 0x91, 0xd1, 0xf6, 0x69, 0xa7, 0xd7, 0x3c, 0x77, 0xa3, 0xee, + 0xf3, 0x76, 0xd4, 0x8e, 0x9e, 0x73, 0x6f, 0xb3, 0xd7, 0xe2, 0x27, 0x7e, 0xe0, 0xbf, 0x04, 0x4b, + 0xff, 0x7e, 0x1d, 0xac, 0x9a, 0xd4, 0xf5, 0xe0, 0xe7, 0x60, 0xb5, 0xe2, 0x74, 0xb1, 0x9a, 0x3b, + 0xc9, 0x9d, 0x16, 0x4a, 0xbb, 0xa3, 0xa1, 0xb6, 0x39, 0x70, 0xba, 0xc1, 0x57, 0x7a, 0xe8, 0x74, + 0xb1, 0x8e, 0xb8, 0x13, 0x1a, 0x60, 0xfd, 0xc2, 0xa1, 0xce, 0x85, 0x4f, 0xd4, 0x65, 0x8e, 0xdb, + 0x1b, 0x0d, 0xb5, 0x5d, 0x81, 0xf3, 0x1c, 0xea, 0x18, 0x9e, 0x4f, 0x74, 0x34, 0xc6, 0xc0, 0x33, + 0xb0, 0xf6, 0xae, 0x68, 0x31, 0xf4, 0x0a, 0x47, 0xc3, 0xd1, 0x50, 0xdb, 0x11, 0xe8, 0x6f, 0x9d, + 0x40, 0x80, 0x53, 0x04, 0xbc, 0x02, 0x8a, 0xe5, 0x27, 0x14, 0x87, 0xe5, 0xc0, 0xc7, 0x21, 0xbd, + 0x41, 0x56, 0xa2, 0xae, 0x9e, 0xac, 0x9c, 0x16, 0x4a, 0x4f, 0x47, 0x43, 0xed, 0x91, 0x60, 0x05, + 0x1c, 0x61, 0xb8, 0x1c, 0x62, 0xf4, 0x48, 0x90, 0xe8, 0x68, 0x8e, 0x06, 0x11, 0xd8, 0x2b, 0x7a, + 0x77, 0x98, 0x50, 0x3f, 0xc1, 0x92, 0x5a, 0x9e, 0xab, 0x9d, 0x8c, 0x86, 0xda, 0x13, 0xa1, 0xe6, + 0x8c, 0x41, 0x59, 0xc1, 0x45, 0x64, 0x58, 0x06, 0x3b, 0xe2, 0x39, 0x35, 0x8c, 0x09, 0x97, 0x5b, + 0xe3, 0x72, 0x8f, 0x47, 0x43, 0xed, 0x30, 0x13, 0x5c, 0x8c, 0x31, 0x49, 0x95, 0x66, 0x28, 0xb0, + 0x09, 0xd4, 0xab, 0xd0, 0xa7, 0xbe, 0x13, 0x4c, 0x1e, 0x31, 0x91, 0x5b, 0xe7, 0x72, 0xcf, 0x46, + 0x43, 0x4d, 0x17, 0x72, 0xbe, 0x40, 0x1a, 0xd3, 0x28, 0x25, 0xe5, 0x7b, 0x75, 0x60, 0x09, 0xec, + 0xa4, 0xbe, 0x72, 0xd0, 0x4b, 0x28, 0x26, 0xea, 0x06, 0xcf, 0xfd, 0xd1, 0x68, 0xa8, 0x3d, 0xcc, + 0x2a, 0xbb, 0x02, 0xa0, 0xa3, 0x19, 0x06, 0x4b, 0x60, 0xd6, 0x52, 0xa7, 0x0e, 0xc5, 0x6a, 0x81, + 0x0b, 0x49, 0x09, 0x9c, 0x11, 0x32, 0x12, 0x06, 0xd3, 0xd1, 0x22, 0xf2, 0xbc, 0xa6, 0x1d, 0xdd, + 0xe2, 0x50, 0x05, 0xff, 0x4f, 0x93, 0x32, 0xd8, 0x9c, 0x26, 0x27, 0xc3, 0x97, 0x60, 0xbb, 0x1e, + 0x3a, 0x71, 0xd2, 0x89, 0x68, 0x39, 0xea, 0x85, 0x54, 0xdd, 0x3c, 0xc9, 0x9d, 0xae, 0x94, 0x1e, + 0x8d, 0x86, 0xda, 0x81, 0x50, 0x4b, 0x52, 0xb7, 0xe1, 0x32, 0xbf, 0x8e, 0xb2, 0x78, 0x68, 0x81, + 0x07, 0x6f, 0x7a, 0x11, 0x75, 0x4a, 0x8e, 0x7b, 0x8b, 0x43, 0xaf, 0x34, 0xa0, 0x38, 0x51, 0xb7, + 0xb8, 0xc8, 0xf1, 0x68, 0xa8, 0x1d, 0x09, 0x91, 0x0f, 0x0c, 0x62, 0x34, 0x05, 0xc6, 0x68, 0x32, + 0x90, 0x8e, 0xe6, 0x89, 0xac, 0x3b, 0x6a, 0x04, 0xbf, 0x8d, 0x28, 0x56, 0xb7, 0x4f, 0x72, 0xa7, + 0x1b, 0x72, 0x77, 0xc4, 0x04, 0x1b, 0x77, 0x11, 0xcb, 0xce, 0x18, 0x23, 0x67, 0x24, 0x22, 0xa4, + 0x17, 0xd3, 0x72, 0x07, 0xbb, 0xb7, 0xea, 0x0e, 0xa7, 0x2e, 0xca, 0x88, 0x40, 0x19, 0x2e, 0x83, + 0x49, 0x19, 0x91, 0xc8, 0xfa, 0xef, 0xf3, 0x60, 0xed, 0x35, 0xee, 0x36, 0x31, 0x81, 0xbf, 0x00, + 0x5b, 0xac, 0xb1, 0xcd, 0x3e, 0x76, 0x6b, 0x0e, 0xed, 0xa4, 0x8d, 0x2d, 0xe5, 0x06, 0x53, 0xd7, + 0x33, 0x70, 0x1f, 0xbb, 0x46, 0xec, 0xd0, 0x8e, 0x8e, 0x32, 0x70, 0xf8, 0x25, 0x28, 0x14, 0xdb, + 0x38, 0xa4, 0x45, 0xcf, 0x23, 0x3c, 0xaf, 0x85, 0xd2, 0xc1, 0x68, 0xa8, 0x3d, 0x48, 0x5b, 0x87, + 0xb9, 0x0c, 0xc7, 0xf3, 0x88, 0x8e, 0xa6, 0x38, 0x96, 0xcf, 0x57, 0x8e, 0x1f, 0xc4, 0x91, 0x1f, + 0xd2, 0x4b, 0xdb, 0xae, 0x71, 0xf2, 0x16, 0x27, 0x4b, 0xf9, 0x6c, 0x8d, 0x21, 0x46, 0x87, 0xd2, + 0x38, 0x55, 0x99, 0x27, 0xb2, 0x7c, 0x96, 0x9c, 0x04, 0xb3, 0xf9, 0x81, 0x67, 0xa7, 0x4d, 0xd3, + 0x49, 0x70, 0x3a, 0x6d, 0x52, 0x0c, 0xfc, 0x0a, 0x6c, 0xb2, 0x37, 0xb0, 0xa2, 0x36, 0x7f, 0xdf, + 0x16, 0xa7, 0xa8, 0xa3, 0xa1, 0xb6, 0x2f, 0xbd, 0x6f, 0x10, 0xb5, 0xd3, 0xd7, 0x95, 0xc1, 0xb0, + 0x08, 0xb6, 0xd9, 0x51, 0x34, 0xbc, 0x6d, 0xd5, 0xd5, 0xbf, 0xe4, 0xf8, 0x67, 0x90, 0xba, 0x86, + 0xd3, 0xd3, 0x41, 0x41, 0x59, 0x0f, 0x66, 0x19, 0xf0, 0x1b, 0xb0, 0x3b, 0x35, 0xd4, 0x48, 0xd4, + 0x1f, 0xa8, 0x7f, 0x15, 0x22, 0x4f, 0x46, 0x43, 0x4d, 0x9d, 0x17, 0x89, 0x19, 0x46, 0x47, 0xb3, + 0xac, 0x71, 0x2c, 0xac, 0xa3, 0x85, 0xcc, 0xdf, 0x16, 0xc7, 0xc2, 0xc7, 0x41, 0x2a, 0x92, 0x65, + 0xc0, 0x1a, 0x80, 0x53, 0x55, 0x33, 0xf4, 0x78, 0x5e, 0xd5, 0xef, 0x45, 0x09, 0x68, 0xa3, 0xa1, + 0xf6, 0x78, 0x3e, 0x1c, 0x9c, 0xc2, 0x74, 0xb4, 0x80, 0x0b, 0x7f, 0x22, 0xd6, 0x84, 0xfa, 0x27, + 0x36, 0xf7, 0x37, 0x5f, 0x6c, 0x9e, 0xf3, 0x6d, 0x73, 0xce, 0x6c, 0xf2, 0xb2, 0x60, 0x82, 0x3a, + 0xe2, 0x50, 0xfd, 0xcf, 0x9b, 0x60, 0xcd, 0xc6, 0x7c, 0xa0, 0xbc, 0x04, 0xdb, 0xe2, 0x57, 0x05, + 0xd3, 0x6f, 0x23, 0x72, 0x3b, 0x5f, 0x8c, 0x94, 0xbb, 0x8d, 0x50, 0xf8, 0x75, 0x94, 0xc5, 0xc3, + 0x9f, 0x01, 0x20, 0x0c, 0xbc, 0xa2, 0xc4, 0xee, 0x79, 0x38, 0x1a, 0x6a, 0x30, 0xc3, 0x16, 0x95, + 0x24, 0x21, 0xd9, 0xd8, 0xbe, 0xc0, 0x81, 0x33, 0xb0, 0x1c, 0x8a, 0x43, 0x77, 0xf0, 0x3a, 0xe1, + 0xa5, 0xbc, 0x2d, 0x8f, 0x6d, 0x8f, 0xf9, 0x8d, 0x40, 0x00, 0x8c, 0x2e, 0x1b, 0xdb, 0x59, 0x0a, + 0xfc, 0x15, 0x50, 0xb2, 0x16, 0x74, 0xc7, 0x8b, 0x7a, 0x5b, 0x2e, 0xea, 0x59, 0x19, 0x83, 0xdc, + 0xe9, 0x68, 0x8e, 0xc7, 0x5e, 0x04, 0x45, 0xbd, 0xd0, 0xb3, 0xfc, 0xae, 0x4f, 0xd5, 0x83, 0x93, + 0xdc, 0x69, 0x5e, 0x7e, 0x11, 0xc2, 0x7c, 0x46, 0xc0, 0x9c, 0x3a, 0x92, 0x90, 0xf0, 0x97, 0x60, + 0xdb, 0xec, 0xfb, 0xb4, 0x1a, 0xb2, 0x36, 0xe9, 0x11, 0xac, 0x3e, 0x9c, 0xab, 0x89, 0xbe, 0x4f, + 0x8d, 0x28, 0x34, 0x5a, 0x02, 0xc0, 0x6a, 0x42, 0x26, 0xc0, 0x4b, 0xa0, 0x94, 0xa3, 0x30, 0xe1, + 0x1b, 0xc9, 0x1d, 0x88, 0x59, 0x73, 0x38, 0x5b, 0x9f, 0xee, 0x14, 0x31, 0x9e, 0x33, 0x73, 0x2c, + 0xf8, 0x73, 0xb0, 0x69, 0x86, 0x4e, 0x33, 0xc0, 0xb5, 0x98, 0x44, 0x2d, 0x55, 0xe5, 0x22, 0x87, + 0xa3, 0xa1, 0xb6, 0x97, 0x46, 0xc2, 0x9d, 0x46, 0xcc, 0xbc, 0xac, 0xcf, 0xa6, 0x58, 0xf8, 0x35, + 0xd8, 0x4a, 0xe3, 0x29, 0x3b, 0x09, 0x4e, 0x54, 0x8d, 0x6f, 0x3d, 0xa9, 0x49, 0xd3, 0xe8, 0x0d, + 0x97, 0xb9, 0x75, 0x94, 0x41, 0xb3, 0xdd, 0x96, 0x9e, 0xeb, 0x9d, 0x5e, 0xab, 0x15, 0x60, 0xf5, + 0x64, 0x36, 0x0b, 0x63, 0x7e, 0x22, 0x00, 0x3a, 0x9a, 0x61, 0xc0, 0x6b, 0x69, 0x44, 0x95, 0xa3, + 0x6e, 0xd7, 0x09, 0xbd, 0x44, 0xfd, 0x6c, 0xf6, 0xa2, 0x31, 0x1d, 0x51, 0x6e, 0x8a, 0x91, 0x27, + 0xd4, 0x98, 0xc7, 0xca, 0x0b, 0xf5, 0xc2, 0x10, 0x93, 0xc9, 0x94, 0xfd, 0x11, 0x2f, 0x4d, 0xa9, + 0xbc, 0x08, 0xf7, 0xcb, 0x73, 0x76, 0x86, 0xc2, 0x6e, 0x3e, 0x66, 0x9f, 0x62, 0x12, 0x3a, 0xc1, + 0x44, 0xe6, 0x8c, 0xcb, 0x48, 0x01, 0xe1, 0x14, 0x21, 0x0b, 0xcd, 0xd1, 0xd8, 0x97, 0xa9, 0x53, + 0x82, 0x93, 0xc4, 0x1e, 0xc4, 0x38, 0x51, 0x31, 0x7f, 0x2d, 0xe9, 0xcb, 0x24, 0xdc, 0x69, 0x50, + 0xe6, 0xd5, 0x91, 0x8c, 0x65, 0x05, 0x26, 0x8e, 0xd7, 0x78, 0x50, 0xf7, 0xbf, 0xc3, 0x7c, 0x7e, + 0xe6, 0xe5, 0xd4, 0xa6, 0xe4, 0x5b, 0x3c, 0x30, 0x12, 0xff, 0x3b, 0x56, 0x60, 0x19, 0x02, 0x1b, + 0x3a, 0x19, 0x83, 0xe5, 0x90, 0x36, 0x56, 0xdb, 0x5c, 0x46, 0x5a, 0x67, 0x33, 0x32, 0x46, 0xc0, + 0x60, 0x3a, 0x5a, 0xc0, 0x85, 0x6f, 0xc1, 0xfe, 0xd4, 0xda, 0x6b, 0xb5, 0xfc, 0x3e, 0x72, 0xc2, + 0x36, 0x56, 0x3b, 0x5c, 0x53, 0x1f, 0x0d, 0xb5, 0xe3, 0x79, 0x4d, 0x8e, 0x33, 0x08, 0x03, 0xea, + 0x68, 0x21, 0x1f, 0xfe, 0x06, 0x1c, 0x2e, 0xb2, 0xdb, 0xfd, 0x50, 0xf5, 0xb9, 0xb4, 0x74, 0x0d, + 0xbb, 0x47, 0xda, 0xa0, 0xfd, 0x50, 0x47, 0xf7, 0xc9, 0xb0, 0x65, 0x30, 0x71, 0xd9, 0xfd, 0xb0, + 0x1a, 0x27, 0xea, 0x6f, 0xb9, 0xb2, 0xf4, 0x49, 0x25, 0x65, 0xda, 0x0f, 0x8d, 0x28, 0x4e, 0x74, + 0x34, 0xcb, 0x9a, 0x7e, 0x16, 0x31, 0x8f, 0x13, 0xb1, 0x98, 0xf2, 0x99, 0x3b, 0x8e, 0xd0, 0x11, + 0x63, 0x3c, 0x99, 0x7c, 0x96, 0x94, 0x00, 0x7f, 0x0a, 0x0a, 0xc2, 0xf0, 0xa6, 0x56, 0x17, 0x1b, + 0x29, 0x2f, 0x6f, 0xf2, 0x94, 0xfd, 0x81, 0x3d, 0x7d, 0x0a, 0xd4, 0x7f, 0x97, 0x03, 0xeb, 0x08, + 0x7f, 0xe8, 0xe1, 0x84, 0xc2, 0x73, 0x50, 0xa8, 0xc6, 0x98, 0x38, 0xd4, 0x8f, 0x42, 0x3e, 0xb9, + 0x77, 0x5e, 0x28, 0xe9, 0xfc, 0x9f, 0xd8, 0xd1, 0x14, 0x02, 0xbf, 0x18, 0xdf, 0x41, 0x54, 0xb1, + 0x2c, 0xb6, 0x53, 0xb0, 0x30, 0xa2, 0xf1, 0x05, 0xe5, 0x8b, 0xf1, 0x7a, 0xe0, 0x7f, 0x1d, 0x4c, + 0x61, 0xc2, 0x88, 0x52, 0xa7, 0xfe, 0x35, 0xd8, 0x40, 0x38, 0x89, 0xa3, 0x30, 0xc1, 0x50, 0x05, + 0xeb, 0xf5, 0x9e, 0xeb, 0xe2, 0x24, 0xe1, 0x71, 0x6c, 0xa0, 0xf1, 0x11, 0x3e, 0x04, 0x6b, 0xec, + 0x9e, 0xd9, 0x4b, 0xc4, 0x72, 0x40, 0xe9, 0xe9, 0xec, 0x9f, 0x39, 0x29, 0x78, 0xb8, 0x03, 0x40, + 0x25, 0xa2, 0x75, 0xea, 0x10, 0x8a, 0x3d, 0x65, 0x09, 0xee, 0x03, 0x25, 0xbd, 0x45, 0x71, 0x1b, + 0x5b, 0x5b, 0x4a, 0x0e, 0xee, 0x82, 0x4d, 0x84, 0x93, 0x89, 0x61, 0x19, 0x6e, 0x81, 0x8d, 0x6b, + 0x3f, 0x08, 0xf8, 0x69, 0x85, 0xb9, 0xd9, 0x24, 0x28, 0x12, 0xb7, 0xe3, 0xdf, 0x61, 0x65, 0x95, + 0xa9, 0x5c, 0xe0, 0x84, 0x92, 0x68, 0xc0, 0x10, 0xfc, 0x36, 0xa4, 0xe4, 0xe1, 0x23, 0x70, 0x50, + 0x0a, 0x1c, 0xf7, 0xb6, 0x13, 0x05, 0xfc, 0x76, 0x5e, 0x8b, 0x08, 0xb5, 0xfb, 0xa8, 0xaf, 0x78, + 0xf0, 0x31, 0x38, 0xbc, 0x09, 0x9b, 0x0b, 0x9d, 0x18, 0x1e, 0x80, 0x07, 0x7c, 0x6b, 0x64, 0xcc, + 0x2d, 0x78, 0x08, 0xf6, 0x6e, 0x42, 0x6f, 0xce, 0xd1, 0x3e, 0xfb, 0xfb, 0x8a, 0x88, 0x27, 0x9d, + 0x92, 0x8c, 0x7f, 0x7d, 0x65, 0x59, 0x8d, 0x6a, 0xc5, 0x6c, 0xbc, 0xaa, 0x5a, 0x56, 0xf5, 0x9d, + 0x89, 0x94, 0x25, 0x16, 0x35, 0x37, 0x5b, 0x66, 0xf1, 0xc2, 0x44, 0x4a, 0x0e, 0x1e, 0x83, 0xa3, + 0x39, 0x5c, 0xe3, 0x55, 0x15, 0x35, 0xac, 0x6a, 0xe5, 0x1b, 0x65, 0x19, 0xaa, 0x60, 0x5f, 0x22, + 0x4c, 0x3d, 0x2b, 0x13, 0xa9, 0x37, 0x37, 0x55, 0x74, 0xf3, 0x5a, 0x59, 0xe5, 0xf9, 0x61, 0x86, + 0xa2, 0x65, 0x29, 0x79, 0x78, 0x06, 0x9e, 0x95, 0xac, 0x62, 0xf9, 0xfa, 0xb2, 0x6a, 0x99, 0x8d, + 0x9a, 0x69, 0xa2, 0x46, 0xad, 0x8a, 0xec, 0x86, 0xfd, 0xbe, 0x81, 0xde, 0x67, 0xa3, 0x5a, 0x83, + 0x3f, 0x00, 0x27, 0xf7, 0x63, 0xd3, 0x50, 0xd7, 0xe1, 0x67, 0xe0, 0xe9, 0xfd, 0x28, 0xf6, 0xd0, + 0x0d, 0xf8, 0x0c, 0xe8, 0x17, 0xa6, 0x55, 0xfc, 0xf5, 0xff, 0x7e, 0x60, 0x01, 0x9e, 0x80, 0x27, + 0x8b, 0x71, 0xe9, 0xc3, 0x00, 0x7c, 0x0a, 0x1e, 0x2d, 0x46, 0xb0, 0x07, 0x6d, 0xb2, 0xef, 0x50, + 0xa9, 0x36, 0x5e, 0x15, 0xaf, 0xac, 0xc6, 0xbb, 0x2b, 0xfb, 0xb2, 0x51, 0xb7, 0x91, 0x59, 0xaf, + 0x2b, 0x1e, 0xfc, 0x21, 0xf8, 0x3c, 0xe3, 0xa8, 0x54, 0x53, 0x9f, 0x48, 0xdd, 0xd5, 0x5b, 0xb3, + 0xc2, 0x80, 0x18, 0xee, 0x02, 0xc0, 0x50, 0xb5, 0xea, 0x55, 0xc5, 0xae, 0x2b, 0x1f, 0x73, 0x70, + 0x1b, 0x6c, 0x98, 0xef, 0x6d, 0x13, 0x55, 0x8a, 0x96, 0xf2, 0xc7, 0xe5, 0xb3, 0x08, 0x80, 0xe9, + 0x60, 0x86, 0x6b, 0x60, 0xf9, 0xfa, 0xad, 0xb2, 0x04, 0x0b, 0x20, 0x6f, 0x99, 0xc5, 0xba, 0xa9, + 0xe4, 0xe0, 0x1e, 0xd8, 0x35, 0x2d, 0xb3, 0x6c, 0x5f, 0x55, 0x2b, 0x0d, 0x74, 0x53, 0xa9, 0x98, + 0x48, 0x59, 0x86, 0x0a, 0xd8, 0x7a, 0x57, 0xb4, 0xcb, 0x97, 0x63, 0xcb, 0x0a, 0x2b, 0x04, 0xab, + 0x5a, 0xbe, 0x6e, 0xa0, 0x62, 0xd9, 0x44, 0x63, 0xf3, 0x2a, 0x03, 0x72, 0xa1, 0xb1, 0x25, 0xff, + 0xe2, 0x25, 0x28, 0xd8, 0xc4, 0x09, 0x93, 0x38, 0x22, 0x14, 0xbe, 0x90, 0x0f, 0x3b, 0x69, 0x4b, + 0xa6, 0x93, 0xe0, 0x68, 0x77, 0x72, 0x16, 0x0d, 0xa9, 0x2f, 0x9d, 0xe6, 0x7e, 0x9c, 0x2b, 0xed, + 0x7f, 0xfc, 0xf7, 0xf1, 0xd2, 0xc7, 0x4f, 0xc7, 0xb9, 0x7f, 0x7c, 0x3a, 0xce, 0xfd, 0xeb, 0xd3, + 0x71, 0xee, 0x0f, 0xff, 0x39, 0x5e, 0x6a, 0xae, 0xf1, 0xff, 0x30, 0x7c, 0xf9, 0xdf, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x71, 0x18, 0x41, 0x02, 0xaa, 0x10, 0x00, 0x00, } diff --git a/tools/functional-tester/rpcpb/rpc.proto b/tools/functional-tester/rpcpb/rpc.proto index dcb9aef37..80d89c58e 100644 --- a/tools/functional-tester/rpcpb/rpc.proto +++ b/tools/functional-tester/rpcpb/rpc.proto @@ -100,23 +100,24 @@ enum FailureCase { DELAY_PEER_PORT_TX_RX_LEADER = 10; DELAY_PEER_PORT_TX_RX_ALL = 11; - FAILPOINTS = 100; - - NO_FAIL = 200; - // TODO: support no-op of liveness duration - // NO_FAIL_LIVENESS = 201; + // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period + // while stressers are still sending requests. + NO_FAIL_WITH_STRESS = 100; + // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection + // with all stressers stopped. + NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS = 101; + FAILPOINTS = 200; EXTERNAL = 300; } enum StressType { KV = 0; LEASE = 1; - NO_STRESS = 2; - ELECTION_RUNNER = 3; - WATCH_RUNNER = 4; - LOCK_RACER_RUNNER = 5; - LEASE_RUNNER = 6; + ELECTION_RUNNER = 2; + WATCH_RUNNER = 3; + LOCK_RACER_RUNNER = 4; + LEASE_RUNNER = 5; } message Tester { @@ -167,8 +168,12 @@ message Tester { int32 StressKeySuffixRangeTxn = 105 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""]; // StressKeyTxnOps is the number of operations per a transaction (max 64). int32 StressKeyTxnOps = 106 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""]; + + // StressClients is the number of concurrent stressing clients + // with "one" shared TCP connection. + int32 StressClients = 201 [(gogoproto.moretags) = "yaml:\"stress-clients\""]; // StressQPS is the maximum number of stresser requests per second. - int32 StressQPS = 107 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; + int32 StressQPS = 202 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; } message Request { From 3510e9b94cd800b82171d446b819b0699a4b34c6 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:03:06 -0700 Subject: [PATCH 02/12] functional-tester/tester: pass "*rpcpb.Member" to stresser Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 4 ++-- tools/functional-tester/tester/stress.go | 21 ++++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 3113e8465..f258b8c16 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -360,8 +360,8 @@ func (clus *Cluster) updateStresserChecker() { ) cs := &compositeStresser{} - for idx := range clus.Members { - cs.stressers = append(cs.stressers, newStresser(clus, idx)) + for _, m := range clus.Members { + cs.stressers = append(cs.stressers, newStresser(clus, m)) } clus.stresser = cs diff --git a/tools/functional-tester/tester/stress.go b/tools/functional-tester/tester/stress.go index 421a6d93f..816189a10 100644 --- a/tools/functional-tester/tester/stress.go +++ b/tools/functional-tester/tester/stress.go @@ -18,6 +18,8 @@ import ( "fmt" "time" + "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "go.uber.org/zap" ) @@ -36,34 +38,31 @@ type Stresser interface { } // newStresser creates stresser from a comma separated list of stresser types. -func newStresser(clus *Cluster, idx int) Stresser { +func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { stressers := make([]Stresser, len(clus.Tester.StressTypes)) for i, stype := range clus.Tester.StressTypes { clus.lg.Info("creating stresser", zap.String("type", stype)) switch stype { - case "NO_STRESS": - stressers[i] = &nopStresser{start: time.Now(), qps: int(clus.rateLimiter.Limit())} - case "KV": // TODO: Too intensive stressing clients can panic etcd member with // 'out of memory' error. Put rate limits in server side. stressers[i] = &keyStresser{ lg: clus.lg, - m: clus.Members[idx], + m: m, keySize: int(clus.Tester.StressKeySize), keyLargeSize: int(clus.Tester.StressKeySizeLarge), keySuffixRange: int(clus.Tester.StressKeySuffixRange), keyTxnSuffixRange: int(clus.Tester.StressKeySuffixRangeTxn), keyTxnOps: int(clus.Tester.StressKeyTxnOps), - N: 100, + clientsN: int(clus.Tester.StressClients), rateLimiter: clus.rateLimiter, } case "LEASE": stressers[i] = &leaseStresser{ lg: clus.lg, - m: clus.Members[idx], + m: m, numLeases: 10, // TODO: configurable keysPerLease: 10, // TODO: configurable rateLimiter: clus.rateLimiter, @@ -75,7 +74,7 @@ func newStresser(clus *Cluster, idx int) Stresser { "election", fmt.Sprintf("%v", time.Now().UnixNano()), // election name as current nano time "--dial-timeout=10s", - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, "--total-client-connections=10", "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), @@ -95,7 +94,7 @@ func newStresser(clus *Cluster, idx int) Stresser { "--total-keys=1", "--total-prefixes=1", "--watch-per-prefix=1", - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), } @@ -106,7 +105,7 @@ func newStresser(clus *Cluster, idx int) Stresser { args := []string{ "lock-racer", fmt.Sprintf("%v", time.Now().UnixNano()), // locker name as current nano time - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, "--total-client-connections=10", "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), @@ -117,7 +116,7 @@ func newStresser(clus *Cluster, idx int) Stresser { args := []string{ "lease-renewer", "--ttl=30", - "--endpoints", clus.Members[idx].EtcdClientEndpoint, + "--endpoints", m.EtcdClientEndpoint, } stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, 0) } From b3fea7ed5385d71ce2eb1946eacec342757012de Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:03:57 -0700 Subject: [PATCH 03/12] functional-tester/tester: refactor "Failure" to support liveness mode Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/failure.go | 95 +++++++++++++------ .../tester/failure_case_external.go | 19 +++- .../tester/failure_case_failpoints.go | 27 ++++-- .../tester/failure_case_kill.go | 18 ++-- .../tester/failure_case_network_blackhole.go | 6 +- .../tester/failure_case_network_slow.go | 10 +- .../tester/failure_case_no_op.go | 19 +++- 7 files changed, 137 insertions(+), 57 deletions(-) diff --git a/tools/functional-tester/tester/failure.go b/tools/functional-tester/tester/failure.go index b0fbb0b86..b40f61245 100644 --- a/tools/functional-tester/tester/failure.go +++ b/tools/functional-tester/tester/failure.go @@ -18,6 +18,8 @@ import ( "fmt" "math/rand" "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) // Failure defines failure injection interface. @@ -33,28 +35,32 @@ type Failure interface { Recover(clus *Cluster) error // Desc returns a description of the failure Desc() string + // FailureCase returns "rpcpb.FailureCase" enum type. + FailureCase() rpcpb.FailureCase } -type description string - -func (d description) Desc() string { return string(d) } - type injectMemberFunc func(*Cluster, int) error type recoverMemberFunc func(*Cluster, int) error type failureByFunc struct { - description + desc + failureCase rpcpb.FailureCase injectMember injectMemberFunc recoverMember recoverMemberFunc } -type failureFollower struct { - failureByFunc - last int - lead int +func (f *failureByFunc) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() } -type failureLeader struct { +func (f *failureByFunc) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +type failureFollower struct { failureByFunc last int lead int @@ -82,22 +88,6 @@ func (f *failureFollower) updateIndex(clus *Cluster) error { return nil } -func (f *failureLeader) updateIndex(clus *Cluster) error { - idx, err := clus.GetLeader() - if err != nil { - return err - } - f.lead = idx - f.last = idx - return nil -} - -type failureQuorum failureByFunc -type failureAll failureByFunc - -// failureUntilSnapshot injects a failure and waits for a snapshot event -type failureUntilSnapshot struct{ Failure } - func (f *failureFollower) Inject(clus *Cluster) error { if err := f.updateIndex(clus); err != nil { return err @@ -109,6 +99,24 @@ func (f *failureFollower) Recover(clus *Cluster) error { return f.recoverMember(clus, f.last) } +func (f *failureFollower) FailureCase() rpcpb.FailureCase { return f.failureCase } + +type failureLeader struct { + failureByFunc + last int + lead int +} + +func (f *failureLeader) updateIndex(clus *Cluster) error { + idx, err := clus.GetLeader() + if err != nil { + return err + } + f.lead = idx + f.last = idx + return nil +} + func (f *failureLeader) Inject(clus *Cluster) error { if err := f.updateIndex(clus); err != nil { return err @@ -120,6 +128,12 @@ func (f *failureLeader) Recover(clus *Cluster) error { return f.recoverMember(clus, f.last) } +func (f *failureLeader) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +type failureQuorum failureByFunc + func (f *failureQuorum) Inject(clus *Cluster) error { for i := range killMap(len(clus.Members), clus.rd) { if err := f.injectMember(clus, i); err != nil { @@ -138,6 +152,10 @@ func (f *failureQuorum) Recover(clus *Cluster) error { return nil } +func (f *failureQuorum) FailureCase() rpcpb.FailureCase { return f.failureCase } + +type failureAll failureByFunc + func (f *failureAll) Inject(clus *Cluster) error { for i := range clus.Members { if err := f.injectMember(clus, i); err != nil { @@ -156,6 +174,18 @@ func (f *failureAll) Recover(clus *Cluster) error { return nil } +func (f *failureAll) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +// failureUntilSnapshot injects a failure and waits for a snapshot event +type failureUntilSnapshot struct { + desc desc + failureCase rpcpb.FailureCase + + Failure +} + const snapshotCount = 10000 func (f *failureUntilSnapshot) Inject(clus *Cluster) error { @@ -190,7 +220,14 @@ func (f *failureUntilSnapshot) Inject(clus *Cluster) error { } func (f *failureUntilSnapshot) Desc() string { - return f.Failure.Desc() + " for a long time and expect it to recover from an incoming snapshot" + if f.desc.Desc() != "" { + return f.desc.Desc() + } + return f.failureCase.String() + " (to trigger snapshot)" +} + +func (f *failureUntilSnapshot) FailureCase() rpcpb.FailureCase { + return f.failureCase } func killMap(size int, seed int) map[int]bool { @@ -204,3 +241,7 @@ func killMap(size int, seed int) map[int]bool { } } } + +type desc string + +func (d desc) Desc() string { return string(d) } diff --git a/tools/functional-tester/tester/failure_case_external.go b/tools/functional-tester/tester/failure_case_external.go index e5b3b6a18..0d73422b1 100644 --- a/tools/functional-tester/tester/failure_case_external.go +++ b/tools/functional-tester/tester/failure_case_external.go @@ -17,13 +17,17 @@ package tester import ( "fmt" "os/exec" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) type failureExternal struct { Failure - description string - scriptPath string + desc string + failureCase rpcpb.FailureCase + + scriptPath string } func (f *failureExternal) Inject(clus *Cluster) error { @@ -34,11 +38,18 @@ func (f *failureExternal) Recover(clus *Cluster) error { return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run() } -func (f *failureExternal) Desc() string { return f.description } +func (f *failureExternal) Desc() string { + return f.desc +} + +func (f *failureExternal) FailureCase() rpcpb.FailureCase { + return f.failureCase +} func newFailureExternal(scriptPath string) Failure { return &failureExternal{ - description: fmt.Sprintf("external fault injector (script: %q)", scriptPath), + desc: fmt.Sprintf("external fault injector (script: %q)", scriptPath), + failureCase: rpcpb.FailureCase_EXTERNAL, scriptPath: scriptPath, } } diff --git a/tools/functional-tester/tester/failure_case_failpoints.go b/tools/functional-tester/tester/failure_case_failpoints.go index ede60916a..f5df8138b 100644 --- a/tools/functional-tester/tester/failure_case_failpoints.go +++ b/tools/functional-tester/tester/failure_case_failpoints.go @@ -21,6 +21,8 @@ import ( "strings" "sync" "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) type failpointStats struct { @@ -42,14 +44,23 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) { if len(fp) == 0 { continue } + fpFails := failuresFromFailpoint(fp, clus.Tester.FailpointCommands) + // wrap in delays so failpoint has time to trigger for i, fpf := range fpFails { if strings.Contains(fp, "Snap") { // hack to trigger snapshot failpoints - fpFails[i] = &failureUntilSnapshot{fpf} + fpFails[i] = &failureUntilSnapshot{ + desc: desc(fpf.Desc()), + failureCase: rpcpb.FailureCase_FAILPOINTS, + Failure: fpf, + } } else { - fpFails[i] = &failureDelay{fpf, 3 * time.Second} + fpFails[i] = &failureDelay{ + Failure: fpf, + delayDuration: 3 * time.Second, + } } } ret = append(ret, fpFails...) @@ -85,7 +96,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure) fs = append(fs, []Failure{ &failureFollower{ failureByFunc: failureByFunc{ - description: description(fmt.Sprintf("failpoint %s (one: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (one: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, @@ -94,7 +106,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure) }, &failureLeader{ failureByFunc: failureByFunc{ - description: description(fmt.Sprintf("failpoint %s (leader: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (leader: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, @@ -102,12 +115,14 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure) lead: -1, }, &failureQuorum{ - description: description(fmt.Sprintf("failpoint %s (quorum: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, &failureAll{ - description: description(fmt.Sprintf("failpoint %s (all: %s)", fp, fcmd)), + desc: desc(fmt.Sprintf("failpoint %q (all: %q)", fp, fcmd)), + failureCase: rpcpb.FailureCase_FAILPOINTS, injectMember: inject, recoverMember: recov, }, diff --git a/tools/functional-tester/tester/failure_case_kill.go b/tools/functional-tester/tester/failure_case_kill.go index d59fddf8f..9e88efe03 100644 --- a/tools/functional-tester/tester/failure_case_kill.go +++ b/tools/functional-tester/tester/failure_case_kill.go @@ -26,7 +26,7 @@ func recoverKill(clus *Cluster, idx int) error { func newFailureKillOneFollower() Failure { ff := failureByFunc{ - description: "kill one follower", + failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER, injectMember: injectKill, recoverMember: recoverKill, } @@ -35,7 +35,7 @@ func newFailureKillOneFollower() Failure { func newFailureKillLeader() Failure { ff := failureByFunc{ - description: "kill leader", + failureCase: rpcpb.FailureCase_KILL_LEADER, injectMember: injectKill, recoverMember: recoverKill, } @@ -44,7 +44,7 @@ func newFailureKillLeader() Failure { func newFailureKillQuorum() Failure { return &failureQuorum{ - description: "kill quorum", + failureCase: rpcpb.FailureCase_KILL_QUORUM, injectMember: injectKill, recoverMember: recoverKill, } @@ -52,16 +52,22 @@ func newFailureKillQuorum() Failure { func newFailureKillAll() Failure { return &failureAll{ - description: "kill all", + failureCase: rpcpb.FailureCase_KILL_ALL, injectMember: injectKill, recoverMember: recoverKill, } } func newFailureKillOneFollowerForLongTime() Failure { - return &failureUntilSnapshot{newFailureKillOneFollower()} + return &failureUntilSnapshot{ + failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER_FOR_LONG, + Failure: newFailureKillOneFollower(), + } } func newFailureKillLeaderForLongTime() Failure { - return &failureUntilSnapshot{newFailureKillLeader()} + return &failureUntilSnapshot{ + failureCase: rpcpb.FailureCase_KILL_LEADER_FOR_LONG, + Failure: newFailureKillLeader(), + } } diff --git a/tools/functional-tester/tester/failure_case_network_blackhole.go b/tools/functional-tester/tester/failure_case_network_blackhole.go index 6951c892f..f57c20477 100644 --- a/tools/functional-tester/tester/failure_case_network_blackhole.go +++ b/tools/functional-tester/tester/failure_case_network_blackhole.go @@ -26,7 +26,7 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error { func newFailureBlackholePeerPortTxRxOneFollower() Failure { ff := failureByFunc{ - description: "blackhole peer port on one follower", + failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER, injectMember: injectBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx, } @@ -39,7 +39,7 @@ func newFailureBlackholePeerPortTxRxOneFollower() Failure { func newFailureBlackholePeerPortTxRxLeader() Failure { ff := failureByFunc{ - description: "blackhole peer port on leader", + failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER, injectMember: injectBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx, } @@ -52,7 +52,7 @@ func newFailureBlackholePeerPortTxRxLeader() Failure { func newFailureBlackholePeerPortTxRxAll() Failure { f := &failureAll{ - description: "blackhole peer port on all", + failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL, injectMember: injectBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx, } diff --git a/tools/functional-tester/tester/failure_case_network_slow.go b/tools/functional-tester/tester/failure_case_network_slow.go index 840d2101d..9fadab67c 100644 --- a/tools/functional-tester/tester/failure_case_network_slow.go +++ b/tools/functional-tester/tester/failure_case_network_slow.go @@ -15,7 +15,6 @@ package tester import ( - "fmt" "time" "github.com/coreos/etcd/tools/functional-tester/rpcpb" @@ -43,9 +42,8 @@ func recoverDelayPeerPortTxRx(clus *Cluster, idx int) error { } func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure { - desc := fmt.Sprintf("delay follower peer port by %d ms", clus.Tester.DelayLatencyMs) ff := failureByFunc{ - description: description(desc), + failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER, injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } @@ -57,9 +55,8 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure { } func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure { - desc := fmt.Sprintf("delay leader peer port by %d ms", clus.Tester.DelayLatencyMs) ff := failureByFunc{ - description: description(desc), + failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_LEADER, injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } @@ -71,9 +68,8 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure { } func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure { - desc := fmt.Sprintf("delay all peer port by %d ms", clus.Tester.DelayLatencyMs) f := &failureAll{ - description: description(desc), + failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ALL, injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } diff --git a/tools/functional-tester/tester/failure_case_no_op.go b/tools/functional-tester/tester/failure_case_no_op.go index fdcddd306..21512214b 100644 --- a/tools/functional-tester/tester/failure_case_no_op.go +++ b/tools/functional-tester/tester/failure_case_no_op.go @@ -14,13 +14,24 @@ package tester +import ( + "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" +) + type failureNoOp failureByFunc -func (f *failureNoOp) Inject(clus *Cluster) error { return nil } -func (f *failureNoOp) Recover(clus *Cluster) error { return nil } +func (f *failureNoOp) Inject(clus *Cluster) error { return nil } +func (f *failureNoOp) Recover(clus *Cluster) error { return nil } +func (f *failureNoOp) FailureCase() rpcpb.FailureCase { return f.failureCase } func newFailureNoOp() Failure { - return &failureNoOp{ - description: "no failure", + f := &failureNoOp{ + failureCase: rpcpb.FailureCase_NO_FAIL_WITH_STRESS, + } + return &failureDelay{ + Failure: f, + delayDuration: 5 * time.Second, } } From 808966a7e9e620ea1eb995d57a5aa971f2f9e0c0 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:04:21 -0700 Subject: [PATCH 04/12] functional-tester/tester: refactor cluster code to support liveness mode Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 5 ++- .../functional-tester/tester/cluster_test.go | 3 ++ .../tester/cluster_tester.go | 24 +++++++---- .../functional-tester/tester/local-test.yaml | 5 ++- tools/functional-tester/tester/stress_key.go | 42 +++++++++---------- .../functional-tester/tester/stress_lease.go | 10 ++--- 6 files changed, 51 insertions(+), 38 deletions(-) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index f258b8c16..a1d6f22f8 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -293,7 +293,10 @@ func (clus *Cluster) updateFailures() { clus.lg.Info("no failpoints found!", zap.Error(fperr)) } clus.failures = append(clus.failures, fpFailures...) - case "NO_FAIL": + case "NO_FAIL_WITH_STRESS": + clus.failures = append(clus.failures, newFailureNoOp()) + case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": + // TODO clus.failures = append(clus.failures, newFailureNoOp()) case "EXTERNAL": clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath)) diff --git a/tools/functional-tester/tester/cluster_test.go b/tools/functional-tester/tester/cluster_test.go index a6ac44c21..2653544b0 100644 --- a/tools/functional-tester/tester/cluster_test.go +++ b/tools/functional-tester/tester/cluster_test.go @@ -131,6 +131,8 @@ func Test_newCluster(t *testing.T) { "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", "DELAY_PEER_PORT_TX_RX_LEADER", "DELAY_PEER_PORT_TX_RX_ALL", + "NO_FAIL_WITH_STRESS", + "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", }, FailureShuffle: true, FailpointCommands: []string{`panic("etcd-tester")`}, @@ -142,6 +144,7 @@ func Test_newCluster(t *testing.T) { StressKeySuffixRange: 250000, StressKeySuffixRangeTxn: 100, StressKeyTxnOps: 10, + StressClients: 100, StressQPS: 1000, }, } diff --git a/tools/functional-tester/tester/cluster_tester.go b/tools/functional-tester/tester/cluster_tester.go index 9d6928ce8..e657f0dba 100644 --- a/tools/functional-tester/tester/cluster_tester.go +++ b/tools/functional-tester/tester/cluster_tester.go @@ -116,30 +116,34 @@ func (clus *Cluster) doRound() error { zap.Int("round", clus.rd), zap.Strings("failures", clus.failureStrings()), ) - for i, f := range clus.failures { + + for i, fa := range clus.failures { clus.cs = i - caseTotalCounter.WithLabelValues(f.Desc()).Inc() + caseTotalCounter.WithLabelValues(fa.Desc()).Inc() clus.lg.Info("wait health before injecting failures") if err := clus.WaitHealth(); err != nil { return fmt.Errorf("wait full health error: %v", err) } + // TODO: "NO_FAIL_WITH_STRESS" + // TODO: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS" + clus.lg.Info( "injecting failure", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) - if err := f.Inject(clus); err != nil { + if err := fa.Inject(clus); err != nil { return fmt.Errorf("injection error: %v", err) } clus.lg.Info( "injected failure", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) // if run local, recovering server may conflict @@ -149,16 +153,16 @@ func (clus *Cluster) doRound() error { "recovering failure", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) - if err := f.Recover(clus); err != nil { + if err := fa.Recover(clus); err != nil { return fmt.Errorf("recovery error: %v", err) } clus.lg.Info( "recovered failure", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) clus.lg.Info("pausing stresser after failure recovery, before wait health") @@ -168,6 +172,7 @@ func (clus *Cluster) doRound() error { if err := clus.WaitHealth(); err != nil { return fmt.Errorf("wait full health error: %v", err) } + clus.lg.Info("check consistency after recovering failures") if err := clus.checkConsistency(); err != nil { return fmt.Errorf("tt.checkConsistency error (%v)", err) @@ -177,7 +182,7 @@ func (clus *Cluster) doRound() error { "failure case passed", zap.Int("round", clus.rd), zap.Int("case", clus.cs), - zap.String("desc", f.Desc()), + zap.String("desc", fa.Desc()), ) } @@ -186,6 +191,7 @@ func (clus *Cluster) doRound() error { zap.Int("round", clus.rd), zap.Strings("failures", clus.failureStrings()), ) + return nil } diff --git a/tools/functional-tester/tester/local-test.yaml b/tools/functional-tester/tester/local-test.yaml index 4104691d4..09f631cfc 100644 --- a/tools/functional-tester/tester/local-test.yaml +++ b/tools/functional-tester/tester/local-test.yaml @@ -97,6 +97,8 @@ tester-config: - DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER - DELAY_PEER_PORT_TX_RX_LEADER - DELAY_PEER_PORT_TX_RX_ALL + - NO_FAIL_WITH_STRESS + - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS failure-shuffle: true failpoint-commands: @@ -110,7 +112,6 @@ tester-config: stress-types: - KV - LEASE - # - NO_STRESS # - ELECTION_RUNNER # - WATCH_RUNNER # - LOCK_RACER_RUNNER @@ -121,4 +122,6 @@ tester-config: stress-key-suffix-range: 250000 stress-key-suffix-range-txn: 100 stress-key-txn-ops: 10 + + stress-clients: 100 stress-qps: 1000 diff --git a/tools/functional-tester/tester/stress_key.go b/tools/functional-tester/tester/stress_key.go index e3868c2ae..2359e87e1 100644 --- a/tools/functional-tester/tester/stress_key.go +++ b/tools/functional-tester/tester/stress_key.go @@ -44,14 +44,15 @@ type keyStresser struct { keyTxnSuffixRange int keyTxnOps int - N int - rateLimiter *rate.Limiter - wg sync.WaitGroup + wg sync.WaitGroup + clientsN int + ctx context.Context cancel func() cli *clientv3.Client + // atomicModifiedKeys records the number of keys created and deleted by the stresser. atomicModifiedKeys int64 @@ -59,40 +60,37 @@ type keyStresser struct { } func (s *keyStresser) Stress() error { - // TODO: add backoff option - cli, err := s.m.CreateEtcdClient() + var err error + s.cli, err = s.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(1 * time.Second)) if err != nil { return fmt.Errorf("%v (%q)", err, s.m.EtcdClientEndpoint) } - ctx, cancel := context.WithCancel(context.Background()) - - s.wg.Add(s.N) - s.cli = cli - s.cancel = cancel + s.ctx, s.cancel = context.WithCancel(context.Background()) + s.wg.Add(s.clientsN) var stressEntries = []stressEntry{ - {weight: 0.7, f: newStressPut(cli, s.keySuffixRange, s.keySize)}, + {weight: 0.7, f: newStressPut(s.cli, s.keySuffixRange, s.keySize)}, { weight: 0.7 * float32(s.keySize) / float32(s.keyLargeSize), - f: newStressPut(cli, s.keySuffixRange, s.keyLargeSize), + f: newStressPut(s.cli, s.keySuffixRange, s.keyLargeSize), }, - {weight: 0.07, f: newStressRange(cli, s.keySuffixRange)}, - {weight: 0.07, f: newStressRangeInterval(cli, s.keySuffixRange)}, - {weight: 0.07, f: newStressDelete(cli, s.keySuffixRange)}, - {weight: 0.07, f: newStressDeleteInterval(cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressRange(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressRangeInterval(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressDelete(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressDeleteInterval(s.cli, s.keySuffixRange)}, } if s.keyTxnSuffixRange > 0 { // adjust to make up ±70% of workloads with writes stressEntries[0].weight = 0.35 stressEntries = append(stressEntries, stressEntry{ weight: 0.35, - f: newStressTxn(cli, s.keyTxnSuffixRange, s.keyTxnOps), + f: newStressTxn(s.cli, s.keyTxnSuffixRange, s.keyTxnOps), }) } s.stressTable = createStressTable(stressEntries) - for i := 0; i < s.N; i++ { - go s.run(ctx) + for i := 0; i < s.clientsN; i++ { + go s.run() } s.lg.Info( @@ -102,18 +100,18 @@ func (s *keyStresser) Stress() error { return nil } -func (s *keyStresser) run(ctx context.Context) { +func (s *keyStresser) run() { defer s.wg.Done() for { - if err := s.rateLimiter.Wait(ctx); err == context.Canceled { + if err := s.rateLimiter.Wait(s.ctx); err == context.Canceled { return } // TODO: 10-second is enough timeout to cover leader failure // and immediate leader election. Find out what other cases this // could be timed out. - sctx, scancel := context.WithTimeout(ctx, 10*time.Second) + sctx, scancel := context.WithTimeout(s.ctx, 10*time.Second) err, modifiedKeys := s.stressTable.choose()(sctx) scancel() if err == nil { diff --git a/tools/functional-tester/tester/stress_lease.go b/tools/functional-tester/tester/stress_lease.go index 264f3ee99..ed65a9ec7 100644 --- a/tools/functional-tester/tester/stress_lease.go +++ b/tools/functional-tester/tester/stress_lease.go @@ -33,8 +33,8 @@ import ( const ( // time to live for lease - TTL = 120 - TTLShort = 2 + defaultTTL = 120 + defaultTTLShort = 2 ) type leaseStresser struct { @@ -201,7 +201,7 @@ func (ls *leaseStresser) createAliveLeases() { wg.Add(1) go func() { defer wg.Done() - leaseID, err := ls.createLeaseWithKeys(TTL) + leaseID, err := ls.createLeaseWithKeys(defaultTTL) if err != nil { ls.lg.Debug( "createLeaseWithKeys failed", @@ -228,7 +228,7 @@ func (ls *leaseStresser) createShortLivedLeases() { wg.Add(1) go func() { defer wg.Done() - leaseID, err := ls.createLeaseWithKeys(TTLShort) + leaseID, err := ls.createLeaseWithKeys(defaultTTLShort) if err != nil { return } @@ -323,7 +323,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { // if it is renewed, this means that invariant checking have at least ttl/2 time before lease exipres which is long enough for the checking to finish. // if it is not renewed, we remove the lease from the alive map so that the lease doesn't exipre during invariant checking renewTime, ok := ls.aliveLeases.read(leaseID) - if ok && renewTime.Add(TTL/2*time.Second).Before(time.Now()) { + if ok && renewTime.Add(defaultTTL/2*time.Second).Before(time.Now()) { ls.aliveLeases.remove(leaseID) ls.lg.Debug( "keepLeaseAlive lease has not been renewed, dropped it", From 0c20902d13097b17de92712a36ba15fab60059af Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:07:45 -0700 Subject: [PATCH 05/12] functional-tester/tester: add "newFailureNoFailWithNoStressForLiveness" Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 4 +-- .../tester/failure_case_no_op.go | 28 +++++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index a1d6f22f8..176525e84 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -294,10 +294,10 @@ func (clus *Cluster) updateFailures() { } clus.failures = append(clus.failures, fpFailures...) case "NO_FAIL_WITH_STRESS": - clus.failures = append(clus.failures, newFailureNoOp()) + clus.failures = append(clus.failures, newFailureNoFailWithStress()) case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": // TODO - clus.failures = append(clus.failures, newFailureNoOp()) + clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness()) case "EXTERNAL": clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath)) } diff --git a/tools/functional-tester/tester/failure_case_no_op.go b/tools/functional-tester/tester/failure_case_no_op.go index 21512214b..9a9ace462 100644 --- a/tools/functional-tester/tester/failure_case_no_op.go +++ b/tools/functional-tester/tester/failure_case_no_op.go @@ -20,14 +20,14 @@ import ( "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) -type failureNoOp failureByFunc +type failureNoFailWithStress failureByFunc -func (f *failureNoOp) Inject(clus *Cluster) error { return nil } -func (f *failureNoOp) Recover(clus *Cluster) error { return nil } -func (f *failureNoOp) FailureCase() rpcpb.FailureCase { return f.failureCase } +func (f *failureNoFailWithStress) Inject(clus *Cluster) error { return nil } +func (f *failureNoFailWithStress) Recover(clus *Cluster) error { return nil } +func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase { return f.failureCase } -func newFailureNoOp() Failure { - f := &failureNoOp{ +func newFailureNoFailWithStress() Failure { + f := &failureNoFailWithStress{ failureCase: rpcpb.FailureCase_NO_FAIL_WITH_STRESS, } return &failureDelay{ @@ -35,3 +35,19 @@ func newFailureNoOp() Failure { delayDuration: 5 * time.Second, } } + +type failureNoFailWithNoStressForLiveness failureByFunc + +func (f *failureNoFailWithNoStressForLiveness) Inject(clus *Cluster) error { return nil } +func (f *failureNoFailWithNoStressForLiveness) Recover(clus *Cluster) error { return nil } +func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase { return f.failureCase } + +func newFailureNoFailWithNoStressForLiveness() Failure { + f := &failureNoFailWithNoStressForLiveness{ + failureCase: rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS, + } + return &failureDelay{ + Failure: f, + delayDuration: 7 * time.Second, + } +} From ca310c5918d549a476a15dd4e0d7916c2acb73f2 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:11:01 -0700 Subject: [PATCH 06/12] functional-tester/rpcpb: add "FailureDelayMs" Signed-off-by: Gyuho Lee --- tools/functional-tester/rpcpb/rpc.pb.go | 278 +++++++++++++----------- tools/functional-tester/rpcpb/rpc.proto | 8 +- 2 files changed, 160 insertions(+), 126 deletions(-) diff --git a/tools/functional-tester/rpcpb/rpc.pb.go b/tools/functional-tester/rpcpb/rpc.pb.go index f28d0b8d3..2ff92826c 100644 --- a/tools/functional-tester/rpcpb/rpc.pb.go +++ b/tools/functional-tester/rpcpb/rpc.pb.go @@ -259,12 +259,14 @@ type Tester struct { EnablePprof bool `protobuf:"varint,24,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"` // FailureCases is the selected test cases to schedule. // If empty, run all failure cases. - // TODO: support no-op FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"` + // FailureDelayMs is the delay duration after failure is injected. + // Useful when triggering snapshot or no-op failure cases. + FailureDelayMs uint32 `protobuf:"varint,32,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"` // FailureShuffle is true to randomize failure injecting order. - FailureShuffle bool `protobuf:"varint,32,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"` + FailureShuffle bool `protobuf:"varint,33,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"` // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). - FailpointCommands []string `protobuf:"bytes,33,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"` + FailpointCommands []string `protobuf:"bytes,34,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"` // RunnerExecPath is a path of etcd-runner binary. RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"` // ExternalExecPath is a path of script for enabling/disabling an external fault injector. @@ -781,11 +783,18 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.FailureShuffle { + if m.FailureDelayMs != 0 { dAtA[i] = 0x80 i++ dAtA[i] = 0x2 i++ + i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs)) + } + if m.FailureShuffle { + dAtA[i] = 0x88 + i++ + dAtA[i] = 0x2 + i++ if m.FailureShuffle { dAtA[i] = 1 } else { @@ -795,7 +804,7 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { } if len(m.FailpointCommands) > 0 { for _, s := range m.FailpointCommands { - dAtA[i] = 0x8a + dAtA[i] = 0x92 i++ dAtA[i] = 0x2 i++ @@ -1125,6 +1134,9 @@ func (m *Tester) Size() (n int) { n += 2 + l + sovRpc(uint64(l)) } } + if m.FailureDelayMs != 0 { + n += 2 + sovRpc(uint64(m.FailureDelayMs)) + } if m.FailureShuffle { n += 3 } @@ -2184,6 +2196,25 @@ func (m *Tester) Unmarshal(dAtA []byte) error { m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 32: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType) + } + m.FailureDelayMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FailureDelayMs |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 33: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType) } @@ -2203,7 +2234,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error { } } m.FailureShuffle = bool(v != 0) - case 33: + case 34: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType) } @@ -2815,121 +2846,122 @@ var ( func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 1851 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x58, 0xff, 0x72, 0xdb, 0x4a, - 0x15, 0x8e, 0x93, 0x38, 0x89, 0x37, 0xbf, 0xd4, 0x4d, 0xd2, 0xa8, 0x69, 0x1b, 0xe5, 0xea, 0x72, - 0x4b, 0xc8, 0x8c, 0x52, 0xe8, 0x65, 0x98, 0xe1, 0xce, 0x65, 0x8a, 0xed, 0xa8, 0x37, 0x21, 0xaa, - 0xed, 0xae, 0x95, 0xb6, 0xfc, 0x65, 0x64, 0x69, 0x6d, 0x8b, 0xc8, 0x92, 0xba, 0x5a, 0xe7, 0xda, - 0xf7, 0x05, 0x18, 0xde, 0x80, 0x87, 0x80, 0x17, 0xe0, 0x09, 0x0a, 0xdc, 0x61, 0x78, 0x02, 0x03, - 0xe5, 0x0d, 0xfc, 0x04, 0xcc, 0xee, 0xca, 0xf6, 0xca, 0x76, 0xe0, 0x3f, 0xef, 0x39, 0xdf, 0xf7, - 0xe9, 0xe8, 0xe8, 0xfc, 0xd8, 0x04, 0xec, 0x92, 0xd8, 0x8d, 0x9b, 0xcf, 0x49, 0xec, 0x9e, 0xc7, - 0x24, 0xa2, 0x11, 0xcc, 0x73, 0xc3, 0x91, 0xd1, 0xf6, 0x69, 0xa7, 0xd7, 0x3c, 0x77, 0xa3, 0xee, - 0xf3, 0x76, 0xd4, 0x8e, 0x9e, 0x73, 0x6f, 0xb3, 0xd7, 0xe2, 0x27, 0x7e, 0xe0, 0xbf, 0x04, 0x4b, - 0xff, 0x7e, 0x1d, 0xac, 0x9a, 0xd4, 0xf5, 0xe0, 0xe7, 0x60, 0xb5, 0xe2, 0x74, 0xb1, 0x9a, 0x3b, - 0xc9, 0x9d, 0x16, 0x4a, 0xbb, 0xa3, 0xa1, 0xb6, 0x39, 0x70, 0xba, 0xc1, 0x57, 0x7a, 0xe8, 0x74, - 0xb1, 0x8e, 0xb8, 0x13, 0x1a, 0x60, 0xfd, 0xc2, 0xa1, 0xce, 0x85, 0x4f, 0xd4, 0x65, 0x8e, 0xdb, - 0x1b, 0x0d, 0xb5, 0x5d, 0x81, 0xf3, 0x1c, 0xea, 0x18, 0x9e, 0x4f, 0x74, 0x34, 0xc6, 0xc0, 0x33, - 0xb0, 0xf6, 0xae, 0x68, 0x31, 0xf4, 0x0a, 0x47, 0xc3, 0xd1, 0x50, 0xdb, 0x11, 0xe8, 0x6f, 0x9d, - 0x40, 0x80, 0x53, 0x04, 0xbc, 0x02, 0x8a, 0xe5, 0x27, 0x14, 0x87, 0xe5, 0xc0, 0xc7, 0x21, 0xbd, - 0x41, 0x56, 0xa2, 0xae, 0x9e, 0xac, 0x9c, 0x16, 0x4a, 0x4f, 0x47, 0x43, 0xed, 0x91, 0x60, 0x05, - 0x1c, 0x61, 0xb8, 0x1c, 0x62, 0xf4, 0x48, 0x90, 0xe8, 0x68, 0x8e, 0x06, 0x11, 0xd8, 0x2b, 0x7a, - 0x77, 0x98, 0x50, 0x3f, 0xc1, 0x92, 0x5a, 0x9e, 0xab, 0x9d, 0x8c, 0x86, 0xda, 0x13, 0xa1, 0xe6, - 0x8c, 0x41, 0x59, 0xc1, 0x45, 0x64, 0x58, 0x06, 0x3b, 0xe2, 0x39, 0x35, 0x8c, 0x09, 0x97, 0x5b, - 0xe3, 0x72, 0x8f, 0x47, 0x43, 0xed, 0x30, 0x13, 0x5c, 0x8c, 0x31, 0x49, 0x95, 0x66, 0x28, 0xb0, - 0x09, 0xd4, 0xab, 0xd0, 0xa7, 0xbe, 0x13, 0x4c, 0x1e, 0x31, 0x91, 0x5b, 0xe7, 0x72, 0xcf, 0x46, - 0x43, 0x4d, 0x17, 0x72, 0xbe, 0x40, 0x1a, 0xd3, 0x28, 0x25, 0xe5, 0x7b, 0x75, 0x60, 0x09, 0xec, - 0xa4, 0xbe, 0x72, 0xd0, 0x4b, 0x28, 0x26, 0xea, 0x06, 0xcf, 0xfd, 0xd1, 0x68, 0xa8, 0x3d, 0xcc, - 0x2a, 0xbb, 0x02, 0xa0, 0xa3, 0x19, 0x06, 0x4b, 0x60, 0xd6, 0x52, 0xa7, 0x0e, 0xc5, 0x6a, 0x81, - 0x0b, 0x49, 0x09, 0x9c, 0x11, 0x32, 0x12, 0x06, 0xd3, 0xd1, 0x22, 0xf2, 0xbc, 0xa6, 0x1d, 0xdd, - 0xe2, 0x50, 0x05, 0xff, 0x4f, 0x93, 0x32, 0xd8, 0x9c, 0x26, 0x27, 0xc3, 0x97, 0x60, 0xbb, 0x1e, - 0x3a, 0x71, 0xd2, 0x89, 0x68, 0x39, 0xea, 0x85, 0x54, 0xdd, 0x3c, 0xc9, 0x9d, 0xae, 0x94, 0x1e, - 0x8d, 0x86, 0xda, 0x81, 0x50, 0x4b, 0x52, 0xb7, 0xe1, 0x32, 0xbf, 0x8e, 0xb2, 0x78, 0x68, 0x81, - 0x07, 0x6f, 0x7a, 0x11, 0x75, 0x4a, 0x8e, 0x7b, 0x8b, 0x43, 0xaf, 0x34, 0xa0, 0x38, 0x51, 0xb7, - 0xb8, 0xc8, 0xf1, 0x68, 0xa8, 0x1d, 0x09, 0x91, 0x0f, 0x0c, 0x62, 0x34, 0x05, 0xc6, 0x68, 0x32, - 0x90, 0x8e, 0xe6, 0x89, 0xac, 0x3b, 0x6a, 0x04, 0xbf, 0x8d, 0x28, 0x56, 0xb7, 0x4f, 0x72, 0xa7, - 0x1b, 0x72, 0x77, 0xc4, 0x04, 0x1b, 0x77, 0x11, 0xcb, 0xce, 0x18, 0x23, 0x67, 0x24, 0x22, 0xa4, - 0x17, 0xd3, 0x72, 0x07, 0xbb, 0xb7, 0xea, 0x0e, 0xa7, 0x2e, 0xca, 0x88, 0x40, 0x19, 0x2e, 0x83, - 0x49, 0x19, 0x91, 0xc8, 0xfa, 0xef, 0xf3, 0x60, 0xed, 0x35, 0xee, 0x36, 0x31, 0x81, 0xbf, 0x00, - 0x5b, 0xac, 0xb1, 0xcd, 0x3e, 0x76, 0x6b, 0x0e, 0xed, 0xa4, 0x8d, 0x2d, 0xe5, 0x06, 0x53, 0xd7, - 0x33, 0x70, 0x1f, 0xbb, 0x46, 0xec, 0xd0, 0x8e, 0x8e, 0x32, 0x70, 0xf8, 0x25, 0x28, 0x14, 0xdb, - 0x38, 0xa4, 0x45, 0xcf, 0x23, 0x3c, 0xaf, 0x85, 0xd2, 0xc1, 0x68, 0xa8, 0x3d, 0x48, 0x5b, 0x87, - 0xb9, 0x0c, 0xc7, 0xf3, 0x88, 0x8e, 0xa6, 0x38, 0x96, 0xcf, 0x57, 0x8e, 0x1f, 0xc4, 0x91, 0x1f, - 0xd2, 0x4b, 0xdb, 0xae, 0x71, 0xf2, 0x16, 0x27, 0x4b, 0xf9, 0x6c, 0x8d, 0x21, 0x46, 0x87, 0xd2, - 0x38, 0x55, 0x99, 0x27, 0xb2, 0x7c, 0x96, 0x9c, 0x04, 0xb3, 0xf9, 0x81, 0x67, 0xa7, 0x4d, 0xd3, - 0x49, 0x70, 0x3a, 0x6d, 0x52, 0x0c, 0xfc, 0x0a, 0x6c, 0xb2, 0x37, 0xb0, 0xa2, 0x36, 0x7f, 0xdf, - 0x16, 0xa7, 0xa8, 0xa3, 0xa1, 0xb6, 0x2f, 0xbd, 0x6f, 0x10, 0xb5, 0xd3, 0xd7, 0x95, 0xc1, 0xb0, - 0x08, 0xb6, 0xd9, 0x51, 0x34, 0xbc, 0x6d, 0xd5, 0xd5, 0xbf, 0xe4, 0xf8, 0x67, 0x90, 0xba, 0x86, - 0xd3, 0xd3, 0x41, 0x41, 0x59, 0x0f, 0x66, 0x19, 0xf0, 0x1b, 0xb0, 0x3b, 0x35, 0xd4, 0x48, 0xd4, - 0x1f, 0xa8, 0x7f, 0x15, 0x22, 0x4f, 0x46, 0x43, 0x4d, 0x9d, 0x17, 0x89, 0x19, 0x46, 0x47, 0xb3, - 0xac, 0x71, 0x2c, 0xac, 0xa3, 0x85, 0xcc, 0xdf, 0x16, 0xc7, 0xc2, 0xc7, 0x41, 0x2a, 0x92, 0x65, - 0xc0, 0x1a, 0x80, 0x53, 0x55, 0x33, 0xf4, 0x78, 0x5e, 0xd5, 0xef, 0x45, 0x09, 0x68, 0xa3, 0xa1, - 0xf6, 0x78, 0x3e, 0x1c, 0x9c, 0xc2, 0x74, 0xb4, 0x80, 0x0b, 0x7f, 0x22, 0xd6, 0x84, 0xfa, 0x27, - 0x36, 0xf7, 0x37, 0x5f, 0x6c, 0x9e, 0xf3, 0x6d, 0x73, 0xce, 0x6c, 0xf2, 0xb2, 0x60, 0x82, 0x3a, - 0xe2, 0x50, 0xfd, 0xcf, 0x9b, 0x60, 0xcd, 0xc6, 0x7c, 0xa0, 0xbc, 0x04, 0xdb, 0xe2, 0x57, 0x05, - 0xd3, 0x6f, 0x23, 0x72, 0x3b, 0x5f, 0x8c, 0x94, 0xbb, 0x8d, 0x50, 0xf8, 0x75, 0x94, 0xc5, 0xc3, - 0x9f, 0x01, 0x20, 0x0c, 0xbc, 0xa2, 0xc4, 0xee, 0x79, 0x38, 0x1a, 0x6a, 0x30, 0xc3, 0x16, 0x95, - 0x24, 0x21, 0xd9, 0xd8, 0xbe, 0xc0, 0x81, 0x33, 0xb0, 0x1c, 0x8a, 0x43, 0x77, 0xf0, 0x3a, 0xe1, - 0xa5, 0xbc, 0x2d, 0x8f, 0x6d, 0x8f, 0xf9, 0x8d, 0x40, 0x00, 0x8c, 0x2e, 0x1b, 0xdb, 0x59, 0x0a, - 0xfc, 0x15, 0x50, 0xb2, 0x16, 0x74, 0xc7, 0x8b, 0x7a, 0x5b, 0x2e, 0xea, 0x59, 0x19, 0x83, 0xdc, - 0xe9, 0x68, 0x8e, 0xc7, 0x5e, 0x04, 0x45, 0xbd, 0xd0, 0xb3, 0xfc, 0xae, 0x4f, 0xd5, 0x83, 0x93, - 0xdc, 0x69, 0x5e, 0x7e, 0x11, 0xc2, 0x7c, 0x46, 0xc0, 0x9c, 0x3a, 0x92, 0x90, 0xf0, 0x97, 0x60, - 0xdb, 0xec, 0xfb, 0xb4, 0x1a, 0xb2, 0x36, 0xe9, 0x11, 0xac, 0x3e, 0x9c, 0xab, 0x89, 0xbe, 0x4f, - 0x8d, 0x28, 0x34, 0x5a, 0x02, 0xc0, 0x6a, 0x42, 0x26, 0xc0, 0x4b, 0xa0, 0x94, 0xa3, 0x30, 0xe1, - 0x1b, 0xc9, 0x1d, 0x88, 0x59, 0x73, 0x38, 0x5b, 0x9f, 0xee, 0x14, 0x31, 0x9e, 0x33, 0x73, 0x2c, - 0xf8, 0x73, 0xb0, 0x69, 0x86, 0x4e, 0x33, 0xc0, 0xb5, 0x98, 0x44, 0x2d, 0x55, 0xe5, 0x22, 0x87, - 0xa3, 0xa1, 0xb6, 0x97, 0x46, 0xc2, 0x9d, 0x46, 0xcc, 0xbc, 0xac, 0xcf, 0xa6, 0x58, 0xf8, 0x35, - 0xd8, 0x4a, 0xe3, 0x29, 0x3b, 0x09, 0x4e, 0x54, 0x8d, 0x6f, 0x3d, 0xa9, 0x49, 0xd3, 0xe8, 0x0d, - 0x97, 0xb9, 0x75, 0x94, 0x41, 0xb3, 0xdd, 0x96, 0x9e, 0xeb, 0x9d, 0x5e, 0xab, 0x15, 0x60, 0xf5, - 0x64, 0x36, 0x0b, 0x63, 0x7e, 0x22, 0x00, 0x3a, 0x9a, 0x61, 0xc0, 0x6b, 0x69, 0x44, 0x95, 0xa3, - 0x6e, 0xd7, 0x09, 0xbd, 0x44, 0xfd, 0x6c, 0xf6, 0xa2, 0x31, 0x1d, 0x51, 0x6e, 0x8a, 0x91, 0x27, - 0xd4, 0x98, 0xc7, 0xca, 0x0b, 0xf5, 0xc2, 0x10, 0x93, 0xc9, 0x94, 0xfd, 0x11, 0x2f, 0x4d, 0xa9, - 0xbc, 0x08, 0xf7, 0xcb, 0x73, 0x76, 0x86, 0xc2, 0x6e, 0x3e, 0x66, 0x9f, 0x62, 0x12, 0x3a, 0xc1, - 0x44, 0xe6, 0x8c, 0xcb, 0x48, 0x01, 0xe1, 0x14, 0x21, 0x0b, 0xcd, 0xd1, 0xd8, 0x97, 0xa9, 0x53, - 0x82, 0x93, 0xc4, 0x1e, 0xc4, 0x38, 0x51, 0x31, 0x7f, 0x2d, 0xe9, 0xcb, 0x24, 0xdc, 0x69, 0x50, - 0xe6, 0xd5, 0x91, 0x8c, 0x65, 0x05, 0x26, 0x8e, 0xd7, 0x78, 0x50, 0xf7, 0xbf, 0xc3, 0x7c, 0x7e, - 0xe6, 0xe5, 0xd4, 0xa6, 0xe4, 0x5b, 0x3c, 0x30, 0x12, 0xff, 0x3b, 0x56, 0x60, 0x19, 0x02, 0x1b, - 0x3a, 0x19, 0x83, 0xe5, 0x90, 0x36, 0x56, 0xdb, 0x5c, 0x46, 0x5a, 0x67, 0x33, 0x32, 0x46, 0xc0, - 0x60, 0x3a, 0x5a, 0xc0, 0x85, 0x6f, 0xc1, 0xfe, 0xd4, 0xda, 0x6b, 0xb5, 0xfc, 0x3e, 0x72, 0xc2, - 0x36, 0x56, 0x3b, 0x5c, 0x53, 0x1f, 0x0d, 0xb5, 0xe3, 0x79, 0x4d, 0x8e, 0x33, 0x08, 0x03, 0xea, - 0x68, 0x21, 0x1f, 0xfe, 0x06, 0x1c, 0x2e, 0xb2, 0xdb, 0xfd, 0x50, 0xf5, 0xb9, 0xb4, 0x74, 0x0d, - 0xbb, 0x47, 0xda, 0xa0, 0xfd, 0x50, 0x47, 0xf7, 0xc9, 0xb0, 0x65, 0x30, 0x71, 0xd9, 0xfd, 0xb0, - 0x1a, 0x27, 0xea, 0x6f, 0xb9, 0xb2, 0xf4, 0x49, 0x25, 0x65, 0xda, 0x0f, 0x8d, 0x28, 0x4e, 0x74, - 0x34, 0xcb, 0x9a, 0x7e, 0x16, 0x31, 0x8f, 0x13, 0xb1, 0x98, 0xf2, 0x99, 0x3b, 0x8e, 0xd0, 0x11, - 0x63, 0x3c, 0x99, 0x7c, 0x96, 0x94, 0x00, 0x7f, 0x0a, 0x0a, 0xc2, 0xf0, 0xa6, 0x56, 0x17, 0x1b, - 0x29, 0x2f, 0x6f, 0xf2, 0x94, 0xfd, 0x81, 0x3d, 0x7d, 0x0a, 0xd4, 0x7f, 0x97, 0x03, 0xeb, 0x08, - 0x7f, 0xe8, 0xe1, 0x84, 0xc2, 0x73, 0x50, 0xa8, 0xc6, 0x98, 0x38, 0xd4, 0x8f, 0x42, 0x3e, 0xb9, - 0x77, 0x5e, 0x28, 0xe9, 0xfc, 0x9f, 0xd8, 0xd1, 0x14, 0x02, 0xbf, 0x18, 0xdf, 0x41, 0x54, 0xb1, - 0x2c, 0xb6, 0x53, 0xb0, 0x30, 0xa2, 0xf1, 0x05, 0xe5, 0x8b, 0xf1, 0x7a, 0xe0, 0x7f, 0x1d, 0x4c, - 0x61, 0xc2, 0x88, 0x52, 0xa7, 0xfe, 0x35, 0xd8, 0x40, 0x38, 0x89, 0xa3, 0x30, 0xc1, 0x50, 0x05, - 0xeb, 0xf5, 0x9e, 0xeb, 0xe2, 0x24, 0xe1, 0x71, 0x6c, 0xa0, 0xf1, 0x11, 0x3e, 0x04, 0x6b, 0xec, - 0x9e, 0xd9, 0x4b, 0xc4, 0x72, 0x40, 0xe9, 0xe9, 0xec, 0x9f, 0x39, 0x29, 0x78, 0xb8, 0x03, 0x40, - 0x25, 0xa2, 0x75, 0xea, 0x10, 0x8a, 0x3d, 0x65, 0x09, 0xee, 0x03, 0x25, 0xbd, 0x45, 0x71, 0x1b, - 0x5b, 0x5b, 0x4a, 0x0e, 0xee, 0x82, 0x4d, 0x84, 0x93, 0x89, 0x61, 0x19, 0x6e, 0x81, 0x8d, 0x6b, - 0x3f, 0x08, 0xf8, 0x69, 0x85, 0xb9, 0xd9, 0x24, 0x28, 0x12, 0xb7, 0xe3, 0xdf, 0x61, 0x65, 0x95, - 0xa9, 0x5c, 0xe0, 0x84, 0x92, 0x68, 0xc0, 0x10, 0xfc, 0x36, 0xa4, 0xe4, 0xe1, 0x23, 0x70, 0x50, - 0x0a, 0x1c, 0xf7, 0xb6, 0x13, 0x05, 0xfc, 0x76, 0x5e, 0x8b, 0x08, 0xb5, 0xfb, 0xa8, 0xaf, 0x78, - 0xf0, 0x31, 0x38, 0xbc, 0x09, 0x9b, 0x0b, 0x9d, 0x18, 0x1e, 0x80, 0x07, 0x7c, 0x6b, 0x64, 0xcc, - 0x2d, 0x78, 0x08, 0xf6, 0x6e, 0x42, 0x6f, 0xce, 0xd1, 0x3e, 0xfb, 0xfb, 0x8a, 0x88, 0x27, 0x9d, - 0x92, 0x8c, 0x7f, 0x7d, 0x65, 0x59, 0x8d, 0x6a, 0xc5, 0x6c, 0xbc, 0xaa, 0x5a, 0x56, 0xf5, 0x9d, - 0x89, 0x94, 0x25, 0x16, 0x35, 0x37, 0x5b, 0x66, 0xf1, 0xc2, 0x44, 0x4a, 0x0e, 0x1e, 0x83, 0xa3, - 0x39, 0x5c, 0xe3, 0x55, 0x15, 0x35, 0xac, 0x6a, 0xe5, 0x1b, 0x65, 0x19, 0xaa, 0x60, 0x5f, 0x22, - 0x4c, 0x3d, 0x2b, 0x13, 0xa9, 0x37, 0x37, 0x55, 0x74, 0xf3, 0x5a, 0x59, 0xe5, 0xf9, 0x61, 0x86, - 0xa2, 0x65, 0x29, 0x79, 0x78, 0x06, 0x9e, 0x95, 0xac, 0x62, 0xf9, 0xfa, 0xb2, 0x6a, 0x99, 0x8d, - 0x9a, 0x69, 0xa2, 0x46, 0xad, 0x8a, 0xec, 0x86, 0xfd, 0xbe, 0x81, 0xde, 0x67, 0xa3, 0x5a, 0x83, - 0x3f, 0x00, 0x27, 0xf7, 0x63, 0xd3, 0x50, 0xd7, 0xe1, 0x67, 0xe0, 0xe9, 0xfd, 0x28, 0xf6, 0xd0, - 0x0d, 0xf8, 0x0c, 0xe8, 0x17, 0xa6, 0x55, 0xfc, 0xf5, 0xff, 0x7e, 0x60, 0x01, 0x9e, 0x80, 0x27, - 0x8b, 0x71, 0xe9, 0xc3, 0x00, 0x7c, 0x0a, 0x1e, 0x2d, 0x46, 0xb0, 0x07, 0x6d, 0xb2, 0xef, 0x50, - 0xa9, 0x36, 0x5e, 0x15, 0xaf, 0xac, 0xc6, 0xbb, 0x2b, 0xfb, 0xb2, 0x51, 0xb7, 0x91, 0x59, 0xaf, - 0x2b, 0x1e, 0xfc, 0x21, 0xf8, 0x3c, 0xe3, 0xa8, 0x54, 0x53, 0x9f, 0x48, 0xdd, 0xd5, 0x5b, 0xb3, - 0xc2, 0x80, 0x18, 0xee, 0x02, 0xc0, 0x50, 0xb5, 0xea, 0x55, 0xc5, 0xae, 0x2b, 0x1f, 0x73, 0x70, - 0x1b, 0x6c, 0x98, 0xef, 0x6d, 0x13, 0x55, 0x8a, 0x96, 0xf2, 0xc7, 0xe5, 0xb3, 0x08, 0x80, 0xe9, - 0x60, 0x86, 0x6b, 0x60, 0xf9, 0xfa, 0xad, 0xb2, 0x04, 0x0b, 0x20, 0x6f, 0x99, 0xc5, 0xba, 0xa9, - 0xe4, 0xe0, 0x1e, 0xd8, 0x35, 0x2d, 0xb3, 0x6c, 0x5f, 0x55, 0x2b, 0x0d, 0x74, 0x53, 0xa9, 0x98, - 0x48, 0x59, 0x86, 0x0a, 0xd8, 0x7a, 0x57, 0xb4, 0xcb, 0x97, 0x63, 0xcb, 0x0a, 0x2b, 0x04, 0xab, - 0x5a, 0xbe, 0x6e, 0xa0, 0x62, 0xd9, 0x44, 0x63, 0xf3, 0x2a, 0x03, 0x72, 0xa1, 0xb1, 0x25, 0xff, - 0xe2, 0x25, 0x28, 0xd8, 0xc4, 0x09, 0x93, 0x38, 0x22, 0x14, 0xbe, 0x90, 0x0f, 0x3b, 0x69, 0x4b, - 0xa6, 0x93, 0xe0, 0x68, 0x77, 0x72, 0x16, 0x0d, 0xa9, 0x2f, 0x9d, 0xe6, 0x7e, 0x9c, 0x2b, 0xed, - 0x7f, 0xfc, 0xf7, 0xf1, 0xd2, 0xc7, 0x4f, 0xc7, 0xb9, 0x7f, 0x7c, 0x3a, 0xce, 0xfd, 0xeb, 0xd3, - 0x71, 0xee, 0x0f, 0xff, 0x39, 0x5e, 0x6a, 0xae, 0xf1, 0xff, 0x30, 0x7c, 0xf9, 0xdf, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x71, 0x18, 0x41, 0x02, 0xaa, 0x10, 0x00, 0x00, + // 1870 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x58, 0x7f, 0x72, 0xdb, 0xb8, + 0x15, 0xb6, 0x6c, 0xcb, 0xb6, 0xe0, 0x5f, 0x0c, 0x6c, 0xc7, 0x8c, 0x93, 0x98, 0x5e, 0x6e, 0x37, + 0x75, 0x3d, 0x43, 0xa7, 0xcd, 0x76, 0x3a, 0xd3, 0x9d, 0xed, 0xa4, 0x92, 0xcc, 0xac, 0x5d, 0x33, + 0x92, 0x02, 0xc9, 0x49, 0xfa, 0x97, 0x4a, 0x51, 0x90, 0xc4, 0x9a, 0x22, 0x19, 0x00, 0xf2, 0x4a, + 0x7b, 0x81, 0x4e, 0x6f, 0xd0, 0x43, 0xb4, 0xf7, 0x48, 0xdb, 0x9d, 0x4e, 0x4f, 0xa0, 0xb6, 0xe9, + 0x0d, 0x74, 0x81, 0x76, 0x00, 0x50, 0x12, 0x28, 0xc9, 0xed, 0x7f, 0xc2, 0x7b, 0xdf, 0xf7, 0xf1, + 0xe1, 0x01, 0xef, 0x3d, 0xd8, 0x60, 0x97, 0xc4, 0x5e, 0xdc, 0x78, 0x4e, 0x62, 0xef, 0x3c, 0x26, + 0x11, 0x8b, 0x60, 0x56, 0x18, 0x8e, 0xac, 0xb6, 0xcf, 0x3a, 0xbd, 0xc6, 0xb9, 0x17, 0x75, 0x9f, + 0xb7, 0xa3, 0x76, 0xf4, 0x5c, 0x78, 0x1b, 0xbd, 0x96, 0x58, 0x89, 0x85, 0xf8, 0x25, 0x59, 0xe6, + 0xf7, 0xeb, 0x60, 0xd5, 0x66, 0x5e, 0x13, 0x7e, 0x0e, 0x56, 0x4b, 0x6e, 0x17, 0xeb, 0x99, 0x93, + 0xcc, 0x69, 0xae, 0xb0, 0x3b, 0x1a, 0x1a, 0x9b, 0x03, 0xb7, 0x1b, 0x7c, 0x65, 0x86, 0x6e, 0x17, + 0x9b, 0x48, 0x38, 0xa1, 0x05, 0xd6, 0x2f, 0x5c, 0xe6, 0x5e, 0xf8, 0x44, 0x5f, 0x16, 0xb8, 0xbd, + 0xd1, 0xd0, 0xd8, 0x95, 0xb8, 0xa6, 0xcb, 0x5c, 0xab, 0xe9, 0x13, 0x13, 0x8d, 0x31, 0xf0, 0x0c, + 0xac, 0xbd, 0xcb, 0x3b, 0x1c, 0xbd, 0x22, 0xd0, 0x70, 0x34, 0x34, 0x76, 0x24, 0xfa, 0x5b, 0x37, + 0x90, 0xe0, 0x04, 0x01, 0xaf, 0x80, 0xe6, 0xf8, 0x94, 0xe1, 0xb0, 0x18, 0xf8, 0x38, 0x64, 0x37, + 0xc8, 0xa1, 0xfa, 0xea, 0xc9, 0xca, 0x69, 0xae, 0xf0, 0x74, 0x34, 0x34, 0x1e, 0x49, 0x56, 0x20, + 0x10, 0x96, 0x27, 0x20, 0x56, 0x8f, 0x04, 0xd4, 0x44, 0x73, 0x34, 0x88, 0xc0, 0x5e, 0xbe, 0x79, + 0x87, 0x09, 0xf3, 0x29, 0x56, 0xd4, 0xb2, 0x42, 0xed, 0x64, 0x34, 0x34, 0x9e, 0x48, 0x35, 0x77, + 0x0c, 0x4a, 0x0b, 0x2e, 0x22, 0xc3, 0x22, 0xd8, 0x91, 0xdf, 0xa9, 0x60, 0x4c, 0x84, 0xdc, 0x9a, + 0x90, 0x7b, 0x3c, 0x1a, 0x1a, 0x87, 0xa9, 0xe0, 0x62, 0x8c, 0x49, 0xa2, 0x34, 0x43, 0x81, 0x0d, + 0xa0, 0x5f, 0x85, 0x3e, 0xf3, 0xdd, 0x60, 0xf2, 0x89, 0x89, 0xdc, 0xba, 0x90, 0x7b, 0x36, 0x1a, + 0x1a, 0xa6, 0x94, 0xf3, 0x25, 0xd2, 0x9a, 0x46, 0xa9, 0x28, 0xdf, 0xab, 0x03, 0x0b, 0x60, 0x27, + 0xf1, 0x15, 0x83, 0x1e, 0x65, 0x98, 0xe8, 0x1b, 0x22, 0xf7, 0x47, 0xa3, 0xa1, 0xf1, 0x30, 0xad, + 0xec, 0x49, 0x80, 0x89, 0x66, 0x18, 0x3c, 0x81, 0x69, 0x4b, 0x95, 0xb9, 0x0c, 0xeb, 0x39, 0x21, + 0xa4, 0x24, 0x70, 0x46, 0xc8, 0xa2, 0x1c, 0x66, 0xa2, 0x45, 0xe4, 0x79, 0xcd, 0x5a, 0x74, 0x8b, + 0x43, 0x1d, 0xfc, 0x3f, 0x4d, 0xc6, 0x61, 0x73, 0x9a, 0x82, 0x0c, 0x5f, 0x82, 0xed, 0x6a, 0xe8, + 0xc6, 0xb4, 0x13, 0xb1, 0x62, 0xd4, 0x0b, 0x99, 0xbe, 0x79, 0x92, 0x39, 0x5d, 0x29, 0x3c, 0x1a, + 0x0d, 0x8d, 0x03, 0xa9, 0x46, 0x13, 0xb7, 0xe5, 0x71, 0xbf, 0x89, 0xd2, 0x78, 0xe8, 0x80, 0x07, + 0x6f, 0x7a, 0x11, 0x73, 0x0b, 0xae, 0x77, 0x8b, 0xc3, 0x66, 0x61, 0xc0, 0x30, 0xd5, 0xb7, 0x84, + 0xc8, 0xf1, 0x68, 0x68, 0x1c, 0x49, 0x91, 0x0f, 0x1c, 0x62, 0x35, 0x24, 0xc6, 0x6a, 0x70, 0x90, + 0x89, 0xe6, 0x89, 0xbc, 0x3a, 0x2a, 0x04, 0xbf, 0x8d, 0x18, 0xd6, 0xb7, 0x4f, 0x32, 0xa7, 0x1b, + 0x6a, 0x75, 0xc4, 0x04, 0x5b, 0x77, 0x11, 0xcf, 0xce, 0x18, 0xa3, 0x66, 0x24, 0x22, 0xa4, 0x17, + 0xb3, 0x62, 0x07, 0x7b, 0xb7, 0xfa, 0x8e, 0xa0, 0x2e, 0xca, 0x88, 0x44, 0x59, 0x1e, 0x87, 0x29, + 0x19, 0x51, 0xc8, 0xe6, 0xef, 0xb3, 0x60, 0xed, 0x35, 0xee, 0x36, 0x30, 0x81, 0xbf, 0x00, 0x5b, + 0xbc, 0xb0, 0xed, 0x3e, 0xf6, 0x2a, 0x2e, 0xeb, 0x24, 0x85, 0xad, 0xe4, 0x06, 0x33, 0xaf, 0x69, + 0xe1, 0x3e, 0xf6, 0xac, 0xd8, 0x65, 0x1d, 0x13, 0xa5, 0xe0, 0xf0, 0x4b, 0x90, 0xcb, 0xb7, 0x71, + 0xc8, 0xf2, 0xcd, 0x26, 0x11, 0x79, 0xcd, 0x15, 0x0e, 0x46, 0x43, 0xe3, 0x41, 0x52, 0x3a, 0xdc, + 0x65, 0xb9, 0xcd, 0x26, 0x31, 0xd1, 0x14, 0xc7, 0xf3, 0xf9, 0xca, 0xf5, 0x83, 0x38, 0xf2, 0x43, + 0x76, 0x59, 0xab, 0x55, 0x04, 0x79, 0x4b, 0x90, 0x95, 0x7c, 0xb6, 0xc6, 0x10, 0xab, 0xc3, 0x58, + 0x9c, 0xa8, 0xcc, 0x13, 0x79, 0x3e, 0x0b, 0x2e, 0xc5, 0xbc, 0x7f, 0xe0, 0xd9, 0x6e, 0xd3, 0x70, + 0x29, 0x4e, 0xba, 0x4d, 0x82, 0x81, 0x5f, 0x81, 0x4d, 0xbe, 0x03, 0x27, 0x6a, 0x8b, 0xfd, 0xb6, + 0x04, 0x45, 0x1f, 0x0d, 0x8d, 0x7d, 0x65, 0xbf, 0x41, 0xd4, 0x4e, 0xb6, 0xab, 0x82, 0x61, 0x1e, + 0x6c, 0xf3, 0xa5, 0x2c, 0xf8, 0x9a, 0x53, 0xd5, 0xff, 0x9c, 0x11, 0xc7, 0xa0, 0x54, 0x8d, 0xa0, + 0x27, 0x8d, 0x82, 0xf1, 0x1a, 0x4c, 0x33, 0xe0, 0x37, 0x60, 0x77, 0x6a, 0xa8, 0x90, 0xa8, 0x3f, + 0xd0, 0xff, 0x22, 0x45, 0x9e, 0x8c, 0x86, 0x86, 0x3e, 0x2f, 0x12, 0x73, 0x8c, 0x89, 0x66, 0x59, + 0xe3, 0x58, 0x78, 0x45, 0x4b, 0x99, 0xbf, 0x2e, 0x8e, 0x45, 0xb4, 0x83, 0x44, 0x24, 0xcd, 0x80, + 0x15, 0x00, 0xa7, 0xaa, 0x76, 0xd8, 0x14, 0x79, 0xd5, 0xbf, 0x97, 0x57, 0xc0, 0x18, 0x0d, 0x8d, + 0xc7, 0xf3, 0xe1, 0xe0, 0x04, 0x66, 0xa2, 0x05, 0x5c, 0xf8, 0x13, 0x39, 0x26, 0xf4, 0x3f, 0xf1, + 0xbe, 0xbf, 0xf9, 0x62, 0xf3, 0x5c, 0x4c, 0x9b, 0x73, 0x6e, 0x53, 0x87, 0x05, 0x17, 0x34, 0x91, + 0x80, 0x9a, 0xff, 0xd9, 0x04, 0x6b, 0x35, 0x2c, 0x1a, 0xca, 0x4b, 0xb0, 0x2d, 0x7f, 0x95, 0x30, + 0xfb, 0x36, 0x22, 0xb7, 0xf3, 0x97, 0x91, 0x09, 0xb7, 0x15, 0x4a, 0xbf, 0x89, 0xd2, 0x78, 0xf8, + 0x33, 0x00, 0xa4, 0x41, 0xdc, 0x28, 0x39, 0x7b, 0x1e, 0x8e, 0x86, 0x06, 0x4c, 0xb1, 0xe5, 0x4d, + 0x52, 0x90, 0xbc, 0x6d, 0x5f, 0xe0, 0xc0, 0x1d, 0x38, 0x2e, 0xc3, 0xa1, 0x37, 0x78, 0x4d, 0xc5, + 0x55, 0xde, 0x56, 0xdb, 0x76, 0x93, 0xfb, 0xad, 0x40, 0x02, 0xac, 0x2e, 0x6f, 0xdb, 0x69, 0x0a, + 0xfc, 0x15, 0xd0, 0xd2, 0x16, 0x74, 0x27, 0x2e, 0xf5, 0xb6, 0x7a, 0xa9, 0x67, 0x65, 0x2c, 0x72, + 0x67, 0xa2, 0x39, 0x1e, 0xdf, 0x08, 0x8a, 0x7a, 0x61, 0xd3, 0xf1, 0xbb, 0x3e, 0xd3, 0x0f, 0x4e, + 0x32, 0xa7, 0x59, 0x75, 0x23, 0x84, 0xfb, 0xac, 0x80, 0x3b, 0x4d, 0xa4, 0x20, 0xe1, 0x2f, 0xc1, + 0xb6, 0xdd, 0xf7, 0x59, 0x39, 0xe4, 0x65, 0xd2, 0x23, 0x58, 0x7f, 0x38, 0x77, 0x27, 0xfa, 0x3e, + 0xb3, 0xa2, 0xd0, 0x6a, 0x49, 0x00, 0xbf, 0x13, 0x2a, 0x01, 0x5e, 0x02, 0xad, 0x18, 0x85, 0x54, + 0x4c, 0x24, 0x6f, 0x20, 0x7b, 0xcd, 0xe1, 0xec, 0xfd, 0xf4, 0xa6, 0x88, 0x71, 0x9f, 0x99, 0x63, + 0xc1, 0x9f, 0x83, 0x4d, 0x3b, 0x74, 0x1b, 0x01, 0xae, 0xc4, 0x24, 0x6a, 0xe9, 0xba, 0x10, 0x39, + 0x1c, 0x0d, 0x8d, 0xbd, 0x24, 0x12, 0xe1, 0xb4, 0x62, 0xee, 0xe5, 0x75, 0x36, 0xc5, 0xc2, 0xaf, + 0xc1, 0x56, 0x12, 0x4f, 0xd1, 0xa5, 0x98, 0xea, 0x86, 0x98, 0x7a, 0x4a, 0x91, 0x26, 0xd1, 0x5b, + 0x1e, 0x77, 0x9b, 0x28, 0x85, 0xe6, 0xa7, 0x99, 0xac, 0x45, 0x5e, 0x5f, 0x53, 0xfd, 0x64, 0xf6, + 0x34, 0xc7, 0x7c, 0x79, 0x1c, 0xe2, 0x34, 0xd3, 0x14, 0x3e, 0x20, 0x13, 0x4b, 0xb5, 0xd3, 0x6b, + 0xb5, 0x02, 0xac, 0x7f, 0x36, 0x9b, 0xca, 0xb1, 0x08, 0x95, 0x80, 0xa9, 0x46, 0xc2, 0x80, 0xd7, + 0x4a, 0x9f, 0x2b, 0x46, 0xdd, 0xae, 0x1b, 0x36, 0xa9, 0x6e, 0xce, 0xbe, 0x56, 0xa6, 0x7d, 0xce, + 0x4b, 0x30, 0x6a, 0x9b, 0x1b, 0xf3, 0xf8, 0xae, 0x50, 0x2f, 0x0c, 0x31, 0x99, 0xb4, 0xea, 0x1f, + 0x89, 0xfb, 0xad, 0xec, 0x8a, 0x08, 0xbf, 0xda, 0xac, 0x67, 0x28, 0xfc, 0xf9, 0x64, 0xf7, 0x19, + 0x26, 0xa1, 0x1b, 0x4c, 0x64, 0xce, 0x84, 0x8c, 0x12, 0x10, 0x4e, 0x10, 0xaa, 0xd0, 0x1c, 0x8d, + 0x1f, 0x6f, 0x95, 0x11, 0x4c, 0x69, 0x6d, 0x10, 0x63, 0xaa, 0x63, 0xb1, 0x2d, 0xe5, 0x78, 0xa9, + 0x70, 0x5a, 0x8c, 0x7b, 0x4d, 0xa4, 0x62, 0xf9, 0x2d, 0x95, 0xcb, 0x6b, 0x3c, 0xa8, 0xfa, 0xdf, + 0x61, 0xd1, 0x84, 0xb3, 0x6a, 0x6a, 0x13, 0xf2, 0x2d, 0x1e, 0x58, 0xd4, 0xff, 0x8e, 0xdf, 0xd2, + 0x14, 0x81, 0x77, 0xae, 0x94, 0xc1, 0x71, 0x49, 0x1b, 0xeb, 0x6d, 0x21, 0xa3, 0xcc, 0xc4, 0x19, + 0x19, 0x2b, 0xe0, 0x30, 0x13, 0x2d, 0xe0, 0xc2, 0xb7, 0x60, 0x7f, 0x6a, 0xed, 0xb5, 0x5a, 0x7e, + 0x1f, 0xb9, 0x61, 0x1b, 0xeb, 0x1d, 0xa1, 0x69, 0x8e, 0x86, 0xc6, 0xf1, 0xbc, 0xa6, 0xc0, 0x59, + 0x84, 0x03, 0x4d, 0xb4, 0x90, 0x0f, 0x7f, 0x03, 0x0e, 0x17, 0xd9, 0x6b, 0xfd, 0x50, 0xf7, 0x85, + 0xb4, 0xf2, 0x96, 0xbb, 0x47, 0xda, 0x62, 0xfd, 0xd0, 0x44, 0xf7, 0xc9, 0xf0, 0x89, 0x32, 0x71, + 0xd5, 0xfa, 0x61, 0x39, 0xa6, 0xfa, 0x6f, 0x85, 0xb2, 0x72, 0xa4, 0x8a, 0x32, 0xeb, 0x87, 0x56, + 0x14, 0x53, 0x13, 0xcd, 0xb2, 0xa6, 0xc7, 0x22, 0x9b, 0x3a, 0x95, 0xd3, 0x2d, 0x9b, 0x7a, 0x28, + 0x49, 0x1d, 0x39, 0x0b, 0xe8, 0xe4, 0x58, 0x12, 0x02, 0xfc, 0x29, 0xc8, 0x49, 0xc3, 0x9b, 0x4a, + 0x55, 0x8e, 0xb5, 0xac, 0xfa, 0x1c, 0x48, 0xd8, 0x1f, 0xf8, 0xd7, 0xa7, 0x40, 0xf3, 0x77, 0x19, + 0xb0, 0x8e, 0xf0, 0x87, 0x1e, 0xa6, 0x0c, 0x9e, 0x83, 0x5c, 0x39, 0xc6, 0xc4, 0x65, 0x7e, 0x14, + 0x8a, 0xf6, 0xbf, 0xf3, 0x42, 0x4b, 0x86, 0xc8, 0xc4, 0x8e, 0xa6, 0x10, 0xf8, 0xc5, 0xf8, 0x21, + 0xa3, 0xcb, 0x89, 0xb3, 0x9d, 0x80, 0xa5, 0x11, 0x8d, 0x5f, 0x39, 0x5f, 0x8c, 0x67, 0x8c, 0xf8, + 0x13, 0x63, 0x0a, 0x93, 0x46, 0x94, 0x38, 0xcd, 0xaf, 0xc1, 0x06, 0xc2, 0x34, 0x8e, 0x42, 0x8a, + 0xa1, 0x0e, 0xd6, 0xab, 0x3d, 0xcf, 0xc3, 0x94, 0x8a, 0x38, 0x36, 0xd0, 0x78, 0x09, 0x1f, 0x82, + 0x35, 0xfe, 0x58, 0xed, 0x51, 0x39, 0x61, 0x50, 0xb2, 0x3a, 0xfb, 0x47, 0x46, 0x09, 0x1e, 0xee, + 0x00, 0x50, 0x8a, 0x58, 0x95, 0xb9, 0x84, 0xe1, 0xa6, 0xb6, 0x04, 0xf7, 0x81, 0x96, 0x3c, 0xc5, + 0x84, 0x8d, 0xcf, 0x3e, 0x2d, 0x03, 0x77, 0xc1, 0x26, 0xc2, 0x74, 0x62, 0x58, 0x86, 0x5b, 0x60, + 0xe3, 0xda, 0x0f, 0x02, 0xb1, 0x5a, 0xe1, 0x6e, 0xde, 0x09, 0xf2, 0xc4, 0xeb, 0xf8, 0x77, 0x58, + 0x5b, 0xe5, 0x2a, 0x17, 0x98, 0x32, 0x12, 0x0d, 0x38, 0x42, 0x3c, 0xa9, 0xb4, 0x2c, 0x7c, 0x04, + 0x0e, 0x0a, 0x81, 0xeb, 0xdd, 0x76, 0xa2, 0x40, 0x3c, 0xf1, 0x2b, 0x11, 0x61, 0xb5, 0x3e, 0xea, + 0x6b, 0x4d, 0xf8, 0x18, 0x1c, 0xde, 0x84, 0x8d, 0x85, 0x4e, 0x0c, 0x0f, 0xc0, 0x03, 0xd1, 0xef, + 0x52, 0xe6, 0x16, 0x3c, 0x04, 0x7b, 0x37, 0x61, 0x73, 0xce, 0xd1, 0x3e, 0xfb, 0xdb, 0x8a, 0x8c, + 0x27, 0x69, 0xb5, 0x9c, 0x7f, 0x7d, 0xe5, 0x38, 0xf5, 0x72, 0xc9, 0xae, 0xbf, 0x2a, 0x3b, 0x4e, + 0xf9, 0x9d, 0x8d, 0xb4, 0x25, 0x1e, 0xb5, 0x30, 0x3b, 0x76, 0xfe, 0xc2, 0x46, 0x5a, 0x06, 0x1e, + 0x83, 0xa3, 0x39, 0x5c, 0xfd, 0x55, 0x19, 0xd5, 0x9d, 0x72, 0xe9, 0x1b, 0x6d, 0x19, 0xea, 0x60, + 0x5f, 0x21, 0x4c, 0x3d, 0x2b, 0x13, 0xa9, 0x37, 0x37, 0x65, 0x74, 0xf3, 0x5a, 0x5b, 0x15, 0xf9, + 0xe1, 0x86, 0xbc, 0xe3, 0x68, 0x59, 0x78, 0x06, 0x9e, 0x15, 0x9c, 0x7c, 0xf1, 0xfa, 0xb2, 0xec, + 0xd8, 0xf5, 0x8a, 0x6d, 0xa3, 0x7a, 0xa5, 0x8c, 0x6a, 0xf5, 0xda, 0xfb, 0x3a, 0x7a, 0x9f, 0x8e, + 0x6a, 0x0d, 0xfe, 0x00, 0x9c, 0xdc, 0x8f, 0x4d, 0x42, 0x5d, 0x87, 0x9f, 0x81, 0xa7, 0xf7, 0xa3, + 0xf8, 0x47, 0x37, 0xe0, 0x33, 0x60, 0x5e, 0xd8, 0x4e, 0xfe, 0xd7, 0xff, 0xfb, 0x83, 0x39, 0x78, + 0x02, 0x9e, 0x2c, 0xc6, 0x25, 0x1f, 0x03, 0xf0, 0x29, 0x78, 0xb4, 0x18, 0xc1, 0x3f, 0xb4, 0xc9, + 0xcf, 0xa1, 0x54, 0xae, 0xbf, 0xca, 0x5f, 0x39, 0xf5, 0x77, 0x57, 0xb5, 0xcb, 0x7a, 0xb5, 0x86, + 0xec, 0x6a, 0x55, 0x6b, 0xc2, 0x1f, 0x82, 0xcf, 0x53, 0x8e, 0x52, 0x39, 0xf1, 0xc9, 0xd4, 0x5d, + 0xbd, 0xb5, 0x4b, 0x1c, 0x88, 0xe1, 0x2e, 0x00, 0x1c, 0x55, 0x29, 0x5f, 0x95, 0x6a, 0x55, 0xed, + 0x63, 0x06, 0x6e, 0x83, 0x0d, 0xfb, 0x7d, 0xcd, 0x46, 0xa5, 0xbc, 0xa3, 0xfd, 0x71, 0xf9, 0x2c, + 0x02, 0x60, 0xda, 0x98, 0xe1, 0x1a, 0x58, 0xbe, 0x7e, 0xab, 0x2d, 0xc1, 0x1c, 0xc8, 0x3a, 0x76, + 0xbe, 0x6a, 0x6b, 0x19, 0xb8, 0x07, 0x76, 0x6d, 0xc7, 0x2e, 0xd6, 0xae, 0xca, 0xa5, 0x3a, 0xba, + 0x29, 0x95, 0x6c, 0xa4, 0x2d, 0x43, 0x0d, 0x6c, 0xbd, 0xcb, 0xd7, 0x8a, 0x97, 0x63, 0xcb, 0x0a, + 0xbf, 0x08, 0x4e, 0xb9, 0x78, 0x5d, 0x47, 0xf9, 0xa2, 0x8d, 0xc6, 0xe6, 0x55, 0x0e, 0x14, 0x42, + 0x63, 0x4b, 0xf6, 0xc5, 0x4b, 0x90, 0xab, 0x11, 0x37, 0xa4, 0x71, 0x44, 0x18, 0x7c, 0xa1, 0x2e, + 0x76, 0x92, 0x92, 0x4c, 0x3a, 0xc1, 0xd1, 0xee, 0x64, 0x2d, 0x0b, 0xd2, 0x5c, 0x3a, 0xcd, 0xfc, + 0x38, 0x53, 0xd8, 0xff, 0xf8, 0xaf, 0xe3, 0xa5, 0x8f, 0x9f, 0x8e, 0x33, 0x7f, 0xff, 0x74, 0x9c, + 0xf9, 0xe7, 0xa7, 0xe3, 0xcc, 0x1f, 0xfe, 0x7d, 0xbc, 0xd4, 0x58, 0x13, 0xff, 0xa6, 0xf8, 0xf2, + 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x33, 0x62, 0x54, 0x24, 0xef, 0x10, 0x00, 0x00, } diff --git a/tools/functional-tester/rpcpb/rpc.proto b/tools/functional-tester/rpcpb/rpc.proto index 80d89c58e..cbe206d56 100644 --- a/tools/functional-tester/rpcpb/rpc.proto +++ b/tools/functional-tester/rpcpb/rpc.proto @@ -141,12 +141,14 @@ message Tester { // FailureCases is the selected test cases to schedule. // If empty, run all failure cases. - // TODO: support no-op repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""]; + // FailureDelayMs is the delay duration after failure is injected. + // Useful when triggering snapshot or no-op failure cases. + uint32 FailureDelayMs = 32 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""]; // FailureShuffle is true to randomize failure injecting order. - bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; + bool FailureShuffle = 33 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). - repeated string FailpointCommands = 33 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; + repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; // RunnerExecPath is a path of etcd-runner binary. string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""]; From 33753d4ae625aadb4a5c2bed81c839cd88e44745 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:15:31 -0700 Subject: [PATCH 07/12] functional-tester/tester: add "failure-delay-ms" for configurable triggerElectionDur Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 6 +++--- tools/functional-tester/tester/cluster_test.go | 1 + .../tester/failure_case_failpoints.go | 2 +- .../tester/failure_case_network_blackhole.go | 18 +++++++++++------- .../tester/failure_case_network_slow.go | 9 +++------ tools/functional-tester/tester/local-test.yaml | 1 + 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 176525e84..5abaf5b4b 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -276,11 +276,11 @@ func (clus *Cluster) updateFailures() { case "KILL_ALL": clus.failures = append(clus.failures, newFailureKillAll()) case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_LEADER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ALL": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus)) case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus)) case "DELAY_PEER_PORT_TX_RX_LEADER": diff --git a/tools/functional-tester/tester/cluster_test.go b/tools/functional-tester/tester/cluster_test.go index 2653544b0..e0627453e 100644 --- a/tools/functional-tester/tester/cluster_test.go +++ b/tools/functional-tester/tester/cluster_test.go @@ -134,6 +134,7 @@ func Test_newCluster(t *testing.T) { "NO_FAIL_WITH_STRESS", "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", }, + FailureDelayMs: 7000, FailureShuffle: true, FailpointCommands: []string{`panic("etcd-tester")`}, RunnerExecPath: "/etcd-runner", diff --git a/tools/functional-tester/tester/failure_case_failpoints.go b/tools/functional-tester/tester/failure_case_failpoints.go index f5df8138b..f468a53c2 100644 --- a/tools/functional-tester/tester/failure_case_failpoints.go +++ b/tools/functional-tester/tester/failure_case_failpoints.go @@ -59,7 +59,7 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) { } else { fpFails[i] = &failureDelay{ Failure: fpf, - delayDuration: 3 * time.Second, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } } diff --git a/tools/functional-tester/tester/failure_case_network_blackhole.go b/tools/functional-tester/tester/failure_case_network_blackhole.go index f57c20477..c3a0f00c5 100644 --- a/tools/functional-tester/tester/failure_case_network_blackhole.go +++ b/tools/functional-tester/tester/failure_case_network_blackhole.go @@ -14,7 +14,11 @@ package tester -import "github.com/coreos/etcd/tools/functional-tester/rpcpb" +import ( + "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" +) func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx) @@ -24,7 +28,7 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx) } -func newFailureBlackholePeerPortTxRxOneFollower() Failure { +func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER, injectMember: injectBlackholePeerPortTxRx, @@ -33,11 +37,11 @@ func newFailureBlackholePeerPortTxRxOneFollower() Failure { f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } -func newFailureBlackholePeerPortTxRxLeader() Failure { +func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER, injectMember: injectBlackholePeerPortTxRx, @@ -46,11 +50,11 @@ func newFailureBlackholePeerPortTxRxLeader() Failure { f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } -func newFailureBlackholePeerPortTxRxAll() Failure { +func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure { f := &failureAll{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL, injectMember: injectBlackholePeerPortTxRx, @@ -58,6 +62,6 @@ func newFailureBlackholePeerPortTxRxAll() Failure { } return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } diff --git a/tools/functional-tester/tester/failure_case_network_slow.go b/tools/functional-tester/tester/failure_case_network_slow.go index 9fadab67c..db28206ad 100644 --- a/tools/functional-tester/tester/failure_case_network_slow.go +++ b/tools/functional-tester/tester/failure_case_network_slow.go @@ -21,9 +21,6 @@ import ( ) const ( - // delay duration to trigger leader election (default election timeout 1s) - triggerElectionDur = 5 * time.Second - // Wait more when it recovers from slow network, because network layer // needs extra time to propagate traffic control (tc command) change. // Otherwise, we get different hash values from the previous revision. @@ -50,7 +47,7 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure { f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } @@ -63,7 +60,7 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure { f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } @@ -75,6 +72,6 @@ func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure { } return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } diff --git a/tools/functional-tester/tester/local-test.yaml b/tools/functional-tester/tester/local-test.yaml index 09f631cfc..f5068c236 100644 --- a/tools/functional-tester/tester/local-test.yaml +++ b/tools/functional-tester/tester/local-test.yaml @@ -100,6 +100,7 @@ tester-config: - NO_FAIL_WITH_STRESS - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS + failure-delay-ms: 7000 failure-shuffle: true failpoint-commands: - panic("etcd-tester") From a43bd84631fc6780b61c61d4f77e37341f0e4244 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:29:24 -0700 Subject: [PATCH 08/12] functional-tester/tester: implement liveness mode failure case Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 15 ++++-- .../tester/cluster_tester.go | 14 +++-- tools/functional-tester/tester/failure.go | 2 +- .../tester/failure_case_failpoints.go | 3 +- .../tester/failure_case_network_blackhole.go | 8 ++- .../tester/failure_case_network_slow.go | 6 +-- ..._case_no_op.go => failure_case_no_fail.go} | 52 +++++++++++++++---- 7 files changed, 71 insertions(+), 29 deletions(-) rename tools/functional-tester/tester/{failure_case_no_op.go => failure_case_no_fail.go} (60%) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 5abaf5b4b..01d6d36c6 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -294,10 +294,9 @@ func (clus *Cluster) updateFailures() { } clus.failures = append(clus.failures, fpFailures...) case "NO_FAIL_WITH_STRESS": - clus.failures = append(clus.failures, newFailureNoFailWithStress()) + clus.failures = append(clus.failures, newFailureNoFailWithStress(clus)) case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": - // TODO - clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness()) + clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness(clus)) case "EXTERNAL": clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath)) } @@ -762,4 +761,12 @@ func (clus *Cluster) defrag() error { return nil } -func (clus *Cluster) Report() int64 { return clus.stresser.ModifiedKeys() } +// GetFailureDelayDuration computes failure delay duration. +func (clus *Cluster) GetFailureDelayDuration() time.Duration { + return time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond +} + +// Report reports the number of modified keys. +func (clus *Cluster) Report() int64 { + return clus.stresser.ModifiedKeys() +} diff --git a/tools/functional-tester/tester/cluster_tester.go b/tools/functional-tester/tester/cluster_tester.go index e657f0dba..032d44f0b 100644 --- a/tools/functional-tester/tester/cluster_tester.go +++ b/tools/functional-tester/tester/cluster_tester.go @@ -19,6 +19,8 @@ import ( "os" "time" + "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "go.uber.org/zap" ) @@ -127,8 +129,10 @@ func (clus *Cluster) doRound() error { return fmt.Errorf("wait full health error: %v", err) } - // TODO: "NO_FAIL_WITH_STRESS" - // TODO: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS" + if fa.FailureCase() == rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { + clus.lg.Info("pausing stresser after before injecting failures") + clus.pauseStresser() + } clus.lg.Info( "injecting failure", @@ -165,8 +169,10 @@ func (clus *Cluster) doRound() error { zap.String("desc", fa.Desc()), ) - clus.lg.Info("pausing stresser after failure recovery, before wait health") - clus.pauseStresser() + if fa.FailureCase() != rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { + clus.lg.Info("pausing stresser after failure recovery, before wait health") + clus.pauseStresser() + } clus.lg.Info("wait health after recovering failures") if err := clus.WaitHealth(); err != nil { diff --git a/tools/functional-tester/tester/failure.go b/tools/functional-tester/tester/failure.go index b40f61245..3aec58e3e 100644 --- a/tools/functional-tester/tester/failure.go +++ b/tools/functional-tester/tester/failure.go @@ -223,7 +223,7 @@ func (f *failureUntilSnapshot) Desc() string { if f.desc.Desc() != "" { return f.desc.Desc() } - return f.failureCase.String() + " (to trigger snapshot)" + return f.failureCase.String() } func (f *failureUntilSnapshot) FailureCase() rpcpb.FailureCase { diff --git a/tools/functional-tester/tester/failure_case_failpoints.go b/tools/functional-tester/tester/failure_case_failpoints.go index f468a53c2..c55e34577 100644 --- a/tools/functional-tester/tester/failure_case_failpoints.go +++ b/tools/functional-tester/tester/failure_case_failpoints.go @@ -20,7 +20,6 @@ import ( "net/http" "strings" "sync" - "time" "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) @@ -59,7 +58,7 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) { } else { fpFails[i] = &failureDelay{ Failure: fpf, - delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, + delayDuration: clus.GetFailureDelayDuration(), } } } diff --git a/tools/functional-tester/tester/failure_case_network_blackhole.go b/tools/functional-tester/tester/failure_case_network_blackhole.go index c3a0f00c5..0f7581280 100644 --- a/tools/functional-tester/tester/failure_case_network_blackhole.go +++ b/tools/functional-tester/tester/failure_case_network_blackhole.go @@ -15,8 +15,6 @@ package tester import ( - "time" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" ) @@ -37,7 +35,7 @@ func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure { f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, + delayDuration: clus.GetFailureDelayDuration(), } } @@ -50,7 +48,7 @@ func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure { f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, + delayDuration: clus.GetFailureDelayDuration(), } } @@ -62,6 +60,6 @@ func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure { } return &failureDelay{ Failure: f, - delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, + delayDuration: clus.GetFailureDelayDuration(), } } diff --git a/tools/functional-tester/tester/failure_case_network_slow.go b/tools/functional-tester/tester/failure_case_network_slow.go index db28206ad..274ba6383 100644 --- a/tools/functional-tester/tester/failure_case_network_slow.go +++ b/tools/functional-tester/tester/failure_case_network_slow.go @@ -47,7 +47,7 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure { f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, + delayDuration: clus.GetFailureDelayDuration(), } } @@ -60,7 +60,7 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure { f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, + delayDuration: clus.GetFailureDelayDuration(), } } @@ -72,6 +72,6 @@ func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure { } return &failureDelay{ Failure: f, - delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, + delayDuration: clus.GetFailureDelayDuration(), } } diff --git a/tools/functional-tester/tester/failure_case_no_op.go b/tools/functional-tester/tester/failure_case_no_fail.go similarity index 60% rename from tools/functional-tester/tester/failure_case_no_op.go rename to tools/functional-tester/tester/failure_case_no_fail.go index 9a9ace462..f7b0dcad3 100644 --- a/tools/functional-tester/tester/failure_case_no_op.go +++ b/tools/functional-tester/tester/failure_case_no_fail.go @@ -18,36 +18,68 @@ import ( "time" "github.com/coreos/etcd/tools/functional-tester/rpcpb" + + "go.uber.org/zap" ) type failureNoFailWithStress failureByFunc -func (f *failureNoFailWithStress) Inject(clus *Cluster) error { return nil } -func (f *failureNoFailWithStress) Recover(clus *Cluster) error { return nil } -func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase { return f.failureCase } +func (f *failureNoFailWithStress) Inject(clus *Cluster) error { + return nil +} -func newFailureNoFailWithStress() Failure { +func (f *failureNoFailWithStress) Recover(clus *Cluster) error { + return nil +} + +func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +func newFailureNoFailWithStress(clus *Cluster) Failure { f := &failureNoFailWithStress{ failureCase: rpcpb.FailureCase_NO_FAIL_WITH_STRESS, } return &failureDelay{ Failure: f, - delayDuration: 5 * time.Second, + delayDuration: clus.GetFailureDelayDuration(), } } type failureNoFailWithNoStressForLiveness failureByFunc -func (f *failureNoFailWithNoStressForLiveness) Inject(clus *Cluster) error { return nil } -func (f *failureNoFailWithNoStressForLiveness) Recover(clus *Cluster) error { return nil } -func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase { return f.failureCase } +func (f *failureNoFailWithNoStressForLiveness) Inject(clus *Cluster) error { + clus.lg.Info( + "extra delay for liveness mode with no stresser", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", f.Desc()), + ) + time.Sleep(clus.GetFailureDelayDuration()) -func newFailureNoFailWithNoStressForLiveness() Failure { + clus.lg.Info( + "wait health in liveness mode", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", f.Desc()), + ) + return clus.WaitHealth() +} + +func (f *failureNoFailWithNoStressForLiveness) Recover(clus *Cluster) error { + return nil +} + +func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase { + return f.failureCase +} + +func newFailureNoFailWithNoStressForLiveness(clus *Cluster) Failure { f := &failureNoFailWithNoStressForLiveness{ failureCase: rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS, } return &failureDelay{ Failure: f, - delayDuration: 7 * time.Second, + delayDuration: clus.GetFailureDelayDuration(), } } From 83739dc9cbb9af9eb2e2687041fded1499ecd703 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:42:30 -0700 Subject: [PATCH 09/12] functional-tester/tester: clean up stresser logic for liveness mode Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 44 ------------- .../tester/cluster_tester.go | 64 +++++++++---------- 2 files changed, 32 insertions(+), 76 deletions(-) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 01d6d36c6..303b93f97 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -383,49 +383,6 @@ func (clus *Cluster) updateStresserChecker() { ) } -func (clus *Cluster) startStresser() (err error) { - clus.lg.Info( - "starting stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - err = clus.stresser.Stress() - clus.lg.Info( - "started stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - return err -} - -func (clus *Cluster) closeStresser() { - clus.lg.Info( - "closing stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - clus.stresser.Close() - clus.lg.Info( - "closed stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) -} - -func (clus *Cluster) pauseStresser() { - clus.lg.Info( - "pausing stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) - clus.stresser.Pause() - clus.lg.Info( - "paused stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) -} - func (clus *Cluster) checkConsistency() (err error) { defer func() { if err != nil { @@ -438,7 +395,6 @@ func (clus *Cluster) checkConsistency() (err error) { ) return } - err = clus.startStresser() }() clus.lg.Info( diff --git a/tools/functional-tester/tester/cluster_tester.go b/tools/functional-tester/tester/cluster_tester.go index 032d44f0b..453556462 100644 --- a/tools/functional-tester/tester/cluster_tester.go +++ b/tools/functional-tester/tester/cluster_tester.go @@ -31,7 +31,6 @@ const compactQPS = 50000 // StartTester starts tester. func (clus *Cluster) StartTester() { // TODO: upate status - clus.startStresser() var preModifiedKey int64 for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ { @@ -123,19 +122,34 @@ func (clus *Cluster) doRound() error { clus.cs = i caseTotalCounter.WithLabelValues(fa.Desc()).Inc() + clus.lg.Info( + "failure case START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", fa.Desc()), + ) clus.lg.Info("wait health before injecting failures") if err := clus.WaitHealth(); err != nil { return fmt.Errorf("wait full health error: %v", err) } - if fa.FailureCase() == rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { - clus.lg.Info("pausing stresser after before injecting failures") - clus.pauseStresser() + stressStarted := false + if fa.FailureCase() != rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { + clus.lg.Info( + "starting stressers before injecting failures", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", fa.Desc()), + ) + if err := clus.stresser.Stress(); err != nil { + return fmt.Errorf("start stresser error: %v", err) + } + stressStarted = true } clus.lg.Info( - "injecting failure", + "injecting", zap.Int("round", clus.rd), zap.Int("case", clus.cs), zap.String("desc", fa.Desc()), @@ -143,18 +157,12 @@ func (clus *Cluster) doRound() error { if err := fa.Inject(clus); err != nil { return fmt.Errorf("injection error: %v", err) } - clus.lg.Info( - "injected failure", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - zap.String("desc", fa.Desc()), - ) // if run local, recovering server may conflict // with stressing client ports // TODO: use unix for local tests clus.lg.Info( - "recovering failure", + "recovering", zap.Int("round", clus.rd), zap.Int("case", clus.cs), zap.String("desc", fa.Desc()), @@ -162,30 +170,24 @@ func (clus *Cluster) doRound() error { if err := fa.Recover(clus); err != nil { return fmt.Errorf("recovery error: %v", err) } - clus.lg.Info( - "recovered failure", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - zap.String("desc", fa.Desc()), - ) - if fa.FailureCase() != rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { + if stressStarted { clus.lg.Info("pausing stresser after failure recovery, before wait health") - clus.pauseStresser() + clus.stresser.Pause() } - clus.lg.Info("wait health after recovering failures") + clus.lg.Info("wait health after recover") if err := clus.WaitHealth(); err != nil { return fmt.Errorf("wait full health error: %v", err) } - clus.lg.Info("check consistency after recovering failures") + clus.lg.Info("check consistency after recover") if err := clus.checkConsistency(); err != nil { return fmt.Errorf("tt.checkConsistency error (%v)", err) } clus.lg.Info( - "failure case passed", + "failure case PASS", zap.Int("round", clus.rd), zap.Int("case", clus.cs), zap.String("desc", fa.Desc()), @@ -216,14 +218,6 @@ func (clus *Cluster) updateRevision() error { } func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) { - clus.lg.Info("pausing stresser before compact") - clus.pauseStresser() - defer func() { - if err == nil { - err = clus.startStresser() - } - }() - clus.lg.Info( "compacting storage", zap.Int64("current-revision", clus.currentRevision), @@ -285,7 +279,13 @@ func (clus *Cluster) cleanup() error { } caseFailedTotalCounter.WithLabelValues(desc).Inc() - clus.closeStresser() + clus.lg.Info( + "closing stressers before archiving failure data", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + ) + clus.stresser.Close() + if err := clus.FailArchive(); err != nil { clus.lg.Warn( "cleanup failed", From d7b468ec6f6de1ddb220c333ca41fafd0e20f92c Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:45:12 -0700 Subject: [PATCH 10/12] test: bump up test timeout to 20m Recently, we've added bunch of tests... Signed-off-by: Gyuho Lee --- test | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test b/test index db2ce994a..2bee76399 100755 --- a/test +++ b/test @@ -234,7 +234,7 @@ function cov_pass { mkdir -p "$COVERDIR" # run code coverage for unit and integration tests - GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 15m" + GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 20m" # shellcheck disable=SC2206 GOCOVFLAGS=($GOCOVFLAGS) failed="" @@ -292,7 +292,7 @@ function e2e_pass { # expectation could be different USERTIMEOUT="" if [ -z "${TIMEOUT}" ]; then - USERTIMEOUT="15m" + USERTIMEOUT="20m" else USERTIMEOUT="${TIMEOUT}" fi @@ -303,9 +303,9 @@ function e2e_pass { function integration_e2e_pass { echo "Running integration and e2e tests..." - go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" & + go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" & e2epid="$!" - go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" & + go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" & intpid="$!" wait $e2epid wait $intpid @@ -315,7 +315,7 @@ function integration_e2e_pass { function grpcproxy_pass { go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration" go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration" - go test -timeout 15m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e" + go test -timeout 20m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e" } function release_pass { From a7ec9f28e3bac1b68e19d817ec844e0fb44de9ce Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 13:53:43 -0700 Subject: [PATCH 11/12] functional-tester/tester: fix "Desc" calls Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 12 ++++---- tools/functional-tester/tester/failure.go | 29 +++++++++++++++++-- .../tester/failure_case_no_fail.go | 14 +++++++++ 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 303b93f97..cc747a800 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -287,18 +287,18 @@ func (clus *Cluster) updateFailures() { clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeader(clus)) case "DELAY_PEER_PORT_TX_RX_ALL": clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxAll(clus)) - case "FAILPOINTS": - fpFailures, fperr := failpointFailures(clus) - if len(fpFailures) == 0 { - clus.lg.Info("no failpoints found!", zap.Error(fperr)) - } - clus.failures = append(clus.failures, fpFailures...) case "NO_FAIL_WITH_STRESS": clus.failures = append(clus.failures, newFailureNoFailWithStress(clus)) case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness(clus)) case "EXTERNAL": clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath)) + case "FAILPOINTS": + fpFailures, fperr := failpointFailures(clus) + if len(fpFailures) == 0 { + clus.lg.Info("no failpoints found!", zap.Error(fperr)) + } + clus.failures = append(clus.failures, fpFailures...) } } } diff --git a/tools/functional-tester/tester/failure.go b/tools/functional-tester/tester/failure.go index 3aec58e3e..f9c205512 100644 --- a/tools/functional-tester/tester/failure.go +++ b/tools/functional-tester/tester/failure.go @@ -99,7 +99,16 @@ func (f *failureFollower) Recover(clus *Cluster) error { return f.recoverMember(clus, f.last) } -func (f *failureFollower) FailureCase() rpcpb.FailureCase { return f.failureCase } +func (f *failureFollower) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() +} + +func (f *failureFollower) FailureCase() rpcpb.FailureCase { + return f.failureCase +} type failureLeader struct { failureByFunc @@ -152,7 +161,16 @@ func (f *failureQuorum) Recover(clus *Cluster) error { return nil } -func (f *failureQuorum) FailureCase() rpcpb.FailureCase { return f.failureCase } +func (f *failureQuorum) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() +} + +func (f *failureQuorum) FailureCase() rpcpb.FailureCase { + return f.failureCase +} type failureAll failureByFunc @@ -174,6 +192,13 @@ func (f *failureAll) Recover(clus *Cluster) error { return nil } +func (f *failureAll) Desc() string { + if string(f.desc) != "" { + return string(f.desc) + } + return f.failureCase.String() +} + func (f *failureAll) FailureCase() rpcpb.FailureCase { return f.failureCase } diff --git a/tools/functional-tester/tester/failure_case_no_fail.go b/tools/functional-tester/tester/failure_case_no_fail.go index f7b0dcad3..221929960 100644 --- a/tools/functional-tester/tester/failure_case_no_fail.go +++ b/tools/functional-tester/tester/failure_case_no_fail.go @@ -32,6 +32,13 @@ func (f *failureNoFailWithStress) Recover(clus *Cluster) error { return nil } +func (f *failureNoFailWithStress) Desc() string { + if f.desc.Desc() != "" { + return f.desc.Desc() + } + return f.failureCase.String() +} + func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase { return f.failureCase } @@ -70,6 +77,13 @@ func (f *failureNoFailWithNoStressForLiveness) Recover(clus *Cluster) error { return nil } +func (f *failureNoFailWithNoStressForLiveness) Desc() string { + if f.desc.Desc() != "" { + return f.desc.Desc() + } + return f.failureCase.String() +} + func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase { return f.failureCase } From 394189d399dd2cdffbe96fb436b7bd7e7bace5c7 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 4 Apr 2018 14:13:38 -0700 Subject: [PATCH 12/12] functional-tester/tester: remove unused "nopStresser" Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/stress_nop.go | 31 -------------------- 1 file changed, 31 deletions(-) delete mode 100644 tools/functional-tester/tester/stress_nop.go diff --git a/tools/functional-tester/tester/stress_nop.go b/tools/functional-tester/tester/stress_nop.go deleted file mode 100644 index 8254a12d7..000000000 --- a/tools/functional-tester/tester/stress_nop.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tester - -import "time" - -// nopStresser implements Stresser that does nothing -type nopStresser struct { - start time.Time - qps int -} - -func (s *nopStresser) Stress() error { return nil } -func (s *nopStresser) Pause() {} -func (s *nopStresser) Close() {} -func (s *nopStresser) ModifiedKeys() int64 { - return 0 -} -func (s *nopStresser) Checker() Checker { return nil }