vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

*: rename, clean up functional tests 

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
release-3.4
Gyuho Lee 2018-04-09 07:09:48 -07:00
parent b7770cd9fe
commit 85e050a120
53 changed files with 668 additions and 620 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile-functional-tester → Dockerfile-functional
View File

@ -22,7 +22,7 @@ RUN rm -rf ${GOROOT} \
RUN mkdir -p ${GOPATH}/src/github.com/coreos/etcd
ADD . ${GOPATH}/src/github.com/coreos/etcd
ADD ./tools/functional-tester/tester/local-test.yaml /local-test.yaml
ADD ./functional.yaml /functional.yaml
RUN go get -v github.com/coreos/gofail \
&& pushd ${GOPATH}/src/github.com/coreos/etcd \
@ -32,7 +32,7 @@ RUN go get -v github.com/coreos/gofail \
&& cp ./bin/etcdctl /bin/etcdctl \
&& GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build \
&& cp ./bin/etcd /bin/etcd-failpoints \
&& ./tools/functional-tester/build \
&& ./functional/build \
&& cp ./bin/etcd-agent /bin/etcd-agent \
&& cp ./bin/etcd-tester /bin/etcd-tester \
&& cp ./bin/etcd-runner /bin/etcd-runner \

26
Makefile
View File

@ -477,23 +477,23 @@ build-etcd-test-proxy:
# Example:
# make build-docker-functional-tester
# make push-docker-functional-tester
# make pull-docker-functional-tester
# make build-docker-functional
# make push-docker-functional
# make pull-docker-functional
build-docker-functional-tester:
build-docker-functional:
$(info GO_VERSION: $(GO_VERSION))
$(info ETCD_VERSION: $(ETCD_VERSION))
@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-functional-tester
@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-functional
docker build \
--tag gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \
--file ./Dockerfile-functional-tester \
--tag gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) \
--file ./Dockerfile-functional \
.
@mv ./Dockerfile-functional-tester.bak ./Dockerfile-functional-tester
@mv ./Dockerfile-functional.bak ./Dockerfile-functional
docker run \
--rm \
gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \
gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) \
/bin/bash -c "./bin/etcd --version && \
./bin/etcd-failpoints --version && \
ETCDCTL_API=3 ./bin/etcdctl version && \
@ -503,12 +503,12 @@ build-docker-functional-tester:
./bin/benchmark --help || true && \
./bin/etcd-test-proxy -help || true"
push-docker-functional-tester:
push-docker-functional:
$(info GO_VERSION: $(GO_VERSION))
$(info ETCD_VERSION: $(ETCD_VERSION))
gcloud docker -- push gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION)
gcloud docker -- push gcr.io/etcd-development/etcd-functional:go$(GO_VERSION)
pull-docker-functional-tester:
pull-docker-functional:
$(info GO_VERSION: $(GO_VERSION))
$(info ETCD_VERSION: $(ETCD_VERSION))
docker pull gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION)
docker pull gcr.io/etcd-development/etcd-functional:go$(GO_VERSION)

13
tools/functional-tester/tester/local-test.yaml → functional.yaml
View File

@ -106,9 +106,9 @@ agent-configs:
initial-corrupt-check: true
tester-config:
tester-data-dir: /tmp/etcd-tester-data
tester-network: tcp
tester-addr: 127.0.0.1:9028
data-dir: /tmp/etcd-tester-data
network: tcp
addr: 127.0.0.1:9028
# slow enough to trigger election
delay-latency-ms: 5000
@ -119,6 +119,8 @@ tester-config:
consistency-check: true
enable-pprof: true
failure-delay-ms: 7000
failure-shuffle: true
failure-cases:
- KILL_ONE_FOLLOWER
- KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT
@ -147,14 +149,11 @@ tester-config:
- NO_FAIL_WITH_STRESS
- NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS
failure-delay-ms: 7000
failure-shuffle: true
failpoint-commands:
- panic("etcd-tester")
# failpoint-commands:
# - panic("etcd-tester"),1*sleep(1000)
runner-exec-path: /etcd-runner
runner-exec-path: ./bin/etcd-runner
external-exec-path: ""
stress-types:

31
functional/README.md Normal file
View File

@ -0,0 +1,31 @@
# etcd Functional Testing
`functional` verifies the correct behavior of etcd under various system and network malfunctions. It sets up an etcd cluster under high pressure loads and continuously injects failures into the cluster. Then it expects the etcd cluster to recover within a few seconds. This has been extremely helpful to find critical bugs.
See [functional.yaml](../functional.yaml) for an example configuration.
### Run locally
```bash
PASSES=functional ./test
```
### Run with Docker
```bash
pushd ../..
make build-docker-functional
popd
```
And run [example scripts](./scripts).
```bash
# run 3 agents for 3-node local etcd cluster
./scripts/docker-local-agent.sh 1
./scripts/docker-local-agent.sh 2
./scripts/docker-local-agent.sh 3
# to run only 1 tester round
./scripts/docker-local-tester.sh
```

0
tools/functional-tester/agent/doc.go → functional/agent/doc.go
View File

3
tools/functional-tester/agent/handler.go → functional/agent/handler.go
View File

@ -25,9 +25,9 @@ import (
"syscall"
"time"
"github.com/coreos/etcd/functional/rpcpb"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/pkg/proxy"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"go.uber.org/zap"
)
@ -118,7 +118,6 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
}, nil
}
// TODO: support TLS
func (srv *Server) startProxy() error {
if srv.Member.EtcdClientProxy {
advertiseClientURL, advertiseClientURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertiseClientURLs[0])

2
tools/functional-tester/agent/server.go → functional/agent/server.go
View File

@ -21,8 +21,8 @@ import (
"os/exec"
"strings"
"github.com/coreos/etcd/functional/rpcpb"
"github.com/coreos/etcd/pkg/proxy"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"go.uber.org/zap"
"google.golang.org/grpc"

0
tools/functional-tester/agent/utils.go → functional/agent/utils.go
View File

0
tools/functional-tester/agent/utils_test.go → functional/agent/utils_test.go
View File

8
tools/functional-tester/build → functional/build
View File

@ -1,10 +1,10 @@
#!/usr/bin/env bash
if ! [[ "$0" =~ "tools/functional-tester/build" ]]; then
if ! [[ "$0" =~ "functional/build" ]]; then
echo "must be run from repository root"
exit 255
fi
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-agent ./tools/functional-tester/cmd/etcd-agent
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-tester ./tools/functional-tester/cmd/etcd-tester
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-runner ./tools/functional-tester/cmd/etcd-runner
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-agent ./functional/cmd/etcd-agent
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-tester ./functional/cmd/etcd-tester
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-runner ./functional/cmd/etcd-runner

2
tools/functional-tester/cmd/etcd-agent/main.go → functional/cmd/etcd-agent/main.go
View File

@ -18,7 +18,7 @@ package main
import (
"flag"
"github.com/coreos/etcd/tools/functional-tester/agent"
"github.com/coreos/etcd/functional/agent"
"go.uber.org/zap"
)

5
tools/functional-tester/cmd/etcd-runner/main.go → functional/cmd/etcd-runner/main.go
View File

@ -12,10 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// etcd-runner is a program for testing etcd clientv3 features against a fault injected cluster.
// etcd-runner is a program for testing etcd clientv3 features
// against a fault injected cluster.
package main
import "github.com/coreos/etcd/tools/functional-tester/runner"
import "github.com/coreos/etcd/functional/runner"
func main() {
runner.Start()

4
tools/functional-tester/cmd/etcd-tester/main.go → functional/cmd/etcd-tester/main.go
View File

@ -18,7 +18,7 @@ package main
import (
"flag"
"github.com/coreos/etcd/tools/functional-tester/tester"
"github.com/coreos/etcd/functional/tester"
"go.uber.org/zap"
)
@ -56,5 +56,5 @@ func main() {
logger.Fatal("WaitHealth failed", zap.Error(err))
}
clus.StartTester()
clus.Run()
}

0
tools/functional-tester/rpcpb/etcd_config.go → functional/rpcpb/etcd_config.go
View File

0
tools/functional-tester/rpcpb/etcd_config_test.go → functional/rpcpb/etcd_config_test.go
View File

0
tools/functional-tester/rpcpb/member.go → functional/rpcpb/member.go
View File

498
tools/functional-tester/rpcpb/rpc.pb.go → functional/rpcpb/rpc.pb.go
View File

@ -310,9 +310,9 @@ func (*Member) ProtoMessage() {}
func (*Member) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} }
type Tester struct {
TesterDataDir string `protobuf:"bytes,1,opt,name=TesterDataDir,proto3" json:"TesterDataDir,omitempty" yaml:"tester-data-dir"`
TesterNetwork string `protobuf:"bytes,2,opt,name=TesterNetwork,proto3" json:"TesterNetwork,omitempty" yaml:"tester-network"`
TesterAddr string `protobuf:"bytes,3,opt,name=TesterAddr,proto3" json:"TesterAddr,omitempty" yaml:"tester-addr"`
DataDir string `protobuf:"bytes,1,opt,name=DataDir,proto3" json:"DataDir,omitempty" yaml:"data-dir"`
Network string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty" yaml:"network"`
Addr string `protobuf:"bytes,3,opt,name=Addr,proto3" json:"Addr,omitempty" yaml:"addr"`
// DelayLatencyMsRv is the delay latency in milliseconds,
// to inject to simulated slow network.
DelayLatencyMs uint32 `protobuf:"varint,11,opt,name=DelayLatencyMs,proto3" json:"DelayLatencyMs,omitempty" yaml:"delay-latency-ms"`
@ -330,15 +330,15 @@ type Tester struct {
ConsistencyCheck bool `protobuf:"varint,23,opt,name=ConsistencyCheck,proto3" json:"ConsistencyCheck,omitempty" yaml:"consistency-check"`
// EnablePprof is true to enable profiler.
EnablePprof bool `protobuf:"varint,24,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"`
// FailureCases is the selected test cases to schedule.
// If empty, run all failure cases.
FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"`
// FailureDelayMs is the delay duration after failure is injected.
// Useful when triggering snapshot or no-op failure cases.
FailureDelayMs uint32 `protobuf:"varint,32,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"`
FailureDelayMs uint32 `protobuf:"varint,31,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"`
// FailureShuffle is true to randomize failure injecting order.
FailureShuffle bool `protobuf:"varint,33,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"`
// FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
FailureShuffle bool `protobuf:"varint,32,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"`
// FailureCases is the selected test cases to schedule.
// If empty, run all failure cases.
FailureCases []string `protobuf:"bytes,33,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"`
// Failpoinommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)
FailpointCommands []string `protobuf:"bytes,34,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"`
// RunnerExecPath is a path of etcd-runner binary.
RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"`
@ -990,23 +990,23 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.TesterDataDir) > 0 {
if len(m.DataDir) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintRpc(dAtA, i, uint64(len(m.TesterDataDir)))
i += copy(dAtA[i:], m.TesterDataDir)
i = encodeVarintRpc(dAtA, i, uint64(len(m.DataDir)))
i += copy(dAtA[i:], m.DataDir)
}
if len(m.TesterNetwork) > 0 {
if len(m.Network) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintRpc(dAtA, i, uint64(len(m.TesterNetwork)))
i += copy(dAtA[i:], m.TesterNetwork)
i = encodeVarintRpc(dAtA, i, uint64(len(m.Network)))
i += copy(dAtA[i:], m.Network)
}
if len(m.TesterAddr) > 0 {
if len(m.Addr) > 0 {
dAtA[i] = 0x1a
i++
i = encodeVarintRpc(dAtA, i, uint64(len(m.TesterAddr)))
i += copy(dAtA[i:], m.TesterAddr)
i = encodeVarintRpc(dAtA, i, uint64(len(m.Addr)))
i += copy(dAtA[i:], m.Addr)
}
if m.DelayLatencyMs != 0 {
dAtA[i] = 0x58
@ -1066,11 +1066,30 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.FailureDelayMs != 0 {
dAtA[i] = 0xf8
i++
dAtA[i] = 0x1
i++
i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs))
}
if m.FailureShuffle {
dAtA[i] = 0x80
i++
dAtA[i] = 0x2
i++
if m.FailureShuffle {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.FailureCases) > 0 {
for _, s := range m.FailureCases {
dAtA[i] = 0xfa
dAtA[i] = 0x8a
i++
dAtA[i] = 0x1
dAtA[i] = 0x2
i++
l = len(s)
for l >= 1<<7 {
@ -1083,25 +1102,6 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
i += copy(dAtA[i:], s)
}
}
if m.FailureDelayMs != 0 {
dAtA[i] = 0x80
i++
dAtA[i] = 0x2
i++
i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs))
}
if m.FailureShuffle {
dAtA[i] = 0x88
i++
dAtA[i] = 0x2
i++
if m.FailureShuffle {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.FailpointCommands) > 0 {
for _, s := range m.FailpointCommands {
dAtA[i] = 0x92
@ -1499,15 +1499,15 @@ func (m *Member) Size() (n int) {
func (m *Tester) Size() (n int) {
var l int
_ = l
l = len(m.TesterDataDir)
l = len(m.DataDir)
if l > 0 {
n += 1 + l + sovRpc(uint64(l))
}
l = len(m.TesterNetwork)
l = len(m.Network)
if l > 0 {
n += 1 + l + sovRpc(uint64(l))
}
l = len(m.TesterAddr)
l = len(m.Addr)
if l > 0 {
n += 1 + l + sovRpc(uint64(l))
}
@ -1532,18 +1532,18 @@ func (m *Tester) Size() (n int) {
if m.EnablePprof {
n += 3
}
if len(m.FailureCases) > 0 {
for _, s := range m.FailureCases {
l = len(s)
n += 2 + l + sovRpc(uint64(l))
}
}
if m.FailureDelayMs != 0 {
n += 2 + sovRpc(uint64(m.FailureDelayMs))
}
if m.FailureShuffle {
n += 3
}
if len(m.FailureCases) > 0 {
for _, s := range m.FailureCases {
l = len(s)
n += 2 + l + sovRpc(uint64(l))
}
}
if len(m.FailpointCommands) > 0 {
for _, s := range m.FailpointCommands {
l = len(s)
@ -3021,7 +3021,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field TesterDataDir", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field DataDir", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -3046,11 +3046,11 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.TesterDataDir = string(dAtA[iNdEx:postIndex])
m.DataDir = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field TesterNetwork", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Network", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -3075,11 +3075,11 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.TesterNetwork = string(dAtA[iNdEx:postIndex])
m.Network = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field TesterAddr", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Addr", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -3104,7 +3104,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.TesterAddr = string(dAtA[iNdEx:postIndex])
m.Addr = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 11:
if wireType != 0 {
@ -3243,6 +3243,45 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
}
m.EnablePprof = bool(v != 0)
case 31:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType)
}
m.FailureDelayMs = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FailureDelayMs |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 32:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.FailureShuffle = bool(v != 0)
case 33:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field FailureCases", wireType)
}
@ -3271,45 +3310,6 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
}
m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 32:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType)
}
m.FailureDelayMs = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FailureDelayMs |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 33:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.FailureShuffle = bool(v != 0)
case 34:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType)
@ -3955,160 +3955,160 @@ var (
func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) }
var fileDescriptorRpc = []byte{
// 2480 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0x5b, 0x57, 0xdb, 0xd8,
0x15, 0x46, 0x38, 0x30, 0x70, 0xb8, 0x99, 0x43, 0x08, 0x4a, 0x32, 0xc1, 0x44, 0x99, 0xa4, 0x84,
0x56, 0xa4, 0x4d, 0x66, 0xb5, 0x4d, 0xe6, 0x92, 0x18, 0xa3, 0x04, 0x17, 0x61, 0x3b, 0xc7, 0x22,
0xc9, 0x3c, 0xb9, 0x42, 0x3e, 0xc6, 0x2a, 0x42, 0x72, 0xa4, 0x63, 0xc6, 0xcc, 0x1f, 0xe8, 0x6b,
0xef, 0xab, 0x0f, 0x5d, 0xfd, 0x05, 0x9d, 0xfe, 0x8e, 0xcc, 0xf4, 0x36, 0x6d, 0xdf, 0xdd, 0x36,
0x5d, 0xfd, 0x03, 0x5e, 0xbd, 0x4d, 0x9f, 0xba, 0xce, 0xc5, 0xf6, 0x91, 0x6c, 0x43, 0xde, 0x7c,
0xf6, 0xfe, 0xbe, 0x4f, 0xfb, 0xec, 0x2d, 0x9d, 0xbd, 0x0f, 0x80, 0x85, 0xb0, 0xe1, 0x34, 0x0e,
0xee, 0x84, 0x0d, 0x67, 0xb3, 0x11, 0x06, 0x24, 0x80, 0x13, 0xcc, 0x70, 0x45, 0x3f, 0x74, 0x49,
0xbd, 0x79, 0xb0, 0xe9, 0x04, 0xc7, 0x77, 0x0e, 0x83, 0xc3, 0xe0, 0x0e, 0xf3, 0x1e, 0x34, 0x6b,
0x6c, 0xc5, 0x16, 0xec, 0x17, 0x67, 0x69, 0x5f, 0xce, 0x81, 0x0b, 0x06, 0x71, 0xaa, 0xf0, 0x06,
0xb8, 0x50, 0xb0, 0x8f, 0xb1, 0xaa, 0xac, 0x29, 0xeb, 0xd3, 0x5b, 0x0b, 0x9d, 0x76, 0x66, 0xe6,
0xd4, 0x3e, 0xf6, 0x1e, 0x68, 0xbe, 0x7d, 0x8c, 0x35, 0xc4, 0x9c, 0x50, 0x07, 0x6f, 0x6d, 0xdb,
0xc4, 0xde, 0x76, 0x43, 0x75, 0x9c, 0xe1, 0x96, 0x3a, 0xed, 0xcc, 0x02, 0xc7, 0x55, 0x6d, 0x62,
0xeb, 0x55, 0x37, 0xd4, 0x50, 0x17, 0x03, 0x37, 0xc0, 0xe4, 0xf3, 0xac, 0x49, 0xd1, 0x29, 0x86,
0x86, 0x9d, 0x76, 0x66, 0x9e, 0xa3, 0x3f, 0xb6, 0x3d, 0x0e, 0x16, 0x08, 0x58, 0x04, 0x4b, 0x3b,
0xd8, 0x0e, 0xc9, 0x01, 0xb6, 0x49, 0xde, 0x27, 0x38, 0x3c, 0xb1, 0xbd, 0xbd, 0x48, 0x9d, 0x59,
0x53, 0xd6, 0x53, 0x5b, 0xd7, 0x3a, 0xed, 0xcc, 0x65, 0x4e, 0xac, 0x77, 0x41, 0xba, 0x2b, 0x50,
0x1a, 0x1a, 0xc6, 0x84, 0x79, 0xb0, 0x68, 0x78, 0xd8, 0x21, 0x6e, 0xe0, 0x5b, 0xee, 0x31, 0x0e,
0x9a, 0x64, 0x2f, 0x52, 0x67, 0x99, 0xdc, 0xd5, 0x4e, 0x3b, 0xb3, 0xc2, 0xe5, 0xb0, 0x80, 0xe8,
0x84, 0x63, 0x34, 0x34, 0xc8, 0x82, 0x79, 0x90, 0x36, 0xdd, 0x88, 0x60, 0x3f, 0xe7, 0xb9, 0xd8,
0x27, 0xfb, 0xc8, 0x8c, 0xd4, 0xe5, 0xb5, 0xd4, 0xfa, 0xb4, 0x1c, 0x98, 0xc7, 0x10, 0xba, 0xc3,
0x20, 0x7a, 0x33, 0xf4, 0x22, 0x0d, 0x0d, 0xd0, 0x20, 0x02, 0x4b, 0xd9, 0xea, 0x09, 0x0e, 0x89,
0x1b, 0x61, 0x49, 0xed, 0x12, 0x53, 0x5b, 0xeb, 0xb4, 0x33, 0x6f, 0x73, 0x35, 0xbb, 0x0b, 0x8a,
0x0b, 0x0e, 0x23, 0xc3, 0xfb, 0x60, 0x8e, 0xaf, 0xb2, 0x4d, 0x12, 0x58, 0x66, 0x59, 0x5d, 0x59,
0x53, 0xd6, 0xa7, 0xe4, 0xda, 0xd8, 0x4d, 0x12, 0xe8, 0x84, 0x0a, 0xc4, 0x91, 0x30, 0x07, 0xe6,
0xb9, 0x21, 0x87, 0x43, 0x6a, 0xac, 0xab, 0x2a, 0xe3, 0x4a, 0x19, 0x12, 0xcf, 0x77, 0x70, 0x48,
0x74, 0xbb, 0x49, 0xea, 0x1a, 0x4a, 0x50, 0xe0, 0xfb, 0xb2, 0xc8, 0x63, 0xd7, 0xc3, 0xea, 0x65,
0x56, 0xee, 0x8b, 0x9d, 0x76, 0x26, 0x2d, 0x44, 0x28, 0xbb, 0xe6, 0x7a, 0x38, 0xc6, 0xa6, 0xd8,
0x7e, 0xf4, 0xbb, 0xf8, 0x94, 0x91, 0xaf, 0x24, 0xdf, 0xac, 0x23, 0x7c, 0x2a, 0xb8, 0x71, 0x24,
0x34, 0xc1, 0x12, 0x37, 0x58, 0x61, 0x33, 0x22, 0xb8, 0x9a, 0xcb, 0x32, 0x81, 0xab, 0x4c, 0xe0,
0x4a, 0xa7, 0x9d, 0xb9, 0xc4, 0x05, 0x08, 0x77, 0xeb, 0x8e, 0x2d, 0x74, 0x86, 0xd1, 0x68, 0x2e,
0x78, 0xb9, 0x4a, 0x18, 0x87, 0xac, 0x2a, 0x19, 0x56, 0x15, 0x29, 0x17, 0xa2, 0xc6, 0x0d, 0x8c,
0x43, 0x51, 0x90, 0x04, 0x05, 0x5a, 0x60, 0xb1, 0x57, 0xa2, 0x9e, 0xce, 0x1a, 0xd3, 0xb9, 0xd5,
0x69, 0x67, 0x34, 0xae, 0xe3, 0xfa, 0x2e, 0x71, 0x6d, 0x4f, 0xef, 0x57, 0x59, 0x92, 0x1c, 0x14,
0x80, 0x0f, 0xc0, 0x0c, 0xfd, 0xdd, 0xad, 0xef, 0x75, 0x56, 0x23, 0xb5, 0xd3, 0xce, 0x5c, 0xe4,
0x7a, 0x8c, 0xdd, 0x2f, 0xb2, 0x0c, 0x86, 0x25, 0x00, 0xe9, 0x32, 0x51, 0x66, 0x8d, 0x49, 0x48,
0x2f, 0x1c, 0x93, 0x18, 0xac, 0xf5, 0x10, 0x2e, 0xfc, 0x00, 0xcc, 0x32, 0x6b, 0xb7, 0xda, 0x37,
0x58, 0xbe, 0x2f, 0x77, 0xda, 0x99, 0x65, 0x59, 0xab, 0x5f, 0xf2, 0x18, 0xbc, 0xbb, 0x99, 0x6e,
0xb9, 0xdf, 0x61, 0xec, 0xe4, 0x66, 0xfa, 0x35, 0x97, 0xc1, 0x70, 0x0f, 0x2c, 0xd2, 0x65, 0xbc,
0xde, 0x37, 0x99, 0x42, 0xa6, 0xd3, 0xce, 0x5c, 0x95, 0x14, 0x06, 0x8a, 0x3e, 0xc8, 0x84, 0x5b,
0x60, 0x3e, 0xcf, 0x4b, 0x91, 0xf3, 0xa8, 0x3d, 0x54, 0x6f, 0x27, 0xdf, 0x9d, 0x6e, 0xa9, 0x1c,
0x0e, 0xd0, 0x50, 0x82, 0x41, 0xbf, 0xe8, 0xb8, 0xa5, 0x4c, 0x6c, 0x82, 0xd5, 0x0d, 0x26, 0x24,
0x25, 0x38, 0x21, 0xa4, 0x47, 0x14, 0xa6, 0xa1, 0x61, 0xe4, 0x41, 0x4d, 0x2b, 0x38, 0xc2, 0xbe,
0xfa, 0xd5, 0xf3, 0x34, 0x09, 0x85, 0x0d, 0x68, 0x32, 0x32, 0x7c, 0x08, 0xe6, 0xca, 0xbe, 0xdd,
0x88, 0xea, 0x01, 0xc9, 0x05, 0x4d, 0x9f, 0xa8, 0xf7, 0xd8, 0x59, 0x28, 0x95, 0x2d, 0x12, 0x6e,
0xdd, 0xa1, 0x7e, 0x0d, 0xc5, 0xf1, 0xd0, 0x04, 0x8b, 0x4f, 0x9b, 0x01, 0xb1, 0xb7, 0x6c, 0xe7,
0x08, 0xfb, 0xd5, 0xad, 0x53, 0x82, 0x23, 0xf5, 0x5d, 0x26, 0xb2, 0xda, 0x69, 0x67, 0xae, 0x70,
0x91, 0x97, 0x14, 0xa2, 0x1f, 0x70, 0x8c, 0x7e, 0x40, 0x41, 0x1a, 0x1a, 0x24, 0xd2, 0x56, 0x52,
0x0a, 0xf1, 0xb3, 0x80, 0x60, 0xf5, 0x61, 0xf2, 0xb8, 0x6a, 0x84, 0x58, 0x3f, 0x09, 0x68, 0x76,
0xba, 0x18, 0x39, 0x23, 0x41, 0x18, 0x36, 0x1b, 0x24, 0x57, 0xc7, 0xce, 0x91, 0xfa, 0x28, 0xf9,
0x1a, 0xf7, 0x32, 0xc2, 0x51, 0xba, 0x43, 0x61, 0x52, 0x46, 0x24, 0xb2, 0xf6, 0xcb, 0x19, 0x30,
0xb9, 0x87, 0x8f, 0x0f, 0x70, 0x48, 0x5f, 0x69, 0xda, 0x05, 0x8d, 0x16, 0x76, 0x4a, 0x36, 0xa9,
0x8b, 0x2e, 0x28, 0xe5, 0x06, 0x13, 0xa7, 0xaa, 0xe3, 0x16, 0x76, 0xf4, 0x86, 0x4d, 0xbf, 0x8b,
0x18, 0x1c, 0xde, 0x03, 0xd3, 0xd9, 0x43, 0x7a, 0xac, 0x56, 0xab, 0x21, 0x6b, 0x59, 0xd3, 0x5b,
0xcb, 0x9d, 0x76, 0x66, 0x51, 0x9c, 0xbe, 0xd4, 0xa5, 0xdb, 0xd5, 0x6a, 0xa8, 0xa1, 0x3e, 0x8e,
0xe6, 0xf3, 0xb1, 0xed, 0x7a, 0x8d, 0xc0, 0xf5, 0xc9, 0x8e, 0x65, 0x95, 0x18, 0x79, 0x96, 0x91,
0xa5, 0x7c, 0xd6, 0xba, 0x10, 0xbd, 0x4e, 0x48, 0x43, 0xa8, 0x0c, 0x12, 0x69, 0x3e, 0xb7, 0xec,
0x08, 0xd3, 0x66, 0x8b, 0x93, 0x07, 0xe8, 0x81, 0x1d, 0x61, 0xd1, 0x9a, 0x05, 0x86, 0x7e, 0x84,
0x74, 0x07, 0x66, 0x70, 0xc8, 0xf6, 0x5b, 0x4b, 0x7e, 0x84, 0x6c, 0xbf, 0x5e, 0x70, 0x28, 0xb6,
0x2b, 0x83, 0xe1, 0x13, 0xb0, 0x40, 0x97, 0xfc, 0x54, 0x28, 0x85, 0x41, 0xeb, 0x54, 0xfd, 0x4c,
0x61, 0x85, 0x78, 0xbb, 0xd3, 0xce, 0xa8, 0x92, 0x80, 0x38, 0x4f, 0x1a, 0x14, 0xa3, 0xa1, 0x24,
0x0b, 0x66, 0xc1, 0x1c, 0x35, 0xd1, 0xef, 0x92, 0xcb, 0x7c, 0xce, 0x65, 0xa4, 0xcf, 0x8f, 0xc9,
0xb0, 0xef, 0x59, 0x88, 0xc4, 0x19, 0xf4, 0x74, 0xeb, 0xab, 0x1a, 0x7e, 0x95, 0x25, 0x45, 0xfd,
0x74, 0x3c, 0x79, 0x24, 0xc8, 0xe1, 0x60, 0x01, 0xd3, 0xd0, 0x10, 0x2e, 0xfc, 0x06, 0x1f, 0x88,
0xd4, 0x5f, 0x53, 0x8d, 0x99, 0xbb, 0x33, 0x9b, 0x6c, 0xae, 0xda, 0xa4, 0x36, 0x79, 0x2c, 0xa2,
0x82, 0x1a, 0xe2, 0xb3, 0xd3, 0xb6, 0xdc, 0x00, 0xe9, 0xf0, 0xa3, 0xfe, 0x90, 0x0f, 0x3c, 0x23,
0xda, 0x28, 0x1d, 0x95, 0x62, 0x8d, 0x90, 0x72, 0xe2, 0x2a, 0xac, 0x2a, 0x3f, 0x3a, 0x53, 0x85,
0x57, 0x26, 0xc1, 0xa1, 0x39, 0xed, 0x35, 0x49, 0x16, 0xca, 0x8f, 0x53, 0xc9, 0x23, 0x4d, 0x88,
0xd0, 0x23, 0x96, 0x47, 0x12, 0x67, 0xc4, 0x24, 0x58, 0x1c, 0x3f, 0x39, 0x4b, 0x82, 0x87, 0x11,
0x67, 0x40, 0x6b, 0xa0, 0x33, 0xb3, 0x58, 0x7e, 0xca, 0x85, 0xae, 0x77, 0xda, 0x99, 0x6b, 0x31,
0x21, 0xe9, 0xb0, 0xe6, 0x21, 0x0d, 0xa3, 0x0f, 0x51, 0x65, 0xe1, 0xfd, 0xec, 0x0d, 0x54, 0x79,
0x94, 0xc3, 0xe8, 0xf0, 0xc3, 0x7e, 0x3b, 0x63, 0x41, 0xfe, 0x2b, 0x35, 0xba, 0x9f, 0xf1, 0xe0,
0x62, 0x78, 0x99, 0xcf, 0xc2, 0xf9, 0xf7, 0x19, 0x7c, 0x71, 0x78, 0xc8, 0x78, 0xf8, 0x5e, 0xaf,
0x1f, 0xb2, 0xc7, 0xff, 0x27, 0x35, 0xb2, 0x21, 0xf2, 0xa7, 0xcb, 0x68, 0x89, 0xcc, 0x9e, 0xfd,
0xdf, 0xd1, 0x64, 0xf1, 0x21, 0x4b, 0x68, 0x58, 0x48, 0x74, 0x53, 0xf6, 0xfc, 0x2f, 0x53, 0xe7,
0xb5, 0x53, 0x1e, 0xc6, 0x20, 0x75, 0x40, 0x8f, 0x85, 0xf4, 0xbf, 0x73, 0xf5, 0x78, 0x64, 0x83,
0x54, 0xed, 0x17, 0x73, 0x60, 0xd2, 0xc2, 0xac, 0xcb, 0x3e, 0x02, 0x73, 0xfc, 0x57, 0xf7, 0xfe,
0xa1, 0x0c, 0x0c, 0x79, 0xcc, 0xad, 0xf7, 0xaf, 0x21, 0x71, 0x02, 0xed, 0x7f, 0xdc, 0x50, 0xc0,
0xe4, 0xe3, 0x20, 0x3c, 0x12, 0x37, 0x18, 0xa9, 0x4c, 0x42, 0xc1, 0xe7, 0xfe, 0x9e, 0x80, 0xc0,
0xc3, 0x6f, 0x02, 0xc0, 0x0d, 0xec, 0xa0, 0xe6, 0xbb, 0xba, 0xd4, 0x69, 0x67, 0x60, 0x8c, 0xcd,
0x0f, 0x68, 0x09, 0x49, 0xe7, 0xca, 0x6d, 0xec, 0xd9, 0xa7, 0xa6, 0x4d, 0xb0, 0xef, 0x9c, 0x8a,
0x4b, 0xcd, 0x9c, 0xfc, 0x59, 0x57, 0xa9, 0x5f, 0xf7, 0x38, 0x40, 0x3f, 0xa6, 0x73, 0x65, 0x9c,
0x02, 0xbf, 0x03, 0xd2, 0x71, 0x0b, 0x3a, 0x61, 0xbd, 0x62, 0x4e, 0xee, 0x15, 0x49, 0x19, 0x3d,
0x3c, 0xd1, 0xd0, 0x00, 0x0f, 0x7e, 0x04, 0x96, 0xf7, 0x1b, 0x55, 0x9b, 0xe0, 0x6a, 0x22, 0xae,
0x39, 0x26, 0x78, 0xa3, 0xd3, 0xce, 0x64, 0xb8, 0x60, 0x93, 0xc3, 0xf4, 0xc1, 0xf8, 0x86, 0x2b,
0xd0, 0x1c, 0xa1, 0xa0, 0xe9, 0x57, 0x4d, 0xf7, 0xd8, 0x25, 0xea, 0xf2, 0x9a, 0xb2, 0x3e, 0x21,
0xe7, 0x28, 0xa4, 0x3e, 0xdd, 0xa3, 0x4e, 0x0d, 0x49, 0x48, 0x5a, 0x5e, 0xa3, 0xe5, 0x92, 0xa2,
0x4f, 0x1b, 0x5b, 0x33, 0xc4, 0xea, 0xa5, 0x81, 0x46, 0xd0, 0x72, 0x89, 0x1e, 0xf8, 0x7a, 0x8d,
0x03, 0x68, 0x23, 0x90, 0x09, 0x70, 0x07, 0xa4, 0x73, 0x81, 0x1f, 0xb1, 0x69, 0xdc, 0x39, 0xe5,
0xd3, 0xc1, 0x4a, 0xb2, 0x29, 0x39, 0x7d, 0x44, 0x77, 0x32, 0x18, 0x60, 0xc1, 0xfb, 0x60, 0xc6,
0xf0, 0xed, 0x03, 0x0f, 0x97, 0x1a, 0x61, 0x50, 0x13, 0x17, 0xa2, 0x95, 0x4e, 0x3b, 0xb3, 0x24,
0x22, 0x61, 0x4e, 0xbd, 0x41, 0xbd, 0xb4, 0x33, 0xf6, 0xb1, 0xf0, 0x7d, 0x30, 0x2b, 0xe2, 0xc9,
0xd9, 0x11, 0xee, 0x5e, 0x20, 0xa4, 0xaf, 0x51, 0x44, 0xaf, 0x3b, 0xd4, 0xad, 0xa1, 0x18, 0x9a,
0xbe, 0x28, 0x62, 0xcd, 0xb2, 0xba, 0x47, 0x2f, 0x0e, 0x89, 0x17, 0xa5, 0xcb, 0xe7, 0x05, 0x61,
0x2f, 0x4a, 0x9c, 0x42, 0x47, 0x5a, 0x61, 0x29, 0xd7, 0x9b, 0xb5, 0x9a, 0x87, 0xc5, 0x6d, 0x41,
0x4a, 0x65, 0x57, 0x24, 0xe2, 0x80, 0xbe, 0x86, 0x60, 0xc0, 0x5d, 0x69, 0x32, 0xc9, 0x05, 0xc7,
0xc7, 0xb6, 0x5f, 0x8d, 0x54, 0x2d, 0x79, 0xe1, 0xed, 0x4f, 0x26, 0x8e, 0xc0, 0xc8, 0x83, 0x49,
0x97, 0x47, 0x77, 0x85, 0x9a, 0xbe, 0x8f, 0xc3, 0xde, 0x70, 0x75, 0x3b, 0xd9, 0xd5, 0x42, 0xe6,
0x97, 0xc7, 0xab, 0x04, 0x85, 0xde, 0xc0, 0x8d, 0x16, 0xc1, 0xa1, 0x6f, 0x7b, 0x3d, 0x19, 0x3e,
0x61, 0x4b, 0x01, 0x61, 0x81, 0x90, 0x85, 0x06, 0x68, 0xb4, 0xbc, 0x65, 0x12, 0xe2, 0x28, 0xb2,
0x4e, 0x1b, 0x38, 0x52, 0x31, 0xdb, 0x96, 0x54, 0xde, 0x88, 0x39, 0x75, 0x42, 0xbd, 0x1a, 0x92,
0xb1, 0xf4, 0x2d, 0xe5, 0xcb, 0x5d, 0x7c, 0x5a, 0x76, 0x3f, 0xc1, 0x6c, 0x6c, 0x9a, 0x90, 0x53,
0x2b, 0xc8, 0xf4, 0xbc, 0x8d, 0xdc, 0x4f, 0xe8, 0x5b, 0x1a, 0x23, 0xd0, 0x71, 0x25, 0x66, 0x30,
0xed, 0xf0, 0x10, 0xab, 0x87, 0x4c, 0x46, 0x9a, 0x62, 0x13, 0x32, 0xba, 0x47, 0x61, 0x1a, 0x1a,
0xc2, 0x85, 0xcf, 0xc0, 0xc5, 0xbe, 0xb5, 0x59, 0xab, 0xb9, 0x2d, 0x64, 0xfb, 0x87, 0x58, 0xad,
0x33, 0x4d, 0xad, 0xd3, 0xce, 0xac, 0x0e, 0x6a, 0x32, 0x9c, 0x1e, 0x52, 0xa0, 0x86, 0x86, 0xf2,
0xe1, 0x77, 0xc1, 0xca, 0x30, 0xbb, 0xd5, 0xf2, 0x55, 0x97, 0x49, 0x4b, 0xd7, 0xd9, 0x11, 0xd2,
0x3a, 0x69, 0xf9, 0x1a, 0x1a, 0x25, 0x43, 0xc7, 0xc8, 0x9e, 0xcb, 0x6a, 0xf9, 0xc5, 0x46, 0xa4,
0x7e, 0x8f, 0x29, 0x4b, 0x25, 0x95, 0x94, 0x49, 0xcb, 0xd7, 0x83, 0x46, 0xa4, 0xa1, 0x24, 0xab,
0x5f, 0x16, 0xde, 0xdd, 0x23, 0x3e, 0x8d, 0x4e, 0xc4, 0xae, 0x36, 0x5c, 0x87, 0xcf, 0x05, 0x51,
0xaf, 0x2c, 0x82, 0x00, 0xdf, 0x05, 0xd3, 0xdc, 0xf0, 0xb4, 0x54, 0xe6, 0x43, 0xe8, 0x84, 0x3c,
0xc0, 0x0b, 0xf6, 0x4b, 0xfa, 0xf4, 0x3e, 0x50, 0xfb, 0xbe, 0x02, 0xde, 0x42, 0xf8, 0x65, 0x13,
0x47, 0x04, 0x6e, 0x82, 0xe9, 0x62, 0x03, 0x87, 0x36, 0x71, 0x03, 0x9f, 0xf5, 0xa6, 0xf9, 0xbb,
0x69, 0x31, 0x39, 0xf6, 0xec, 0xa8, 0x0f, 0x81, 0x37, 0xbb, 0x57, 0x0f, 0x95, 0x8f, 0x99, 0x73,
0x02, 0xcc, 0x8d, 0xa8, 0x7b, 0x2f, 0xb9, 0xd9, 0x6d, 0x80, 0xac, 0xdf, 0xf4, 0x61, 0xdc, 0x88,
0x84, 0x53, 0x73, 0xc0, 0x14, 0xc2, 0x51, 0x23, 0xf0, 0x23, 0x0c, 0x55, 0xf0, 0x56, 0xb9, 0xe9,
0x38, 0x38, 0x8a, 0x58, 0x1c, 0x53, 0xa8, 0xbb, 0x84, 0x97, 0xc0, 0x24, 0xbd, 0x5e, 0x36, 0x23,
0xde, 0xfa, 0x90, 0x58, 0x49, 0xb1, 0xa4, 0xce, 0x88, 0x65, 0xe3, 0x2f, 0x8a, 0xb4, 0x47, 0x38,
0x0f, 0x40, 0x21, 0x20, 0x65, 0x62, 0x87, 0x04, 0x57, 0xd3, 0x63, 0xf0, 0x22, 0x48, 0x8b, 0x3b,
0x16, 0xb3, 0xd1, 0xb9, 0x38, 0xad, 0xc0, 0x05, 0x30, 0x83, 0x70, 0xd4, 0x33, 0x8c, 0xc3, 0x59,
0x30, 0xb5, 0xeb, 0x7a, 0x1e, 0x5b, 0xa5, 0xa8, 0x9b, 0x1e, 0x18, 0xd9, 0xd0, 0xa9, 0xbb, 0x27,
0x38, 0x7d, 0x81, 0xaa, 0x6c, 0xe3, 0x88, 0x84, 0xc1, 0x29, 0x45, 0xb0, 0xbb, 0x52, 0x7a, 0x02,
0x5e, 0x06, 0xcb, 0x5b, 0x9e, 0xed, 0x1c, 0xd5, 0x03, 0x8f, 0xfd, 0x4d, 0xa4, 0x14, 0x84, 0xc4,
0x6a, 0xa1, 0x56, 0xba, 0x0a, 0xaf, 0x82, 0x95, 0x7d, 0xff, 0x60, 0xa8, 0x13, 0xc3, 0x65, 0xb0,
0xc8, 0x8e, 0xc5, 0x98, 0xb9, 0x06, 0x57, 0xc0, 0xd2, 0xbe, 0x5f, 0x1d, 0x70, 0x1c, 0x6e, 0xfc,
0x63, 0x8a, 0xc7, 0x23, 0x4e, 0x64, 0xca, 0xdf, 0xcd, 0x9b, 0x66, 0xa5, 0x58, 0x30, 0x2a, 0x8f,
0x8b, 0xa6, 0x59, 0x7c, 0x6e, 0xa0, 0xf4, 0x18, 0xfc, 0x1a, 0x58, 0x1f, 0x30, 0x57, 0xf6, 0x0b,
0x56, 0xde, 0xac, 0x58, 0x28, 0xff, 0xe4, 0x89, 0x81, 0x2a, 0xe5, 0x42, 0xb6, 0x54, 0xde, 0x29,
0x5a, 0x3c, 0x05, 0x0c, 0x6d, 0x1a, 0xd9, 0x6d, 0x03, 0xa5, 0xc7, 0xe1, 0x2d, 0xa0, 0x49, 0x86,
0x51, 0xc4, 0x54, 0x8f, 0xf8, 0x74, 0xbf, 0x88, 0xf6, 0xf7, 0xd2, 0x17, 0x58, 0xee, 0xa8, 0x21,
0x6b, 0x9a, 0xe9, 0x09, 0xb8, 0x01, 0x6e, 0x6d, 0x99, 0xd9, 0xdc, 0xee, 0x4e, 0xd1, 0x34, 0x2a,
0x25, 0xc3, 0x40, 0x95, 0x52, 0x11, 0x59, 0x15, 0xeb, 0x45, 0x05, 0xbd, 0x88, 0x47, 0x5c, 0x85,
0x59, 0xf0, 0xc1, 0x9b, 0x61, 0x47, 0x45, 0x83, 0xe1, 0x3b, 0x60, 0x6d, 0xb4, 0x84, 0xd8, 0x5b,
0x0d, 0xbe, 0x07, 0xbe, 0x75, 0x1e, 0x6a, 0xd4, 0x23, 0x0e, 0xcf, 0x7e, 0x84, 0xc8, 0x42, 0x1d,
0x5e, 0x07, 0xd7, 0x46, 0xa3, 0x68, 0x6a, 0x5c, 0xf8, 0x15, 0xa0, 0x6d, 0x1b, 0x66, 0xf6, 0xa3,
0xb3, 0xd3, 0xf2, 0x4a, 0x81, 0x9b, 0xe0, 0x36, 0xca, 0x16, 0xb6, 0x8b, 0x7b, 0x95, 0x37, 0xc0,
0x7f, 0xa6, 0xc0, 0x0f, 0xc1, 0xfd, 0xf3, 0x81, 0xa3, 0x36, 0xf8, 0xb9, 0x02, 0x0d, 0xf0, 0xe8,
0x8d, 0x9f, 0x37, 0x4a, 0xe6, 0x37, 0x0a, 0xbc, 0x0e, 0xde, 0x1e, 0xce, 0x17, 0x75, 0xf8, 0xad,
0x02, 0xd7, 0xc1, 0x8d, 0x33, 0x9f, 0x24, 0x90, 0xbf, 0x53, 0xe0, 0xb7, 0xc1, 0xbd, 0xb3, 0x20,
0xa3, 0xc2, 0xf8, 0xbd, 0x02, 0x1f, 0x82, 0x07, 0x6f, 0xf0, 0x8c, 0x51, 0x02, 0x7f, 0x38, 0x63,
0x1f, 0xa2, 0xd8, 0x5f, 0x9c, 0xbf, 0x0f, 0x81, 0xfc, 0xa3, 0x02, 0x57, 0xc1, 0xe5, 0xe1, 0x10,
0xfa, 0x4e, 0xfc, 0x49, 0x81, 0x37, 0xc1, 0xda, 0x99, 0x4a, 0x14, 0xf6, 0x67, 0x05, 0xaa, 0x60,
0xa9, 0x50, 0xac, 0x3c, 0xce, 0xe6, 0xcd, 0xca, 0xf3, 0xbc, 0xb5, 0x53, 0x29, 0x5b, 0xc8, 0x28,
0x97, 0xd3, 0xbf, 0x1a, 0xa7, 0xa1, 0xc4, 0x3c, 0x85, 0xa2, 0x70, 0x56, 0x1e, 0x17, 0x51, 0xc5,
0xcc, 0x3f, 0x33, 0x0a, 0x14, 0xf9, 0xe9, 0x38, 0x5c, 0x00, 0x80, 0xc2, 0x4a, 0xc5, 0x7c, 0xc1,
0x2a, 0xa7, 0x7f, 0x90, 0x82, 0x73, 0x60, 0xca, 0x78, 0x61, 0x19, 0xa8, 0x90, 0x35, 0xd3, 0xff,
0x4c, 0x6d, 0x04, 0x00, 0xf4, 0xc7, 0x0a, 0x38, 0x09, 0xc6, 0x77, 0x9f, 0xa5, 0xc7, 0xe0, 0x34,
0x98, 0x30, 0x8d, 0x6c, 0xd9, 0x48, 0x2b, 0x70, 0x09, 0x2c, 0x18, 0xa6, 0x91, 0xb3, 0xf2, 0xc5,
0x42, 0x05, 0xed, 0x17, 0x0a, 0xec, 0xdc, 0x48, 0x83, 0xd9, 0xe7, 0x59, 0x2b, 0xb7, 0xd3, 0xb5,
0xa4, 0xe8, 0xf9, 0x64, 0x16, 0x73, 0xbb, 0x15, 0x94, 0xcd, 0x19, 0xa8, 0x6b, 0xbe, 0x40, 0x81,
0x4c, 0xa8, 0x6b, 0x99, 0xb8, 0xfb, 0x10, 0x4c, 0x5b, 0xa1, 0xed, 0x47, 0x8d, 0x20, 0x24, 0xf0,
0xae, 0xbc, 0x98, 0x17, 0x67, 0xbd, 0xe8, 0x63, 0x57, 0x16, 0x7a, 0x6b, 0xde, 0x4e, 0xb4, 0xb1,
0x75, 0xe5, 0xeb, 0xca, 0xd6, 0xc5, 0x57, 0x7f, 0x5b, 0x1d, 0x7b, 0xf5, 0x7a, 0x55, 0xf9, 0xe2,
0xf5, 0xaa, 0xf2, 0xd7, 0xd7, 0xab, 0xca, 0xcf, 0xff, 0xbe, 0x3a, 0x76, 0x30, 0xc9, 0xfe, 0x87,
0x74, 0xef, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x10, 0x53, 0x17, 0x6b, 0x8c, 0x1a, 0x00, 0x00,
// 2468 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0xdb, 0x76, 0xdb, 0xc6,
0xd5, 0x16, 0x44, 0x4b, 0x96, 0x46, 0x27, 0x6a, 0x64, 0x59, 0xb0, 0x9d, 0x88, 0x32, 0x1c, 0xe7,
0x57, 0xf4, 0x07, 0x72, 0x6b, 0x67, 0xb5, 0xb5, 0x73, 0xb0, 0x29, 0x0a, 0xb6, 0x58, 0x41, 0x24,
0x3d, 0x84, 0x6c, 0xe7, 0x8a, 0x85, 0xc0, 0xa1, 0x88, 0x0a, 0x02, 0x68, 0x60, 0xa8, 0x50, 0x79,
0x81, 0xde, 0xf6, 0xbc, 0x7a, 0xd5, 0x27, 0x68, 0xfa, 0x1c, 0x4e, 0x7a, 0x4a, 0xdb, 0x7b, 0xb6,
0x75, 0x56, 0x5f, 0x80, 0xab, 0xa7, 0xf4, 0xaa, 0x6b, 0x0e, 0x24, 0x07, 0x20, 0x29, 0xe9, 0x4e,
0xd8, 0xfb, 0xfb, 0x3e, 0xec, 0xd9, 0x9b, 0xd8, 0x7b, 0x8f, 0x0d, 0x16, 0xc2, 0x86, 0xd3, 0x38,
0xb8, 0x13, 0x36, 0x9c, 0xcd, 0x46, 0x18, 0x90, 0x00, 0x4e, 0x30, 0xc3, 0x75, 0xfd, 0xd0, 0x25,
0xf5, 0xe6, 0xc1, 0xa6, 0x13, 0x1c, 0xdf, 0x39, 0x0c, 0x0e, 0x83, 0x3b, 0xcc, 0x7b, 0xd0, 0xac,
0xb1, 0x27, 0xf6, 0xc0, 0xfe, 0xe2, 0x2c, 0xed, 0xeb, 0x39, 0x70, 0xc9, 0x20, 0x4e, 0x15, 0xde,
0x02, 0x97, 0x0a, 0xf6, 0x31, 0x56, 0x95, 0x35, 0x65, 0x7d, 0x7a, 0x6b, 0xa1, 0xd3, 0xce, 0xcc,
0x9c, 0xda, 0xc7, 0xde, 0x03, 0xcd, 0xb7, 0x8f, 0xb1, 0x86, 0x98, 0x13, 0xea, 0xe0, 0xf2, 0xb6,
0x4d, 0xec, 0x6d, 0x37, 0x54, 0xc7, 0x19, 0x6e, 0xa9, 0xd3, 0xce, 0x2c, 0x70, 0x5c, 0xd5, 0x26,
0xb6, 0x5e, 0x75, 0x43, 0x0d, 0x75, 0x31, 0x70, 0x03, 0x4c, 0x3e, 0xcf, 0x9a, 0x14, 0x9d, 0x62,
0x68, 0xd8, 0x69, 0x67, 0xe6, 0x39, 0xfa, 0x13, 0xdb, 0xe3, 0x60, 0x81, 0x80, 0x45, 0xb0, 0xb4,
0x83, 0xed, 0x90, 0x1c, 0x60, 0x9b, 0xe4, 0x7d, 0x82, 0xc3, 0x13, 0xdb, 0xdb, 0x8b, 0xd4, 0x99,
0x35, 0x65, 0x3d, 0xb5, 0xf5, 0x66, 0xa7, 0x9d, 0xb9, 0xc6, 0x89, 0xf5, 0x2e, 0x48, 0x77, 0x05,
0x4a, 0x43, 0xc3, 0x98, 0x30, 0x0f, 0x16, 0x0d, 0x0f, 0x3b, 0xc4, 0x0d, 0x7c, 0xcb, 0x3d, 0xc6,
0x41, 0x93, 0xec, 0x45, 0xea, 0x2c, 0x93, 0xbb, 0xd1, 0x69, 0x67, 0x56, 0xb8, 0x1c, 0x16, 0x10,
0x9d, 0x70, 0x8c, 0x86, 0x06, 0x59, 0x30, 0x0f, 0xd2, 0xa6, 0x1b, 0x11, 0xec, 0xe7, 0x3c, 0x17,
0xfb, 0x64, 0x1f, 0x99, 0x91, 0xba, 0xbc, 0x96, 0x5a, 0x9f, 0x96, 0x03, 0xf3, 0x18, 0x42, 0x77,
0x18, 0x44, 0x6f, 0x86, 0x5e, 0xa4, 0xa1, 0x01, 0x1a, 0x44, 0x60, 0x29, 0x5b, 0x3d, 0xc1, 0x21,
0x71, 0x23, 0x2c, 0xa9, 0x5d, 0x65, 0x6a, 0x6b, 0x9d, 0x76, 0xe6, 0x0d, 0xae, 0x66, 0x77, 0x41,
0x71, 0xc1, 0x61, 0x64, 0x78, 0x1f, 0xcc, 0xf1, 0xa7, 0x6c, 0x93, 0x04, 0x96, 0x59, 0x56, 0x57,
0xd6, 0x94, 0xf5, 0x29, 0xb9, 0x36, 0x76, 0x93, 0x04, 0x3a, 0xa1, 0x02, 0x71, 0x24, 0xcc, 0x81,
0x79, 0x6e, 0xc8, 0xe1, 0x90, 0x1a, 0xeb, 0xaa, 0xca, 0xb8, 0x52, 0x86, 0xc4, 0xfb, 0x1d, 0x1c,
0x12, 0xdd, 0x6e, 0x92, 0xba, 0x86, 0x12, 0x14, 0xf8, 0x81, 0x2c, 0xf2, 0xd8, 0xf5, 0xb0, 0x7a,
0x8d, 0x95, 0xfb, 0x4a, 0xa7, 0x9d, 0x49, 0x0b, 0x11, 0xca, 0xae, 0xb9, 0x1e, 0x8e, 0xb1, 0x29,
0xb6, 0x1f, 0xfd, 0x2e, 0x3e, 0x65, 0xe4, 0xeb, 0xc9, 0x5f, 0xd6, 0x11, 0x3e, 0x15, 0xdc, 0x38,
0x12, 0x9a, 0x60, 0x89, 0x1b, 0xac, 0xb0, 0x19, 0x11, 0x5c, 0xcd, 0x65, 0x99, 0xc0, 0x0d, 0x26,
0x70, 0xbd, 0xd3, 0xce, 0x5c, 0xe5, 0x02, 0x84, 0xbb, 0x75, 0xc7, 0x16, 0x3a, 0xc3, 0x68, 0x34,
0x17, 0xbc, 0x5c, 0x25, 0x8c, 0x43, 0x56, 0x95, 0x0c, 0xab, 0x8a, 0x94, 0x0b, 0x51, 0xe3, 0x06,
0xc6, 0xa1, 0x28, 0x48, 0x82, 0x02, 0x2d, 0xb0, 0xd8, 0x2b, 0x51, 0x4f, 0x67, 0x8d, 0xe9, 0xbc,
0xdd, 0x69, 0x67, 0x34, 0xae, 0xe3, 0xfa, 0x2e, 0x71, 0x6d, 0x4f, 0xef, 0x57, 0x59, 0x92, 0x1c,
0x14, 0x80, 0x0f, 0xc0, 0x0c, 0xfd, 0xbb, 0x5b, 0xdf, 0x9b, 0xac, 0x46, 0x6a, 0xa7, 0x9d, 0xb9,
0xc2, 0xf5, 0x18, 0xbb, 0x5f, 0x64, 0x19, 0x0c, 0x4b, 0x00, 0xd2, 0xc7, 0x44, 0x99, 0x35, 0x26,
0x21, 0xfd, 0xe0, 0x98, 0xc4, 0x60, 0xad, 0x87, 0x70, 0xe1, 0x87, 0x60, 0x96, 0x59, 0xbb, 0xd5,
0xbe, 0xc5, 0xf2, 0x7d, 0xad, 0xd3, 0xce, 0x2c, 0xcb, 0x5a, 0xfd, 0x92, 0xc7, 0xe0, 0xdd, 0xc3,
0x74, 0xcb, 0xfd, 0x16, 0x63, 0x27, 0x0f, 0xd3, 0xaf, 0xb9, 0x0c, 0x86, 0x7b, 0x60, 0x91, 0x3e,
0xc6, 0xeb, 0x7d, 0x9b, 0x29, 0x64, 0x3a, 0xed, 0xcc, 0x0d, 0x49, 0x61, 0xa0, 0xe8, 0x83, 0x4c,
0xb8, 0x05, 0xe6, 0xf3, 0xbc, 0x14, 0x39, 0x8f, 0xda, 0x43, 0xf5, 0x9d, 0xe4, 0x6f, 0xa7, 0x5b,
0x2a, 0x87, 0x03, 0x34, 0x94, 0x60, 0xd0, 0x2f, 0x3a, 0x6e, 0x29, 0x13, 0x9b, 0x60, 0x75, 0x83,
0x09, 0x49, 0x09, 0x4e, 0x08, 0xe9, 0x11, 0x85, 0x69, 0x68, 0x18, 0x79, 0x50, 0xd3, 0x0a, 0x8e,
0xb0, 0xaf, 0xfe, 0xff, 0x79, 0x9a, 0x84, 0xc2, 0x06, 0x34, 0x19, 0x19, 0x3e, 0x04, 0x73, 0x65,
0xdf, 0x6e, 0x44, 0xf5, 0x80, 0xe4, 0x82, 0xa6, 0x4f, 0xd4, 0x7b, 0xac, 0x17, 0x4a, 0x65, 0x8b,
0x84, 0x5b, 0x77, 0xa8, 0x5f, 0x43, 0x71, 0x3c, 0x34, 0xc1, 0xe2, 0xd3, 0x66, 0x40, 0xec, 0x2d,
0xdb, 0x39, 0xc2, 0x7e, 0x75, 0xeb, 0x94, 0xe0, 0x48, 0x7d, 0x8f, 0x89, 0xac, 0x76, 0xda, 0x99,
0xeb, 0x5c, 0xe4, 0x25, 0x85, 0xe8, 0x07, 0x1c, 0xa3, 0x1f, 0x50, 0x90, 0x86, 0x06, 0x89, 0x74,
0x94, 0x94, 0x42, 0xfc, 0x2c, 0x20, 0x58, 0x7d, 0x98, 0x6c, 0x57, 0x8d, 0x10, 0xeb, 0x27, 0x01,
0xcd, 0x4e, 0x17, 0x23, 0x67, 0x24, 0x08, 0xc3, 0x66, 0x83, 0xe4, 0xea, 0xd8, 0x39, 0x52, 0x1f,
0x25, 0x7f, 0xc6, 0xbd, 0x8c, 0x70, 0x94, 0xee, 0x50, 0x98, 0x94, 0x11, 0x89, 0xac, 0xfd, 0x72,
0x06, 0x4c, 0xee, 0xe1, 0xe3, 0x03, 0x1c, 0xd2, 0x9f, 0x34, 0x9d, 0x82, 0x46, 0x0b, 0x3b, 0x25,
0x9b, 0xd4, 0xc5, 0x14, 0x94, 0x72, 0x83, 0x89, 0x53, 0xd5, 0x71, 0x0b, 0x3b, 0x7a, 0xc3, 0xa6,
0xdf, 0x45, 0x0c, 0x0e, 0xef, 0x81, 0xe9, 0xec, 0x21, 0x6d, 0xab, 0xd5, 0x6a, 0xc8, 0x46, 0xd6,
0xf4, 0xd6, 0x72, 0xa7, 0x9d, 0x59, 0x14, 0xdd, 0x97, 0xba, 0x74, 0xbb, 0x5a, 0x0d, 0x35, 0xd4,
0xc7, 0xd1, 0x7c, 0x3e, 0xb6, 0x5d, 0xaf, 0x11, 0xb8, 0x3e, 0xd9, 0xb1, 0xac, 0x12, 0x23, 0xcf,
0x32, 0xb2, 0x94, 0xcf, 0x5a, 0x17, 0xa2, 0xd7, 0x09, 0x69, 0x08, 0x95, 0x41, 0x22, 0xcd, 0xe7,
0x96, 0x1d, 0x61, 0x3a, 0x6c, 0x71, 0xb2, 0x81, 0x1e, 0xd8, 0x11, 0x16, 0xa3, 0x59, 0x60, 0xe8,
0x47, 0x48, 0x4f, 0x60, 0x06, 0x87, 0xec, 0xbc, 0xb5, 0xe4, 0x47, 0xc8, 0xce, 0xeb, 0x05, 0x87,
0xe2, 0xb8, 0x32, 0x18, 0x3e, 0x01, 0x0b, 0xf4, 0x91, 0x77, 0x85, 0x52, 0x18, 0xb4, 0x4e, 0xd5,
0xcf, 0x15, 0x56, 0x88, 0x37, 0x3a, 0xed, 0x8c, 0x2a, 0x09, 0x88, 0x7e, 0xd2, 0xa0, 0x18, 0x0d,
0x25, 0x59, 0x30, 0x0b, 0xe6, 0xa8, 0x89, 0x7e, 0x97, 0x5c, 0xe6, 0x0b, 0x2e, 0x23, 0x7d, 0x7e,
0x4c, 0x86, 0x7d, 0xcf, 0x42, 0x24, 0xce, 0xa0, 0xdd, 0xad, 0xaf, 0x6a, 0xf8, 0x55, 0x96, 0x14,
0xf5, 0xb3, 0xf1, 0x64, 0x4b, 0x90, 0xc3, 0xc1, 0x02, 0xa6, 0xa1, 0x21, 0x5c, 0xf8, 0x4d, 0xbe,
0x10, 0xa9, 0xbf, 0xa6, 0x1a, 0x33, 0x77, 0x67, 0x36, 0xd9, 0x5e, 0xb5, 0x49, 0x6d, 0xf2, 0x5a,
0x44, 0x05, 0x35, 0xc4, 0x77, 0xa7, 0x6d, 0x79, 0x00, 0xd2, 0xe5, 0x47, 0xfd, 0x11, 0x5f, 0x78,
0x46, 0x8c, 0x51, 0xba, 0x2a, 0xc5, 0x06, 0x21, 0xe5, 0xc4, 0x55, 0x58, 0x55, 0x7e, 0x7c, 0xa6,
0x0a, 0xaf, 0x4c, 0x82, 0x43, 0x73, 0xda, 0x1b, 0x92, 0x2c, 0x94, 0x9f, 0xa4, 0x92, 0x2d, 0x4d,
0x88, 0xd0, 0x16, 0xcb, 0x23, 0x89, 0x33, 0x62, 0x12, 0x2c, 0x8e, 0x9f, 0x9e, 0x25, 0xc1, 0xc3,
0x88, 0x33, 0xa0, 0x35, 0x30, 0x99, 0x59, 0x2c, 0x3f, 0xe3, 0x42, 0x37, 0x3b, 0xed, 0xcc, 0x9b,
0x31, 0x21, 0xa9, 0x59, 0xf3, 0x90, 0x86, 0xd1, 0x87, 0xa8, 0xb2, 0xf0, 0x7e, 0x7e, 0x01, 0x55,
0x1e, 0xe5, 0x30, 0x3a, 0xfc, 0xa8, 0x3f, 0xce, 0x58, 0x90, 0xff, 0x4c, 0x8d, 0x9e, 0x67, 0x3c,
0xb8, 0x18, 0x5e, 0xe6, 0xb3, 0x70, 0xfe, 0x75, 0x06, 0x5f, 0x34, 0x0f, 0x19, 0x0f, 0xdf, 0xef,
0xcd, 0x43, 0xf6, 0xfa, 0x7f, 0xa7, 0x46, 0x0e, 0x44, 0xfe, 0x76, 0x19, 0x2d, 0x91, 0xd9, 0xbb,
0xff, 0x33, 0x9a, 0x2c, 0x3e, 0x64, 0x09, 0x0d, 0x0b, 0x89, 0x69, 0xca, 0xde, 0xff, 0x75, 0xea,
0xbc, 0x71, 0xca, 0xc3, 0x18, 0xa4, 0x0e, 0xe8, 0xb1, 0x90, 0xfe, 0x7b, 0xae, 0x1e, 0x8f, 0x6c,
0x90, 0xaa, 0x7d, 0x35, 0x0b, 0x26, 0x2d, 0xcc, 0xa6, 0xac, 0x74, 0xf3, 0x50, 0x2e, 0x70, 0xf3,
0x78, 0x17, 0x5c, 0x2e, 0x60, 0xf2, 0x49, 0x10, 0x1e, 0x89, 0x8b, 0x8a, 0x74, 0xf5, 0xf0, 0xb9,
0x43, 0x43, 0x5d, 0x08, 0xbd, 0xfb, 0xb0, 0xe6, 0x9b, 0x4a, 0xde, 0x7d, 0x78, 0xb7, 0x65, 0x4e,
0xba, 0x1e, 0x6e, 0x63, 0xcf, 0x3e, 0x35, 0x6d, 0x82, 0x7d, 0xe7, 0x54, 0xdc, 0x4d, 0xe6, 0xe4,
0xaf, 0xb3, 0x4a, 0xfd, 0xba, 0xc7, 0x01, 0xfa, 0x31, 0x5d, 0x0f, 0xe3, 0x14, 0xf8, 0x5d, 0x90,
0x8e, 0x5b, 0xd0, 0x09, 0x6b, 0xf9, 0x73, 0x72, 0xcb, 0x4f, 0xca, 0xe8, 0xe1, 0x89, 0x86, 0x06,
0x78, 0xf0, 0x63, 0xb0, 0xbc, 0xdf, 0xa8, 0xda, 0x04, 0x57, 0x13, 0x71, 0xcd, 0x31, 0xc1, 0x5b,
0x9d, 0x76, 0x26, 0xc3, 0x05, 0x9b, 0x1c, 0xa6, 0x0f, 0xc6, 0x37, 0x5c, 0x01, 0x7e, 0x0b, 0x00,
0x14, 0x34, 0xfd, 0xaa, 0xe9, 0x1e, 0xbb, 0x44, 0x5d, 0x5e, 0x53, 0xd6, 0x27, 0xb6, 0xae, 0x76,
0xda, 0x19, 0xc8, 0xf5, 0x42, 0xea, 0xd3, 0x3d, 0xea, 0xd4, 0x90, 0x84, 0x84, 0x8f, 0xc0, 0x9c,
0xd1, 0x72, 0x49, 0xd1, 0xa7, 0xf3, 0xa9, 0x19, 0x62, 0xf5, 0xea, 0x40, 0x3f, 0x6f, 0xb9, 0x44,
0x0f, 0x7c, 0xbd, 0xc6, 0x01, 0xb4, 0x9f, 0xcb, 0x04, 0xb8, 0x03, 0xd2, 0xb9, 0xc0, 0x8f, 0xd8,
0x52, 0xed, 0x9c, 0xf2, 0x21, 0xbf, 0x92, 0x9c, 0x2d, 0x4e, 0x1f, 0xd1, 0x1d, 0xf0, 0x03, 0x2c,
0x78, 0x1f, 0xcc, 0x18, 0xbe, 0x7d, 0xe0, 0xe1, 0x52, 0x23, 0x0c, 0x6a, 0xe2, 0x5e, 0xb3, 0xd2,
0x69, 0x67, 0x96, 0x44, 0x24, 0xcc, 0xa9, 0x37, 0xa8, 0x97, 0x0e, 0xb8, 0x3e, 0x96, 0x96, 0x5a,
0xc4, 0xc3, 0xf2, 0xb2, 0x47, 0x6f, 0x02, 0x89, 0x52, 0x8b, 0xf8, 0x45, 0x4a, 0x59, 0xa9, 0xe3,
0x14, 0xba, 0x5b, 0x0a, 0x4b, 0xb9, 0xde, 0xac, 0xd5, 0x3c, 0xac, 0xae, 0x25, 0x93, 0xd1, 0x15,
0x89, 0x38, 0xa0, 0xaf, 0x21, 0x18, 0xf0, 0x03, 0x30, 0x2b, 0x2c, 0x39, 0x3b, 0xc2, 0x91, 0x7a,
0x93, 0x5d, 0x24, 0xa4, 0xaf, 0xbb, 0xab, 0xe0, 0x50, 0xb7, 0x86, 0x62, 0x68, 0xb8, 0x2b, 0x2d,
0x18, 0xb9, 0xe0, 0xf8, 0xd8, 0xf6, 0xab, 0x91, 0xaa, 0x25, 0xef, 0xad, 0xfd, 0x05, 0xc3, 0x11,
0x18, 0x79, 0xbf, 0xe8, 0xf2, 0x68, 0x4e, 0x50, 0xd3, 0xf7, 0x71, 0xd8, 0xdb, 0x91, 0xde, 0x49,
0x0e, 0xa7, 0x90, 0xf9, 0xe5, 0x2d, 0x29, 0x41, 0xa1, 0x17, 0x69, 0xa3, 0x45, 0x70, 0xe8, 0xdb,
0x5e, 0x4f, 0x86, 0x2f, 0xca, 0x52, 0x40, 0x58, 0x20, 0x64, 0xa1, 0x01, 0x1a, 0x2d, 0x6f, 0x99,
0x84, 0x38, 0x8a, 0xac, 0xd3, 0x06, 0x8e, 0x54, 0xcc, 0x8e, 0x25, 0x95, 0x37, 0x62, 0x4e, 0x9d,
0x50, 0xaf, 0x86, 0x64, 0x2c, 0xfd, 0x95, 0xf2, 0xc7, 0x5d, 0x7c, 0x5a, 0x76, 0x3f, 0xc5, 0x6c,
0xfb, 0x99, 0x90, 0x0b, 0x23, 0xc8, 0xb4, 0x6d, 0x46, 0xee, 0xa7, 0xf4, 0x57, 0x1a, 0x23, 0xd0,
0xad, 0x23, 0x66, 0x30, 0xed, 0xf0, 0x10, 0xab, 0x87, 0x4c, 0x46, 0x5a, 0x46, 0x13, 0x32, 0xba,
0x47, 0x61, 0x1a, 0x1a, 0xc2, 0x85, 0xcf, 0xc0, 0x95, 0xbe, 0xb5, 0x59, 0xab, 0xb9, 0x2d, 0x64,
0xfb, 0x87, 0x58, 0xad, 0x33, 0x4d, 0xad, 0xd3, 0xce, 0xac, 0x0e, 0x6a, 0x32, 0x9c, 0x1e, 0x52,
0xa0, 0x86, 0x86, 0xf2, 0xe1, 0xf7, 0xc0, 0xca, 0x30, 0xbb, 0xd5, 0xf2, 0x55, 0x97, 0x49, 0x4b,
0xb7, 0xd2, 0x11, 0xd2, 0x3a, 0x69, 0xf9, 0x1a, 0x1a, 0x25, 0x43, 0xb7, 0xc1, 0x9e, 0xcb, 0x6a,
0xf9, 0xc5, 0x46, 0xa4, 0x7e, 0x9f, 0x29, 0x4b, 0x25, 0x95, 0x94, 0x49, 0xcb, 0xd7, 0x83, 0x46,
0xa4, 0xa1, 0x24, 0xab, 0x5f, 0x16, 0x3e, 0xa4, 0x23, 0xbe, 0x54, 0x4e, 0xc4, 0x6e, 0x28, 0x5c,
0x87, 0x8f, 0xf7, 0xa8, 0x57, 0x16, 0x41, 0x80, 0xef, 0x81, 0x69, 0x6e, 0x78, 0x5a, 0x2a, 0xf3,
0x5d, 0x72, 0x42, 0xde, 0xc3, 0x05, 0xfb, 0x25, 0x7d, 0x7b, 0x1f, 0xa8, 0xfd, 0x40, 0x01, 0x97,
0x11, 0x7e, 0xd9, 0xc4, 0x11, 0x81, 0x9b, 0x60, 0xba, 0xd8, 0xc0, 0xa1, 0x4d, 0xdc, 0xc0, 0x67,
0x83, 0x66, 0xfe, 0x6e, 0x5a, 0x2c, 0x80, 0x3d, 0x3b, 0xea, 0x43, 0xe0, 0xed, 0xee, 0x0d, 0x42,
0xe5, 0xdb, 0xe2, 0x9c, 0x00, 0x73, 0x23, 0xea, 0x5e, 0x2f, 0x6e, 0x77, 0xe7, 0x18, 0x1b, 0x31,
0x7d, 0x18, 0x37, 0x22, 0xe1, 0xd4, 0x1c, 0x30, 0x85, 0x70, 0xd4, 0x08, 0xfc, 0x08, 0x43, 0x15,
0x5c, 0x2e, 0x37, 0x1d, 0x07, 0x47, 0x11, 0x8b, 0x63, 0x0a, 0x75, 0x1f, 0xe1, 0x55, 0x30, 0x49,
0x6f, 0x89, 0xcd, 0x88, 0x8f, 0x36, 0x24, 0x9e, 0xa4, 0x58, 0x52, 0x67, 0xc4, 0xb2, 0xf1, 0x17,
0x45, 0x3a, 0x23, 0x9c, 0x07, 0xa0, 0x10, 0x90, 0x32, 0xb1, 0x43, 0x82, 0xab, 0xe9, 0x31, 0x78,
0x05, 0xa4, 0xc5, 0x55, 0x89, 0xd9, 0xe8, 0x7a, 0x9b, 0x56, 0xe0, 0x02, 0x98, 0x41, 0x38, 0xea,
0x19, 0xc6, 0xe1, 0x2c, 0x98, 0xda, 0x75, 0x3d, 0x8f, 0x3d, 0xa5, 0xa8, 0x9b, 0x36, 0x8c, 0x6c,
0xe8, 0xd4, 0xdd, 0x13, 0x9c, 0xbe, 0x44, 0x55, 0xb6, 0x71, 0x44, 0xc2, 0xe0, 0x94, 0x22, 0xd8,
0x95, 0x27, 0x3d, 0x01, 0xaf, 0x81, 0xe5, 0x2d, 0xcf, 0x76, 0x8e, 0xea, 0x81, 0xc7, 0xfe, 0x69,
0xa3, 0x14, 0x84, 0xc4, 0x6a, 0xa1, 0x56, 0xba, 0x0a, 0x6f, 0x80, 0x95, 0x7d, 0xff, 0x60, 0xa8,
0x13, 0xc3, 0x65, 0xb0, 0xc8, 0x9a, 0x6a, 0xcc, 0x5c, 0x83, 0x2b, 0x60, 0x69, 0xdf, 0xaf, 0x0e,
0x38, 0x0e, 0x37, 0xfe, 0x3e, 0xc5, 0xe3, 0x11, 0x8d, 0x90, 0xf2, 0x77, 0xf3, 0xa6, 0x59, 0x29,
0x16, 0x8c, 0xca, 0xe3, 0xa2, 0x69, 0x16, 0x9f, 0x1b, 0x28, 0x3d, 0x06, 0xdf, 0x05, 0xeb, 0x03,
0xe6, 0xca, 0x7e, 0xc1, 0xca, 0x9b, 0x15, 0x0b, 0xe5, 0x9f, 0x3c, 0x31, 0x50, 0xa5, 0x5c, 0xc8,
0x96, 0xca, 0x3b, 0x45, 0x8b, 0xa7, 0x80, 0xa1, 0x4d, 0x23, 0xbb, 0x6d, 0xa0, 0xf4, 0x38, 0x7c,
0x1b, 0x68, 0x92, 0x61, 0x14, 0x31, 0xd5, 0x23, 0x3e, 0xdd, 0x2f, 0xa2, 0xfd, 0xbd, 0xf4, 0x25,
0x96, 0x3b, 0x6a, 0xc8, 0x9a, 0x66, 0x7a, 0x02, 0x6e, 0x80, 0xb7, 0xb7, 0xcc, 0x6c, 0x6e, 0x77,
0xa7, 0x68, 0x1a, 0x95, 0x92, 0x61, 0xa0, 0x4a, 0xa9, 0x88, 0xac, 0x8a, 0xf5, 0xa2, 0x82, 0x5e,
0xc4, 0x23, 0xae, 0xc2, 0x2c, 0xf8, 0xf0, 0x62, 0xd8, 0x51, 0xd1, 0x60, 0xf8, 0x16, 0x58, 0x1b,
0x2d, 0x21, 0xce, 0x56, 0x83, 0xef, 0x83, 0x6f, 0x9f, 0x87, 0x1a, 0xf5, 0x8a, 0xc3, 0xb3, 0x5f,
0x21, 0xb2, 0x50, 0x87, 0x37, 0xc1, 0x9b, 0xa3, 0x51, 0x34, 0x35, 0x2e, 0xfc, 0x3f, 0xa0, 0x6d,
0x1b, 0x66, 0xf6, 0xe3, 0xb3, 0xd3, 0xf2, 0x4a, 0x81, 0x9b, 0xe0, 0x1d, 0x94, 0x2d, 0x6c, 0x17,
0xf7, 0x2a, 0x17, 0xc0, 0x7f, 0xae, 0xc0, 0x8f, 0xc0, 0xfd, 0xf3, 0x81, 0xa3, 0x0e, 0xf8, 0x85,
0x02, 0x0d, 0xf0, 0xe8, 0xc2, 0xef, 0x1b, 0x25, 0xf3, 0x1b, 0x05, 0xde, 0x04, 0x6f, 0x0c, 0xe7,
0x8b, 0x3a, 0xfc, 0x56, 0x81, 0xeb, 0xe0, 0xd6, 0x99, 0x6f, 0x12, 0xc8, 0xdf, 0x29, 0xf0, 0x3b,
0xe0, 0xde, 0x59, 0x90, 0x51, 0x61, 0xfc, 0x5e, 0x81, 0x0f, 0xc1, 0x83, 0x0b, 0xbc, 0x63, 0x94,
0xc0, 0x1f, 0xce, 0x38, 0x87, 0x28, 0xf6, 0x97, 0xe7, 0x9f, 0x43, 0x20, 0xff, 0xa8, 0xc0, 0x55,
0x70, 0x6d, 0x38, 0x84, 0xfe, 0x26, 0xfe, 0xa4, 0xc0, 0xdb, 0x60, 0xed, 0x4c, 0x25, 0x0a, 0xfb,
0xb3, 0x02, 0x55, 0xb0, 0x54, 0x28, 0x56, 0x1e, 0x67, 0xf3, 0x66, 0xe5, 0x79, 0xde, 0xda, 0xa9,
0x94, 0x2d, 0x64, 0x94, 0xcb, 0xe9, 0x5f, 0x8d, 0xd3, 0x50, 0x62, 0x9e, 0x42, 0x51, 0x38, 0x2b,
0x8f, 0x8b, 0xa8, 0x62, 0xe6, 0x9f, 0x19, 0x05, 0x8a, 0xfc, 0x6c, 0x1c, 0x2e, 0x00, 0x40, 0x61,
0xa5, 0x62, 0xbe, 0x60, 0x95, 0xd3, 0x3f, 0x4c, 0xc1, 0x39, 0x30, 0x65, 0xbc, 0xb0, 0x0c, 0x54,
0xc8, 0x9a, 0xe9, 0x7f, 0xa4, 0x36, 0x02, 0x00, 0xfa, 0x6b, 0x05, 0x9c, 0x04, 0xe3, 0xbb, 0xcf,
0xd2, 0x63, 0x70, 0x1a, 0x4c, 0x98, 0x46, 0xb6, 0x6c, 0xa4, 0x15, 0xb8, 0x04, 0x16, 0x0c, 0xd3,
0xc8, 0x59, 0xf9, 0x62, 0xa1, 0x82, 0xf6, 0x0b, 0x05, 0xd6, 0x37, 0xd2, 0x60, 0xf6, 0x79, 0xd6,
0xca, 0xed, 0x74, 0x2d, 0x29, 0xda, 0x9f, 0xcc, 0x62, 0x6e, 0xb7, 0x82, 0xb2, 0x39, 0x03, 0x75,
0xcd, 0x97, 0x28, 0x90, 0x09, 0x75, 0x2d, 0x13, 0x77, 0x1f, 0x82, 0x69, 0x2b, 0xb4, 0xfd, 0xa8,
0x11, 0x84, 0x04, 0xde, 0x95, 0x1f, 0xe6, 0x45, 0xaf, 0x17, 0x73, 0xec, 0xfa, 0x42, 0xef, 0x99,
0x8f, 0x13, 0x6d, 0x6c, 0x5d, 0xf9, 0x86, 0xb2, 0x75, 0xe5, 0xd5, 0xdf, 0x56, 0xc7, 0x5e, 0xbd,
0x5e, 0x55, 0xbe, 0x7c, 0xbd, 0xaa, 0xfc, 0xf5, 0xf5, 0xaa, 0xf2, 0x8b, 0xaf, 0x56, 0xc7, 0x0e,
0x26, 0xd9, 0x7f, 0x05, 0xdd, 0xfb, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd4, 0x42, 0xd4, 0x02,
0x53, 0x1a, 0x00, 0x00,
}

18
tools/functional-tester/rpcpb/rpc.proto → functional/rpcpb/rpc.proto
View File

@ -171,9 +171,9 @@ enum StressType {
}
message Tester {
string TesterDataDir = 1 [(gogoproto.moretags) = "yaml:\"tester-data-dir\""];
string TesterNetwork = 2 [(gogoproto.moretags) = "yaml:\"tester-network\""];
string TesterAddr = 3 [(gogoproto.moretags) = "yaml:\"tester-addr\""];
string DataDir = 1 [(gogoproto.moretags) = "yaml:\"data-dir\""];
string Network = 2 [(gogoproto.moretags) = "yaml:\"network\""];
string Addr = 3 [(gogoproto.moretags) = "yaml:\"addr\""];
// DelayLatencyMsRv is the delay latency in milliseconds,
// to inject to simulated slow network.
@ -194,15 +194,15 @@ message Tester {
// EnablePprof is true to enable profiler.
bool EnablePprof = 24 [(gogoproto.moretags) = "yaml:\"enable-pprof\""];
// FailureCases is the selected test cases to schedule.
// If empty, run all failure cases.
repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""];
// FailureDelayMs is the delay duration after failure is injected.
// Useful when triggering snapshot or no-op failure cases.
uint32 FailureDelayMs = 32 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""];
uint32 FailureDelayMs = 31 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""];
// FailureShuffle is true to randomize failure injecting order.
bool FailureShuffle = 33 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
// FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
// FailureCases is the selected test cases to schedule.
// If empty, run all failure cases.
repeated string FailureCases = 33 [(gogoproto.moretags) = "yaml:\"failure-cases\""];
// Failpoinommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)
repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""];
// RunnerExecPath is a path of etcd-runner binary.

0
tools/functional-tester/runner/election_command.go → functional/runner/election_command.go
View File

0
tools/functional-tester/runner/error.go → functional/runner/error.go
View File

0
tools/functional-tester/runner/global.go → functional/runner/global.go
View File

0
tools/functional-tester/runner/help.go → functional/runner/help.go
View File

0
tools/functional-tester/runner/lease_renewer_command.go → functional/runner/lease_renewer_command.go
View File

0
tools/functional-tester/runner/lock_racer_command.go → functional/runner/lock_racer_command.go
View File

0
tools/functional-tester/runner/root.go → functional/runner/root.go
View File

0
tools/functional-tester/runner/watch_command.go → functional/runner/watch_command.go
View File

2
tools/functional-tester/scripts/docker-local-agent.sh → functional/scripts/docker-local-agent.sh
View File

@ -8,7 +8,7 @@
COMMENT
if ! [[ "${0}" =~ "scripts/docker-local-agent.sh" ]]; then
echo "must be run from tools/functional-tester"
echo "must be run from functional"
exit 255
fi

4
tools/functional-tester/scripts/docker-local-tester.sh → functional/scripts/docker-local-tester.sh
View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash
if ! [[ "${0}" =~ "scripts/docker-local-tester.sh" ]]; then
echo "must be run from tools/functional-tester"
echo "must be run from functional"
exit 255
fi
@ -15,4 +15,4 @@ docker run \
--net=host \
--name tester \
gcr.io/etcd-development/etcd-functional-tester:go${GO_VERSION} \
/bin/bash -c "./bin/etcd-tester --config ./local-test.yaml"
/bin/bash -c "./bin/etcd-tester --config ./functional.yaml"

0
tools/functional-tester/scripts/genproto.sh → functional/scripts/genproto.sh
View File

2
tools/functional-tester/tester/checks.go → functional/tester/checks.go
View File

@ -21,7 +21,7 @@ import (
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
"google.golang.org/grpc"

272
tools/functional-tester/tester/cluster.go → functional/tester/cluster.go
View File

@ -26,15 +26,14 @@ import (
"strings"
"time"
"github.com/coreos/etcd/functional/rpcpb"
"github.com/coreos/etcd/pkg/debugutil"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
"golang.org/x/time/rate"
"google.golang.org/grpc"
yaml "gopkg.in/yaml.v2"
)
// Cluster defines tester cluster.
@ -62,221 +61,6 @@ type Cluster struct {
cs int
}
func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
bts, err := ioutil.ReadFile(fpath)
if err != nil {
return nil, err
}
lg.Info("opened configuration file", zap.String("path", fpath))
clus := &Cluster{lg: lg}
if err = yaml.Unmarshal(bts, clus); err != nil {
return nil, err
}
for i, mem := range clus.Members {
if mem.BaseDir == "" {
return nil, fmt.Errorf("Members[i].BaseDir cannot be empty (got %q)", mem.BaseDir)
}
if mem.EtcdLogPath == "" {
return nil, fmt.Errorf("Members[i].EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath)
}
if mem.Etcd.Name == "" {
return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem)
}
if mem.Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem)
}
if mem.Etcd.SnapshotCount == 0 {
return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount)
}
if mem.Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir)
}
if mem.Etcd.WALDir == "" {
clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal")
}
if mem.Etcd.HeartbeatIntervalMs == 0 {
return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd)
}
if mem.Etcd.ElectionTimeoutMs == 0 {
return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd)
}
if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs {
return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs)
}
port := ""
listenClientPorts := make([]string, len(clus.Members))
for i, u := range mem.Etcd.ListenClientURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u)
}
listenClientPorts[i], err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u)
}
}
for i, u := range mem.Etcd.AdvertiseClientURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u)
}
port, err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u)
}
if mem.EtcdClientProxy && listenClientPorts[i] == port {
return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i])
}
}
listenPeerPorts := make([]string, len(clus.Members))
for i, u := range mem.Etcd.ListenPeerURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u)
}
listenPeerPorts[i], err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u)
}
}
for j, u := range mem.Etcd.AdvertisePeerURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u)
}
port, err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u)
}
if mem.EtcdPeerProxy && listenPeerPorts[j] == port {
return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j])
}
}
if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) {
return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath)
}
if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) {
return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir)
}
// TODO: support separate WALDir that can be handled via failure-archive
if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) {
return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir)
}
// TODO: only support generated certs with TLS generator
// deprecate auto TLS
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertAuth {
return nil, fmt.Errorf("Etcd.ClientAutoTLS and Etcd.ClientCertAuth are both 'true'")
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile)
}
if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" {
return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile)
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerClientCertAuth {
return nil, fmt.Errorf("Etcd.PeerAutoTLS and Etcd.PeerClientCertAuth are both 'true'")
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" {
return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile)
}
if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" {
for _, cu := range mem.Etcd.ListenClientURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
}
}
for _, cu := range mem.Etcd.AdvertiseClientURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
}
}
}
if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" {
for _, cu := range mem.Etcd.ListenPeerURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
}
}
for _, cu := range mem.Etcd.AdvertisePeerURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
}
}
}
}
if len(clus.Tester.FailureCases) == 0 {
return nil, errors.New("FailureCases not found")
}
if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 {
return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv)
}
if clus.Tester.UpdatedDelayLatencyMs == 0 {
clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs
}
for _, v := range clus.Tester.FailureCases {
if _, ok := rpcpb.FailureCase_value[v]; !ok {
return nil, fmt.Errorf("%q is not defined in 'rpcpb.FailureCase_value'", v)
}
}
for _, v := range clus.Tester.StressTypes {
if _, ok := rpcpb.StressType_value[v]; !ok {
return nil, fmt.Errorf("StressType is unknown; got %q", v)
}
}
if clus.Tester.StressKeySuffixRangeTxn > 100 {
return nil, fmt.Errorf("StressKeySuffixRangeTxn maximum value is 100, got %v", clus.Tester.StressKeySuffixRangeTxn)
}
if clus.Tester.StressKeyTxnOps > 64 {
return nil, fmt.Errorf("StressKeyTxnOps maximum value is 64, got %v", clus.Tester.StressKeyTxnOps)
}
return clus, err
}
var dialOpts = []grpc.DialOption{
grpc.WithInsecure(),
grpc.WithTimeout(5 * time.Second),
@ -285,7 +69,7 @@ var dialOpts = []grpc.DialOption{
// NewCluster creates a client from a tester configuration.
func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
clus, err := newCluster(lg, fpath)
clus, err := read(lg, fpath)
if err != nil {
return nil, err
}
@ -320,7 +104,7 @@ func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
}
}
clus.testerHTTPServer = &http.Server{
Addr: clus.Tester.TesterAddr,
Addr: clus.Tester.Addr,
Handler: mux,
}
go clus.serveTesterServer()
@ -340,12 +124,12 @@ func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
func (clus *Cluster) serveTesterServer() {
clus.lg.Info(
"started tester HTTP server",
zap.String("tester-address", clus.Tester.TesterAddr),
zap.String("tester-address", clus.Tester.Addr),
)
err := clus.testerHTTPServer.ListenAndServe()
clus.lg.Info(
"tester HTTP server returned",
zap.String("tester-address", clus.Tester.TesterAddr),
zap.String("tester-address", clus.Tester.Addr),
zap.Error(err),
)
if err != nil && err != http.ErrServerClosed {
@ -444,48 +228,6 @@ func (clus *Cluster) UpdateDelayLatencyMs() {
}
}
func (clus *Cluster) shuffleFailures() {
rand.Seed(time.Now().UnixNano())
offset := rand.Intn(1000)
n := len(clus.failures)
cp := coprime(n)
fs := make([]Failure, n)
for i := 0; i < n; i++ {
fs[i] = clus.failures[(cp*i+offset)%n]
}
clus.failures = fs
clus.lg.Info("shuffled test failure cases", zap.Int("total", n))
}
/*
x and y of GCD 1 are coprime to each other
x1 = ( coprime of n * idx1 + offset ) % n
x2 = ( coprime of n * idx2 + offset ) % n
(x2 - x1) = coprime of n * (idx2 - idx1) % n
= (idx2 - idx1) = 1
Consecutive x's are guaranteed to be distinct
*/
func coprime(n int) int {
coprime := 1
for i := n / 2; i < n; i++ {
if gcd(i, n) == 1 {
coprime = i
break
}
}
return coprime
}
func gcd(x, y int) int {
if y == 0 {
return x
}
return gcd(y, x%y)
}
func (clus *Cluster) updateStresserChecker() {
cs := &compositeStresser{}
for _, m := range clus.Members {
@ -641,7 +383,7 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error {
// store TLS assets from agents/servers onto disk
if secure && (op == rpcpb.Operation_InitialStartEtcd || op == rpcpb.Operation_RestartEtcd) {
dirClient := filepath.Join(
clus.Tester.TesterDataDir,
clus.Tester.DataDir,
clus.Members[idx].Etcd.Name,
"fixtures",
"client",
@ -717,7 +459,7 @@ func (clus *Cluster) DestroyEtcdAgents() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
err := clus.testerHTTPServer.Shutdown(ctx)
cancel()
clus.lg.Info("closed tester HTTP server", zap.String("tester-address", clus.Tester.TesterAddr), zap.Error(err))
clus.lg.Info("closed tester HTTP server", zap.String("tester-address", clus.Tester.Addr), zap.Error(err))
}
}

244
functional/tester/cluster_read_config.go Normal file
View File

@ -0,0 +1,244 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tester
import (
"errors"
"fmt"
"io/ioutil"
"net/url"
"path/filepath"
"strings"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
yaml "gopkg.in/yaml.v2"
)
func read(lg *zap.Logger, fpath string) (*Cluster, error) {
bts, err := ioutil.ReadFile(fpath)
if err != nil {
return nil, err
}
lg.Info("opened configuration file", zap.String("path", fpath))
clus := &Cluster{lg: lg}
if err = yaml.Unmarshal(bts, clus); err != nil {
return nil, err
}
for i, mem := range clus.Members {
if mem.BaseDir == "" {
return nil, fmt.Errorf("BaseDir cannot be empty (got %q)", mem.BaseDir)
}
if mem.EtcdLogPath == "" {
return nil, fmt.Errorf("EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath)
}
if mem.Etcd.Name == "" {
return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem)
}
if mem.Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem)
}
if mem.Etcd.SnapshotCount == 0 {
return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount)
}
if mem.Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir)
}
if mem.Etcd.WALDir == "" {
clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal")
}
if mem.Etcd.HeartbeatIntervalMs == 0 {
return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd)
}
if mem.Etcd.ElectionTimeoutMs == 0 {
return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd)
}
if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs {
return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs)
}
port := ""
listenClientPorts := make([]string, len(clus.Members))
for i, u := range mem.Etcd.ListenClientURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u)
}
listenClientPorts[i], err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u)
}
}
for i, u := range mem.Etcd.AdvertiseClientURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u)
}
port, err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u)
}
if mem.EtcdClientProxy && listenClientPorts[i] == port {
return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i])
}
}
listenPeerPorts := make([]string, len(clus.Members))
for i, u := range mem.Etcd.ListenPeerURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u)
}
listenPeerPorts[i], err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u)
}
}
for j, u := range mem.Etcd.AdvertisePeerURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u)
}
port, err = getPort(u)
if err != nil {
return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u)
}
if mem.EtcdPeerProxy && listenPeerPorts[j] == port {
return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j])
}
}
if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) {
return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath)
}
if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) {
return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir)
}
// TODO: support separate WALDir that can be handled via failure-archive
if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) {
return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir)
}
// TODO: only support generated certs with TLS generator
// deprecate auto TLS
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertAuth {
return nil, fmt.Errorf("Etcd.ClientAutoTLS and Etcd.ClientCertAuth are both 'true'")
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile)
}
if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" {
return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile)
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerClientCertAuth {
return nil, fmt.Errorf("Etcd.PeerAutoTLS and Etcd.PeerClientCertAuth are both 'true'")
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" {
return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile)
}
if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" {
for _, cu := range mem.Etcd.ListenClientURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
}
}
for _, cu := range mem.Etcd.AdvertiseClientURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
}
}
}
if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" {
for _, cu := range mem.Etcd.ListenPeerURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
}
}
for _, cu := range mem.Etcd.AdvertisePeerURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
}
}
}
}
if len(clus.Tester.FailureCases) == 0 {
return nil, errors.New("FailureCases not found")
}
if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 {
return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv)
}
if clus.Tester.UpdatedDelayLatencyMs == 0 {
clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs
}
for _, v := range clus.Tester.FailureCases {
if _, ok := rpcpb.FailureCase_value[v]; !ok {
return nil, fmt.Errorf("%q is not defined in 'rpcpb.FailureCase_value'", v)
}
}
for _, v := range clus.Tester.StressTypes {
if _, ok := rpcpb.StressType_value[v]; !ok {
return nil, fmt.Errorf("StressType is unknown; got %q", v)
}
}
if clus.Tester.StressKeySuffixRangeTxn > 100 {
return nil, fmt.Errorf("StressKeySuffixRangeTxn maximum value is 100, got %v", clus.Tester.StressKeySuffixRangeTxn)
}
if clus.Tester.StressKeyTxnOps > 64 {
return nil, fmt.Errorf("StressKeyTxnOps maximum value is 64, got %v", clus.Tester.StressKeyTxnOps)
}
return clus, err
}

10
tools/functional-tester/tester/cluster_tester.go → functional/tester/cluster_run.go
View File

@ -19,8 +19,8 @@ import (
"os"
"time"
"github.com/coreos/etcd/functional/rpcpb"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"go.uber.org/zap"
)
@ -29,12 +29,12 @@ import (
// Previous tests showed etcd can compact about 60,000 entries per second.
const compactQPS = 50000
// StartTester starts tester.
func (clus *Cluster) StartTester() {
if err := fileutil.TouchDirAll(clus.Tester.TesterDataDir); err != nil {
// Run starts tester.
func (clus *Cluster) Run() {
if err := fileutil.TouchDirAll(clus.Tester.DataDir); err != nil {
clus.lg.Panic(
"failed to create test data directory",
zap.String("dir", clus.Tester.TesterDataDir),
zap.String("dir", clus.Tester.DataDir),
zap.Error(err),
)
}

64
functional/tester/cluster_shuffle.go Normal file
View File

@ -0,0 +1,64 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tester
import (
"math/rand"
"time"
"go.uber.org/zap"
)
func (clus *Cluster) shuffleFailures() {
rand.Seed(time.Now().UnixNano())
offset := rand.Intn(1000)
n := len(clus.failures)
cp := coprime(n)
fs := make([]Failure, n)
for i := 0; i < n; i++ {
fs[i] = clus.failures[(cp*i+offset)%n]
}
clus.failures = fs
clus.lg.Info("shuffled test failure cases", zap.Int("total", n))
}
/*
x and y of GCD 1 are coprime to each other
x1 = ( coprime of n * idx1 + offset ) % n
x2 = ( coprime of n * idx2 + offset ) % n
(x2 - x1) = coprime of n * (idx2 - idx1) % n
= (idx2 - idx1) = 1
Consecutive x's are guaranteed to be distinct
*/
func coprime(n int) int {
coprime := 1
for i := n / 2; i < n; i++ {
if gcd(i, n) == 1 {
coprime = i
break
}
}
return coprime
}
func gcd(x, y int) int {
if y == 0 {
return x
}
return gcd(y, x%y)
}

19
tools/functional-tester/tester/cluster_test.go → functional/tester/cluster_test.go
View File

@ -19,12 +19,12 @@ import (
"sort"
"testing"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
)
func Test_newCluster(t *testing.T) {
func Test_read(t *testing.T) {
exp := &Cluster{
Members: []*rpcpb.Member{
{
@ -143,9 +143,9 @@ func Test_newCluster(t *testing.T) {
},
},
Tester: &rpcpb.Tester{
TesterDataDir: "/tmp/etcd-tester-data",
TesterNetwork: "tcp",
TesterAddr: "127.0.0.1:9028",
DataDir: "/tmp/etcd-tester-data",
Network: "tcp",
Addr: "127.0.0.1:9028",
DelayLatencyMs: 5000,
DelayLatencyMsRv: 500,
UpdatedDelayLatencyMs: 5000,
@ -153,6 +153,8 @@ func Test_newCluster(t *testing.T) {
ExitOnFailure: true,
ConsistencyCheck: true,
EnablePprof: true,
FailureDelayMs: 7000,
FailureShuffle: true,
FailureCases: []string{
"KILL_ONE_FOLLOWER",
"KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
@ -181,10 +183,8 @@ func Test_newCluster(t *testing.T) {
"NO_FAIL_WITH_STRESS",
"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
},
FailureDelayMs: 7000,
FailureShuffle: true,
FailpointCommands: []string{`panic("etcd-tester")`},
RunnerExecPath: "/etcd-runner",
RunnerExecPath: "./bin/etcd-runner",
ExternalExecPath: "",
StressTypes: []string{"KV", "LEASE"},
StressKeySize: 100,
@ -203,7 +203,7 @@ func Test_newCluster(t *testing.T) {
}
defer logger.Sync()
cfg, err := newCluster(logger, "./local-test.yaml")
cfg, err := read(logger, "../../functional.yaml")
if err != nil {
t.Fatal(err)
}
@ -235,6 +235,7 @@ func Test_newCluster(t *testing.T) {
sort.Strings(fs1)
sort.Strings(fs2)
sort.Strings(fs3)
if !reflect.DeepEqual(fs1, fs2) {
t.Fatalf("expected %q, got %q", fs1, fs2)
}

0
tools/functional-tester/tester/doc.go → functional/tester/doc.go
View File

2
tools/functional-tester/tester/failure.go → functional/tester/failure.go
View File

@ -19,7 +19,7 @@ import (
"math/rand"
"time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
)

0
tools/functional-tester/tester/failure_case_delay.go → functional/tester/failure_case_delay.go
View File

2
tools/functional-tester/tester/failure_case_external.go → functional/tester/failure_case_external.go
View File

@ -18,7 +18,7 @@ import (
"fmt"
"os/exec"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
)
type failureExternal struct {

2
tools/functional-tester/tester/failure_case_failpoints.go → functional/tester/failure_case_failpoints.go
View File

@ -21,7 +21,7 @@ import (
"strings"
"sync"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
)
type failpointStats struct {

2
tools/functional-tester/tester/failure_case_kill.go → functional/tester/failure_case_kill.go
View File

@ -14,7 +14,7 @@
package tester
import "github.com/coreos/etcd/tools/functional-tester/rpcpb"
import "github.com/coreos/etcd/functional/rpcpb"
func injectKill(clus *Cluster, idx int) error {
return clus.sendOperation(idx, rpcpb.Operation_KillEtcd)

2
tools/functional-tester/tester/failure_case_network_blackhole.go → functional/tester/failure_case_network_blackhole.go
View File

@ -14,7 +14,7 @@
package tester
import "github.com/coreos/etcd/tools/functional-tester/rpcpb"
import "github.com/coreos/etcd/functional/rpcpb"
func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error {
return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx)

2
tools/functional-tester/tester/failure_case_network_delay.go → functional/tester/failure_case_network_delay.go
View File

@ -17,7 +17,7 @@ package tester
import (
"time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
)

2
tools/functional-tester/tester/failure_case_no_fail.go → functional/tester/failure_case_no_fail.go
View File

@ -17,7 +17,7 @@ package tester
import (
"time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
)

0
tools/functional-tester/tester/metrics.go → functional/tester/metrics.go
View File

2
tools/functional-tester/tester/stress.go → functional/tester/stress.go
View File

@ -18,7 +18,7 @@ import (
"fmt"
"time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
)

0
tools/functional-tester/tester/stress_composite.go → functional/tester/stress_composite.go
View File

2
tools/functional-tester/tester/stress_key.go → functional/tester/stress_key.go
View File

@ -25,7 +25,7 @@ import (
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/etcdserver"
"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
"golang.org/x/time/rate"

2
tools/functional-tester/tester/stress_lease.go → functional/tester/stress_lease.go
View File

@ -24,7 +24,7 @@ import (
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/coreos/etcd/functional/rpcpb"
"go.uber.org/zap"
"golang.org/x/time/rate"

0
tools/functional-tester/tester/stress_runner.go → functional/tester/stress_runner.go
View File

0
tools/functional-tester/tester/utils.go → functional/tester/utils.go
View File

4
test
View File

@ -37,7 +37,7 @@ source ./build
# build before setting up test GOPATH
if [[ "${PASSES}" == *"functional"* ]]; then
./tools/functional-tester/build
./functional/build
fi
if [ -z "$PASSES" ]; then
@ -196,7 +196,7 @@ function functional_pass {
done
echo "Starting 'etcd-tester'"
./bin/etcd-tester --config ./tools/functional-tester/tester/local-test.yaml && echo "'etcd-tester' succeeded"
./bin/etcd-tester --config ./functional.yaml && echo "'etcd-tester' succeeded"
ETCD_TESTER_EXIT_CODE=$?
echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}

33
tools/functional-tester/README.md
View File

@ -1,33 +0,0 @@
# etcd functional test suite
etcd functional test suite tests the functionality of an etcd cluster with a focus on failure resistance under high pressure. It sets up an etcd cluster and inject failures into the cluster by killing the process or isolate the network of the process. It expects the etcd cluster to recover within a short amount of time after fixing the fault.
etcd functional test suite has two components: etcd-agent and etcd-tester. etcd-agent runs on every test machine, and etcd-tester is a single controller of the test. tester controls agents: start etcd process, stop, terminate, inject failures, and so on.
### Run locally
```bash
PASSES=functional ./test
```
### Run with Docker
To run locally, first build tester image:
```bash
pushd ../..
make build-docker-functional-tester
popd
```
And run [example scripts](./scripts).
```bash
# run 3 agents for 3-node local etcd cluster
./scripts/docker-local-agent.sh 1
./scripts/docker-local-agent.sh 2
./scripts/docker-local-agent.sh 3
# to run only 1 tester round
./scripts/docker-local-tester.sh
```