vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #9534 from gyuho/test-tls 

functional-tester: enable TLS, phase 1
release-3.4
Gyuho Lee 2018-04-05 17:07:35 -07:00 committed by GitHub
parent 72ba5572a2 a0b094ca89
commit c91a61be00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1895 additions and 452 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG-3.4.md
View File

@ -30,7 +30,7 @@ See [code changes](https://github.com/coreos/etcd/compare/v3.3.0...v3.4.0) and [
- Futhermore, when `--auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute.
- Improve [lease expire/revoke operation performance](https://github.com/coreos/etcd/pull/9418), address [lease scalability issue](https://github.com/coreos/etcd/issues/9496).
- Make [Lease `Lookup` non-blocking with concurrent `Grant`/`Revoke`](https://github.com/coreos/etcd/pull/9229).
- Improve functional tester coverage: use [proxy layer to run network fault tests in CIs](https://github.com/coreos/etcd/pull/9081), enable [TLS](https://github.com/coreos/etcd/issues/8943), add [liveness mode](https://github.com/coreos/etcd/issues/9230), [shuffle test sequence](https://github.com/coreos/etcd/issues/9381).
- Improve [functional tester](https://github.com/coreos/etcd/tree/master/tools/functional-tester) coverage: use [proxy layer to run network fault tests in CI](https://github.com/coreos/etcd/pull/9081), enable [TLS both for server and client](https://github.com/coreos/etcd/pull/9534), add [liveness mode](https://github.com/coreos/etcd/issues/9230), and [shuffle test sequence](https://github.com/coreos/etcd/issues/9381).
### Breaking Changes

14
tools/functional-tester/README.md
View File

@ -2,19 +2,7 @@
etcd functional test suite tests the functionality of an etcd cluster with a focus on failure resistance under high pressure. It sets up an etcd cluster and inject failures into the cluster by killing the process or isolate the network of the process. It expects the etcd cluster to recover within a short amount of time after fixing the fault.
etcd functional test suite has two components: etcd-agent and etcd-tester. etcd-agent runs on every test machines and etcd-tester is a single controller of the test. etcd-tester controls all the etcd-agent to start etcd clusters and simulate various failure cases.
## Requirements
The environment of the cluster must be stable enough, so etcd test suite can assume that most of the failures are generated by itself.
## etcd agent
etcd agent is a daemon on each machines. It can start, stop, restart, isolate and terminate an etcd process. The agent exposes these functionality via HTTP RPC.
## etcd tester
etcd functional tester control the progress of the functional tests. It calls the RPC of the etcd agent to simulate various test cases. For example, it can start a three members cluster by sending three start RPC calls to three different etcd agents. It can make one of the member failed by sending stop RPC call to one etcd agent.
etcd functional test suite has two components: etcd-agent and etcd-tester. etcd-agent runs on every test machine, and etcd-tester is a single controller of the test. tester controls agents: start etcd process, stop, terminate, inject failures, and so on.
### Run locally

161
tools/functional-tester/agent/handler.go
View File

@ -17,9 +17,11 @@ package agent
import (
"errors"
"fmt"
"io/ioutil"
"net/url"
"os"
"os/exec"
"path/filepath"
"syscall"
"time"
@ -72,6 +74,7 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
return &rpcpb.Response{
Success: false,
Status: fmt.Sprintf("%q is not valid; last server operation was %q", rpcpb.Operation_InitialStartEtcd.String(), srv.last.String()),
Member: req.Member,
}, nil
}
@ -84,16 +87,22 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
}
srv.lg.Info("created base directory", zap.String("path", srv.Member.BaseDir))
if err = srv.createEtcdFile(); err != nil {
if err = srv.saveEtcdLogFile(); err != nil {
return nil, err
}
srv.creatEtcdCmd()
err = srv.startEtcdCmd()
if err != nil {
if err = srv.saveTLSAssets(); err != nil {
return nil, err
}
if err = srv.startEtcdCmd(); err != nil {
return nil, err
}
srv.lg.Info("started etcd", zap.String("command-path", srv.etcdCmd.Path))
if err = srv.loadAutoTLSAssets(); err != nil {
return nil, err
}
// wait some time for etcd listener start
// before setting up proxy
@ -104,10 +113,12 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
return &rpcpb.Response{
Success: true,
Status: "successfully started etcd!",
Status: "start etcd PASS",
Member: srv.Member,
}, nil
}
// TODO: support TLS
func (srv *Server) startProxy() error {
if srv.Member.EtcdClientProxy {
advertiseClientURL, advertiseClientURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertiseClientURLs[0])
@ -133,7 +144,7 @@ func (srv *Server) startProxy() error {
}
if srv.Member.EtcdPeerProxy {
advertisePeerURL, advertisePeerURLPort, err := getURLAndPort(srv.Member.Etcd.InitialAdvertisePeerURLs[0])
advertisePeerURL, advertisePeerURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertisePeerURLs[0])
if err != nil {
return err
}
@ -200,7 +211,7 @@ func (srv *Server) stopProxy() {
}
}
func (srv *Server) createEtcdFile() error {
func (srv *Server) saveEtcdLogFile() error {
var err error
srv.etcdLogFile, err = os.Create(srv.Member.EtcdLogPath)
if err != nil {
@ -225,6 +236,128 @@ func (srv *Server) creatEtcdCmd() {
srv.etcdCmd.Stderr = srv.etcdLogFile
}
func (srv *Server) saveTLSAssets() error {
// if started with manual TLS, stores TLS assets
// from tester/client to disk before starting etcd process
// TODO: not implemented yet
if !srv.Member.Etcd.ClientAutoTLS {
if srv.Member.Etcd.ClientCertAuth {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientCertAuth is %v", srv.Member.Etcd.ClientCertAuth)
}
if srv.Member.Etcd.ClientCertFile != "" {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientCertFile is %q", srv.Member.Etcd.ClientCertFile)
}
if srv.Member.Etcd.ClientKeyFile != "" {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientKeyFile is %q", srv.Member.Etcd.ClientKeyFile)
}
if srv.Member.Etcd.ClientTrustedCAFile != "" {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientTrustedCAFile is %q", srv.Member.Etcd.ClientTrustedCAFile)
}
}
if !srv.Member.Etcd.PeerAutoTLS {
if srv.Member.Etcd.PeerClientCertAuth {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerClientCertAuth is %v", srv.Member.Etcd.PeerClientCertAuth)
}
if srv.Member.Etcd.PeerCertFile != "" {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerCertFile is %q", srv.Member.Etcd.PeerCertFile)
}
if srv.Member.Etcd.PeerKeyFile != "" {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerKeyFile is %q", srv.Member.Etcd.PeerKeyFile)
}
if srv.Member.Etcd.PeerTrustedCAFile != "" {
return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerTrustedCAFile is %q", srv.Member.Etcd.PeerTrustedCAFile)
}
}
// TODO
return nil
}
func (srv *Server) loadAutoTLSAssets() error {
// if started with auto TLS, sends back TLS assets to tester/client
if srv.Member.Etcd.ClientAutoTLS {
// in case of slow disk
time.Sleep(time.Second)
fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "client")
srv.lg.Info(
"loading client TLS assets",
zap.String("dir", fdir),
zap.String("endpoint", srv.EtcdClientEndpoint),
)
certPath := filepath.Join(fdir, "cert.pem")
if !fileutil.Exist(certPath) {
return fmt.Errorf("cannot find %q", certPath)
}
certData, err := ioutil.ReadFile(certPath)
if err != nil {
return fmt.Errorf("cannot read %q (%v)", certPath, err)
}
srv.Member.ClientCertData = string(certData)
keyPath := filepath.Join(fdir, "key.pem")
if !fileutil.Exist(keyPath) {
return fmt.Errorf("cannot find %q", keyPath)
}
keyData, err := ioutil.ReadFile(keyPath)
if err != nil {
return fmt.Errorf("cannot read %q (%v)", keyPath, err)
}
srv.Member.ClientKeyData = string(keyData)
srv.lg.Info(
"loaded client TLS assets",
zap.String("peer-cert-path", certPath),
zap.Int("peer-cert-length", len(certData)),
zap.String("peer-key-path", keyPath),
zap.Int("peer-key-length", len(keyData)),
)
}
if srv.Member.Etcd.ClientAutoTLS {
// in case of slow disk
time.Sleep(time.Second)
fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "peer")
srv.lg.Info(
"loading client TLS assets",
zap.String("dir", fdir),
zap.String("endpoint", srv.EtcdClientEndpoint),
)
certPath := filepath.Join(fdir, "cert.pem")
if !fileutil.Exist(certPath) {
return fmt.Errorf("cannot find %q", certPath)
}
certData, err := ioutil.ReadFile(certPath)
if err != nil {
return fmt.Errorf("cannot read %q (%v)", certPath, err)
}
srv.Member.PeerCertData = string(certData)
keyPath := filepath.Join(fdir, "key.pem")
if !fileutil.Exist(keyPath) {
return fmt.Errorf("cannot find %q", keyPath)
}
keyData, err := ioutil.ReadFile(keyPath)
if err != nil {
return fmt.Errorf("cannot read %q (%v)", keyPath, err)
}
srv.Member.PeerKeyData = string(keyData)
srv.lg.Info(
"loaded peer TLS assets",
zap.String("peer-cert-path", certPath),
zap.Int("peer-cert-length", len(certData)),
zap.String("peer-key-path", keyPath),
zap.Int("peer-key-length", len(keyData)),
)
}
return nil
}
// start but do not wait for it to complete
func (srv *Server) startEtcdCmd() error {
return srv.etcdCmd.Start()
@ -233,12 +366,17 @@ func (srv *Server) startEtcdCmd() error {
func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) {
srv.creatEtcdCmd()
srv.lg.Info("restarting etcd")
err := srv.startEtcdCmd()
if err != nil {
var err error
if err = srv.saveTLSAssets(); err != nil {
return nil, err
}
if err = srv.startEtcdCmd(); err != nil {
return nil, err
}
srv.lg.Info("restarted etcd", zap.String("command-path", srv.etcdCmd.Path))
if err = srv.loadAutoTLSAssets(); err != nil {
return nil, err
}
// wait some time for etcd listener start
// before setting up proxy
@ -251,7 +389,8 @@ func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) {
return &rpcpb.Response{
Success: true,
Status: "successfully restarted etcd!",
Status: "restart etcd PASS",
Member: srv.Member,
}, nil
}
@ -293,7 +432,7 @@ func (srv *Server) handleFailArchive() (*rpcpb.Response, error) {
}
srv.lg.Info("archived data", zap.String("base-dir", srv.Member.BaseDir))
if err = srv.createEtcdFile(); err != nil {
if err = srv.saveEtcdLogFile(); err != nil {
return nil, err
}

20
tools/functional-tester/rpcpb/etcd_config.go
View File

@ -30,8 +30,19 @@ var etcdFields = []string{
"ListenClientURLs",
"AdvertiseClientURLs",
"ClientAutoTLS",
"ClientCertAuth",
"ClientCertFile",
"ClientKeyFile",
"ClientTrustedCAFile",
"ListenPeerURLs",
"InitialAdvertisePeerURLs",
"AdvertisePeerURLs",
"PeerAutoTLS",
"PeerClientCertAuth",
"PeerCertFile",
"PeerKeyFile",
"PeerTrustedCAFile",
"InitialCluster",
"InitialClusterState",
@ -72,12 +83,17 @@ func (cfg *Etcd) Flags() (fs []string) {
default:
panic(fmt.Errorf("field %q (%v) cannot be parsed", name, fv.Type().Kind()))
}
fname := field.Tag.Get("yaml")
// TODO: remove this
if fname == "initial-corrupt-check" {
fname = "experimental-" + fname
}
fs = append(fs, fmt.Sprintf("--%s=%s", fname, sv))
if sv != "" {
fs = append(fs, fmt.Sprintf("--%s=%s", fname, sv))
}
}
return fs
}

63
tools/functional-tester/rpcpb/etcd_config_test.go
View File

@ -21,34 +21,55 @@ import (
func TestEtcdFlags(t *testing.T) {
cfg := &Etcd{
Name: "s1",
DataDir: "/tmp/etcd-agent-data-1/etcd.data",
WALDir: "/tmp/etcd-agent-data-1/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"127.0.0.1:1379"},
AdvertiseClientURLs: []string{"127.0.0.1:13790"},
ListenPeerURLs: []string{"127.0.0.1:1380"},
InitialAdvertisePeerURLs: []string{"127.0.0.1:13800"},
InitialCluster: "s1=127.0.0.1:13800,s2=127.0.0.1:23800,s3=127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
Name: "s1",
DataDir: "/tmp/etcd-agent-data-1/etcd.data",
WALDir: "/tmp/etcd-agent-data-1/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"https://127.0.0.1:1379"},
AdvertiseClientURLs: []string{"https://127.0.0.1:13790"},
ClientAutoTLS: true,
ClientCertAuth: false,
ClientCertFile: "",
ClientKeyFile: "",
ClientTrustedCAFile: "",
ListenPeerURLs: []string{"https://127.0.0.1:1380"},
AdvertisePeerURLs: []string{"https://127.0.0.1:13800"},
PeerAutoTLS: true,
PeerClientCertAuth: false,
PeerCertFile: "",
PeerKeyFile: "",
PeerTrustedCAFile: "",
InitialCluster: "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
}
exp := []string{
"--name=s1",
"--data-dir=/tmp/etcd-agent-data-1/etcd.data",
"--wal-dir=/tmp/etcd-agent-data-1/etcd.data/member/wal",
"--heartbeat-interval=100",
"--election-timeout=1000",
"--listen-client-urls=127.0.0.1:1379",
"--advertise-client-urls=127.0.0.1:13790",
"--listen-peer-urls=127.0.0.1:1380",
"--initial-advertise-peer-urls=127.0.0.1:13800",
"--initial-cluster=s1=127.0.0.1:13800,s2=127.0.0.1:23800,s3=127.0.0.1:33800",
"--listen-client-urls=https://127.0.0.1:1379",
"--advertise-client-urls=https://127.0.0.1:13790",
"--auto-tls=true",
"--client-cert-auth=false",
"--listen-peer-urls=https://127.0.0.1:1380",
"--initial-advertise-peer-urls=https://127.0.0.1:13800",
"--peer-auto-tls=true",
"--peer-client-cert-auth=false",
"--initial-cluster=s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
"--initial-cluster-state=new",
"--initial-cluster-token=tkn",
"--snapshot-count=10000",

80
tools/functional-tester/rpcpb/member.go
View File

@ -17,39 +17,93 @@ package rpcpb
import (
"context"
"fmt"
"net/url"
"time"
"github.com/coreos/etcd/clientv3"
pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
"github.com/coreos/etcd/pkg/transport"
grpc "google.golang.org/grpc"
"google.golang.org/grpc/credentials"
)
var dialOpts = []grpc.DialOption{
grpc.WithInsecure(),
grpc.WithTimeout(5 * time.Second),
grpc.WithBlock(),
}
// DialEtcdGRPCServer creates a raw gRPC connection to an etcd member.
func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) {
if m.EtcdClientTLS {
// TODO: support TLS
panic("client TLS not supported yet")
dialOpts := []grpc.DialOption{
grpc.WithTimeout(5 * time.Second),
grpc.WithBlock(),
}
return grpc.Dial(m.EtcdClientEndpoint, append(dialOpts, opts...)...)
secure := false
for _, cu := range m.Etcd.AdvertiseClientURLs {
u, err := url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme == "https" { // TODO: handle unix
secure = true
}
}
if secure {
// assume save TLS assets are already stord on disk
tlsInfo := transport.TLSInfo{
CertFile: m.ClientCertPath,
KeyFile: m.ClientKeyPath,
TrustedCAFile: m.ClientTrustedCAPath,
// TODO: remove this with generated certs
// only need it for auto TLS
InsecureSkipVerify: true,
}
tlsConfig, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
}
creds := credentials.NewTLS(tlsConfig)
dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds))
} else {
dialOpts = append(dialOpts, grpc.WithInsecure())
}
dialOpts = append(dialOpts, opts...)
return grpc.Dial(m.EtcdClientEndpoint, dialOpts...)
}
// CreateEtcdClient creates a client from member.
func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) {
secure := false
for _, cu := range m.Etcd.AdvertiseClientURLs {
u, err := url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme == "https" { // TODO: handle unix
secure = true
}
}
cfg := clientv3.Config{
Endpoints: []string{m.EtcdClientEndpoint},
DialTimeout: 5 * time.Second,
DialOptions: opts,
}
if m.EtcdClientTLS {
// TODO: support TLS
panic("client TLS not supported yet")
if secure {
// assume save TLS assets are already stord on disk
tlsInfo := transport.TLSInfo{
CertFile: m.ClientCertPath,
KeyFile: m.ClientKeyPath,
TrustedCAFile: m.ClientTrustedCAPath,
// TODO: remove this with generated certs
// only need it for auto TLS
InsecureSkipVerify: true,
}
tlsConfig, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
}
cfg.TLS = tlsConfig
}
return clientv3.New(cfg)
}

1501
tools/functional-tester/rpcpb/rpc.pb.go
View File

File diff suppressed because it is too large Load Diff

73
tools/functional-tester/rpcpb/rpc.proto
View File

@ -48,20 +48,29 @@ message Etcd {
repeated string ListenClientURLs = 21 [(gogoproto.moretags) = "yaml:\"listen-client-urls\""];
repeated string AdvertiseClientURLs = 22 [(gogoproto.moretags) = "yaml:\"advertise-client-urls\""];
repeated string ListenPeerURLs = 23 [(gogoproto.moretags) = "yaml:\"listen-peer-urls\""];
repeated string InitialAdvertisePeerURLs = 24 [(gogoproto.moretags) = "yaml:\"initial-advertise-peer-urls\""];
bool ClientAutoTLS = 23 [(gogoproto.moretags) = "yaml:\"auto-tls\""];
bool ClientCertAuth = 24 [(gogoproto.moretags) = "yaml:\"client-cert-auth\""];
string ClientCertFile = 25 [(gogoproto.moretags) = "yaml:\"cert-file\""];
string ClientKeyFile = 26 [(gogoproto.moretags) = "yaml:\"key-file\""];
string ClientTrustedCAFile = 27 [(gogoproto.moretags) = "yaml:\"trusted-ca-file\""];
string InitialCluster = 31 [(gogoproto.moretags) = "yaml:\"initial-cluster\""];
string InitialClusterState = 32 [(gogoproto.moretags) = "yaml:\"initial-cluster-state\""];
string InitialClusterToken = 33 [(gogoproto.moretags) = "yaml:\"initial-cluster-token\""];
repeated string ListenPeerURLs = 31 [(gogoproto.moretags) = "yaml:\"listen-peer-urls\""];
repeated string AdvertisePeerURLs = 32 [(gogoproto.moretags) = "yaml:\"initial-advertise-peer-urls\""];
bool PeerAutoTLS = 33 [(gogoproto.moretags) = "yaml:\"peer-auto-tls\""];
bool PeerClientCertAuth = 34 [(gogoproto.moretags) = "yaml:\"peer-client-cert-auth\""];
string PeerCertFile = 35 [(gogoproto.moretags) = "yaml:\"peer-cert-file\""];
string PeerKeyFile = 36 [(gogoproto.moretags) = "yaml:\"peer-key-file\""];
string PeerTrustedCAFile = 37 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-file\""];
int64 SnapshotCount = 41 [(gogoproto.moretags) = "yaml:\"snapshot-count\""];
int64 QuotaBackendBytes = 42 [(gogoproto.moretags) = "yaml:\"quota-backend-bytes\""];
string InitialCluster = 41 [(gogoproto.moretags) = "yaml:\"initial-cluster\""];
string InitialClusterState = 42 [(gogoproto.moretags) = "yaml:\"initial-cluster-state\""];
string InitialClusterToken = 43 [(gogoproto.moretags) = "yaml:\"initial-cluster-token\""];
bool PreVote = 43 [(gogoproto.moretags) = "yaml:\"pre-vote\""];
bool InitialCorruptCheck = 44 [(gogoproto.moretags) = "yaml:\"initial-corrupt-check\""];
int64 SnapshotCount = 51 [(gogoproto.moretags) = "yaml:\"snapshot-count\""];
int64 QuotaBackendBytes = 52 [(gogoproto.moretags) = "yaml:\"quota-backend-bytes\""];
// TODO: support TLS
bool PreVote = 63 [(gogoproto.moretags) = "yaml:\"pre-vote\""];
bool InitialCorruptCheck = 64 [(gogoproto.moretags) = "yaml:\"initial-corrupt-check\""];
}
message Member {
@ -80,19 +89,37 @@ message Member {
// EtcdLogPath is the log file to store current etcd server logs.
string EtcdLogPath = 102 [(gogoproto.moretags) = "yaml:\"etcd-log-path\""];
// EtcdClientTLS is true when client traffic needs to be encrypted.
bool EtcdClientTLS = 201 [(gogoproto.moretags) = "yaml:\"etcd-client-tls\""];
// EtcdClientProxy is true when client traffic needs to be proxied.
// If true, listen client URL port must be different than advertise client URL port.
bool EtcdClientProxy = 202 [(gogoproto.moretags) = "yaml:\"etcd-client-proxy\""];
bool EtcdClientProxy = 201 [(gogoproto.moretags) = "yaml:\"etcd-client-proxy\""];
// EtcdPeerProxy is true when peer traffic needs to be proxied.
// If true, listen peer URL port must be different than advertise peer URL port.
bool EtcdPeerProxy = 203 [(gogoproto.moretags) = "yaml:\"etcd-peer-proxy\""];
// EtcdClientEndpoint is the etcd client endpoint.
string EtcdClientEndpoint = 204 [(gogoproto.moretags) = "yaml:\"etcd-client-endpoint\""];
bool EtcdPeerProxy = 202 [(gogoproto.moretags) = "yaml:\"etcd-peer-proxy\""];
// EtcdClientEndpoint is the etcd client endpoint.
string EtcdClientEndpoint = 301 [(gogoproto.moretags) = "yaml:\"etcd-client-endpoint\""];
// Etcd defines etcd binary configuration flags.
Etcd Etcd = 301 [(gogoproto.moretags) = "yaml:\"etcd\""];
Etcd Etcd = 302 [(gogoproto.moretags) = "yaml:\"etcd\""];
// ClientCertData contains cert file contents from this member's etcd server.
string ClientCertData = 401 [(gogoproto.moretags) = "yaml:\"client-cert-data\""];
string ClientCertPath = 402 [(gogoproto.moretags) = "yaml:\"client-cert-path\""];
// ClientKeyData contains key file contents from this member's etcd server.
string ClientKeyData = 403 [(gogoproto.moretags) = "yaml:\"client-key-data\""];
string ClientKeyPath = 404 [(gogoproto.moretags) = "yaml:\"client-key-path\""];
// ClientTrustedCAData contains trusted CA file contents from this member's etcd server.
string ClientTrustedCAData = 405 [(gogoproto.moretags) = "yaml:\"client-trusted-ca-data\""];
string ClientTrustedCAPath = 406 [(gogoproto.moretags) = "yaml:\"client-trusted-ca-path\""];
// PeerCertData contains cert file contents from this member's etcd server.
string PeerCertData = 501 [(gogoproto.moretags) = "yaml:\"peer-cert-data\""];
string PeerCertPath = 502 [(gogoproto.moretags) = "yaml:\"peer-cert-path\""];
// PeerKeyData contains key file contents from this member's etcd server.
string PeerKeyData = 503 [(gogoproto.moretags) = "yaml:\"peer-key-data\""];
string PeerKeyPath = 504 [(gogoproto.moretags) = "yaml:\"peer-key-path\""];
// PeerTrustedCAData contains trusted CA file contents from this member's etcd server.
string PeerTrustedCAData = 505 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-data\""];
string PeerTrustedCAPath = 506 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-path\""];
}
enum FailureCase {
@ -144,8 +171,9 @@ enum StressType {
}
message Tester {
string TesterNetwork = 1 [(gogoproto.moretags) = "yaml:\"tester-network\""];
string TesterAddr = 2 [(gogoproto.moretags) = "yaml:\"tester-addr\""];
string TesterDataDir = 1 [(gogoproto.moretags) = "yaml:\"tester-data-dir\""];
string TesterNetwork = 2 [(gogoproto.moretags) = "yaml:\"tester-network\""];
string TesterAddr = 3 [(gogoproto.moretags) = "yaml:\"tester-addr\""];
// DelayLatencyMsRv is the delay latency in milliseconds,
// to inject to simulated slow network.
@ -207,14 +235,15 @@ message Tester {
message Request {
Operation Operation = 1;
// Member contains the same Member object from tester configuration.
Member Member = 2;
// Tester contains tester configuration.
Tester Tester = 3;
}
message Response {
bool Success = 1;
string Status = 2;
// TODO: support TLS
// Member contains the same Member object from tester request.
Member Member = 3;
}

227
tools/functional-tester/tester/cluster.go
View File

@ -21,11 +21,13 @@ import (
"io/ioutil"
"math/rand"
"net/http"
"net/url"
"path/filepath"
"strings"
"time"
"github.com/coreos/etcd/pkg/debugutil"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"github.com/prometheus/client_golang/prometheus/promhttp"
@ -72,43 +74,43 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
return nil, err
}
for i := range clus.Members {
if clus.Members[i].BaseDir == "" {
return nil, fmt.Errorf("Members[i].BaseDir cannot be empty (got %q)", clus.Members[i].BaseDir)
for i, mem := range clus.Members {
if mem.BaseDir == "" {
return nil, fmt.Errorf("Members[i].BaseDir cannot be empty (got %q)", mem.BaseDir)
}
if clus.Members[i].EtcdLogPath == "" {
return nil, fmt.Errorf("Members[i].EtcdLogPath cannot be empty (got %q)", clus.Members[i].EtcdLogPath)
if mem.EtcdLogPath == "" {
return nil, fmt.Errorf("Members[i].EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath)
}
if clus.Members[i].Etcd.Name == "" {
return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", clus.Members[i])
if mem.Etcd.Name == "" {
return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem)
}
if clus.Members[i].Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", clus.Members[i])
if mem.Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem)
}
if clus.Members[i].Etcd.SnapshotCount == 0 {
return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", clus.Members[i].Etcd.SnapshotCount)
if mem.Etcd.SnapshotCount == 0 {
return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount)
}
if clus.Members[i].Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", clus.Members[i].Etcd.DataDir)
if mem.Etcd.DataDir == "" {
return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir)
}
if clus.Members[i].Etcd.WALDir == "" {
clus.Members[i].Etcd.WALDir = filepath.Join(clus.Members[i].Etcd.DataDir, "member", "wal")
if mem.Etcd.WALDir == "" {
clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal")
}
if clus.Members[i].Etcd.HeartbeatIntervalMs == 0 {
return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", clus.Members[i].Etcd)
if mem.Etcd.HeartbeatIntervalMs == 0 {
return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd)
}
if clus.Members[i].Etcd.ElectionTimeoutMs == 0 {
return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", clus.Members[i].Etcd)
if mem.Etcd.ElectionTimeoutMs == 0 {
return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd)
}
if int64(clus.Tester.DelayLatencyMs) <= clus.Members[i].Etcd.ElectionTimeoutMs {
return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, clus.Members[i].Etcd.ElectionTimeoutMs)
if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs {
return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs)
}
port := ""
listenClientPorts := make([]string, len(clus.Members))
for i, u := range clus.Members[i].Etcd.ListenClientURLs {
for i, u := range mem.Etcd.ListenClientURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u)
}
@ -117,7 +119,7 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u)
}
}
for i, u := range clus.Members[i].Etcd.AdvertiseClientURLs {
for i, u := range mem.Etcd.AdvertiseClientURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u)
}
@ -125,13 +127,13 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
if err != nil {
return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u)
}
if clus.Members[i].EtcdClientProxy && listenClientPorts[i] == port {
if mem.EtcdClientProxy && listenClientPorts[i] == port {
return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i])
}
}
listenPeerPorts := make([]string, len(clus.Members))
for i, u := range clus.Members[i].Etcd.ListenPeerURLs {
for i, u := range mem.Etcd.ListenPeerURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u)
}
@ -140,7 +142,7 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u)
}
}
for i, u := range clus.Members[i].Etcd.InitialAdvertisePeerURLs {
for j, u := range mem.Etcd.AdvertisePeerURLs {
if !isValidURL(u) {
return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u)
}
@ -148,28 +150,105 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
if err != nil {
return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u)
}
if clus.Members[i].EtcdPeerProxy && listenPeerPorts[i] == port {
return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[i])
if mem.EtcdPeerProxy && listenPeerPorts[j] == port {
return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j])
}
}
if !strings.HasPrefix(clus.Members[i].EtcdLogPath, clus.Members[i].BaseDir) {
return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", clus.Members[i].EtcdLogPath)
if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) {
return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath)
}
if !strings.HasPrefix(clus.Members[i].Etcd.DataDir, clus.Members[i].BaseDir) {
return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", clus.Members[i].Etcd.DataDir)
if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) {
return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir)
}
// TODO: support separate WALDir that can be handled via failure-archive
if !strings.HasPrefix(clus.Members[i].Etcd.WALDir, clus.Members[i].BaseDir) {
return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", clus.Members[i].Etcd.WALDir)
if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) {
return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir)
}
if len(clus.Tester.FailureCases) == 0 {
return nil, errors.New("FailureCases not found")
// TODO: only support generated certs with TLS generator
// deprecate auto TLS
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertAuth {
return nil, fmt.Errorf("Etcd.ClientAutoTLS and Etcd.ClientCertAuth are both 'true'")
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile)
}
if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" {
return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile)
}
if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" {
return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerClientCertAuth {
return nil, fmt.Errorf("Etcd.PeerAutoTLS and Etcd.PeerClientCertAuth are both 'true'")
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" {
return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile)
}
if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" {
return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile)
}
if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" {
for _, cu := range mem.Etcd.ListenClientURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
}
}
for _, cu := range mem.Etcd.AdvertiseClientURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
}
}
}
if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" {
for _, cu := range mem.Etcd.ListenPeerURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
}
}
for _, cu := range mem.Etcd.AdvertisePeerURLs {
var u *url.URL
u, err = url.Parse(cu)
if err != nil {
return nil, err
}
if u.Scheme != "https" { // TODO: support unix
return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
}
}
}
}
if len(clus.Tester.FailureCases) == 0 {
return nil, errors.New("FailureCases not found")
}
if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 {
return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv)
}
@ -198,8 +277,6 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
return clus, err
}
// TODO: status handler
var dialOpts = []grpc.DialOption{
grpc.WithInsecure(),
grpc.WithTimeout(5 * time.Second),
@ -547,9 +624,79 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error {
}
if !resp.Success {
err = errors.New(resp.Status)
return errors.New(resp.Status)
}
return err
m, secure := clus.Members[idx], false
for _, cu := range m.Etcd.AdvertiseClientURLs {
u, err := url.Parse(cu)
if err != nil {
return err
}
if u.Scheme == "https" { // TODO: handle unix
secure = true
}
}
// store TLS assets from agents/servers onto disk
if secure && (op == rpcpb.Operation_InitialStartEtcd || op == rpcpb.Operation_RestartEtcd) {
dirClient := filepath.Join(
clus.Tester.TesterDataDir,
clus.Members[idx].Etcd.Name,
"fixtures",
"client",
)
if err = fileutil.TouchDirAll(dirClient); err != nil {
return err
}
clientCertData := []byte(resp.Member.ClientCertData)
if len(clientCertData) == 0 {
return fmt.Errorf("got empty client cert from %q", m.EtcdClientEndpoint)
}
clientCertPath := filepath.Join(dirClient, "cert.pem")
if err = ioutil.WriteFile(clientCertPath, clientCertData, 0644); err != nil { // overwrite if exists
return err
}
resp.Member.ClientCertPath = clientCertPath
clus.lg.Info(
"saved client cert file",
zap.String("path", clientCertPath),
)
clientKeyData := []byte(resp.Member.ClientKeyData)
if len(clientKeyData) == 0 {
return fmt.Errorf("got empty client key from %q", m.EtcdClientEndpoint)
}
clientKeyPath := filepath.Join(dirClient, "key.pem")
if err = ioutil.WriteFile(clientKeyPath, clientKeyData, 0644); err != nil { // overwrite if exists
return err
}
resp.Member.ClientKeyPath = clientKeyPath
clus.lg.Info(
"saved client key file",
zap.String("path", clientKeyPath),
)
clientTrustedCAData := []byte(resp.Member.ClientTrustedCAData)
if len(clientTrustedCAData) != 0 {
// TODO: disable this when auto TLS is deprecated
clientTrustedCAPath := filepath.Join(dirClient, "ca.pem")
if err = ioutil.WriteFile(clientTrustedCAPath, clientTrustedCAData, 0644); err != nil { // overwrite if exists
return err
}
resp.Member.ClientTrustedCAPath = clientTrustedCAPath
clus.lg.Info(
"saved client trusted CA file",
zap.String("path", clientTrustedCAPath),
)
}
// no need to store peer certs for tester clients
clus.Members[idx] = resp.Member
}
return nil
}
// DestroyEtcdAgents terminates all tester connections to agents and etcd servers.

130
tools/functional-tester/tester/cluster_test.go
View File

@ -33,27 +33,36 @@ func Test_newCluster(t *testing.T) {
FailpointHTTPAddr: "http://127.0.0.1:7381",
BaseDir: "/tmp/etcd-agent-data-1",
EtcdLogPath: "/tmp/etcd-agent-data-1/current-etcd.log",
EtcdClientTLS: false,
EtcdClientProxy: false,
EtcdPeerProxy: true,
EtcdClientEndpoint: "127.0.0.1:1379",
Etcd: &rpcpb.Etcd{
Name: "s1",
DataDir: "/tmp/etcd-agent-data-1/etcd.data",
WALDir: "/tmp/etcd-agent-data-1/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"http://127.0.0.1:1379"},
AdvertiseClientURLs: []string{"http://127.0.0.1:1379"},
ListenPeerURLs: []string{"http://127.0.0.1:1380"},
InitialAdvertisePeerURLs: []string{"http://127.0.0.1:13800"},
InitialCluster: "s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
Name: "s1",
DataDir: "/tmp/etcd-agent-data-1/etcd.data",
WALDir: "/tmp/etcd-agent-data-1/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"https://127.0.0.1:1379"},
AdvertiseClientURLs: []string{"https://127.0.0.1:1379"},
ClientAutoTLS: true,
ClientCertAuth: false,
ClientCertFile: "",
ClientKeyFile: "",
ClientTrustedCAFile: "",
ListenPeerURLs: []string{"https://127.0.0.1:1380"},
AdvertisePeerURLs: []string{"https://127.0.0.1:13800"},
PeerAutoTLS: true,
PeerClientCertAuth: false,
PeerCertFile: "",
PeerKeyFile: "",
PeerTrustedCAFile: "",
InitialCluster: "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
},
},
{
@ -62,27 +71,36 @@ func Test_newCluster(t *testing.T) {
FailpointHTTPAddr: "http://127.0.0.1:7382",
BaseDir: "/tmp/etcd-agent-data-2",
EtcdLogPath: "/tmp/etcd-agent-data-2/current-etcd.log",
EtcdClientTLS: false,
EtcdClientProxy: false,
EtcdPeerProxy: true,
EtcdClientEndpoint: "127.0.0.1:2379",
Etcd: &rpcpb.Etcd{
Name: "s2",
DataDir: "/tmp/etcd-agent-data-2/etcd.data",
WALDir: "/tmp/etcd-agent-data-2/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"http://127.0.0.1:2379"},
AdvertiseClientURLs: []string{"http://127.0.0.1:2379"},
ListenPeerURLs: []string{"http://127.0.0.1:2380"},
InitialAdvertisePeerURLs: []string{"http://127.0.0.1:23800"},
InitialCluster: "s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
Name: "s2",
DataDir: "/tmp/etcd-agent-data-2/etcd.data",
WALDir: "/tmp/etcd-agent-data-2/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"https://127.0.0.1:2379"},
AdvertiseClientURLs: []string{"https://127.0.0.1:2379"},
ClientAutoTLS: true,
ClientCertAuth: false,
ClientCertFile: "",
ClientKeyFile: "",
ClientTrustedCAFile: "",
ListenPeerURLs: []string{"https://127.0.0.1:2380"},
AdvertisePeerURLs: []string{"https://127.0.0.1:23800"},
PeerAutoTLS: true,
PeerClientCertAuth: false,
PeerCertFile: "",
PeerKeyFile: "",
PeerTrustedCAFile: "",
InitialCluster: "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
},
},
{
@ -91,31 +109,41 @@ func Test_newCluster(t *testing.T) {
FailpointHTTPAddr: "http://127.0.0.1:7383",
BaseDir: "/tmp/etcd-agent-data-3",
EtcdLogPath: "/tmp/etcd-agent-data-3/current-etcd.log",
EtcdClientTLS: false,
EtcdClientProxy: false,
EtcdPeerProxy: true,
EtcdClientEndpoint: "127.0.0.1:3379",
Etcd: &rpcpb.Etcd{
Name: "s3",
DataDir: "/tmp/etcd-agent-data-3/etcd.data",
WALDir: "/tmp/etcd-agent-data-3/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"http://127.0.0.1:3379"},
AdvertiseClientURLs: []string{"http://127.0.0.1:3379"},
ListenPeerURLs: []string{"http://127.0.0.1:3380"},
InitialAdvertisePeerURLs: []string{"http://127.0.0.1:33800"},
InitialCluster: "s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
Name: "s3",
DataDir: "/tmp/etcd-agent-data-3/etcd.data",
WALDir: "/tmp/etcd-agent-data-3/etcd.data/member/wal",
HeartbeatIntervalMs: 100,
ElectionTimeoutMs: 1000,
ListenClientURLs: []string{"https://127.0.0.1:3379"},
AdvertiseClientURLs: []string{"https://127.0.0.1:3379"},
ClientAutoTLS: true,
ClientCertAuth: false,
ClientCertFile: "",
ClientKeyFile: "",
ClientTrustedCAFile: "",
ListenPeerURLs: []string{"https://127.0.0.1:3380"},
AdvertisePeerURLs: []string{"https://127.0.0.1:33800"},
PeerAutoTLS: true,
PeerClientCertAuth: false,
PeerCertFile: "",
PeerKeyFile: "",
PeerTrustedCAFile: "",
InitialCluster: "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
InitialClusterState: "new",
InitialClusterToken: "tkn",
SnapshotCount: 10000,
QuotaBackendBytes: 10740000000,
PreVote: true,
InitialCorruptCheck: true,
},
},
},
Tester: &rpcpb.Tester{
TesterDataDir: "/tmp/etcd-tester-data",
TesterNetwork: "tcp",
TesterAddr: "127.0.0.1:9028",
DelayLatencyMs: 5000,

12
tools/functional-tester/tester/cluster_tester.go
View File

@ -19,6 +19,7 @@ import (
"os"
"time"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"go.uber.org/zap"
@ -30,7 +31,13 @@ const compactQPS = 50000
// StartTester starts tester.
func (clus *Cluster) StartTester() {
// TODO: upate status
if err := fileutil.TouchDirAll(clus.Tester.TesterDataDir); err != nil {
clus.lg.Panic(
"failed to create test data directory",
zap.String("dir", clus.Tester.TesterDataDir),
zap.Error(err),
)
}
var preModifiedKey int64
for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
@ -124,6 +131,7 @@ func (clus *Cluster) doRound() error {
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", fa.Desc()),
zap.Int("total-failures", len(clus.failures)),
)
clus.lg.Info("wait health before injecting failures")
@ -208,6 +216,7 @@ func (clus *Cluster) doRound() error {
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", fa.Desc()),
zap.Int("total-failures", len(clus.failures)),
zap.Duration("took", time.Since(caseNow)),
)
}
@ -216,6 +225,7 @@ func (clus *Cluster) doRound() error {
"round ALL PASS",
zap.Int("round", clus.rd),
zap.Strings("failures", clus.failureStrings()),
zap.Int("total-failures", len(clus.failures)),
zap.Duration("took", time.Since(roundNow)),
)
return nil

64
tools/functional-tester/tester/local-test.yaml
View File

@ -4,7 +4,6 @@ agent-configs:
failpoint-http-addr: http://127.0.0.1:7381
base-dir: /tmp/etcd-agent-data-1
etcd-log-path: /tmp/etcd-agent-data-1/current-etcd.log
etcd-client-tls: false
etcd-client-proxy: false
etcd-peer-proxy: true
etcd-client-endpoint: 127.0.0.1:1379
@ -14,11 +13,21 @@ agent-configs:
wal-dir: /tmp/etcd-agent-data-1/etcd.data/member/wal
heartbeat-interval: 100
election-timeout: 1000
listen-client-urls: ["http://127.0.0.1:1379"]
advertise-client-urls: ["http://127.0.0.1:1379"]
listen-peer-urls: ["http://127.0.0.1:1380"]
initial-advertise-peer-urls: ["http://127.0.0.1:13800"]
initial-cluster: s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800
listen-client-urls: ["https://127.0.0.1:1379"]
advertise-client-urls: ["https://127.0.0.1:1379"]
auto-tls: true
client-cert-auth: false
cert-file: ""
key-file: ""
trusted-ca-file: ""
listen-peer-urls: ["https://127.0.0.1:1380"]
initial-advertise-peer-urls: ["https://127.0.0.1:13800"]
peer-auto-tls: true
peer-client-cert-auth: false
peer-cert-file: ""
peer-key-file: ""
peer-trusted-ca-file: ""
initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
initial-cluster-state: new
initial-cluster-token: tkn
snapshot-count: 10000
@ -30,7 +39,6 @@ agent-configs:
failpoint-http-addr: http://127.0.0.1:7382
base-dir: /tmp/etcd-agent-data-2
etcd-log-path: /tmp/etcd-agent-data-2/current-etcd.log
etcd-client-tls: false
etcd-client-proxy: false
etcd-peer-proxy: true
etcd-client-endpoint: 127.0.0.1:2379
@ -40,11 +48,21 @@ agent-configs:
wal-dir: /tmp/etcd-agent-data-2/etcd.data/member/wal
heartbeat-interval: 100
election-timeout: 1000
listen-client-urls: ["http://127.0.0.1:2379"]
advertise-client-urls: ["http://127.0.0.1:2379"]
listen-peer-urls: ["http://127.0.0.1:2380"]
initial-advertise-peer-urls: ["http://127.0.0.1:23800"]
initial-cluster: s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800
listen-client-urls: ["https://127.0.0.1:2379"]
advertise-client-urls: ["https://127.0.0.1:2379"]
auto-tls: true
client-cert-auth: false
cert-file: ""
key-file: ""
trusted-ca-file: ""
listen-peer-urls: ["https://127.0.0.1:2380"]
initial-advertise-peer-urls: ["https://127.0.0.1:23800"]
peer-auto-tls: true
peer-client-cert-auth: false
peer-cert-file: ""
peer-key-file: ""
peer-trusted-ca-file: ""
initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
initial-cluster-state: new
initial-cluster-token: tkn
snapshot-count: 10000
@ -56,7 +74,6 @@ agent-configs:
failpoint-http-addr: http://127.0.0.1:7383
base-dir: /tmp/etcd-agent-data-3
etcd-log-path: /tmp/etcd-agent-data-3/current-etcd.log
etcd-client-tls: false
etcd-client-proxy: false
etcd-peer-proxy: true
etcd-client-endpoint: 127.0.0.1:3379
@ -66,11 +83,21 @@ agent-configs:
wal-dir: /tmp/etcd-agent-data-3/etcd.data/member/wal
heartbeat-interval: 100
election-timeout: 1000
listen-client-urls: ["http://127.0.0.1:3379"]
advertise-client-urls: ["http://127.0.0.1:3379"]
listen-peer-urls: ["http://127.0.0.1:3380"]
initial-advertise-peer-urls: ["http://127.0.0.1:33800"]
initial-cluster: s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800
listen-client-urls: ["https://127.0.0.1:3379"]
advertise-client-urls: ["https://127.0.0.1:3379"]
auto-tls: true
client-cert-auth: false
cert-file: ""
key-file: ""
trusted-ca-file: ""
listen-peer-urls: ["https://127.0.0.1:3380"]
initial-advertise-peer-urls: ["https://127.0.0.1:33800"]
peer-auto-tls: true
peer-client-cert-auth: false
peer-cert-file: ""
peer-key-file: ""
peer-trusted-ca-file: ""
initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
initial-cluster-state: new
initial-cluster-token: tkn
snapshot-count: 10000
@ -79,6 +106,7 @@ agent-configs:
initial-corrupt-check: true
tester-config:
tester-data-dir: /tmp/etcd-tester-data
tester-network: tcp
tester-addr: 127.0.0.1:9028