tests: Add triggering random go failpoints to linearizability tests
Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>dependabot/go_modules/go.uber.org/atomic-1.10.0
parent
e614eec948
commit
219278b298
|
@ -21,18 +21,40 @@ import (
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
clientv3 "go.etcd.io/etcd/client/v3"
|
||||||
"go.etcd.io/etcd/tests/v3/framework/e2e"
|
"go.etcd.io/etcd/tests/v3/framework/e2e"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
KillFailpoint Failpoint = killFailpoint{}
|
KillFailpoint Failpoint = killFailpoint{}
|
||||||
RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic"}
|
DefragBeforeCopyPanic Failpoint = goFailpoint{"backend/defragBeforeCopy", "panic", triggerDefrag}
|
||||||
|
DefragBeforeRenamePanic Failpoint = goFailpoint{"backend/defragBeforeRename", "panic", triggerDefrag}
|
||||||
|
BeforeCommitPanic Failpoint = goFailpoint{"backend/beforeCommit", "panic", nil}
|
||||||
|
AfterCommitPanic Failpoint = goFailpoint{"backend/afterCommit", "panic", nil}
|
||||||
|
RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic", nil}
|
||||||
|
RaftAfterSavePanic Failpoint = goFailpoint{"etcdserver/raftAfterSave", "panic", nil}
|
||||||
|
RandomFailpoint Failpoint = randomFailpoint{[]Failpoint{
|
||||||
|
KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic,
|
||||||
|
RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic,
|
||||||
|
}}
|
||||||
|
// TODO: Figure out how to reliably trigger below failpoints and add them to RandomFailpoint
|
||||||
|
raftBeforeLeaderSendPanic Failpoint = goFailpoint{"etcdserver/raftBeforeLeaderSend", "panic", nil}
|
||||||
|
raftBeforeApplySnapPanic Failpoint = goFailpoint{"etcdserver/raftBeforeApplySnap", "panic", nil}
|
||||||
|
raftAfterApplySnapPanic Failpoint = goFailpoint{"etcdserver/raftAfterApplySnap", "panic", nil}
|
||||||
|
raftAfterWALReleasePanic Failpoint = goFailpoint{"etcdserver/raftAfterWALRelease", "panic", nil}
|
||||||
|
raftBeforeFollowerSendPanic Failpoint = goFailpoint{"etcdserver/raftBeforeFollowerSend", "panic", nil}
|
||||||
|
raftBeforeSaveSnapPanic Failpoint = goFailpoint{"etcdserver/raftBeforeSaveSnap", "panic", nil}
|
||||||
|
raftAfterSaveSnapPanic Failpoint = goFailpoint{"etcdserver/raftAfterSaveSnap", "panic", nil}
|
||||||
)
|
)
|
||||||
|
|
||||||
type Failpoint interface {
|
type Failpoint interface {
|
||||||
Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error
|
Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error
|
||||||
|
Name() string
|
||||||
}
|
}
|
||||||
|
|
||||||
type killFailpoint struct{}
|
type killFailpoint struct{}
|
||||||
|
@ -54,30 +76,41 @@ func (f killFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f killFailpoint) Name() string {
|
||||||
|
return "Kill"
|
||||||
|
}
|
||||||
|
|
||||||
type goFailpoint struct {
|
type goFailpoint struct {
|
||||||
failpoint string
|
failpoint string
|
||||||
payload string
|
payload string
|
||||||
|
trigger func(ctx context.Context, member e2e.EtcdProcess) error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f goFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error {
|
func (f goFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error {
|
||||||
member := clus.Procs[rand.Int()%len(clus.Procs)]
|
member := clus.Procs[rand.Int()%len(clus.Procs)]
|
||||||
address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort)
|
address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort)
|
||||||
err := triggerGoFailpoint(address, f.failpoint, f.payload)
|
err := setupGoFailpoint(address, f.failpoint, f.payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to trigger failpoint %q, err: %v", f.failpoint, err)
|
return fmt.Errorf("gofailpoint setup failed: %w", err)
|
||||||
}
|
}
|
||||||
err = clus.Procs[0].Wait()
|
if f.trigger != nil {
|
||||||
|
err = f.trigger(ctx, member)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("triggering gofailpoint failed: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = member.Wait()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
err = clus.Procs[0].Start(ctx)
|
err = member.Start(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func triggerGoFailpoint(host, failpoint, payload string) error {
|
func setupGoFailpoint(host, failpoint, payload string) error {
|
||||||
failpointUrl := url.URL{
|
failpointUrl := url.URL{
|
||||||
Scheme: "http",
|
Scheme: "http",
|
||||||
Host: host,
|
Host: host,
|
||||||
|
@ -98,6 +131,42 @@ func triggerGoFailpoint(host, failpoint, payload string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f goFailpoint) Name() string {
|
||||||
|
return f.failpoint
|
||||||
|
}
|
||||||
|
|
||||||
|
func triggerDefrag(ctx context.Context, member e2e.EtcdProcess) error {
|
||||||
|
cc, err := clientv3.New(clientv3.Config{
|
||||||
|
Endpoints: member.EndpointsV3(),
|
||||||
|
Logger: zap.NewNop(),
|
||||||
|
DialKeepAliveTime: 1 * time.Millisecond,
|
||||||
|
DialKeepAliveTimeout: 5 * time.Millisecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed creating client: %w", err)
|
||||||
|
}
|
||||||
|
defer cc.Close()
|
||||||
|
_, err = cc.Defragment(ctx, member.EndpointsV3()[0])
|
||||||
|
if err != nil && !strings.Contains(err.Error(), "error reading from server: EOF") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
var httpClient = http.Client{
|
var httpClient = http.Client{
|
||||||
Timeout: 10 * time.Millisecond,
|
Timeout: 10 * time.Millisecond,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type randomFailpoint struct {
|
||||||
|
failpoints []Failpoint
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f randomFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error {
|
||||||
|
failpoint := f.failpoints[rand.Int()%len(f.failpoints)]
|
||||||
|
fmt.Printf("Triggering %v failpoint\n", failpoint.Name())
|
||||||
|
return failpoint.Trigger(ctx, clus)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f randomFailpoint) Name() string {
|
||||||
|
return "Random"
|
||||||
|
}
|
||||||
|
|
|
@ -47,17 +47,19 @@ func TestLinearizability(t *testing.T) {
|
||||||
config e2e.EtcdProcessClusterConfig
|
config e2e.EtcdProcessClusterConfig
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "KillClusterOfSize1",
|
name: "ClusterOfSize1",
|
||||||
failpoint: KillFailpoint,
|
failpoint: RandomFailpoint,
|
||||||
config: e2e.EtcdProcessClusterConfig{
|
config: e2e.EtcdProcessClusterConfig{
|
||||||
ClusterSize: 1,
|
ClusterSize: 1,
|
||||||
|
GoFailEnabled: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "KillClusterOfSize3",
|
name: "ClusterOfSize3",
|
||||||
failpoint: KillFailpoint,
|
failpoint: RandomFailpoint,
|
||||||
config: e2e.EtcdProcessClusterConfig{
|
config: e2e.EtcdProcessClusterConfig{
|
||||||
ClusterSize: 3,
|
ClusterSize: 3,
|
||||||
|
GoFailEnabled: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -96,7 +98,7 @@ func testLinearizability(ctx context.Context, t *testing.T, config e2e.EtcdProce
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
go func() {
|
go func() {
|
||||||
defer cancel()
|
defer cancel()
|
||||||
err := triggerFailpoints(ctx, clus, failpoint)
|
err := triggerFailpoints(ctx, t, clus, failpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
@ -120,7 +122,7 @@ func testLinearizability(ctx context.Context, t *testing.T, config e2e.EtcdProce
|
||||||
t.Logf("saving visualization to %q", path)
|
t.Logf("saving visualization to %q", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func triggerFailpoints(ctx context.Context, clus *e2e.EtcdProcessCluster, config FailpointConfig) error {
|
func triggerFailpoints(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, config FailpointConfig) error {
|
||||||
var err error
|
var err error
|
||||||
successes := 0
|
successes := 0
|
||||||
failures := 0
|
failures := 0
|
||||||
|
@ -128,6 +130,7 @@ func triggerFailpoints(ctx context.Context, clus *e2e.EtcdProcessCluster, config
|
||||||
for successes < config.count && failures < config.count {
|
for successes < config.count && failures < config.count {
|
||||||
err = config.failpoint.Trigger(ctx, clus)
|
err = config.failpoint.Trigger(ctx, clus)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
t.Logf("Failed to trigger failpoint %q, err: %v\n", config.failpoint.Name(), err)
|
||||||
failures++
|
failures++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue