vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #14897 from serathius/linearizability-trigger-exit 

Allow failpoint requests to fail assuming that process exists within 1 second
dependabot/go_modules/go.uber.org/atomic-1.10.0
Marek Siarkowicz 2022-12-06 09:31:21 +01:00 committed by GitHub
parent 2263315871 f2bc0823f6
commit 42bb543315
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 114 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tests/framework/e2e/cluster_proxy.go
View File

@ -116,8 +116,12 @@ func (p *proxyEtcdProcess) Kill() error {
return p.etcdProc.Kill()
}
func (p *proxyEtcdProcess) Wait() error {
return p.etcdProc.Wait()
func (p *proxyEtcdProcess) IsRunning() bool {
return p.etcdProc.IsRunning()
}
func (p *proxyEtcdProcess) Wait(ctx context.Context) error {
return p.etcdProc.Wait(ctx)
}
type proxyProc struct {

33
tests/framework/e2e/etcd_process.go
View File

@ -42,7 +42,8 @@ type EtcdProcess interface {
EndpointsMetrics() []string
Client(opts ...config.ClientOption) *EtcdctlV3
Wait() error
IsRunning() bool
Wait(ctx context.Context) error
Start(ctx context.Context) error
Restart(ctx context.Context) error
Stop() error
@ -201,11 +202,35 @@ func (ep *EtcdServerProcess) Kill() error {
return ep.proc.Signal(syscall.SIGKILL)
}
func (ep *EtcdServerProcess) Wait() error {
ep.proc.Wait()
func (ep *EtcdServerProcess) Wait(ctx context.Context) error {
ch := make(chan struct{})
go func() {
defer close(ch)
if ep.proc != nil {
ep.proc.Wait()
ep.cfg.lg.Info("server exited", zap.String("name", ep.cfg.Name))
}
}()
select {
case <-ch:
ep.proc = nil
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (ep *EtcdServerProcess) IsRunning() bool {
if ep.proc == nil {
return false
}
_, err := ep.proc.ExitCode()
if err == expect.ErrProcessRunning {
return true
}
ep.cfg.lg.Info("server exited", zap.String("name", ep.cfg.Name))
ep.proc = nil
return nil
return false
}
func AssertProcessLogs(t *testing.T, ep EtcdProcess, expectLog string) {

139
tests/linearizability/failpoints.go
View File

@ -31,27 +31,31 @@ import (
"go.etcd.io/etcd/tests/v3/framework/e2e"
)
const (
triggerTimeout = time.Second
)
var (
KillFailpoint Failpoint = killFailpoint{}
DefragBeforeCopyPanic Failpoint = goFailpoint{"backend/defragBeforeCopy", "panic", triggerDefrag, AnyMember}
DefragBeforeRenamePanic Failpoint = goFailpoint{"backend/defragBeforeRename", "panic", triggerDefrag, AnyMember}
BeforeCommitPanic Failpoint = goFailpoint{"backend/beforeCommit", "panic", nil, AnyMember}
AfterCommitPanic Failpoint = goFailpoint{"backend/afterCommit", "panic", nil, AnyMember}
RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic", nil, AnyMember}
RaftAfterSavePanic Failpoint = goFailpoint{"etcdserver/raftAfterSave", "panic", nil, AnyMember}
BackendBeforePreCommitHookPanic Failpoint = goFailpoint{"backend/commitBeforePreCommitHook", "panic", nil, AnyMember}
BackendAfterPreCommitHookPanic Failpoint = goFailpoint{"backend/commitAfterPreCommitHook", "panic", nil, AnyMember}
BackendBeforeStartDBTxnPanic Failpoint = goFailpoint{"backend/beforeStartDBTxn", "panic", nil, AnyMember}
BackendAfterStartDBTxnPanic Failpoint = goFailpoint{"backend/afterStartDBTxn", "panic", nil, AnyMember}
BackendBeforeWritebackBufPanic Failpoint = goFailpoint{"backend/beforeWritebackBuf", "panic", nil, AnyMember}
BackendAfterWritebackBufPanic Failpoint = goFailpoint{"backend/afterWritebackBuf", "panic", nil, AnyMember}
CompactBeforeCommitScheduledCompactPanic Failpoint = goFailpoint{"mvcc/compactBeforeCommitScheduledCompact", "panic", triggerCompact, AnyMember}
CompactAfterCommitScheduledCompactPanic Failpoint = goFailpoint{"mvcc/compactAfterCommitScheduledCompact", "panic", triggerCompact, AnyMember}
CompactBeforeSetFinishedCompactPanic Failpoint = goFailpoint{"mvcc/compactBeforeSetFinishedCompact", "panic", triggerCompact, AnyMember}
CompactAfterSetFinishedCompactPanic Failpoint = goFailpoint{"mvcc/compactAfterSetFinishedCompact", "panic", triggerCompact, AnyMember}
CompactBeforeCommitBatchPanic Failpoint = goFailpoint{"mvcc/compactBeforeCommitBatch", "panic", triggerCompact, AnyMember}
CompactAfterCommitBatchPanic Failpoint = goFailpoint{"mvcc/compactAfterCommitBatch", "panic", triggerCompact, AnyMember}
RaftBeforeLeaderSendPanic Failpoint = goFailpoint{"etcdserver/raftBeforeLeaderSend", "panic", nil, Leader}
DefragBeforeCopyPanic Failpoint = goPanicFailpoint{"backend/defragBeforeCopy", triggerDefrag, AnyMember}
DefragBeforeRenamePanic Failpoint = goPanicFailpoint{"backend/defragBeforeRename", triggerDefrag, AnyMember}
BeforeCommitPanic Failpoint = goPanicFailpoint{"backend/beforeCommit", nil, AnyMember}
AfterCommitPanic Failpoint = goPanicFailpoint{"backend/afterCommit", nil, AnyMember}
RaftBeforeSavePanic Failpoint = goPanicFailpoint{"etcdserver/raftBeforeSave", nil, AnyMember}
RaftAfterSavePanic Failpoint = goPanicFailpoint{"etcdserver/raftAfterSave", nil, AnyMember}
BackendBeforePreCommitHookPanic Failpoint = goPanicFailpoint{"backend/commitBeforePreCommitHook", nil, AnyMember}
BackendAfterPreCommitHookPanic Failpoint = goPanicFailpoint{"backend/commitAfterPreCommitHook", nil, AnyMember}
BackendBeforeStartDBTxnPanic Failpoint = goPanicFailpoint{"backend/beforeStartDBTxn", nil, AnyMember}
BackendAfterStartDBTxnPanic Failpoint = goPanicFailpoint{"backend/afterStartDBTxn", nil, AnyMember}
BackendBeforeWritebackBufPanic Failpoint = goPanicFailpoint{"backend/beforeWritebackBuf", nil, AnyMember}
BackendAfterWritebackBufPanic Failpoint = goPanicFailpoint{"backend/afterWritebackBuf", nil, AnyMember}
CompactBeforeCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"mvcc/compactBeforeCommitScheduledCompact", triggerCompact, AnyMember}
CompactAfterCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"mvcc/compactAfterCommitScheduledCompact", triggerCompact, AnyMember}
CompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{"mvcc/compactBeforeSetFinishedCompact", triggerCompact, AnyMember}
CompactAfterSetFinishedCompactPanic Failpoint = goPanicFailpoint{"mvcc/compactAfterSetFinishedCompact", triggerCompact, AnyMember}
CompactBeforeCommitBatchPanic Failpoint = goPanicFailpoint{"mvcc/compactBeforeCommitBatch", triggerCompact, AnyMember}
CompactAfterCommitBatchPanic Failpoint = goPanicFailpoint{"mvcc/compactAfterCommitBatch", triggerCompact, AnyMember}
RaftBeforeLeaderSendPanic Failpoint = goPanicFailpoint{"etcdserver/raftBeforeLeaderSend", nil, Leader}
RandomFailpoint Failpoint = randomFailpoint{[]Failpoint{
KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic,
RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic,
@ -64,12 +68,12 @@ var (
RaftBeforeLeaderSendPanic,
}}
// TODO: Figure out how to reliably trigger below failpoints and add them to RandomFailpoint
raftBeforeApplySnapPanic Failpoint = goFailpoint{"etcdserver/raftBeforeApplySnap", "panic", nil, AnyMember}
raftAfterApplySnapPanic Failpoint = goFailpoint{"etcdserver/raftAfterApplySnap", "panic", nil, AnyMember}
raftAfterWALReleasePanic Failpoint = goFailpoint{"etcdserver/raftAfterWALRelease", "panic", nil, AnyMember}
raftBeforeFollowerSendPanic Failpoint = goFailpoint{"etcdserver/raftBeforeFollowerSend", "panic", nil, AnyMember}
raftBeforeSaveSnapPanic Failpoint = goFailpoint{"etcdserver/raftBeforeSaveSnap", "panic", nil, AnyMember}
raftAfterSaveSnapPanic Failpoint = goFailpoint{"etcdserver/raftAfterSaveSnap", "panic", nil, AnyMember}
raftBeforeApplySnapPanic Failpoint = goPanicFailpoint{"etcdserver/raftBeforeApplySnap", nil, AnyMember}
raftAfterApplySnapPanic Failpoint = goPanicFailpoint{"etcdserver/raftAfterApplySnap", nil, AnyMember}
raftAfterWALReleasePanic Failpoint = goPanicFailpoint{"etcdserver/raftAfterWALRelease", nil, AnyMember}
raftBeforeFollowerSendPanic Failpoint = goPanicFailpoint{"etcdserver/raftBeforeFollowerSend", nil, AnyMember}
raftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"etcdserver/raftBeforeSaveSnap", nil, AnyMember}
raftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"etcdserver/raftAfterSaveSnap", nil, AnyMember}
)
type Failpoint interface {
@ -81,15 +85,21 @@ type killFailpoint struct{}
func (f killFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error {
member := clus.Procs[rand.Int()%len(clus.Procs)]
err := member.Kill()
if err != nil {
return err
killCtx, cancel := context.WithTimeout(ctx, triggerTimeout)
defer cancel()
for member.IsRunning() {
err := member.Kill()
if err != nil {
t.Logf("sending kill signal failed: %v", err)
}
err = member.Wait(killCtx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return fmt.Errorf("failed to kill the process within %s, err: %w", triggerTimeout, err)
}
}
err = member.Wait()
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return err
}
err = member.Start(ctx)
err := member.Start(ctx)
if err != nil {
return err
}
@ -100,9 +110,8 @@ func (f killFailpoint) Name() string {
return "Kill"
}
type goFailpoint struct {
type goPanicFailpoint struct {
failpoint string
payload string
trigger func(ctx context.Context, member e2e.EtcdProcess) error
target failpointTarget
}
@ -114,45 +123,55 @@ const (
Leader failpointTarget = "Leader"
)
func (f goFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error {
var member e2e.EtcdProcess
switch f.target {
case AnyMember:
member = clus.Procs[rand.Int()%len(clus.Procs)]
case Leader:
member = clus.Procs[clus.WaitLeader(t)]
default:
panic("unknown target")
}
func (f goPanicFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error {
member := f.pickMember(t, clus)
address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort)
err := setupGoFailpoint(address, f.failpoint, f.payload)
if err != nil {
return fmt.Errorf("gofailpoint setup failed: %w", err)
}
if f.trigger != nil {
err = f.trigger(ctx, member)
triggerCtx, cancel := context.WithTimeout(ctx, triggerTimeout)
defer cancel()
for member.IsRunning() {
err := setupGoFailpoint(triggerCtx, address, f.failpoint, "panic")
if err != nil {
return fmt.Errorf("triggering gofailpoint failed: %w", err)
t.Logf("gofailpoint setup failed: %v", err)
}
if f.trigger != nil {
err = f.trigger(triggerCtx, member)
if err != nil {
t.Logf("triggering gofailpoint failed: %v", err)
}
}
err = member.Wait(triggerCtx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return fmt.Errorf("failed to trigger a process panic within %s, err: %w", triggerTimeout, err)
}
}
err = member.Wait()
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return err
}
err = member.Start(ctx)
err := member.Start(ctx)
if err != nil {
return err
}
return nil
}
func setupGoFailpoint(host, failpoint, payload string) error {
func (f goPanicFailpoint) pickMember(t *testing.T, clus *e2e.EtcdProcessCluster) e2e.EtcdProcess {
switch f.target {
case AnyMember:
return clus.Procs[rand.Int()%len(clus.Procs)]
case Leader:
return clus.Procs[clus.WaitLeader(t)]
default:
panic("unknown target")
}
}
func setupGoFailpoint(ctx context.Context, host, failpoint, payload string) error {
failpointUrl := url.URL{
Scheme: "http",
Host: host,
Path: failpoint,
}
r, err := http.NewRequest("PUT", failpointUrl.String(), bytes.NewBuffer([]byte(payload)))
r, err := http.NewRequestWithContext(ctx, "PUT", failpointUrl.String(), bytes.NewBuffer([]byte(payload)))
if err != nil {
return err
}
@ -167,7 +186,7 @@ func setupGoFailpoint(host, failpoint, payload string) error {
return nil
}
func (f goFailpoint) Name() string {
func (f goPanicFailpoint) Name() string {
return f.failpoint
}