diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 2db5b2cd2..8fc612c14 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -174,6 +174,7 @@ type EtcdProcessClusterConfig struct { CompactHashCheckEnabled bool CompactHashCheckTime time.Duration GoFailEnabled bool + CompactionBatchLimit int } func DefaultConfig() *EtcdProcessClusterConfig { @@ -521,6 +522,9 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in if cfg.CompactHashCheckTime != 0 { args = append(args, "--experimental-compact-hash-check-time", cfg.CompactHashCheckTime.String()) } + if cfg.CompactionBatchLimit != 0 { + args = append(args, "--experimental-compaction-batch-limit", fmt.Sprintf("%d", cfg.CompactionBatchLimit)) + } envVars := map[string]string{} for key, value := range cfg.EnvVars { envVars[key] = value diff --git a/tests/linearizability/failpoints.go b/tests/linearizability/failpoints.go index 7b302ceb3..05e9e1654 100644 --- a/tests/linearizability/failpoints.go +++ b/tests/linearizability/failpoints.go @@ -31,25 +31,34 @@ import ( ) var ( - KillFailpoint Failpoint = killFailpoint{} - DefragBeforeCopyPanic Failpoint = goFailpoint{"backend/defragBeforeCopy", "panic", triggerDefrag} - DefragBeforeRenamePanic Failpoint = goFailpoint{"backend/defragBeforeRename", "panic", triggerDefrag} - BeforeCommitPanic Failpoint = goFailpoint{"backend/beforeCommit", "panic", nil} - AfterCommitPanic Failpoint = goFailpoint{"backend/afterCommit", "panic", nil} - RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic", nil} - RaftAfterSavePanic Failpoint = goFailpoint{"etcdserver/raftAfterSave", "panic", nil} - BackendBeforePreCommitHookPanic Failpoint = goFailpoint{"backend/commitBeforePreCommitHook", "panic", nil} - BackendAfterPreCommitHookPanic Failpoint = goFailpoint{"backend/commitAfterPreCommitHook", "panic", nil} - BackendBeforeStartDBTxnPanic Failpoint = goFailpoint{"backend/beforeStartDBTxn", "panic", nil} - BackendAfterStartDBTxnPanic Failpoint = goFailpoint{"backend/afterStartDBTxn", "panic", nil} - BackendBeforeWritebackBufPanic Failpoint = goFailpoint{"backend/beforeWritebackBuf", "panic", nil} - BackendAfterWritebackBufPanic Failpoint = goFailpoint{"backend/afterWritebackBuf", "panic", nil} - RandomFailpoint Failpoint = randomFailpoint{[]Failpoint{ + KillFailpoint Failpoint = killFailpoint{} + DefragBeforeCopyPanic Failpoint = goFailpoint{"backend/defragBeforeCopy", "panic", triggerDefrag} + DefragBeforeRenamePanic Failpoint = goFailpoint{"backend/defragBeforeRename", "panic", triggerDefrag} + BeforeCommitPanic Failpoint = goFailpoint{"backend/beforeCommit", "panic", nil} + AfterCommitPanic Failpoint = goFailpoint{"backend/afterCommit", "panic", nil} + RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic", nil} + RaftAfterSavePanic Failpoint = goFailpoint{"etcdserver/raftAfterSave", "panic", nil} + BackendBeforePreCommitHookPanic Failpoint = goFailpoint{"backend/commitBeforePreCommitHook", "panic", nil} + BackendAfterPreCommitHookPanic Failpoint = goFailpoint{"backend/commitAfterPreCommitHook", "panic", nil} + BackendBeforeStartDBTxnPanic Failpoint = goFailpoint{"backend/beforeStartDBTxn", "panic", nil} + BackendAfterStartDBTxnPanic Failpoint = goFailpoint{"backend/afterStartDBTxn", "panic", nil} + BackendBeforeWritebackBufPanic Failpoint = goFailpoint{"backend/beforeWritebackBuf", "panic", nil} + BackendAfterWritebackBufPanic Failpoint = goFailpoint{"backend/afterWritebackBuf", "panic", nil} + CompactBeforeCommitScheduledCompactPanic Failpoint = goFailpoint{"mvcc/compactBeforeCommitScheduledCompact", "panic", triggerCompact} + CompactAfterCommitScheduledCompactPanic Failpoint = goFailpoint{"mvcc/compactAfterCommitScheduledCompact", "panic", triggerCompact} + CompactBeforeSetFinishedCompactPanic Failpoint = goFailpoint{"mvcc/compactBeforeSetFinishedCompact", "panic", triggerCompact} + CompactAfterSetFinishedCompactPanic Failpoint = goFailpoint{"mvcc/compactAfterSetFinishedCompact", "panic", triggerCompact} + CompactBeforeCommitBatchPanic Failpoint = goFailpoint{"mvcc/compactBeforeCommitBatch", "panic", triggerCompact} + CompactAfterCommitBatchPanic Failpoint = goFailpoint{"mvcc/compactAfterCommitBatch", "panic", triggerCompact} + RandomFailpoint Failpoint = randomFailpoint{[]Failpoint{ KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic, BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic, BackendAfterWritebackBufPanic, + CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic, + CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, + CompactBeforeCommitBatchPanic, CompactAfterCommitBatchPanic, }} // TODO: Figure out how to reliably trigger below failpoints and add them to RandomFailpoint raftBeforeLeaderSendPanic Failpoint = goFailpoint{"etcdserver/raftBeforeLeaderSend", "panic", nil} @@ -162,6 +171,28 @@ func triggerDefrag(ctx context.Context, member e2e.EtcdProcess) error { return nil } +func triggerCompact(ctx context.Context, member e2e.EtcdProcess) error { + cc, err := clientv3.New(clientv3.Config{ + Endpoints: member.EndpointsV3(), + Logger: zap.NewNop(), + DialKeepAliveTime: 1 * time.Millisecond, + DialKeepAliveTimeout: 5 * time.Millisecond, + }) + if err != nil { + return fmt.Errorf("failed creating client: %w", err) + } + defer cc.Close() + resp, err := cc.Get(ctx, "/") + if err != nil { + return err + } + _, err = cc.Compact(ctx, resp.Header.Revision) + if err != nil && !strings.Contains(err.Error(), "error reading from server: EOF") { + return err + } + return nil +} + var httpClient = http.Client{ Timeout: 10 * time.Millisecond, } diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 854d45676..5beff43b4 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -51,16 +51,18 @@ func TestLinearizability(t *testing.T) { name: "ClusterOfSize1", failpoint: RandomFailpoint, config: e2e.EtcdProcessClusterConfig{ - ClusterSize: 1, - GoFailEnabled: true, + ClusterSize: 1, + GoFailEnabled: true, + CompactionBatchLimit: 100, // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints }, }, { name: "ClusterOfSize3", failpoint: RandomFailpoint, config: e2e.EtcdProcessClusterConfig{ - ClusterSize: 3, - GoFailEnabled: true, + ClusterSize: 3, + GoFailEnabled: true, + CompactionBatchLimit: 100, // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints }, }, {