tests: Add compact failpoints
Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>dependabot/go_modules/go.uber.org/atomic-1.10.0
parent
f96957adba
commit
8ae4420c4c
|
@ -174,6 +174,7 @@ type EtcdProcessClusterConfig struct {
|
||||||
CompactHashCheckEnabled bool
|
CompactHashCheckEnabled bool
|
||||||
CompactHashCheckTime time.Duration
|
CompactHashCheckTime time.Duration
|
||||||
GoFailEnabled bool
|
GoFailEnabled bool
|
||||||
|
CompactionBatchLimit int
|
||||||
}
|
}
|
||||||
|
|
||||||
func DefaultConfig() *EtcdProcessClusterConfig {
|
func DefaultConfig() *EtcdProcessClusterConfig {
|
||||||
|
@ -521,6 +522,9 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
|
||||||
if cfg.CompactHashCheckTime != 0 {
|
if cfg.CompactHashCheckTime != 0 {
|
||||||
args = append(args, "--experimental-compact-hash-check-time", cfg.CompactHashCheckTime.String())
|
args = append(args, "--experimental-compact-hash-check-time", cfg.CompactHashCheckTime.String())
|
||||||
}
|
}
|
||||||
|
if cfg.CompactionBatchLimit != 0 {
|
||||||
|
args = append(args, "--experimental-compaction-batch-limit", fmt.Sprintf("%d", cfg.CompactionBatchLimit))
|
||||||
|
}
|
||||||
envVars := map[string]string{}
|
envVars := map[string]string{}
|
||||||
for key, value := range cfg.EnvVars {
|
for key, value := range cfg.EnvVars {
|
||||||
envVars[key] = value
|
envVars[key] = value
|
||||||
|
|
|
@ -44,12 +44,21 @@ var (
|
||||||
BackendAfterStartDBTxnPanic Failpoint = goFailpoint{"backend/afterStartDBTxn", "panic", nil}
|
BackendAfterStartDBTxnPanic Failpoint = goFailpoint{"backend/afterStartDBTxn", "panic", nil}
|
||||||
BackendBeforeWritebackBufPanic Failpoint = goFailpoint{"backend/beforeWritebackBuf", "panic", nil}
|
BackendBeforeWritebackBufPanic Failpoint = goFailpoint{"backend/beforeWritebackBuf", "panic", nil}
|
||||||
BackendAfterWritebackBufPanic Failpoint = goFailpoint{"backend/afterWritebackBuf", "panic", nil}
|
BackendAfterWritebackBufPanic Failpoint = goFailpoint{"backend/afterWritebackBuf", "panic", nil}
|
||||||
|
CompactBeforeCommitScheduledCompactPanic Failpoint = goFailpoint{"mvcc/compactBeforeCommitScheduledCompact", "panic", triggerCompact}
|
||||||
|
CompactAfterCommitScheduledCompactPanic Failpoint = goFailpoint{"mvcc/compactAfterCommitScheduledCompact", "panic", triggerCompact}
|
||||||
|
CompactBeforeSetFinishedCompactPanic Failpoint = goFailpoint{"mvcc/compactBeforeSetFinishedCompact", "panic", triggerCompact}
|
||||||
|
CompactAfterSetFinishedCompactPanic Failpoint = goFailpoint{"mvcc/compactAfterSetFinishedCompact", "panic", triggerCompact}
|
||||||
|
CompactBeforeCommitBatchPanic Failpoint = goFailpoint{"mvcc/compactBeforeCommitBatch", "panic", triggerCompact}
|
||||||
|
CompactAfterCommitBatchPanic Failpoint = goFailpoint{"mvcc/compactAfterCommitBatch", "panic", triggerCompact}
|
||||||
RandomFailpoint Failpoint = randomFailpoint{[]Failpoint{
|
RandomFailpoint Failpoint = randomFailpoint{[]Failpoint{
|
||||||
KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic,
|
KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic,
|
||||||
RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic,
|
RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic,
|
||||||
BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
|
BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
|
||||||
BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic,
|
BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic,
|
||||||
BackendBeforeWritebackBufPanic, BackendAfterWritebackBufPanic,
|
BackendBeforeWritebackBufPanic, BackendAfterWritebackBufPanic,
|
||||||
|
CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
|
||||||
|
CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic,
|
||||||
|
CompactBeforeCommitBatchPanic, CompactAfterCommitBatchPanic,
|
||||||
}}
|
}}
|
||||||
// TODO: Figure out how to reliably trigger below failpoints and add them to RandomFailpoint
|
// TODO: Figure out how to reliably trigger below failpoints and add them to RandomFailpoint
|
||||||
raftBeforeLeaderSendPanic Failpoint = goFailpoint{"etcdserver/raftBeforeLeaderSend", "panic", nil}
|
raftBeforeLeaderSendPanic Failpoint = goFailpoint{"etcdserver/raftBeforeLeaderSend", "panic", nil}
|
||||||
|
@ -162,6 +171,28 @@ func triggerDefrag(ctx context.Context, member e2e.EtcdProcess) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func triggerCompact(ctx context.Context, member e2e.EtcdProcess) error {
|
||||||
|
cc, err := clientv3.New(clientv3.Config{
|
||||||
|
Endpoints: member.EndpointsV3(),
|
||||||
|
Logger: zap.NewNop(),
|
||||||
|
DialKeepAliveTime: 1 * time.Millisecond,
|
||||||
|
DialKeepAliveTimeout: 5 * time.Millisecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed creating client: %w", err)
|
||||||
|
}
|
||||||
|
defer cc.Close()
|
||||||
|
resp, err := cc.Get(ctx, "/")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = cc.Compact(ctx, resp.Header.Revision)
|
||||||
|
if err != nil && !strings.Contains(err.Error(), "error reading from server: EOF") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
var httpClient = http.Client{
|
var httpClient = http.Client{
|
||||||
Timeout: 10 * time.Millisecond,
|
Timeout: 10 * time.Millisecond,
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,6 +53,7 @@ func TestLinearizability(t *testing.T) {
|
||||||
config: e2e.EtcdProcessClusterConfig{
|
config: e2e.EtcdProcessClusterConfig{
|
||||||
ClusterSize: 1,
|
ClusterSize: 1,
|
||||||
GoFailEnabled: true,
|
GoFailEnabled: true,
|
||||||
|
CompactionBatchLimit: 100, // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -61,6 +62,7 @@ func TestLinearizability(t *testing.T) {
|
||||||
config: e2e.EtcdProcessClusterConfig{
|
config: e2e.EtcdProcessClusterConfig{
|
||||||
ClusterSize: 3,
|
ClusterSize: 3,
|
||||||
GoFailEnabled: true,
|
GoFailEnabled: true,
|
||||||
|
CompactionBatchLimit: 100, // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue