From 2afaddd5b7ef405b25215b920b1af4d65b17689c Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 15 Feb 2023 09:55:03 +0100 Subject: [PATCH 1/6] tests: Refactor getting longest history Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index aa2791542..967df8dd7 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -19,7 +19,6 @@ import ( "encoding/json" "os" "path/filepath" - "sort" "strings" "sync" "testing" @@ -179,10 +178,10 @@ func TestLinearizability(t *testing.T) { forcestopCluster(clus) watchProgressNotifyEnabled := clus.Cfg.WatchProcessNotifyInterval != 0 validateWatchResponses(t, watchResponses, watchProgressNotifyEnabled) - longestHistory, remainingEvents := watchEventHistory(watchResponses) - validateEventsMatch(t, longestHistory, remainingEvents) - operations = patchOperationBasedOnWatchEvents(operations, longestHistory) - checkOperationsAndPersistResults(t, lg, operations, clus) + watchEvents := watchEvents(watchResponses) + validateEventsMatch(t, watchEvents) + patchedOperations := patchOperationBasedOnWatchEvents(operations, longestHistory(watchEvents)) + checkOperationsAndPersistResults(t, lg, patchedOperations, clus) }) } } @@ -383,28 +382,35 @@ type trafficConfig struct { traffic Traffic } -func watchEventHistory(responses [][]watchResponse) (longest []watchEvent, rest [][]watchEvent) { +func watchEvents(responses [][]watchResponse) [][]watchEvent { ops := make([][]watchEvent, len(responses)) for i, resps := range responses { ops[i] = toWatchEvents(resps) } - - sort.Slice(ops, func(i, j int) bool { - return len(ops[i]) > len(ops[j]) - }) - return ops[0], ops[1:] + return ops } -func validateEventsMatch(t *testing.T, longestHistory []watchEvent, other [][]watchEvent) { - for i := 0; i < len(other); i++ { - length := len(other[i]) +func validateEventsMatch(t *testing.T, histories [][]watchEvent) { + longestHistory := longestHistory(histories) + for i := 0; i < len(histories); i++ { + length := len(histories[i]) // We compare prefix of watch events, as we are not guaranteed to collect all events from each node. - if diff := cmp.Diff(longestHistory[:length], other[i][:length], cmpopts.IgnoreFields(watchEvent{}, "Time")); diff != "" { + if diff := cmp.Diff(longestHistory[:length], histories[i][:length], cmpopts.IgnoreFields(watchEvent{}, "Time")); diff != "" { t.Errorf("Events in watches do not match, %s", diff) } } } +func longestHistory(histories [][]watchEvent) []watchEvent { + longestIndex := 0 + for i, history := range histories { + if len(history) > len(histories[longestIndex]) { + longestIndex = i + } + } + return histories[longestIndex] +} + func checkOperationsAndPersistResults(t *testing.T, lg *zap.Logger, operations []porcupine.Operation, clus *e2e.EtcdProcessCluster) { path, err := testResultsDirectory(t) if err != nil { From d0e5c44f673f63bcb51f567bf9c7ff85a801b143 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 15 Feb 2023 09:58:01 +0100 Subject: [PATCH 2/6] tests: Refactor getting test results directory Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 967df8dd7..3312a37e5 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -412,10 +412,7 @@ func longestHistory(histories [][]watchEvent) []watchEvent { } func checkOperationsAndPersistResults(t *testing.T, lg *zap.Logger, operations []porcupine.Operation, clus *e2e.EtcdProcessCluster) { - path, err := testResultsDirectory(t) - if err != nil { - t.Error(err) - } + path := testResultsDirectory(t) linearizable, info := porcupine.CheckOperationsVerbose(model.Etcd, operations, 5*time.Minute) if linearizable == porcupine.Illegal { @@ -431,7 +428,7 @@ func checkOperationsAndPersistResults(t *testing.T, lg *zap.Logger, operations [ visualizationPath := filepath.Join(path, "history.html") lg.Info("Saving visualization", zap.String("path", visualizationPath)) - err = porcupine.VisualizePath(model.Etcd, info, visualizationPath) + err := porcupine.VisualizePath(model.Etcd, info, visualizationPath) if err != nil { t.Errorf("Failed to visualize, err: %v", err) } @@ -470,16 +467,20 @@ func persistMemberDataDir(t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessClu } } -func testResultsDirectory(t *testing.T) (string, error) { +func testResultsDirectory(t *testing.T) string { path, err := filepath.Abs(filepath.Join(resultsDirectory, strings.ReplaceAll(t.Name(), "/", "_"))) if err != nil { - return path, err + t.Fatal(err) + } + err = os.RemoveAll(path) + if err != nil { + t.Fatal(err) } err = os.MkdirAll(path, 0700) if err != nil { - return path, err + t.Fatal(err) } - return path, nil + return path } // forcestopCluster stops the etcd member with signal kill. From a64263cf4937f9bacc37256b15ab301dd6af200f Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 15 Feb 2023 10:01:46 +0100 Subject: [PATCH 3/6] tests: Refactor persisting single member dir Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 3312a37e5..00338dccf 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -423,7 +423,9 @@ func checkOperationsAndPersistResults(t *testing.T, lg *zap.Logger, operations [ } if linearizable != porcupine.Ok { persistOperationHistory(t, lg, path, operations) - persistMemberDataDir(t, lg, clus, path) + for _, member := range clus.Procs { + persistMemberDataDir(t, lg, member, filepath.Join(path, member.Config().Name)) + } } visualizationPath := filepath.Join(path, "history.html") @@ -452,18 +454,11 @@ func persistOperationHistory(t *testing.T, lg *zap.Logger, path string, operatio } } -func persistMemberDataDir(t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, path string) { - for _, member := range clus.Procs { - memberDataDir := filepath.Join(path, member.Config().Name) - err := os.RemoveAll(memberDataDir) - if err != nil { - t.Error(err) - } - lg.Info("Saving member data dir", zap.String("member", member.Config().Name), zap.String("path", memberDataDir)) - err = os.Rename(member.Config().DataDirPath, memberDataDir) - if err != nil { - t.Error(err) - } +func persistMemberDataDir(t *testing.T, lg *zap.Logger, member e2e.EtcdProcess, path string) { + lg.Info("Saving member data dir", zap.String("member", member.Config().Name), zap.String("path", path)) + err := os.Rename(member.Config().DataDirPath, path) + if err != nil { + t.Fatal(err) } } From 58d74e2b7395703464e63eb1982c21995d8f9636 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 15 Feb 2023 10:17:55 +0100 Subject: [PATCH 4/6] test: Refactor TestLinearizability function Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 00338dccf..6e4b13722 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -164,29 +164,35 @@ func TestLinearizability(t *testing.T) { lg := zaptest.NewLogger(t) scenario.config.Logger = lg ctx := context.Background() - clus, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(&scenario.config)) - if err != nil { - t.Fatal(err) - } - defer clus.Close() - operations, watchResponses := testLinearizability(ctx, t, lg, clus, FailpointConfig{ + testLinearizability(ctx, t, lg, scenario.config, scenario.traffic, FailpointConfig{ failpoint: scenario.failpoint, count: 1, retries: 3, waitBetweenTriggers: waitBetweenFailpointTriggers, - }, *scenario.traffic) - forcestopCluster(clus) - watchProgressNotifyEnabled := clus.Cfg.WatchProcessNotifyInterval != 0 - validateWatchResponses(t, watchResponses, watchProgressNotifyEnabled) - watchEvents := watchEvents(watchResponses) - validateEventsMatch(t, watchEvents) - patchedOperations := patchOperationBasedOnWatchEvents(operations, longestHistory(watchEvents)) - checkOperationsAndPersistResults(t, lg, patchedOperations, clus) + }) }) } } -func testLinearizability(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint FailpointConfig, traffic trafficConfig) (operations []porcupine.Operation, responses [][]watchResponse) { +func testLinearizability(ctx context.Context, t *testing.T, lg *zap.Logger, config e2e.EtcdProcessClusterConfig, traffic *trafficConfig, failpoint FailpointConfig) { + clus, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(&config)) + if err != nil { + t.Fatal(err) + } + defer clus.Close() + + operations, watchResponses := runScenario(ctx, t, lg, clus, *traffic, failpoint) + forcestopCluster(clus) + + watchProgressNotifyEnabled := clus.Cfg.WatchProcessNotifyInterval != 0 + validateWatchResponses(t, watchResponses, watchProgressNotifyEnabled) + watchEvents := watchEvents(watchResponses) + validateEventsMatch(t, watchEvents) + patchedOperations := patchOperationBasedOnWatchEvents(operations, longestHistory(watchEvents)) + checkOperationsAndPersistResults(t, lg, patchedOperations, clus) +} + +func runScenario(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, traffic trafficConfig, failpoint FailpointConfig) (operations []porcupine.Operation, responses [][]watchResponse) { // Run multiple test components (traffic, failpoints, etc) in parallel and use canceling context to propagate stop signal. g := errgroup.Group{} trafficCtx, trafficCancel := context.WithCancel(ctx) From d99b1dbdaf3435684ea8dc10912e7df007df32f4 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sat, 11 Feb 2023 12:45:55 +0100 Subject: [PATCH 5/6] tests: Move results reporting to top and add reporting watch histories Signed-off-by: Marek Siarkowicz --- .../workflows/linearizability-template.yaml | 1 + tests/linearizability/linearizability_test.go | 70 ++++++++++++------- tests/linearizability/watch.go | 36 ++++++++++ 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/.github/workflows/linearizability-template.yaml b/.github/workflows/linearizability-template.yaml index 5f97a0010..7d775a343 100644 --- a/.github/workflows/linearizability-template.yaml +++ b/.github/workflows/linearizability-template.yaml @@ -45,6 +45,7 @@ jobs: esac - name: test-linearizability run: | + # Use --failfast to avoid overriding report generated by failed test EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability - uses: actions/upload-artifact@v2 if: always() diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 6e4b13722..4b6bc1ad5 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -175,21 +175,51 @@ func TestLinearizability(t *testing.T) { } func testLinearizability(ctx context.Context, t *testing.T, lg *zap.Logger, config e2e.EtcdProcessClusterConfig, traffic *trafficConfig, failpoint FailpointConfig) { + var responses [][]watchResponse + var events [][]watchEvent + var operations []porcupine.Operation + var patchedOperations []porcupine.Operation + var visualizeHistory func(path string) + clus, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(&config)) if err != nil { t.Fatal(err) } defer clus.Close() - operations, watchResponses := runScenario(ctx, t, lg, clus, *traffic, failpoint) + defer func() { + path := testResultsDirectory(t) + if t.Failed() { + for i, member := range clus.Procs { + memberDataDir := filepath.Join(path, member.Config().Name) + persistMemberDataDir(t, lg, member, memberDataDir) + if responses != nil { + persistWatchResponses(t, lg, filepath.Join(memberDataDir, "responses.json"), responses[i]) + } + if events != nil { + persistWatchEvents(t, lg, filepath.Join(memberDataDir, "events.json"), events[i]) + } + } + if operations != nil { + persistOperationHistory(t, lg, filepath.Join(path, "full-history.json"), operations) + } + if patchedOperations != nil { + persistOperationHistory(t, lg, filepath.Join(path, "patched-history.json"), patchedOperations) + } + } + visualizeHistory(filepath.Join(path, "history.html")) + }() + operations, responses = runScenario(ctx, t, lg, clus, *traffic, failpoint) forcestopCluster(clus) watchProgressNotifyEnabled := clus.Cfg.WatchProcessNotifyInterval != 0 - validateWatchResponses(t, watchResponses, watchProgressNotifyEnabled) - watchEvents := watchEvents(watchResponses) - validateEventsMatch(t, watchEvents) - patchedOperations := patchOperationBasedOnWatchEvents(operations, longestHistory(watchEvents)) - checkOperationsAndPersistResults(t, lg, patchedOperations, clus) + validateWatchResponses(t, responses, watchProgressNotifyEnabled) + + events = watchEvents(responses) + validateEventsMatch(t, events) + + patchedOperations = patchOperationBasedOnWatchEvents(operations, longestHistory(events)) + visualizeHistory = validateOperationHistoryAndReturnVisualize(t, lg, patchedOperations) } func runScenario(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, traffic trafficConfig, failpoint FailpointConfig) (operations []porcupine.Operation, responses [][]watchResponse) { @@ -402,7 +432,7 @@ func validateEventsMatch(t *testing.T, histories [][]watchEvent) { length := len(histories[i]) // We compare prefix of watch events, as we are not guaranteed to collect all events from each node. if diff := cmp.Diff(longestHistory[:length], histories[i][:length], cmpopts.IgnoreFields(watchEvent{}, "Time")); diff != "" { - t.Errorf("Events in watches do not match, %s", diff) + t.Error("Events in watches do not match") } } } @@ -417,9 +447,8 @@ func longestHistory(histories [][]watchEvent) []watchEvent { return histories[longestIndex] } -func checkOperationsAndPersistResults(t *testing.T, lg *zap.Logger, operations []porcupine.Operation, clus *e2e.EtcdProcessCluster) { - path := testResultsDirectory(t) - +// return visualize as porcupine.linearizationInfo used to generate visualization is private +func validateOperationHistoryAndReturnVisualize(t *testing.T, lg *zap.Logger, operations []porcupine.Operation) (visualize func(basepath string)) { linearizable, info := porcupine.CheckOperationsVerbose(model.Etcd, operations, 5*time.Minute) if linearizable == porcupine.Illegal { t.Error("Model is not linearizable") @@ -427,25 +456,18 @@ func checkOperationsAndPersistResults(t *testing.T, lg *zap.Logger, operations [ if linearizable == porcupine.Unknown { t.Error("Linearization timed out") } - if linearizable != porcupine.Ok { - persistOperationHistory(t, lg, path, operations) - for _, member := range clus.Procs { - persistMemberDataDir(t, lg, member, filepath.Join(path, member.Config().Name)) + return func(path string) { + lg.Info("Saving visualization", zap.String("path", path)) + err := porcupine.VisualizePath(model.Etcd, info, path) + if err != nil { + t.Errorf("Failed to visualize, err: %v", err) } } - - visualizationPath := filepath.Join(path, "history.html") - lg.Info("Saving visualization", zap.String("path", visualizationPath)) - err := porcupine.VisualizePath(model.Etcd, info, visualizationPath) - if err != nil { - t.Errorf("Failed to visualize, err: %v", err) - } } func persistOperationHistory(t *testing.T, lg *zap.Logger, path string, operations []porcupine.Operation) { - historyFilePath := filepath.Join(path, "history.json") - lg.Info("Saving operation history", zap.String("path", historyFilePath)) - file, err := os.OpenFile(historyFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + lg.Info("Saving operation history", zap.String("path", path)) + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) if err != nil { t.Errorf("Failed to save operation history: %v", err) return diff --git a/tests/linearizability/watch.go b/tests/linearizability/watch.go index 6d63020f6..e979d5fcf 100644 --- a/tests/linearizability/watch.go +++ b/tests/linearizability/watch.go @@ -16,6 +16,8 @@ package linearizability import ( "context" + "encoding/json" + "os" "sync" "testing" "time" @@ -156,3 +158,37 @@ type watchEvent struct { Revision int64 Time time.Time } + +func persistWatchResponses(t *testing.T, lg *zap.Logger, path string, responses []watchResponse) { + lg.Info("Saving watch responses", zap.String("path", path)) + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + t.Errorf("Failed to save watch history: %v", err) + return + } + defer file.Close() + encoder := json.NewEncoder(file) + for _, resp := range responses { + err := encoder.Encode(resp) + if err != nil { + t.Errorf("Failed to encode response: %v", err) + } + } +} + +func persistWatchEvents(t *testing.T, lg *zap.Logger, path string, events []watchEvent) { + lg.Info("Saving watch events", zap.String("path", path)) + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + t.Errorf("Failed to save watch history: %v", err) + return + } + defer file.Close() + encoder := json.NewEncoder(file) + for _, event := range events { + err := encoder.Encode(event) + if err != nil { + t.Errorf("Failed to encode response: %v", err) + } + } +} From 0cd5c9ca37921e7173c5da85f65ff566fa89eb0f Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 15 Feb 2023 10:23:48 +0100 Subject: [PATCH 6/6] tests: Refactor reporting results Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 86 +++++++++++-------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 4b6bc1ad5..0087b80f8 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -175,51 +175,63 @@ func TestLinearizability(t *testing.T) { } func testLinearizability(ctx context.Context, t *testing.T, lg *zap.Logger, config e2e.EtcdProcessClusterConfig, traffic *trafficConfig, failpoint FailpointConfig) { - var responses [][]watchResponse - var events [][]watchEvent - var operations []porcupine.Operation - var patchedOperations []porcupine.Operation - var visualizeHistory func(path string) - - clus, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(&config)) + r := report{lg: lg} + var err error + r.clus, err = e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(&config)) if err != nil { t.Fatal(err) } - defer clus.Close() + defer r.clus.Close() defer func() { - path := testResultsDirectory(t) - if t.Failed() { - for i, member := range clus.Procs { - memberDataDir := filepath.Join(path, member.Config().Name) - persistMemberDataDir(t, lg, member, memberDataDir) - if responses != nil { - persistWatchResponses(t, lg, filepath.Join(memberDataDir, "responses.json"), responses[i]) - } - if events != nil { - persistWatchEvents(t, lg, filepath.Join(memberDataDir, "events.json"), events[i]) - } + r.Report(t) + }() + r.operations, r.responses = runScenario(ctx, t, lg, r.clus, *traffic, failpoint) + forcestopCluster(r.clus) + + watchProgressNotifyEnabled := r.clus.Cfg.WatchProcessNotifyInterval != 0 + validateWatchResponses(t, r.responses, watchProgressNotifyEnabled) + + r.events = watchEvents(r.responses) + validateEventsMatch(t, r.events) + + r.patchedOperations = patchOperationBasedOnWatchEvents(r.operations, longestHistory(r.events)) + r.visualizeHistory = validateOperationHistoryAndReturnVisualize(t, lg, r.patchedOperations) +} + +type report struct { + lg *zap.Logger + clus *e2e.EtcdProcessCluster + responses [][]watchResponse + events [][]watchEvent + operations []porcupine.Operation + patchedOperations []porcupine.Operation + visualizeHistory func(path string) +} + +func (r *report) Report(t *testing.T) { + path := testResultsDirectory(t) + if t.Failed() { + for i, member := range r.clus.Procs { + memberDataDir := filepath.Join(path, member.Config().Name) + persistMemberDataDir(t, r.lg, member, memberDataDir) + if r.responses != nil { + persistWatchResponses(t, r.lg, filepath.Join(memberDataDir, "responses.json"), r.responses[i]) } - if operations != nil { - persistOperationHistory(t, lg, filepath.Join(path, "full-history.json"), operations) - } - if patchedOperations != nil { - persistOperationHistory(t, lg, filepath.Join(path, "patched-history.json"), patchedOperations) + if r.events != nil { + persistWatchEvents(t, r.lg, filepath.Join(memberDataDir, "events.json"), r.events[i]) } } - visualizeHistory(filepath.Join(path, "history.html")) - }() - operations, responses = runScenario(ctx, t, lg, clus, *traffic, failpoint) - forcestopCluster(clus) - - watchProgressNotifyEnabled := clus.Cfg.WatchProcessNotifyInterval != 0 - validateWatchResponses(t, responses, watchProgressNotifyEnabled) - - events = watchEvents(responses) - validateEventsMatch(t, events) - - patchedOperations = patchOperationBasedOnWatchEvents(operations, longestHistory(events)) - visualizeHistory = validateOperationHistoryAndReturnVisualize(t, lg, patchedOperations) + if r.operations != nil { + persistOperationHistory(t, r.lg, filepath.Join(path, "full-history.json"), r.operations) + } + if r.patchedOperations != nil { + persistOperationHistory(t, r.lg, filepath.Join(path, "patched-history.json"), r.patchedOperations) + } + } + if r.visualizeHistory != nil { + r.visualizeHistory(filepath.Join(path, "history.html")) + } } func runScenario(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, traffic trafficConfig, failpoint FailpointConfig) (operations []porcupine.Operation, responses [][]watchResponse) {