functional_test: stop etcd and cleanup data when test is successful

Signed-off-by: Benjamin Wang <wachao@vmware.com>

scripts/test.sh

@@ -149,7 +149,9 @@ function generic_checker {
 function killall_functional_test {
   log_callout "Killing all etcd-agent and etcd processes..."
   killall -9 etcd-agent
-  killall -9 etcd
+  # When functional test is successful, the etcd processes have already been
+  # stopped by the agent, so we should ignore the error in this case.
+  killall -9 etcd || true
 }
 
 function functional_pass {

tests/functional/cmd/etcd-tester/etcd_tester_test.go

@@ -47,5 +47,8 @@ func TestFunctional(t *testing.T) {
 		t.Fatal("WaitHealth failed", zap.Error(err))
 	}
 
-	clus.Run(t)
+	if err := clus.Run(t); err == nil {
+		// Only stop etcd and cleanup data when test is successful.
+		clus.Send_SIGQUIT_ETCD_AND_REMOVE_DATA()
+	}
 }

tests/functional/tester/cluster.go

@@ -395,11 +395,16 @@ func (clus *Cluster) Send_INITIAL_START_ETCD() error {
 	return clus.broadcast(rpcpb.Operation_INITIAL_START_ETCD)
 }
 
-// send_SIGQUIT_ETCD_AND_ARCHIVE_DATA sends "send_SIGQUIT_ETCD_AND_ARCHIVE_DATA" operation.
+// send_SIGQUIT_ETCD_AND_ARCHIVE_DATA sends "Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA" operation.
 func (clus *Cluster) send_SIGQUIT_ETCD_AND_ARCHIVE_DATA() error {
 	return clus.broadcast(rpcpb.Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA)
 }
 
+// Send_SIGQUIT_ETCD_AND_REMOVE_DATA sends "Operation_SIGQUIT_ETCD_AND_REMOVE_DATA" operation.
+func (clus *Cluster) Send_SIGQUIT_ETCD_AND_REMOVE_DATA() error {
+	return clus.broadcast(rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA)
+}
+
 // send_RESTART_ETCD sends restart operation.
 func (clus *Cluster) send_RESTART_ETCD() error {
 	return clus.broadcast(rpcpb.Operation_RESTART_ETCD)

tests/functional/tester/cluster_run.go

@@ -31,7 +31,7 @@ import (
 const compactQPS = 50000
 
 // Run starts tester.
-func (clus *Cluster) Run(t *testing.T) {
+func (clus *Cluster) Run(t *testing.T) error {
 	defer printReport()
 
 	// updateCases must be executed after etcd is started, because the FAILPOINTS case
@@ -46,17 +46,25 @@ func (clus *Cluster) Run(t *testing.T) {
 		)
 	}
 
-	var preModifiedKey int64
+	var (
+		preModifiedKey int64
+		err            error
+	)
 	for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
-		t.Run(fmt.Sprintf("round:%v", round), func(t *testing.T) {
+		t.Run(fmt.Sprintf("round-%d", round), func(t *testing.T) {
-			preModifiedKey = clus.doRoundAndCompact(t, round, preModifiedKey)
+			preModifiedKey, err = clus.doRoundAndCompact(t, round, preModifiedKey)
 		})
 
+		if err != nil {
+			clus.failed(err)
+			return err
+		}
+
 		if round > 0 && round%500 == 0 { // every 500 rounds
 			t.Logf("Defragmenting in round: %v", round)
 			if err := clus.defrag(); err != nil {
 				clus.failed(err)
-				return
+				return err
 			}
 		}
 	}
@@ -67,13 +75,14 @@ func (clus *Cluster) Run(t *testing.T) {
 		zap.Int("case", clus.cs),
 		zap.Int("case-total", len(clus.cases)),
 	)
+	return nil
 }
 
-func (clus *Cluster) doRoundAndCompact(t *testing.T, round int, preModifiedKey int64) (postModifiedKey int64) {
+func (clus *Cluster) doRoundAndCompact(t *testing.T, round int, preModifiedKey int64) (postModifiedKey int64, err error) {
 	roundTotalCounter.Inc()
 	clus.rd = round
 
-	if err := clus.doRound(t); err != nil {
+	if err = clus.doRound(t); err != nil {
 		clus.lg.Error(
 			"round FAIL",
 			zap.Int("round", clus.rd),
@@ -81,11 +90,15 @@ func (clus *Cluster) doRoundAndCompact(t *testing.T, round int, preModifiedKey i
 			zap.Int("case-total", len(clus.cases)),
 			zap.Error(err),
 		)
-		if clus.cleanup(err) != nil {
+		if cerr := clus.cleanup(err); cerr != nil {
-			return
+			clus.lg.Warn(
+				"cleanup FAIL",
+				zap.Int("round", clus.rd),
+				zap.Int("case", clus.cs),
+				zap.Int("case-total", len(clus.cases)),
+				zap.Error(cerr),
+			)
 		}
-		// reset preModifiedKey after clean up
-		postModifiedKey = 0
 		return
 	}
 
@@ -104,7 +117,7 @@ func (clus *Cluster) doRoundAndCompact(t *testing.T, round int, preModifiedKey i
 		zap.Int("case-total", len(clus.cases)),
 		zap.Duration("timeout", timeout),
 	)
-	if err := clus.compact(revToCompact, timeout); err != nil {
+	if err = clus.compact(revToCompact, timeout); err != nil {
 		clus.lg.Warn(
 			"compact FAIL",
 			zap.Int("round", clus.rd),
@@ -112,20 +125,19 @@ func (clus *Cluster) doRoundAndCompact(t *testing.T, round int, preModifiedKey i
 			zap.Int("case-total", len(clus.cases)),
 			zap.Error(err),
 		)
-		if err = clus.cleanup(err); err != nil {
+		if cerr := clus.cleanup(err); cerr != nil {
 			clus.lg.Warn(
 				"cleanup FAIL",
 				zap.Int("round", clus.rd),
 				zap.Int("case", clus.cs),
 				zap.Int("case-total", len(clus.cases)),
-				zap.Error(err),
+				zap.Error(cerr),
 			)
-			return
 		}
-		// reset preModifiedKey after clean up
+	} else {
-		return 0
+		postModifiedKey = currentModifiedKey
 	}
-	return currentModifiedKey
+	return
 }
 
 func (clus *Cluster) doRound(t *testing.T) error {