From 7ae5766fdb55ffb1f23084eaf464ef902c676500 Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Fri, 16 Feb 2024 01:45:27 +0300
Subject: [PATCH] Wait to clear has_degraded in test_heal - should fix flaps of
 test_heal_* in CI

---
 tests/run_3osds.sh |  2 +-
 tests/test_heal.sh | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh
index 2f6073eb..05b4acf7 100644
--- a/tests/run_3osds.sh
+++ b/tests/run_3osds.sh
@@ -22,7 +22,7 @@ if [ "$IMMEDIATE_COMMIT" != "" ]; then
     NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
     $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 else
-    NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5"
+    NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5 --min_flusher_count 16"
     $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 fi
 
diff --git a/tests/test_heal.sh b/tests/test_heal.sh
index 3604b295..7be2c2a9 100755
--- a/tests/test_heal.sh
+++ b/tests/test_heal.sh
@@ -30,14 +30,16 @@ kill_osds()
     kill -9 $OSD1_PID
     $ETCDCTL del /vitastor/osd/state/1
 
-    for i in $(seq 2 $OSD_COUNT); do
+    for kill_osd in $(seq 2 $OSD_COUNT); do
         sleep 15
-        echo Killing OSD $i and starting OSD $((i-1))
-        p=OSD${i}_PID
+        # Wait for all PGs to clear has_degraded - all data will be at least in 2 copies
+        wait_condition 60 "$ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only |\
+            jq -s -e '[ .[] | select(.state | contains(["'"'"active"'"'"])) | select(.state | contains(["'"'"has_degraded"'"'"]) | not) ] | length == '$PG_COUNT"
+        echo Killing OSD $kill_osd and starting OSD $((kill_osd-1))
+        p=OSD${kill_osd}_PID
         kill -9 ${!p}
-        $ETCDCTL del /vitastor/osd/state/$i
-        start_osd $((i-1))
-        sleep 15
+        $ETCDCTL del /vitastor/osd/state/$kill_osd
+        start_osd $((kill_osd-1))
     done
 
     sleep 5