Wait to clear has_degraded in test_heal - should fix flaps of test_heal_* in CI
parent
f882c7dd87
commit
7ae5766fdb
|
@ -22,7 +22,7 @@ if [ "$IMMEDIATE_COMMIT" != "" ]; then
|
||||||
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
|
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
|
||||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
|
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
|
||||||
else
|
else
|
||||||
NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5"
|
NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5 --min_flusher_count 16"
|
||||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
|
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
@ -30,14 +30,16 @@ kill_osds()
|
||||||
kill -9 $OSD1_PID
|
kill -9 $OSD1_PID
|
||||||
$ETCDCTL del /vitastor/osd/state/1
|
$ETCDCTL del /vitastor/osd/state/1
|
||||||
|
|
||||||
for i in $(seq 2 $OSD_COUNT); do
|
for kill_osd in $(seq 2 $OSD_COUNT); do
|
||||||
sleep 15
|
sleep 15
|
||||||
echo Killing OSD $i and starting OSD $((i-1))
|
# Wait for all PGs to clear has_degraded - all data will be at least in 2 copies
|
||||||
p=OSD${i}_PID
|
wait_condition 60 "$ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only |\
|
||||||
|
jq -s -e '[ .[] | select(.state | contains(["'"'"active"'"'"])) | select(.state | contains(["'"'"has_degraded"'"'"]) | not) ] | length == '$PG_COUNT"
|
||||||
|
echo Killing OSD $kill_osd and starting OSD $((kill_osd-1))
|
||||||
|
p=OSD${kill_osd}_PID
|
||||||
kill -9 ${!p}
|
kill -9 ${!p}
|
||||||
$ETCDCTL del /vitastor/osd/state/$i
|
$ETCDCTL del /vitastor/osd/state/$kill_osd
|
||||||
start_osd $((i-1))
|
start_osd $((kill_osd-1))
|
||||||
sleep 15
|
|
||||||
done
|
done
|
||||||
|
|
||||||
sleep 5
|
sleep 5
|
||||||
|
|
Loading…
Reference in New Issue