Add scrub test

tests/run_3osds.sh

@@ -95,19 +95,29 @@ try_reweight()
     sleep 3
 }
 
+wait_condition()
+{
+    sec=$1
+    check=$2
+    proc=$3
+    i=0
+    while [[ $i -lt $sec ]]; do
+        eval "$check" && break
+        if [ $i -eq $sec ]; then
+            format_error "$proc couldn't finish in $sec seconds"
+        fi
+        sleep 1
+        i=$((i+1))
+    done
+}
+
 wait_finish_rebalance()
 {
     sec=$1
-    i=0
-    while [[ $i -lt $sec ]]; do
-        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT) && \
-            break
-        sleep 1
-        i=$((i+1))
-        if [ $i -eq $sec ]; then
-            format_error "Rebalance couldn't finish in $sec seconds"
-        fi
-    done
+    check=$2
+    check=${check:-'.state == ["active"] or .state == ["active", "left_on_dead"]'}
+    check="$ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select($check) ] | length) == $PG_COUNT'"
+    wait_condition "$sec" "$check" Rebalance
 }
 
 check_qemu()

tests/run_tests.sh

@@ -46,3 +46,10 @@ SCHEME=xor ./test_write.sh
 
 PG_SIZE=2 ./test_heal.sh
 SCHEME=ec ./test_heal.sh
+
+./test_scrub.sh
+ZERO_OSD=2 ./test_scrub.sh
+SCHEME=xor ./test_scrub.sh
+PG_SIZE=3 ./test_scrub.sh
+PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec ./test_scrub.sh
+SCHEME=ec ./test_scrub.sh

tests/test_scrub.sh

@@ -0,0 +1,64 @@
+#!/bin/bash -ex
+# Test for scrub without checksums
+
+ZERO_OSD=${ZERO_OSD:-1}
+
+if [[ ("$SCHEME" = "" || "$SCHEME" = "replicated") && ("$PG_SIZE" = "" || "$PG_SIZE" = 2) ]]; then
+    OSD_COUNT=2
+fi
+
+. `dirname $0`/run_3osds.sh
+
+check_qemu
+
+IMG_SIZE=128
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
+
+# Write
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
+        -mirror_file=./testdata/mirror.bin -end_fsync=1 -rw=write -etcd=$ETCD_URL -image=testimg -runtime=10
+
+# Intentionally corrupt OSD data and restart it
+zero_osd_pid=OSD${ZERO_OSD}_PID
+kill ${!zero_osd_pid}
+sleep 1
+kill -9 ${!zero_osd_pid} || true
+data_offset=$(build/src/vitastor-disk simple-offsets ./testdata/test_osd$ZERO_OSD.bin $OFFSET_ARGS | grep data_offset | awk '{print $2}')
+truncate -s $data_offset ./testdata/test_osd$ZERO_OSD.bin
+dd if=/dev/zero of=./testdata/test_osd$ZERO_OSD.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
+$ETCDCTL del /vitastor/osd/state/$ZERO_OSD
+start_osd $ZERO_OSD
+
+# Wait until start
+wait_up 10
+
+# Trigger scrub
+$ETCDCTL put /vitastor/pg/history/1/1 `$ETCDCTL get --print-value-only /vitastor/pg/history/1/1 | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
+
+# Wait for scrub to finish
+wait_condition 60 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
+
+if [[ ($SCHEME = replicated && $PG_SIZE < 3) || ($SCHEME != replicated && $((PG_SIZE-PG_DATA_SIZE)) < 2) ]]; then
+    # Check that objects are marked as inconsistent if 2 replicas or EC/XOR 2+1
+    build/src/vitastor-cli describe --etcd_address $ETCD_URL --json | jq -e '[ .[] | select(.inconsistent) ] | length == '$((IMG_SIZE * 8 * PG_SIZE / (SCHEME = replicated ? 1 : PG_DATA_SIZE)))
+
+    # Fix objects
+
+    # Read everything back
+elif [[ ($SCHEME = replicated && $PG_SIZE > 2) || ($SCHEME != replicated && $((PG_SIZE-PG_DATA_SIZE)) > 1) ]]; then
+    # Check that everything heals
+    wait_finish_rebalance 60
+
+    build/src/vitastor-cli describe --etcd_address $ETCD_URL --json | jq -e '. | length == 0'
+
+    # Read everything back
+    qemu-img convert -S 4096 -p \
+        -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
+        -O raw ./testdata/read.bin
+
+    diff ./testdata/read.bin ./testdata/mirror.bin
+fi
+
+format_green OK