Test etcd schizophrenia

patches/VitastorPlugin.pm

@@ -410,8 +410,8 @@ sub volume_size_info
     my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/';
     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
     my $info = _process_list($scfg, $storeid, run_cli($scfg, [ 'ls', $prefix.$name ]))->[0];
-    #return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size;
-    return $info->{size};
+    # (size, format, used, parent, ctime)
+    return wantarray ? ($info->{size}, $info->{format}, $info->{size}, $info->{parent}, 0) : $info->{size};
 }
 
 sub volume_resize

src/osd/osd_cluster.cpp

@@ -1051,8 +1051,15 @@ void osd_t::report_pg_states()
     etcd_reporting_pg_state = true;
     st_cli.etcd_txn(json11::Json::object {
         { "compare", checks }, { "success", success }, { "failure", failure }
-    }, st_cli.etcd_quick_timeout, 0, 0, [this, reporting_pgs](std::string err, json11::Json data)
+    }, st_cli.etcd_quick_timeout, 0, 0, [this, reporting_pgs, success_count = success.size(), failure_count = failure.size()](std::string err, json11::Json data)
     {
+        int expected_count = (data["succeeded"].bool_value() ? success_count : failure_count);
+        if (expected_count != data["responses"].array_items().size())
+        {
+            printf("Unexpected response from etcd - 'responses' count (%u) isn't equal to expected (%u), stopping\n",
+                data["responses"].array_items().size(), expected_count);
+            force_stop(1);
+        }
         etcd_reporting_pg_state = false;
         if (!data["succeeded"].bool_value())
         {

src/osd/osd_primary_subops.cpp

@@ -46,11 +46,6 @@ void osd_t::autosync()
 void osd_t::finish_op(osd_op_t *cur_op, int retval)
 {
     inflight_ops--;
-    if (cur_op->req.hdr.opcode == OSD_OP_WRITE)
-    {
-        printf("%jx %jx+%x p%jx v%jx r=%x\n", cur_op->req.rw.inode, cur_op->req.rw.offset, cur_op->req.rw.len,
-            cur_op->op_data ? cur_op->op_data->orig_ver : 0, cur_op->reply.rw.version, retval);
-    }
     if (cur_op->req.hdr.opcode == OSD_OP_READ ||
         cur_op->req.hdr.opcode == OSD_OP_WRITE ||
         cur_op->req.hdr.opcode == OSD_OP_DELETE)

src/osd/osd_scrub.cpp

@@ -505,7 +505,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 if (votes[role] > 0 && votes[role] < votes[best])
                 {
                     printf(
-                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
+                        "[PG %u/%u] Object %jx:%jx v%ju copy on OSD %ju doesn't match %d other copies%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[role].osd_num, votes[best],
@@ -528,7 +528,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 best = -1;
                 inconsistent = true;
                 printf(
-                    "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
+                    "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: copies don't match. Use vitastor-cli fix to fix it\n",
                     INODE_POOL(op_data->oid.inode), op_data->pg_num,
                     op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
                 );
@@ -547,7 +547,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
         {
             inconsistent = true;
             printf(
-                "[PG %u/%u] Scrub error: object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
+                "[PG %u/%u] Object %jx:%jx v%ju is inconsistent: parity chunks don't match data. Use vitastor-cli fix to fix it\n",
                 INODE_POOL(op_data->oid.inode), op_data->pg_num,
                 op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
             );
@@ -574,7 +574,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
                 {
                     op_data->stripes[i].read_error = true;
                     printf(
-                        "[PG %u/%u] Scrub error: object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
+                        "[PG %u/%u] Object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
                         op_data->stripes[i].role, op_data->stripes[i].osd_num,

tests/test_heal.sh

@@ -55,20 +55,6 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bsrange=4k-128k -blockalign=4k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
         -serialize_overlap=1 -randrepeat=0 -refill_buffers=1 -mirror_file=./testdata/bin/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120
 
-# Wait for active/clean
-wait_up 120
-
-# Trigger scrub
-for i in $(seq 1 $PG_COUNT); do
-    $ETCDCTL put /vitastor/pg/history/1/$i `$ETCDCTL get --print-value-only /vitastor/pg/history/1/$i | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
-done
-
-# Wait for scrub to finish
-wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
-
-# Check that everything is ok
-wait_up 1
-
 qemu-img convert -S 4096 -p \
     -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
     -O raw ./testdata/bin/read.bin
@@ -78,7 +64,7 @@ if ! diff -q ./testdata/bin/read.bin ./testdata/bin/mirror.bin; then
     format_error Data lost during self-heal
 fi
 
-if grep -qP 'Checksum mismatch|BUG|Scrub error' ./testdata/osd*.log; then
+if grep -qP 'Checksum mismatch|BUG' ./testdata/osd*.log; then
     format_error Checksum mismatches or BUGs detected during test
 fi