Add ENOSPC handling tests

src/cluster_client.cpp

@@ -397,6 +397,8 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
             client_eio_retry_interval = 10;
         }
     }
+    // client_retry_enospc
+    client_retry_enospc = config["client_retry_enospc"].is_null() ? true : config["client_retry_enospc"].bool_value();
     // log_level
     log_level = config["log_level"].uint64_value();
     msgr.parse_config(config);
@@ -817,7 +819,7 @@ resume_2:
         return 1;
     }
     else if (op->retval != 0 && !(op->flags & OP_FLUSH_BUFFER) &&
-        op->retval != -EPIPE && (op->retval != -EIO || !client_eio_retry_interval) && op->retval != -ENOSPC)
+        op->retval != -EPIPE && (op->retval != -EIO || !client_eio_retry_interval) && (op->retval != -ENOSPC || !client_retry_enospc))
     {
         // Fatal error (neither -EPIPE, -EIO nor -ENOSPC)
         erase_op(op);

src/cluster_client.h

@@ -94,6 +94,7 @@ class cluster_client_t
     int log_level = 0;
     int client_retry_interval = 50; // ms
     int client_eio_retry_interval = 1000; // ms
+    bool client_retry_enospc = true;
 
     int retry_timeout_id = 0;
     int retry_timeout_duration = 0;

tests/run_tests.sh

@@ -62,6 +62,11 @@ TEST_NAME=csum_4k_dmj  OSD_ARGS="--data_csum_type crc32c --inmemory_metadata fal
 TEST_NAME=csum_4k_dj   OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
 TEST_NAME=csum_4k      OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
 
+./test_enospc.sh
+SCHEME=xor ./test_enospc.sh
+IMMEDIATE_COMMIT=1 ./test_enospc.sh
+IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh
+
 ./test_scrub.sh
 ZERO_OSD=2 ./test_scrub.sh
 SCHEME=xor ./test_scrub.sh

tests/test_enospc.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+OSD_SIZE=200
+GLOBAL_CONFIG=',"client_retry_enospc":false'
+
+. `dirname $0`/run_3osds.sh
+
+export LD_PRELOAD="build/src/libfio_vitastor.so"
+
+# Should fail with ENOSPC
+if fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
+    -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
+    format_error "Should get ENOSPC, but didn't"
+fi
+
+# Should fail with ENOSPC too (the idea is to try to overwrite first objects to check their rollback)
+if fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=32 \
+    -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
+    format_error "Should get ENOSPC, but didn't"
+fi
+
+# Should complete OK
+if ! fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 \
+    -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=100M -cluster_log_level=10 -number_ios=4096; then
+    format_error "Should do random writes over ENOSPC correctly, but got an error"
+fi
+
+format_green OK