diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index ad781d0c..2d1c2ad0 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2093,6 +2093,24 @@ jobs: echo "" done + test_enospc_retry_log: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_enospc_retry_log.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_scrub: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 4ea6e002..b10a3837 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -785,6 +785,11 @@ void osd_t::print_slow() { cur_slow_op_primary = 0; cur_slow_op_secondary = 0; + if (enospc_subop_count > 1) + { + printf("%ju more write subops failed with ENOSPC in the last %ju s\n", enospc_subop_count-1, slow_log_interval); + } + enospc_subop_count = 0; char alloc[1024]; timespec now; clock_gettime(CLOCK_REALTIME, &now); diff --git a/src/osd/osd.h b/src/osd/osd.h index 0b890050..dd7147f6 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -200,6 +200,7 @@ class osd_t int print_stats_timer_id = -1, slow_log_timer_id = -1; uint64_t cur_slow_op_primary = 0; uint64_t cur_slow_op_secondary = 0; + uint64_t enospc_subop_count = 0; // peers and PGs diff --git a/src/osd/osd_primary_subops.cpp b/src/osd/osd_primary_subops.cpp index 8b024ea1..a7c63a52 100644 --- a/src/osd/osd_primary_subops.cpp +++ b/src/osd/osd_primary_subops.cpp @@ -462,7 +462,12 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op) { int64_t peer_osd = (msgr.clients.find(subop->client_id) != msgr.clients.end() ? msgr.clients.at(subop->client_id)->osd_num : 0); - if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) + if (retval == -ENOSPC && (opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) && + enospc_subop_count++ > 0) + { + // Writes waiting for free space are retried every second, print_slow() reports the rest + } + else if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) { printf("%s subop to %jx:%jx v%ju failed ", osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version); diff --git a/tests/run_tests.sh b/tests/run_tests.sh index caac41b7..689c3109 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -162,6 +162,7 @@ OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh ./test_recovery_enospc.sh +./test_enospc_retry_log.sh ./test_scrub.sh ZERO_OSD=2 ./test_scrub.sh diff --git a/tests/test_enospc_retry_log.sh b/tests/test_enospc_retry_log.sh new file mode 100755 index 00000000..fea195d3 --- /dev/null +++ b/tests/test_enospc_retry_log.sh @@ -0,0 +1,30 @@ +#!/bin/bash -ex + +# Writes waiting for free space are retried every second, OSDs shouldn't log every failed subop + +OSD_SIZE=200 +GLOBAL_CONFIG=',"client_retry_enospc":false' + +. `dirname $0`/run_3osds.sh + +if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=500M; then + format_error "Should get ENOSPC, but didn't" +fi + +$ETCDCTL put /vitastor/config/global "$($ETCDCTL get /vitastor/config/global --print-value-only | jq -c '.client_retry_enospc = true')" +BEFORE=$(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28' || true) +$VITASTOR_FIO -bs=1M -direct=1 -iodepth=16 -rw=write -pool=1 -inode=1 -size=500M & +FIO_PID=$! +sleep 15 +kill -9 $FIO_PID +wait $FIO_PID || true + +LINES=$(( $(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28') - BEFORE )) +if [[ $LINES -gt 30 ]]; then + format_error "OSDs logged $LINES failed subops in 15 s" +fi +if ! grep -q 'more write subops failed with ENOSPC' ./testdata/osd*.log; then + format_error "No summary of ENOSPC subop failures" +fi + +format_green OK