From cddd6d80af1622af8978416b29ee7cf30bdc5564 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Sun, 22 Nov 2026 19:39:02 +0300 Subject: [PATCH] osd: Log only the first ENOSPC write subop failure per slow_log_interval Clients retry writes that got ENOSPC every client_eio_retry_interval until space is freed (client_retry_enospc), and the primary OSD printed a line for every failed write subop of every retry. That is one line per second for every waiting write, so with many writes waiting the logs can fill the disk long before space is freed. Print only the first ENOSPC write subop failure in each slow_log_interval and report the number of the others from print_slow(). Other subop errors are still printed one by one. --- .gitea/workflows/test.yml | 18 ++++++++++++++++++ src/osd/osd.cpp | 5 +++++ src/osd/osd.h | 1 + src/osd/osd_primary_subops.cpp | 7 ++++++- tests/run_tests.sh | 1 + tests/test_enospc_retry_log.sh | 30 ++++++++++++++++++++++++++++++ 6 files changed, 61 insertions(+), 1 deletion(-) create mode 100755 tests/test_enospc_retry_log.sh diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index ad781d0c..2d1c2ad0 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2093,6 +2093,24 @@ jobs: echo "" done + test_enospc_retry_log: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_enospc_retry_log.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_scrub: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 4ea6e002..b10a3837 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -785,6 +785,11 @@ void osd_t::print_slow() { cur_slow_op_primary = 0; cur_slow_op_secondary = 0; + if (enospc_subop_count > 1) + { + printf("%ju more write subops failed with ENOSPC in the last %ju s\n", enospc_subop_count-1, slow_log_interval); + } + enospc_subop_count = 0; char alloc[1024]; timespec now; clock_gettime(CLOCK_REALTIME, &now); diff --git a/src/osd/osd.h b/src/osd/osd.h index 0b890050..dd7147f6 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -200,6 +200,7 @@ class osd_t int print_stats_timer_id = -1, slow_log_timer_id = -1; uint64_t cur_slow_op_primary = 0; uint64_t cur_slow_op_secondary = 0; + uint64_t enospc_subop_count = 0; // peers and PGs diff --git a/src/osd/osd_primary_subops.cpp b/src/osd/osd_primary_subops.cpp index 8b024ea1..a7c63a52 100644 --- a/src/osd/osd_primary_subops.cpp +++ b/src/osd/osd_primary_subops.cpp @@ -462,7 +462,12 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op) { int64_t peer_osd = (msgr.clients.find(subop->client_id) != msgr.clients.end() ? msgr.clients.at(subop->client_id)->osd_num : 0); - if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) + if (retval == -ENOSPC && (opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) && + enospc_subop_count++ > 0) + { + // Writes waiting for free space are retried every second, print_slow() reports the rest + } + else if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) { printf("%s subop to %jx:%jx v%ju failed ", osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version); diff --git a/tests/run_tests.sh b/tests/run_tests.sh index caac41b7..689c3109 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -162,6 +162,7 @@ OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh ./test_recovery_enospc.sh +./test_enospc_retry_log.sh ./test_scrub.sh ZERO_OSD=2 ./test_scrub.sh diff --git a/tests/test_enospc_retry_log.sh b/tests/test_enospc_retry_log.sh new file mode 100755 index 00000000..fea195d3 --- /dev/null +++ b/tests/test_enospc_retry_log.sh @@ -0,0 +1,30 @@ +#!/bin/bash -ex + +# Writes waiting for free space are retried every second, OSDs shouldn't log every failed subop + +OSD_SIZE=200 +GLOBAL_CONFIG=',"client_retry_enospc":false' + +. `dirname $0`/run_3osds.sh + +if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=500M; then + format_error "Should get ENOSPC, but didn't" +fi + +$ETCDCTL put /vitastor/config/global "$($ETCDCTL get /vitastor/config/global --print-value-only | jq -c '.client_retry_enospc = true')" +BEFORE=$(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28' || true) +$VITASTOR_FIO -bs=1M -direct=1 -iodepth=16 -rw=write -pool=1 -inode=1 -size=500M & +FIO_PID=$! +sleep 15 +kill -9 $FIO_PID +wait $FIO_PID || true + +LINES=$(( $(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28') - BEFORE )) +if [[ $LINES -gt 30 ]]; then + format_error "OSDs logged $LINES failed subops in 15 s" +fi +if ! grep -q 'more write subops failed with ENOSPC' ./testdata/osd*.log; then + format_error "No summary of ENOSPC subop failures" +fi + +format_green OK -- 2.47.3