osd: Log only the first ENOSPC write subop failure per slow_log_interval #150

Open
koolkhel wants to merge 1 commits from koolkhel/vitastor:osd-enospc-subop-log into master
6 changed files with 61 additions and 1 deletions

View File

@@ -2093,6 +2093,24 @@ jobs:
echo ""
done
test_enospc_retry_log:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: /root/vitastor/tests/test_enospc_retry_log.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_scrub:
runs-on: ubuntu-latest
needs: build

View File

@@ -785,6 +785,11 @@ void osd_t::print_slow()
{
cur_slow_op_primary = 0;
cur_slow_op_secondary = 0;
if (enospc_subop_count > 1)
{
printf("%ju more write subops failed with ENOSPC in the last %ju s\n", enospc_subop_count-1, slow_log_interval);
}
enospc_subop_count = 0;
char alloc[1024];
timespec now;
clock_gettime(CLOCK_REALTIME, &now);

View File

@@ -200,6 +200,7 @@ class osd_t
int print_stats_timer_id = -1, slow_log_timer_id = -1;
uint64_t cur_slow_op_primary = 0;
uint64_t cur_slow_op_secondary = 0;
uint64_t enospc_subop_count = 0;
// peers and PGs

View File

@@ -462,7 +462,12 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
{
int64_t peer_osd = (msgr.clients.find(subop->client_id) != msgr.clients.end()
? msgr.clients.at(subop->client_id)->osd_num : 0);
if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
if (retval == -ENOSPC && (opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) &&
enospc_subop_count++ > 0)
{
// Writes waiting for free space are retried every second, print_slow() reports the rest
}
else if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
{
printf("%s subop to %jx:%jx v%ju failed ", osd_op_names[opcode],
subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version);

View File

@@ -162,6 +162,7 @@ OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh
OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh
./test_recovery_enospc.sh
./test_enospc_retry_log.sh
./test_scrub.sh
ZERO_OSD=2 ./test_scrub.sh

30
tests/test_enospc_retry_log.sh Executable file
View File

@@ -0,0 +1,30 @@
#!/bin/bash -ex
# Writes waiting for free space are retried every second, OSDs shouldn't log every failed subop
OSD_SIZE=200
GLOBAL_CONFIG=',"client_retry_enospc":false'
. `dirname $0`/run_3osds.sh
if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=500M; then
format_error "Should get ENOSPC, but didn't"
fi
$ETCDCTL put /vitastor/config/global "$($ETCDCTL get /vitastor/config/global --print-value-only | jq -c '.client_retry_enospc = true')"
BEFORE=$(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28' || true)
$VITASTOR_FIO -bs=1M -direct=1 -iodepth=16 -rw=write -pool=1 -inode=1 -size=500M &
FIO_PID=$!
sleep 15
kill -9 $FIO_PID
wait $FIO_PID || true
LINES=$(( $(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28') - BEFORE ))
if [[ $LINES -gt 30 ]]; then
format_error "OSDs logged $LINES failed subops in 15 s"
fi
if ! grep -q 'more write subops failed with ENOSPC' ./testdata/osd*.log; then
format_error "No summary of ENOSPC subop failures"
fi
format_green OK