osd: Log only the first ENOSPC write subop failure per slow_log_interval #150
@@ -2093,6 +2093,24 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_enospc_retry_log:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||
steps:
|
||||
- name: Run test
|
||||
id: test
|
||||
timeout-minutes: 3
|
||||
run: /root/vitastor/tests/test_enospc_retry_log.sh
|
||||
- name: Print logs
|
||||
if: always() && steps.test.outcome == 'failure'
|
||||
run: |
|
||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||
echo "-------- $i --------"
|
||||
cat $i
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_scrub:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
|
||||
@@ -785,6 +785,11 @@ void osd_t::print_slow()
|
||||
{
|
||||
cur_slow_op_primary = 0;
|
||||
cur_slow_op_secondary = 0;
|
||||
if (enospc_subop_count > 1)
|
||||
{
|
||||
printf("%ju more write subops failed with ENOSPC in the last %ju s\n", enospc_subop_count-1, slow_log_interval);
|
||||
}
|
||||
enospc_subop_count = 0;
|
||||
char alloc[1024];
|
||||
timespec now;
|
||||
clock_gettime(CLOCK_REALTIME, &now);
|
||||
|
||||
@@ -200,6 +200,7 @@ class osd_t
|
||||
int print_stats_timer_id = -1, slow_log_timer_id = -1;
|
||||
uint64_t cur_slow_op_primary = 0;
|
||||
uint64_t cur_slow_op_secondary = 0;
|
||||
uint64_t enospc_subop_count = 0;
|
||||
|
||||
// peers and PGs
|
||||
|
||||
|
||||
@@ -462,7 +462,12 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
|
||||
{
|
||||
int64_t peer_osd = (msgr.clients.find(subop->client_id) != msgr.clients.end()
|
||||
? msgr.clients.at(subop->client_id)->osd_num : 0);
|
||||
if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
|
||||
if (retval == -ENOSPC && (opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE) &&
|
||||
enospc_subop_count++ > 0)
|
||||
{
|
||||
// Writes waiting for free space are retried every second, print_slow() reports the rest
|
||||
}
|
||||
else if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
|
||||
{
|
||||
printf("%s subop to %jx:%jx v%ju failed ", osd_op_names[opcode],
|
||||
subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version);
|
||||
|
||||
@@ -162,6 +162,7 @@ OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh
|
||||
OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh
|
||||
|
||||
./test_recovery_enospc.sh
|
||||
./test_enospc_retry_log.sh
|
||||
|
||||
./test_scrub.sh
|
||||
ZERO_OSD=2 ./test_scrub.sh
|
||||
|
||||
30
tests/test_enospc_retry_log.sh
Executable file
30
tests/test_enospc_retry_log.sh
Executable file
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash -ex
|
||||
|
||||
# Writes waiting for free space are retried every second, OSDs shouldn't log every failed subop
|
||||
|
||||
OSD_SIZE=200
|
||||
GLOBAL_CONFIG=',"client_retry_enospc":false'
|
||||
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=500M; then
|
||||
format_error "Should get ENOSPC, but didn't"
|
||||
fi
|
||||
|
||||
$ETCDCTL put /vitastor/config/global "$($ETCDCTL get /vitastor/config/global --print-value-only | jq -c '.client_retry_enospc = true')"
|
||||
BEFORE=$(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28' || true)
|
||||
$VITASTOR_FIO -bs=1M -direct=1 -iodepth=16 -rw=write -pool=1 -inode=1 -size=500M &
|
||||
FIO_PID=$!
|
||||
sleep 15
|
||||
kill -9 $FIO_PID
|
||||
wait $FIO_PID || true
|
||||
|
||||
LINES=$(( $(cat ./testdata/osd*.log | grep -c 'subop .* retval = -28') - BEFORE ))
|
||||
if [[ $LINES -gt 30 ]]; then
|
||||
format_error "OSDs logged $LINES failed subops in 15 s"
|
||||
fi
|
||||
if ! grep -q 'more write subops failed with ENOSPC' ./testdata/osd*.log; then
|
||||
format_error "No summary of ENOSPC subop failures"
|
||||
fi
|
||||
|
||||
format_green OK
|
||||
Reference in New Issue
Block a user