Add ENOSPC handling tests
Test / buildenv (push) Successful in 10s
Details
Test / build (push) Successful in 3m11s
Details
Test / test_cas (push) Successful in 10s
Details
Test / make_test (push) Failing after 37s
Details
Test / test_change_pg_size (push) Successful in 8s
Details
Test / test_change_pg_count (push) Successful in 39s
Details
Test / test_change_pg_count_ec (push) Successful in 36s
Details
Test / test_create_nomaxid (push) Successful in 7s
Details
Test / test_etcd_fail (push) Successful in 54s
Details
Test / test_interrupted_rebalance_ec_imm (push) Has been cancelled
Details
Test / test_failure_domain (push) Has been cancelled
Details
Test / test_snapshot (push) Has been cancelled
Details
Test / test_snapshot_ec (push) Has been cancelled
Details
Test / test_minsize_1 (push) Has been cancelled
Details
Test / test_move_reappear (push) Has been cancelled
Details
Test / test_rm (push) Has been cancelled
Details
Test / test_snapshot_chain (push) Has been cancelled
Details
Test / test_snapshot_chain_ec (push) Has been cancelled
Details
Test / test_snapshot_down (push) Has been cancelled
Details
Test / test_snapshot_down_ec (push) Has been cancelled
Details
Test / test_splitbrain (push) Has been cancelled
Details
Test / test_rebalance_verify (push) Has been cancelled
Details
Test / test_rebalance_verify_imm (push) Has been cancelled
Details
Test / test_rebalance_verify_ec (push) Has been cancelled
Details
Test / test_rebalance_verify_ec_imm (push) Has been cancelled
Details
Test / test_interrupted_rebalance (push) Has been cancelled
Details
Test / test_switch_primary (push) Has been cancelled
Details
Test / test_interrupted_rebalance_ec (push) Has been cancelled
Details
Test / test_write (push) Has been cancelled
Details
Test / test_write_xor (push) Has been cancelled
Details
Test / test_write_no_same (push) Has been cancelled
Details
Test / test_heal_pg_size_2 (push) Has been cancelled
Details
Test / test_interrupted_rebalance_imm (push) Has been cancelled
Details
Test / test_heal_ec (push) Has been cancelled
Details
Test / test_heal_csum_32k_dmj (push) Has been cancelled
Details
Test / test_heal_csum_32k_dj (push) Has been cancelled
Details
Test / test_heal_csum_32k (push) Has been cancelled
Details
Test / test_heal_csum_4k_dmj (push) Has been cancelled
Details
Test / test_heal_csum_4k_dj (push) Has been cancelled
Details
Test / test_heal_csum_4k (push) Has been cancelled
Details
Test / test_enospc (push) Has been cancelled
Details
Test / test_enospc_xor (push) Has been cancelled
Details
Test / test_enospc_imm (push) Has been cancelled
Details
Test / test_enospc_imm_xor (push) Has been cancelled
Details
Test / test_scrub (push) Has been cancelled
Details
Test / test_scrub_zero_osd_2 (push) Has been cancelled
Details
Test / test_scrub_xor (push) Has been cancelled
Details
Test / test_scrub_pg_size_3 (push) Has been cancelled
Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Has been cancelled
Details
Test / test_scrub_ec (push) Has been cancelled
Details
Test / test_nfs (push) Has been cancelled
Details
Test / test_add_osd (push) Has been cancelled
Details
Test / buildenv (push) Successful in 10s
Details
Test / build (push) Successful in 3m11s
Details
Test / test_cas (push) Successful in 10s
Details
Test / make_test (push) Failing after 37s
Details
Test / test_change_pg_size (push) Successful in 8s
Details
Test / test_change_pg_count (push) Successful in 39s
Details
Test / test_change_pg_count_ec (push) Successful in 36s
Details
Test / test_create_nomaxid (push) Successful in 7s
Details
Test / test_etcd_fail (push) Successful in 54s
Details
Test / test_interrupted_rebalance_ec_imm (push) Has been cancelled
Details
Test / test_failure_domain (push) Has been cancelled
Details
Test / test_snapshot (push) Has been cancelled
Details
Test / test_snapshot_ec (push) Has been cancelled
Details
Test / test_minsize_1 (push) Has been cancelled
Details
Test / test_move_reappear (push) Has been cancelled
Details
Test / test_rm (push) Has been cancelled
Details
Test / test_snapshot_chain (push) Has been cancelled
Details
Test / test_snapshot_chain_ec (push) Has been cancelled
Details
Test / test_snapshot_down (push) Has been cancelled
Details
Test / test_snapshot_down_ec (push) Has been cancelled
Details
Test / test_splitbrain (push) Has been cancelled
Details
Test / test_rebalance_verify (push) Has been cancelled
Details
Test / test_rebalance_verify_imm (push) Has been cancelled
Details
Test / test_rebalance_verify_ec (push) Has been cancelled
Details
Test / test_rebalance_verify_ec_imm (push) Has been cancelled
Details
Test / test_interrupted_rebalance (push) Has been cancelled
Details
Test / test_switch_primary (push) Has been cancelled
Details
Test / test_interrupted_rebalance_ec (push) Has been cancelled
Details
Test / test_write (push) Has been cancelled
Details
Test / test_write_xor (push) Has been cancelled
Details
Test / test_write_no_same (push) Has been cancelled
Details
Test / test_heal_pg_size_2 (push) Has been cancelled
Details
Test / test_interrupted_rebalance_imm (push) Has been cancelled
Details
Test / test_heal_ec (push) Has been cancelled
Details
Test / test_heal_csum_32k_dmj (push) Has been cancelled
Details
Test / test_heal_csum_32k_dj (push) Has been cancelled
Details
Test / test_heal_csum_32k (push) Has been cancelled
Details
Test / test_heal_csum_4k_dmj (push) Has been cancelled
Details
Test / test_heal_csum_4k_dj (push) Has been cancelled
Details
Test / test_heal_csum_4k (push) Has been cancelled
Details
Test / test_enospc (push) Has been cancelled
Details
Test / test_enospc_xor (push) Has been cancelled
Details
Test / test_enospc_imm (push) Has been cancelled
Details
Test / test_enospc_imm_xor (push) Has been cancelled
Details
Test / test_scrub (push) Has been cancelled
Details
Test / test_scrub_zero_osd_2 (push) Has been cancelled
Details
Test / test_scrub_xor (push) Has been cancelled
Details
Test / test_scrub_pg_size_3 (push) Has been cancelled
Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Has been cancelled
Details
Test / test_scrub_ec (push) Has been cancelled
Details
Test / test_nfs (push) Has been cancelled
Details
Test / test_add_osd (push) Has been cancelled
Details
parent
17f61ddd96
commit
e094c30959
|
@ -748,6 +748,78 @@ jobs:
|
||||||
echo ""
|
echo ""
|
||||||
done
|
done
|
||||||
|
|
||||||
|
test_enospc:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||||
|
steps:
|
||||||
|
- name: Run test
|
||||||
|
id: test
|
||||||
|
timeout-minutes: 3
|
||||||
|
run: /root/vitastor/tests/test_enospc.sh
|
||||||
|
- name: Print logs
|
||||||
|
if: always() && steps.test.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||||
|
echo "-------- $i --------"
|
||||||
|
cat $i
|
||||||
|
echo ""
|
||||||
|
done
|
||||||
|
|
||||||
|
test_enospc_xor:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||||
|
steps:
|
||||||
|
- name: Run test
|
||||||
|
id: test
|
||||||
|
timeout-minutes: 3
|
||||||
|
run: SCHEME=xor /root/vitastor/tests/test_enospc.sh
|
||||||
|
- name: Print logs
|
||||||
|
if: always() && steps.test.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||||
|
echo "-------- $i --------"
|
||||||
|
cat $i
|
||||||
|
echo ""
|
||||||
|
done
|
||||||
|
|
||||||
|
test_enospc_imm:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||||
|
steps:
|
||||||
|
- name: Run test
|
||||||
|
id: test
|
||||||
|
timeout-minutes: 3
|
||||||
|
run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_enospc.sh
|
||||||
|
- name: Print logs
|
||||||
|
if: always() && steps.test.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||||
|
echo "-------- $i --------"
|
||||||
|
cat $i
|
||||||
|
echo ""
|
||||||
|
done
|
||||||
|
|
||||||
|
test_enospc_imm_xor:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||||
|
steps:
|
||||||
|
- name: Run test
|
||||||
|
id: test
|
||||||
|
timeout-minutes: 3
|
||||||
|
run: IMMEDIATE_COMMIT=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
|
||||||
|
- name: Print logs
|
||||||
|
if: always() && steps.test.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||||
|
echo "-------- $i --------"
|
||||||
|
cat $i
|
||||||
|
echo ""
|
||||||
|
done
|
||||||
|
|
||||||
test_scrub:
|
test_scrub:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
|
|
|
@ -397,6 +397,8 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
|
||||||
client_eio_retry_interval = 10;
|
client_eio_retry_interval = 10;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// client_retry_enospc
|
||||||
|
client_retry_enospc = config["client_retry_enospc"].is_null() ? true : config["client_retry_enospc"].bool_value();
|
||||||
// log_level
|
// log_level
|
||||||
log_level = config["log_level"].uint64_value();
|
log_level = config["log_level"].uint64_value();
|
||||||
msgr.parse_config(config);
|
msgr.parse_config(config);
|
||||||
|
@ -817,7 +819,7 @@ resume_2:
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
else if (op->retval != 0 && !(op->flags & OP_FLUSH_BUFFER) &&
|
else if (op->retval != 0 && !(op->flags & OP_FLUSH_BUFFER) &&
|
||||||
op->retval != -EPIPE && (op->retval != -EIO || !client_eio_retry_interval) && op->retval != -ENOSPC)
|
op->retval != -EPIPE && (op->retval != -EIO || !client_eio_retry_interval) && (op->retval != -ENOSPC || !client_retry_enospc))
|
||||||
{
|
{
|
||||||
// Fatal error (neither -EPIPE, -EIO nor -ENOSPC)
|
// Fatal error (neither -EPIPE, -EIO nor -ENOSPC)
|
||||||
erase_op(op);
|
erase_op(op);
|
||||||
|
|
|
@ -94,6 +94,7 @@ class cluster_client_t
|
||||||
int log_level = 0;
|
int log_level = 0;
|
||||||
int client_retry_interval = 50; // ms
|
int client_retry_interval = 50; // ms
|
||||||
int client_eio_retry_interval = 1000; // ms
|
int client_eio_retry_interval = 1000; // ms
|
||||||
|
bool client_retry_enospc = true;
|
||||||
|
|
||||||
int retry_timeout_id = 0;
|
int retry_timeout_id = 0;
|
||||||
int retry_timeout_duration = 0;
|
int retry_timeout_duration = 0;
|
||||||
|
|
|
@ -62,6 +62,11 @@ TEST_NAME=csum_4k_dmj OSD_ARGS="--data_csum_type crc32c --inmemory_metadata fal
|
||||||
TEST_NAME=csum_4k_dj OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
|
TEST_NAME=csum_4k_dj OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
|
||||||
TEST_NAME=csum_4k OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
|
TEST_NAME=csum_4k OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
|
||||||
|
|
||||||
|
./test_enospc.sh
|
||||||
|
SCHEME=xor ./test_enospc.sh
|
||||||
|
IMMEDIATE_COMMIT=1 ./test_enospc.sh
|
||||||
|
IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh
|
||||||
|
|
||||||
./test_scrub.sh
|
./test_scrub.sh
|
||||||
ZERO_OSD=2 ./test_scrub.sh
|
ZERO_OSD=2 ./test_scrub.sh
|
||||||
SCHEME=xor ./test_scrub.sh
|
SCHEME=xor ./test_scrub.sh
|
||||||
|
|
|
@ -0,0 +1,28 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
|
||||||
|
OSD_SIZE=200
|
||||||
|
GLOBAL_CONFIG=',"client_retry_enospc":false'
|
||||||
|
|
||||||
|
. `dirname $0`/run_3osds.sh
|
||||||
|
|
||||||
|
export LD_PRELOAD="build/src/libfio_vitastor.so"
|
||||||
|
|
||||||
|
# Should fail with ENOSPC
|
||||||
|
if fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
|
||||||
|
-rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
|
||||||
|
format_error "Should get ENOSPC, but didn't"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Should fail with ENOSPC too (the idea is to try to overwrite first objects to check their rollback)
|
||||||
|
if fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=32 \
|
||||||
|
-rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
|
||||||
|
format_error "Should get ENOSPC, but didn't"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Should complete OK
|
||||||
|
if ! fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 \
|
||||||
|
-rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=100M -cluster_log_level=10 -number_ios=4096; then
|
||||||
|
format_error "Should do random writes over ENOSPC correctly, but got an error"
|
||||||
|
fi
|
||||||
|
|
||||||
|
format_green OK
|
Loading…
Reference in New Issue