diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 421089ed..9bb0816a 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -532,6 +532,24 @@ jobs: echo "" done + test_switch_primary: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_switch_primary.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_write: runs-on: ubuntu-latest needs: build diff --git a/mon/mon.js b/mon/mon.js index 0e852c30..4eb7a763 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -390,7 +390,8 @@ class Mon { constructor(config) { - this.die = (e) => this._die(e); + this.failconnect = (e) => this._die(e, 2); + this.die = (e) => this._die(e, 1); if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf')) { config = { @@ -604,7 +605,7 @@ class Mon } if (!this.ws) { - this.die('Failed to open etcd watch websocket'); + this.failconnect('Failed to open etcd watch websocket'); } const cur_addr = this.selected_etcd_url; this.ws_alive = true; @@ -791,7 +792,7 @@ class Mon const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries); if (!res.result.TTL) { - this.die('Lease expired'); + this.failconnect('Lease expired'); } }, this.config.etcd_mon_timeout); if (!this.signals_set) @@ -1997,14 +1998,14 @@ class Mon return res.json; } } - this.die(); + this.failconnect(); } - _die(err) + _die(err, code) { // In fact we can just try to rejoin console.error(new Error(err || 'Cluster connection failed')); - process.exit(1); + process.exit(code || 2); } local_ips(all) diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh index 5d54676e..2f6073eb 100644 --- a/tests/run_3osds.sh +++ b/tests/run_3osds.sh @@ -10,6 +10,7 @@ SCHEME=${SCHEME:-replicated} # OFFSET_ARGS # PG_SIZE # PG_MINSIZE +# GLOBAL_CONFIG if [ "$SCHEME" = "ec" ]; then OSD_COUNT=${OSD_COUNT:-5} @@ -19,10 +20,10 @@ fi if [ "$IMMEDIATE_COMMIT" != "" ]; then NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"immediate_commit":"all","client_enable_writeback":true}' + $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}' else NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"client_enable_writeback":true}' + $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}' fi start_osd_on() @@ -53,7 +54,7 @@ for i in $(seq 1 $OSD_COUNT); do start_osd $i done -(while true; do node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 || true; done) >>./testdata/mon.log 2>&1 & +(while true; do set +e; node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1; if [[ $? -ne 2 ]]; then break; fi; done) >>./testdata/mon.log 2>&1 & MON_PID=$! if [ "$SCHEME" = "ec" ]; then diff --git a/tests/run_tests.sh b/tests/run_tests.sh index c93058bb..fcf079ec 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -45,6 +45,8 @@ IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh SCHEME=ec ./test_rebalance_verify.sh SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh +./test_switch_primary.sh + ./test_write.sh SCHEME=xor ./test_write.sh diff --git a/tests/test_add_osd.sh b/tests/test_add_osd.sh index de75ed68..42fa996c 100755 --- a/tests/test_add_osd.sh +++ b/tests/test_add_osd.sh @@ -1,7 +1,7 @@ #!/bin/bash -ex PG_COUNT=2048 - +GLOBAL_CONFIG=',"osd_out_time":1' . `dirname $0`/run_3osds.sh LD_PRELOAD="build/src/libfio_vitastor.so" \ diff --git a/tests/test_heal.sh b/tests/test_heal.sh index cf4efd1b..3604b295 100755 --- a/tests/test_heal.sh +++ b/tests/test_heal.sh @@ -9,6 +9,7 @@ if [[ "$SCHEME" = "ec" ]]; then fi OSD_COUNT=${OSD_COUNT:-7} PG_COUNT=32 +GLOBAL_CONFIG=',"osd_out_time":1' . `dirname $0`/run_3osds.sh check_qemu diff --git a/tests/test_minsize_1.sh b/tests/test_minsize_1.sh index 2569bd64..51ef2ab9 100755 --- a/tests/test_minsize_1.sh +++ b/tests/test_minsize_1.sh @@ -2,6 +2,7 @@ PG_MINSIZE=1 SCHEME=replicated +GLOBAL_CONFIG=',"osd_out_time":1' . `dirname $0`/run_3osds.sh diff --git a/tests/test_splitbrain.sh b/tests/test_splitbrain.sh index 9c637d2b..19cf0f49 100755 --- a/tests/test_splitbrain.sh +++ b/tests/test_splitbrain.sh @@ -4,6 +4,7 @@ OSD_COUNT=2 PG_SIZE=2 PG_MINSIZE=1 SCHEME=replicated +GLOBAL_CONFIG=',"osd_out_time":1' . `dirname $0`/run_3osds.sh diff --git a/tests/test_switch_primary.sh b/tests/test_switch_primary.sh new file mode 100755 index 00000000..78dcde78 --- /dev/null +++ b/tests/test_switch_primary.sh @@ -0,0 +1,18 @@ +#!/bin/bash -ex + +. `dirname $0`/run_3osds.sh + +primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary') +primary_pid=OSD${primary}_PID +kill -9 ${!primary_pid} + +sleep 15 +wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/config/pgs | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'" + +newprim=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary') + +if [ "$newprim" = "$primary" ]; then + format_error Primary not switched +fi + +format_green OK