forked from vitalif/vitastor
Add a test for simple master switching without PG reconfiguration
Also use osd_out_time:1 only in select tests and restart mon in tests only on connection errors
parent
3ca3b8a8d8
commit
2aa5aa7ab6
|
@ -532,6 +532,24 @@ jobs:
|
|||
echo ""
|
||||
done
|
||||
|
||||
test_switch_primary:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||
steps:
|
||||
- name: Run test
|
||||
id: test
|
||||
timeout-minutes: 3
|
||||
run: /root/vitastor/tests/test_switch_primary.sh
|
||||
- name: Print logs
|
||||
if: always() && steps.test.outcome == 'failure'
|
||||
run: |
|
||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||
echo "-------- $i --------"
|
||||
cat $i
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_write:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
|
|
13
mon/mon.js
13
mon/mon.js
|
@ -390,7 +390,8 @@ class Mon
|
|||
{
|
||||
constructor(config)
|
||||
{
|
||||
this.die = (e) => this._die(e);
|
||||
this.failconnect = (e) => this._die(e, 2);
|
||||
this.die = (e) => this._die(e, 1);
|
||||
if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf'))
|
||||
{
|
||||
config = {
|
||||
|
@ -604,7 +605,7 @@ class Mon
|
|||
}
|
||||
if (!this.ws)
|
||||
{
|
||||
this.die('Failed to open etcd watch websocket');
|
||||
this.failconnect('Failed to open etcd watch websocket');
|
||||
}
|
||||
const cur_addr = this.selected_etcd_url;
|
||||
this.ws_alive = true;
|
||||
|
@ -791,7 +792,7 @@ class Mon
|
|||
const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
|
||||
if (!res.result.TTL)
|
||||
{
|
||||
this.die('Lease expired');
|
||||
this.failconnect('Lease expired');
|
||||
}
|
||||
}, this.config.etcd_mon_timeout);
|
||||
if (!this.signals_set)
|
||||
|
@ -1997,14 +1998,14 @@ class Mon
|
|||
return res.json;
|
||||
}
|
||||
}
|
||||
this.die();
|
||||
this.failconnect();
|
||||
}
|
||||
|
||||
_die(err)
|
||||
_die(err, code)
|
||||
{
|
||||
// In fact we can just try to rejoin
|
||||
console.error(new Error(err || 'Cluster connection failed'));
|
||||
process.exit(1);
|
||||
process.exit(code || 2);
|
||||
}
|
||||
|
||||
local_ips(all)
|
||||
|
|
|
@ -10,6 +10,7 @@ SCHEME=${SCHEME:-replicated}
|
|||
# OFFSET_ARGS
|
||||
# PG_SIZE
|
||||
# PG_MINSIZE
|
||||
# GLOBAL_CONFIG
|
||||
|
||||
if [ "$SCHEME" = "ec" ]; then
|
||||
OSD_COUNT=${OSD_COUNT:-5}
|
||||
|
@ -19,10 +20,10 @@ fi
|
|||
|
||||
if [ "$IMMEDIATE_COMMIT" != "" ]; then
|
||||
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"immediate_commit":"all","client_enable_writeback":true}'
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
|
||||
else
|
||||
NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5"
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"client_enable_writeback":true}'
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
|
||||
fi
|
||||
|
||||
start_osd_on()
|
||||
|
@ -53,7 +54,7 @@ for i in $(seq 1 $OSD_COUNT); do
|
|||
start_osd $i
|
||||
done
|
||||
|
||||
(while true; do node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 || true; done) >>./testdata/mon.log 2>&1 &
|
||||
(while true; do set +e; node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1; if [[ $? -ne 2 ]]; then break; fi; done) >>./testdata/mon.log 2>&1 &
|
||||
MON_PID=$!
|
||||
|
||||
if [ "$SCHEME" = "ec" ]; then
|
||||
|
|
|
@ -45,6 +45,8 @@ IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
|
|||
SCHEME=ec ./test_rebalance_verify.sh
|
||||
SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
|
||||
|
||||
./test_switch_primary.sh
|
||||
|
||||
./test_write.sh
|
||||
SCHEME=xor ./test_write.sh
|
||||
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash -ex
|
||||
|
||||
PG_COUNT=2048
|
||||
|
||||
GLOBAL_CONFIG=',"osd_out_time":1'
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
||||
|
|
|
@ -9,6 +9,7 @@ if [[ "$SCHEME" = "ec" ]]; then
|
|||
fi
|
||||
OSD_COUNT=${OSD_COUNT:-7}
|
||||
PG_COUNT=32
|
||||
GLOBAL_CONFIG=',"osd_out_time":1'
|
||||
. `dirname $0`/run_3osds.sh
|
||||
check_qemu
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
PG_MINSIZE=1
|
||||
SCHEME=replicated
|
||||
GLOBAL_CONFIG=',"osd_out_time":1'
|
||||
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
|
|
|
@ -4,6 +4,7 @@ OSD_COUNT=2
|
|||
PG_SIZE=2
|
||||
PG_MINSIZE=1
|
||||
SCHEME=replicated
|
||||
GLOBAL_CONFIG=',"osd_out_time":1'
|
||||
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
#!/bin/bash -ex
|
||||
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
|
||||
primary_pid=OSD${primary}_PID
|
||||
kill -9 ${!primary_pid}
|
||||
|
||||
sleep 15
|
||||
wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/config/pgs | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'"
|
||||
|
||||
newprim=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
|
||||
|
||||
if [ "$newprim" = "$primary" ]; then
|
||||
format_error Primary not switched
|
||||
fi
|
||||
|
||||
format_green OK
|
Loading…
Reference in New Issue