forked from vitalif/vitastor
Add a 2/5 etcd failure test
parent
2c930d55fb
commit
8a6b07d8f7
|
@ -23,20 +23,40 @@ trap 'kill -9 $(jobs -p)' EXIT
|
||||||
ETCD=${ETCD:-etcd}
|
ETCD=${ETCD:-etcd}
|
||||||
ETCD_IP=${ETCD_IP:-127.0.0.1}
|
ETCD_IP=${ETCD_IP:-127.0.0.1}
|
||||||
ETCD_PORT=${ETCD_PORT:-12379}
|
ETCD_PORT=${ETCD_PORT:-12379}
|
||||||
|
ETCD_COUNT=${ETCD_COUNT:-1}
|
||||||
|
|
||||||
if [ "$KEEP_DATA" = "" ]; then
|
if [ "$KEEP_DATA" = "" ]; then
|
||||||
rm -rf ./testdata
|
rm -rf ./testdata
|
||||||
mkdir -p ./testdata
|
mkdir -p ./testdata
|
||||||
fi
|
fi
|
||||||
|
|
||||||
$ETCD -name etcd_test --data-dir ./testdata/etcd \
|
ETCD_URL="http://$ETCD_IP:$ETCD_PORT"
|
||||||
--advertise-client-urls http://$ETCD_IP:$ETCD_PORT --listen-client-urls http://$ETCD_IP:$ETCD_PORT \
|
ETCD_CLUSTER="etcd1=http://$ETCD_IP:$((ETCD_PORT+1))"
|
||||||
--initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+1)) \
|
for i in $(seq 2 $ETCD_COUNT); do
|
||||||
--max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd.log &
|
ETCD_URL="$ETCD_URL,http://$ETCD_IP:$((ETCD_PORT+2*i-2))"
|
||||||
ETCD_PID=$!
|
ETCD_CLUSTER="$ETCD_CLUSTER,etcd$i=http://$ETCD_IP:$((ETCD_PORT+2*i-1))"
|
||||||
ETCD_URL=$ETCD_IP:$ETCD_PORT/v3
|
done
|
||||||
ETCDCTL="${ETCD}ctl --endpoints=http://$ETCD_URL"
|
ETCDCTL="${ETCD}ctl --endpoints=http://$ETCD_URL"
|
||||||
|
|
||||||
|
start_etcd()
|
||||||
|
{
|
||||||
|
local i=$1
|
||||||
|
$ETCD -name etcd$i --data-dir ./testdata/etcd$i \
|
||||||
|
--advertise-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) \
|
||||||
|
--initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) \
|
||||||
|
--initial-cluster-token vitastor-tests-etcd --initial-cluster-state new \
|
||||||
|
--initial-cluster "$ETCD_CLUSTER" \
|
||||||
|
--max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd$i.log &
|
||||||
|
eval ETCD${i}_PID=$!
|
||||||
|
}
|
||||||
|
|
||||||
|
for i in $(seq 1 $ETCD_COUNT); do
|
||||||
|
start_etcd $i
|
||||||
|
done
|
||||||
|
if [ $ETCD_COUNT -gt 1 ]; then
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
|
||||||
echo leak:fio >> testdata/lsan-suppress.txt
|
echo leak:fio >> testdata/lsan-suppress.txt
|
||||||
echo leak:tcmalloc >> testdata/lsan-suppress.txt
|
echo leak:tcmalloc >> testdata/lsan-suppress.txt
|
||||||
echo leak:ceph >> testdata/lsan-suppress.txt
|
echo leak:ceph >> testdata/lsan-suppress.txt
|
||||||
|
|
|
@ -18,7 +18,7 @@ done
|
||||||
cd mon
|
cd mon
|
||||||
npm install
|
npm install
|
||||||
cd ..
|
cd ..
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
||||||
MON_PID=$!
|
MON_PID=$!
|
||||||
|
|
||||||
if [ -n "$GLOBAL_CONF" ]; then
|
if [ -n "$GLOBAL_CONF" ]; then
|
||||||
|
|
|
@ -22,7 +22,7 @@ done
|
||||||
cd mon
|
cd mon
|
||||||
npm install
|
npm install
|
||||||
cd ..
|
cd ..
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
|
node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
|
||||||
MON_PID=$!
|
MON_PID=$!
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":16,"failure_domain":"osd"}}'
|
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":16,"failure_domain":"osd"}}'
|
||||||
|
|
|
@ -14,7 +14,7 @@ done
|
||||||
cd mon
|
cd mon
|
||||||
npm install
|
npm install
|
||||||
cd ..
|
cd ..
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
||||||
MON_PID=$!
|
MON_PID=$!
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":3,"pg_minsize":2,"pg_count":16,"failure_domain":"osd"}}'
|
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":3,"pg_minsize":2,"pg_count":16,"failure_domain":"osd"}}'
|
||||||
|
|
|
@ -0,0 +1,33 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
|
||||||
|
# Run 5 etcds and kill them while operating
|
||||||
|
ETCD_COUNT=5
|
||||||
|
|
||||||
|
. `dirname $0`/run_3osds.sh
|
||||||
|
|
||||||
|
LD_PRELOAD=libasan.so.5 \
|
||||||
|
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
|
||||||
|
-etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
|
||||||
|
|
||||||
|
kill_etcds()
|
||||||
|
{
|
||||||
|
sleep 5
|
||||||
|
kill $ETCD1_PID $ETCD2_PID
|
||||||
|
sleep 5
|
||||||
|
kill $ETCD3_PID
|
||||||
|
start_etcd 1
|
||||||
|
sleep 5
|
||||||
|
kill $ETCD4_PID
|
||||||
|
start_etcd 2
|
||||||
|
sleep 5
|
||||||
|
kill $ETCD5_PID
|
||||||
|
start_etcd 3
|
||||||
|
}
|
||||||
|
|
||||||
|
kill_etcds &
|
||||||
|
|
||||||
|
LD_PRELOAD=libasan.so.5 \
|
||||||
|
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
|
||||||
|
-etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30
|
||||||
|
|
||||||
|
format_green OK
|
|
@ -18,7 +18,7 @@ $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicate
|
||||||
cd mon
|
cd mon
|
||||||
npm install
|
npm install
|
||||||
cd ..
|
cd ..
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
||||||
MON_PID=$!
|
MON_PID=$!
|
||||||
|
|
||||||
sleep 2
|
sleep 2
|
||||||
|
|
|
@ -22,7 +22,7 @@ done
|
||||||
cd mon
|
cd mon
|
||||||
npm install
|
npm install
|
||||||
cd ..
|
cd ..
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
|
node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
|
||||||
MON_PID=$!
|
MON_PID=$!
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":32,"failure_domain":"osd"}}'
|
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":32,"failure_domain":"osd"}}'
|
||||||
|
|
|
@ -14,7 +14,7 @@ for i in $(seq 1 $OSD_COUNT); do
|
||||||
eval OSD${i}_PID=$!
|
eval OSD${i}_PID=$!
|
||||||
done
|
done
|
||||||
|
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
||||||
MON_PID=$!
|
MON_PID=$!
|
||||||
|
|
||||||
sleep 3
|
sleep 3
|
||||||
|
|
Loading…
Reference in New Issue