109 lines
2.5 KiB
Bash
Executable File
109 lines
2.5 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
PROCFILE="tools/local-tester/Procfile"
|
|
HTTPFAIL=(127.0.0.1:11180 127.0.0.1:22280 127.0.0.1:33380)
|
|
|
|
function wait_time {
|
|
expr $RANDOM % 10 + 1
|
|
}
|
|
|
|
function cycle {
|
|
for a; do
|
|
echo "cycling $a"
|
|
goreman -f $PROCFILE run stop $a || echo "could not stop $a"
|
|
sleep `wait_time`s
|
|
goreman -f $PROCFILE run restart $a || echo "could not restart $a"
|
|
done
|
|
}
|
|
|
|
function cycle_members {
|
|
cycle etcd1 etcd2 etcd3
|
|
}
|
|
function cycle_pbridge {
|
|
cycle pbridge1 pbridge2 pbridge3
|
|
}
|
|
function cycle_cbridge {
|
|
cycle cbridge1 cbridge2 cbridge3
|
|
}
|
|
function cycle_stresser {
|
|
cycle stress-put
|
|
}
|
|
|
|
function kill_maj {
|
|
idx="etcd"`expr $RANDOM % 3 + 1`
|
|
idx2="$idx"
|
|
while [ "$idx" == "$idx2" ]; do
|
|
idx2="etcd"`expr $RANDOM % 3 + 1`
|
|
done
|
|
echo "kill majority $idx $idx2"
|
|
goreman -f $PROCFILE run stop $idx || echo "could not stop $idx"
|
|
goreman -f $PROCFILE run stop $idx2 || echo "could not stop $idx2"
|
|
sleep `wait_time`s
|
|
goreman -f $PROCFILE run restart $idx || echo "could not restart $idx"
|
|
goreman -f $PROCFILE run restart $idx2 || echo "could not restart $idx2"
|
|
}
|
|
|
|
function kill_all {
|
|
for a in etcd1 etcd2 etcd3; do
|
|
goreman -f $PROCFILE run stop $a || echo "could not stop $a"
|
|
done
|
|
sleep `wait_time`s
|
|
for a in etcd1 etcd2 etcd3; do
|
|
goreman -f $PROCFILE run restart $a || echo "could not restart $a"
|
|
done
|
|
}
|
|
|
|
function rand_fp {
|
|
echo "$FAILPOINTS" | sed `expr $RANDOM % $NUMFPS + 1`"q;d"
|
|
}
|
|
|
|
# fp_activate <http> <fppath> <value>
|
|
function fp_activate {
|
|
curl "$1"/"$2" -XPUT -d "$3" >/dev/null 2>&1
|
|
}
|
|
|
|
function fp_rand_single {
|
|
fp=`rand_fp`
|
|
fp_activate ${HTTPFAIL[`expr $RANDOM % ${#HTTPFAIL[@]}`]} $fp 'panic("'$fp'")'
|
|
sleep `wait_time`s
|
|
}
|
|
|
|
function fp_rand_all {
|
|
fp=`rand_fp`
|
|
for a in `seq ${#HTTPFAIL[@]}`; do fp_activate ${HTTPFAIL[$a]} "$fp" 'panic("'$fp'")'; done
|
|
sleep `wait_time`s
|
|
}
|
|
|
|
function fp_all_rand_fire {
|
|
for fp in $FAILPOINTS; do
|
|
for url in "${HTTPFAIL[@]}"; do
|
|
fp_activate "$url" "$fp" '0.5%panic("0.5%'$fp'")'
|
|
done
|
|
done
|
|
}
|
|
|
|
function choose {
|
|
fault=${FAULTS[`expr $RANDOM % ${#FAULTS[@]}`]}
|
|
echo $fault
|
|
$fault || echo "failed: $fault"
|
|
}
|
|
|
|
sleep 2s
|
|
|
|
FAULTS=(cycle_members kill_maj kill_all cycle_pbridge cycle_cbridge cycle_stresser)
|
|
|
|
# add failpoint faults if available
|
|
FAILPOINTS=`curl http://"${HTTPFAIL[0]}" 2>/dev/null | cut -f1 -d'=' | grep -v "^$"`
|
|
NUMFPS=`echo $(echo "$FAILPOINTS" | wc -l)`
|
|
if [ "$NUMFPS" != "0" ]; then
|
|
FAULTS+=(fp_rand_single)
|
|
FAULTS+=(fp_rand_all)
|
|
fi
|
|
|
|
while [ 1 ]; do
|
|
choose
|
|
# start any nodes that have been killed by failpoints
|
|
for a in etcd1 etcd2 etcd3; do goreman -f $PROCFILE run start $a; done
|
|
fp_all_rand_fire
|
|
done
|