diff --git a/tests/common.sh b/tests/common.sh index cace1f1a..7cec8147 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -53,15 +53,18 @@ start_etcd() eval ETCD${i}_PID=$! } -for i in $(seq 1 $ETCD_COUNT); do - start_etcd $i -done -for i in {1..30}; do - ${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=1s --command-timeout=1s member list >/dev/null && break - if [[ $i = 30 ]]; then - format_error "Failed to start etcd" - fi -done +nodejs mon/tinyraft/antietcd.js &>./testdata/antietcd.log & +ANTIETCD_PID=$! +sleep 1 +#for i in $(seq 1 $ETCD_COUNT); do +# start_etcd $i +#done +#for i in {1..30}; do +# ${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=1s --command-timeout=1s member list >/dev/null && break +# if [[ $i = 30 ]]; then +# format_error "Failed to start etcd" +# fi +#done echo leak:fio >> testdata/lsan-suppress.txt echo leak:tcmalloc >> testdata/lsan-suppress.txt diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh index 05b4acf7..b40bd233 100644 --- a/tests/run_3osds.sh +++ b/tests/run_3osds.sh @@ -18,12 +18,33 @@ else OSD_COUNT=${OSD_COUNT:-3} fi +etcd_put() +{ + local key=$(echo -n "$1" | base64 -w 0) + local value=$(echo -n "$2" | base64 -w 0) + curl -s -XPOST --json '{"key":"'"$key"'","value":"'"$value"'"}' http://localhost:12379/v3/kv/put +} + +etcd_get() +{ + local key=$(echo -n "$1" | base64 -w 0) + curl -s -XPOST --json '{"key":"'"$key"'"}' http://localhost:12379/v3/kv/range | jq -r '.kvs[].value | @base64d' +} + +etcd_get_prefix() +{ + local key=$(echo -n "$1" | base64 -w 0) + local end="$1" + end=$(echo -n "${end%%/}"0 | base64 -w 0) + curl -s -XPOST --json '{"key":"'"$key"'","range_end":"'"$end"'"}' http://localhost:12379/v3/kv/range | jq -r '.kvs[].value | @base64d' +} + if [ "$IMMEDIATE_COMMIT" != "" ]; then NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}' + etcd_put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}' else NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5 --min_flusher_count 16" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}' + etcd_put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}' fi start_osd_on() @@ -74,7 +95,7 @@ else POOLCFG='"scheme":"replicated"' fi POOLCFG='"name":"testpool","failure_domain":"osd",'$POOLCFG -$ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}' +etcd_put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}' wait_up() { @@ -82,10 +103,10 @@ wait_up() local i=0 local configured=0 while [[ $i -lt $sec ]]; do - if $ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] | + if etcd_get /vitastor/config/pgs | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] | select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT; then configured=1 - if $ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then + if etcd_get_prefix /vitastor/pg/state/1/ | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then break fi fi @@ -108,7 +129,7 @@ try_reweight() { osd=$1 w=$2 - $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}' + etcd_put /vitastor/config/osd/$osd '{"reweight":'$w'}' sleep 3 } @@ -133,7 +154,7 @@ wait_finish_rebalance() sec=$1 check=$2 check=${check:-'.state == ["active"] or .state == ["active", "left_on_dead"]'} - check="$ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select($check) ] | length) == $PG_COUNT'" + check="etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select($check) ] | length) == $PG_COUNT'" wait_condition "$sec" "$check" Rebalance } diff --git a/tests/test_add_osd.sh b/tests/test_add_osd.sh index 42fa996c..9a47d50b 100755 --- a/tests/test_add_osd.sh +++ b/tests/test_add_osd.sh @@ -13,14 +13,14 @@ start_osd 4 sleep 2 for i in {1..30}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + (etcd_get /vitastor/config/pgs |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])') && \ - ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT) && \ + (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT) && \ break sleep 1 done -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ +if ! (etcd_get /vitastor/config/pgs |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])'); then format_error "FAILED: OSD NOT ADDED INTO DISTRIBUTION" fi @@ -35,14 +35,14 @@ build/src/vitastor-cli --etcd_address $ETCD_URL rm-osd --force 4 sleep 2 for i in {1..30}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + (etcd_get /vitastor/config/pgs |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \ - ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \ + (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \ break sleep 1 done -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ +if ! (etcd_get /vitastor/config/pgs |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then format_error "FAILED: OSD NOT REMOVED FROM DISTRIBUTION" fi diff --git a/tests/test_change_pg_count.sh b/tests/test_change_pg_count.sh index 2a1ce213..6e0204d0 100755 --- a/tests/test_change_pg_count.sh +++ b/tests/test_change_pg_count.sh @@ -20,31 +20,31 @@ try_change() done echo --- Change PG count to $n --- >>testdata/mon.log - $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}' + etcd_put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}' for i in {1..60}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \ - ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "has_misplaced"]) ] | length) == '$n'') && \ + (etcd_get /vitastor/config/pgs | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \ + (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "has_misplaced"]) ] | length) == '$n'') && \ break sleep 1 done # Wait for the rebalance to finish for i in {1..60}; do - ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \ + (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \ break sleep 1 done - if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then - $ETCDCTL get /vitastor/config/pgs - $ETCDCTL get --prefix /vitastor/pg/state/ + if ! (etcd_get /vitastor/config/pgs | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then + etcd_get /vitastor/config/pgs + etcd_get_prefix /vitastor/pg/state/ format_error "FAILED: $n PGS NOT CONFIGURED" fi - if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then - $ETCDCTL get /vitastor/config/pgs - $ETCDCTL get --prefix /vitastor/pg/state/ + if ! (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then + etcd_get /vitastor/config/pgs + etcd_get_prefix /vitastor/pg/state/ format_error "FAILED: $n PGS NOT UP" fi @@ -53,7 +53,7 @@ try_change() nobj=0 waittime=0 while [[ $nobj -ne $NOBJ && $waittime -lt 7 ]]; do - nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'` + nobj=`etcd_get_prefix '/vitastor/pg/stats/' | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'` if [[ $nobj -ne $NOBJ ]]; then waittime=$((waittime+1)) sleep 1