Run tests with antietcd

tests/common.sh

@@ -53,15 +53,18 @@ start_etcd()
     eval ETCD${i}_PID=$!
 }
 
-for i in $(seq 1 $ETCD_COUNT); do
-    start_etcd $i
-done
-for i in {1..30}; do
-    ${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=1s --command-timeout=1s member list >/dev/null && break
-    if [[ $i = 30 ]]; then
-        format_error "Failed to start etcd"
-    fi
-done
+nodejs mon/tinyraft/antietcd.js &>./testdata/antietcd.log &
+ANTIETCD_PID=$!
+sleep 1
+#for i in $(seq 1 $ETCD_COUNT); do
+#    start_etcd $i
+#done
+#for i in {1..30}; do
+#    ${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=1s --command-timeout=1s member list >/dev/null && break
+#    if [[ $i = 30 ]]; then
+#        format_error "Failed to start etcd"
+#    fi
+#done
 
 echo leak:fio >> testdata/lsan-suppress.txt
 echo leak:tcmalloc >> testdata/lsan-suppress.txt

tests/run_3osds.sh

@@ -18,12 +18,33 @@ else
     OSD_COUNT=${OSD_COUNT:-3}
 fi
 
+etcd_put()
+{
+    local key=$(echo -n "$1" | base64 -w 0)
+    local value=$(echo -n "$2" | base64 -w 0)
+    curl -s -XPOST --json '{"key":"'"$key"'","value":"'"$value"'"}' http://localhost:12379/v3/kv/put
+}
+
+etcd_get()
+{
+    local key=$(echo -n "$1" | base64 -w 0)
+    curl -s -XPOST --json '{"key":"'"$key"'"}' http://localhost:12379/v3/kv/range | jq -r '.kvs[].value | @base64d'
+}
+
+etcd_get_prefix()
+{
+    local key=$(echo -n "$1" | base64 -w 0)
+    local end="$1"
+    end=$(echo -n "${end%%/}"0 | base64 -w 0)
+    curl -s -XPOST --json '{"key":"'"$key"'","range_end":"'"$end"'"}' http://localhost:12379/v3/kv/range | jq -r '.kvs[].value | @base64d'
+}
+
 if [ "$IMMEDIATE_COMMIT" != "" ]; then
     NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
+    etcd_put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 else
     NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5 --min_flusher_count 16"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
+    etcd_put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 fi
 
 start_osd_on()
@@ -74,7 +95,7 @@ else
     POOLCFG='"scheme":"replicated"'
 fi
 POOLCFG='"name":"testpool","failure_domain":"osd",'$POOLCFG
-$ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}'
+etcd_put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}'
 
 wait_up()
 {
@@ -82,10 +103,10 @@ wait_up()
     local i=0
     local configured=0
     while [[ $i -lt $sec ]]; do
-        if $ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] |
+        if etcd_get /vitastor/config/pgs | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] |
             select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT; then
             configured=1
-            if $ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then
+            if etcd_get_prefix /vitastor/pg/state/1/ | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then
                 break
             fi
         fi
@@ -108,7 +129,7 @@ try_reweight()
 {
     osd=$1
     w=$2
-    $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}'
+    etcd_put /vitastor/config/osd/$osd '{"reweight":'$w'}'
     sleep 3
 }
 
@@ -133,7 +154,7 @@ wait_finish_rebalance()
     sec=$1
     check=$2
     check=${check:-'.state == ["active"] or .state == ["active", "left_on_dead"]'}
-    check="$ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select($check) ] | length) == $PG_COUNT'"
+    check="etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select($check) ] | length) == $PG_COUNT'"
     wait_condition "$sec" "$check" Rebalance
 }
 

tests/test_add_osd.sh

@@ -13,14 +13,14 @@ start_osd 4
 sleep 2
 
 for i in {1..30}; do
-    ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    (etcd_get /vitastor/config/pgs |\
         jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])') && \
-        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT) && \
+        (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT) && \
         break
     sleep 1
 done
 
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+if ! (etcd_get /vitastor/config/pgs |\
     jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])'); then
     format_error "FAILED: OSD NOT ADDED INTO DISTRIBUTION"
 fi
@@ -35,14 +35,14 @@ build/src/vitastor-cli --etcd_address $ETCD_URL rm-osd --force 4
 sleep 2
 
 for i in {1..30}; do
-    ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    (etcd_get /vitastor/config/pgs |\
         jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \
-        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \
+        (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \
         break
     sleep 1
 done
 
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+if ! (etcd_get /vitastor/config/pgs |\
     jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then
     format_error "FAILED: OSD NOT REMOVED FROM DISTRIBUTION"
 fi

tests/test_change_pg_count.sh

@@ -20,31 +20,31 @@ try_change()
     done
     echo --- Change PG count to $n --- >>testdata/mon.log
 
-    $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}'
+    etcd_put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}'
 
     for i in {1..60}; do
-        ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \
-            ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "has_misplaced"]) ] | length) == '$n'') && \
+        (etcd_get /vitastor/config/pgs | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \
+            (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "has_misplaced"]) ] | length) == '$n'') && \
             break
         sleep 1
     done
 
     # Wait for the rebalance to finish
     for i in {1..60}; do
-        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \
+        (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \
             break
         sleep 1
     done
 
-    if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then
-        $ETCDCTL get /vitastor/config/pgs
-        $ETCDCTL get --prefix /vitastor/pg/state/
+    if ! (etcd_get /vitastor/config/pgs | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then
+        etcd_get /vitastor/config/pgs
+        etcd_get_prefix /vitastor/pg/state/
         format_error "FAILED: $n PGS NOT CONFIGURED"
     fi
 
-    if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then
-        $ETCDCTL get /vitastor/config/pgs
-        $ETCDCTL get --prefix /vitastor/pg/state/
+    if ! (etcd_get_prefix /vitastor/pg/state/ | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then
+        etcd_get /vitastor/config/pgs
+        etcd_get_prefix /vitastor/pg/state/
         format_error "FAILED: $n PGS NOT UP"
     fi
 
@@ -53,7 +53,7 @@ try_change()
     nobj=0
     waittime=0
     while [[ $nobj -ne $NOBJ && $waittime -lt 7 ]]; do
-        nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
+        nobj=`etcd_get_prefix '/vitastor/pg/stats/' | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
         if [[ $nobj -ne $NOBJ ]]; then
             waittime=$((waittime+1))
             sleep 1