2022-06-03 01:56:36 +03:00
#!/bin/bash
2021-06-14 23:11:50 +03:00
. ` dirname $0 ` /common.sh
OSD_SIZE = ${ OSD_SIZE :- 1024 }
2021-07-19 01:42:35 +03:00
PG_COUNT = ${ PG_COUNT :- 1 }
2022-06-03 01:56:36 +03:00
# OSD_COUNT
SCHEME = ${ SCHEME :- replicated }
# OSD_ARGS
2023-07-17 23:43:56 +03:00
# OFFSET_ARGS
2022-06-03 01:56:36 +03:00
# PG_SIZE
# PG_MINSIZE
2024-01-17 00:16:56 +03:00
# GLOBAL_CONFIG
2022-06-03 01:56:36 +03:00
if [ " $SCHEME " = "ec" ] ; then
OSD_COUNT = ${ OSD_COUNT :- 5 }
else
OSD_COUNT = ${ OSD_COUNT :- 3 }
fi
if [ " $IMMEDIATE_COMMIT " != "" ] ; then
2023-10-27 01:24:33 +03:00
NO_SAME = "--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
2024-01-17 00:16:56 +03:00
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32' $GLOBAL_CONFIG '}'
2022-06-03 01:56:36 +03:00
else
2024-02-16 01:45:27 +03:00
NO_SAME = "--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5 --min_flusher_count 16"
2024-01-17 00:16:56 +03:00
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32' $GLOBAL_CONFIG '}'
2022-06-03 01:56:36 +03:00
fi
2023-07-17 23:43:56 +03:00
start_osd_on( )
2022-06-03 01:56:36 +03:00
{
local i = $1
2023-07-17 23:43:56 +03:00
local dev = $2
2023-10-31 13:11:46 +03:00
build/src/vitastor-osd --osd_num $i --bind_address $ETCD_IP $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
2023-07-17 23:43:56 +03:00
$( build/src/vitastor-disk simple-offsets --format options $OFFSET_ARGS $dev $OFFSET_ARGS 2>/dev/null) \
2022-11-29 01:37:45 +03:00
>>./testdata/osd$i .log 2>& 1 &
2022-06-03 01:56:36 +03:00
eval OSD${ i } _PID = $!
}
2021-06-14 23:11:50 +03:00
2023-07-17 23:43:56 +03:00
if ! type -t osd_dev; then
osd_dev( )
{
local i = $1
[ [ -f ./testdata/test_osd$i .bin ] ] || dd if = /dev/zero of = ./testdata/test_osd$i .bin bs = 1024 count = 1 seek = $(( OSD_SIZE*1024-1))
echo ./testdata/test_osd$i .bin
}
fi
start_osd( )
{
start_osd_on $1 $( osd_dev $1 )
}
2021-11-13 02:48:10 +03:00
for i in $( seq 1 $OSD_COUNT ) ; do
2022-06-03 01:56:36 +03:00
start_osd $i
2021-11-13 02:48:10 +03:00
done
2021-06-14 23:11:50 +03:00
2024-01-17 00:16:56 +03:00
( while true; do set +e; node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1; if [ [ $? -ne 2 ] ] ; then break; fi ; done ) >>./testdata/mon.log 2>& 1 &
2021-06-14 23:11:50 +03:00
MON_PID = $!
2022-06-03 01:56:36 +03:00
if [ " $SCHEME " = "ec" ] ; then
PG_SIZE = ${ PG_SIZE :- 5 }
2023-05-11 01:16:07 +03:00
PG_MINSIZE = ${ PG_MINSIZE :- 4 }
PG_DATA_SIZE = ${ PG_DATA_SIZE :- 3 }
POOLCFG = '"scheme":"ec","parity_chunks":' $(( PG_SIZE-PG_DATA_SIZE))
2022-06-03 01:56:36 +03:00
elif [ " $SCHEME " = "xor" ] ; then
PG_SIZE = ${ PG_SIZE :- 3 }
2023-05-11 01:16:07 +03:00
PG_MINSIZE = ${ PG_MINSIZE :- 3 }
PG_DATA_SIZE = $(( PG_SIZE-1))
POOLCFG = '"scheme":"xor","parity_chunks":1'
2021-11-13 02:48:10 +03:00
else
2022-06-03 01:56:36 +03:00
PG_SIZE = ${ PG_SIZE :- 2 }
PG_MINSIZE = ${ PG_MINSIZE :- 2 }
PG_DATA_SIZE = 1
POOLCFG = '"scheme":"replicated"'
2021-11-13 02:48:10 +03:00
fi
2022-06-03 01:56:36 +03:00
POOLCFG = '"name":"testpool","failure_domain":"osd",' $POOLCFG
$ETCDCTL put /vitastor/config/pools '{"1":{' $POOLCFG ',"pg_size":' $PG_SIZE ',"pg_minsize":' $PG_MINSIZE ',"pg_count":' $PG_COUNT '}}'
2021-06-14 23:11:50 +03:00
2022-12-24 11:48:40 +03:00
wait_up( )
{
local sec = $1
local i = 0
local configured = 0
while [ [ $i -lt $sec ] ] ; do
if $ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e ' ( . | length) != 0 and ( [ .[ 0] .items[ "1" ] [ ] |
select ( ( ( .osd_set | select ( . != 0) | sort | unique) | length) = = '$PG_SIZE' ) ] | length) = = ' $PG_COUNT ; then
configured = 1
if $ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == ' $PG_COUNT ; then
break
fi
fi
sleep 1
i = $(( i+1))
if [ $i -eq $sec ] ; then
if [ [ $configured -ne 0 ] ] ; then
format_error " FAILED: $PG_COUNT PG(s) NOT CONFIGURED "
fi
format_error " FAILED: $PG_COUNT PG(s) NOT UP "
fi
done
}
2021-06-14 23:11:50 +03:00
2023-07-09 01:13:30 +03:00
if [ [ $OSD_COUNT -gt 0 ] ] ; then
2023-07-17 23:43:56 +03:00
wait_up 120
2023-07-09 01:13:30 +03:00
fi
2021-06-14 23:11:50 +03:00
2022-06-03 01:56:36 +03:00
try_reweight( )
{
osd = $1
w = $2
$ETCDCTL put /vitastor/config/osd/$osd '{"reweight":' $w '}'
sleep 3
}
2023-04-19 00:33:28 +03:00
wait_condition( )
2022-06-03 01:56:36 +03:00
{
sec = $1
2023-04-19 00:33:28 +03:00
check = $2
proc = $3
2022-06-03 01:56:36 +03:00
i = 0
while [ [ $i -lt $sec ] ] ; do
2023-04-19 00:33:28 +03:00
eval " $check " && break
2023-05-08 19:37:00 +03:00
if [ $i -eq $sec ] ; then
2023-04-19 00:33:28 +03:00
format_error " $proc couldn't finish in $sec seconds "
2023-05-08 19:37:00 +03:00
fi
2023-04-19 00:33:28 +03:00
sleep 1
i = $(( i+1))
2022-06-03 01:56:36 +03:00
done
}
2023-04-19 00:33:28 +03:00
wait_finish_rebalance( )
{
sec = $1
check = $2
check = ${ check :- '.state == ["active"] or .state == ["active", "left_on_dead"]' }
check = " $ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select( $check ) ] | length) == $PG_COUNT ' "
wait_condition " $sec " " $check " Rebalance
}
2022-06-03 01:56:36 +03:00
check_qemu( )
{
if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then
sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
sudo ln -s " $( realpath .) /build/src/block-vitastor.so " /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
fi
}
2023-05-08 20:37:34 +03:00
check_nbd( )
{
if [ [ -d /sys/module/nbd && ! -e /dev/nbd0 ] ] ; then
max_part = $( cat /sys/module/nbd/parameters/max_part)
nbds_max = $( cat /sys/module/nbd/parameters/nbds_max)
for i in $( seq 1 $nbds_max ) ; do
mknod /dev/nbd$(( i-1)) b 43 $(( ( i-1) * ( max_part+1)) )
done
fi
}