#!/bin/bash -ex

# Run 3 OSDs

. `dirname $0`/run_3osds.sh

# Write inodes 1 and 2

LD_PRELOAD="build/src/libfio_vitastor.so" \
    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
        -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10

LD_PRELOAD="build/src/libfio_vitastor.so" \
    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
        -rw=write -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -runtime=10

LD_PRELOAD="build/src/libfio_vitastor.so" \
    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 \
        -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10 &>/dev/null &

sleep 5

# Stop OSD 1

kill -INT $OSD1_PID
sleep 2

# Remove inode 2

build/src/vitastor-cli rm-data --etcd_address $ETCD_URL --pool 1 --inode 2

# Run 3 more OSDs and move PG to 4,5,6

for i in $(seq 4 6); do
    dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
    eval OSD${i}_PID=$!
done

$ETCDCTL put /vitastor/config/osd/1 '{"reweight":0}'
$ETCDCTL put /vitastor/config/osd/2 '{"reweight":0}'
$ETCDCTL put /vitastor/config/osd/3 '{"reweight":0}'

# Wait for rebalance to finish

wait_finish_rebalance()
{
    local sec=$1
    local st=$2
    local i=0
    while [[ $i -lt $sec ]]; do
        if $ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e \
            '([ .[] | select(.state == ['$st'] and (.peers | contains([1]) | not) and (.peers | contains([2,3]) | not)) ] | length) == '$PG_COUNT; then
            break
        fi
        sleep 1
        i=$((i+1))
        if [ $i -eq $sec ]; then
            format_error "Rebalance couldn't finish in $sec seconds"
        fi
    done
}

wait_finish_rebalance 60 '"active","left_on_dead"'

# Stop OSD 2,3

kill -INT $OSD2_PID
kill -INT $OSD3_PID
sleep 2

# Verify that PGs are still active

if ! ($ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active","left_on_dead"]) ] | length == '$PG_COUNT); then
    format_error "FAILED: $PG_COUNT PG(s) NOT UP"
fi

# Start OSD 1

build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
OSD1_PID=$!

# Verify that inode 2 is removed and inode 1 is in place

wait_repeer_1()
{
    local sec=$1
    local i=0
    while [[ $i -lt $sec ]]; do
        if grep -q 'Repeer because of OSD 1' testdata/osd4.log testdata/osd5.log testdata/osd6.log; then
            break
        fi
        sleep 1
        i=$((i+1))
        if [ $i -eq $sec ]; then
            format_error "OSD 4/5/6 do not peer with older OSD 1"
        fi
    done
}

wait_repeer_1 15

wait_finish_rebalance 15 '"active"'

if [ "$SCHEME" = "replicated" ]; then
    NOBJ=1024
else
    NOBJ=$((1024/(PG_SIZE-1)))
fi

if ! ($ETCDCTL get /vitastor/pg/stats/1/1 --print-value-only | jq -s -e '.[0].object_count == '$NOBJ); then
    format_error "FAILED: PG SHOULD CONTAIN EXACTLY 128 MB OF DATA, BUT IT DOESN'T"
fi

qemu-img convert -S 4096 -p \
    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=1:size=4096" \
    -O raw ./testdata/inode1.bin

qemu-img convert -S 4096 -p \
    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=2:size="$((128*1024*1024)) \
    -O raw ./testdata/inode2.bin

if (dd if=/dev/zero bs=4096 count=1 | diff - ./testdata/inode1.bin); then
    format_error "FAILED: INODE 1 SEEMS LOST"
fi

if ! (dd if=/dev/zero bs=1M count=128 | diff - ./testdata/inode2.bin); then
    format_error "FAILED: INODE 2 SEEMS RESTORED"
fi

format_green OK