forked from vitalif/vitastor
Add a test with OSD kills
parent
2bdf415eb3
commit
82b9f4c52d
|
@ -36,6 +36,7 @@ struct sec_data
|
|||
/* The list of completed io_u structs. */
|
||||
std::vector<io_u*> completed;
|
||||
uint64_t inflight = 0;
|
||||
int mirror_fd = -1;
|
||||
bool trace = false;
|
||||
};
|
||||
|
||||
|
@ -46,6 +47,7 @@ struct sec_options
|
|||
char *etcd_host = NULL;
|
||||
char *etcd_prefix = NULL;
|
||||
char *image = NULL;
|
||||
char *mirror_file = NULL;
|
||||
uint64_t pool = 0;
|
||||
uint64_t inode = 0;
|
||||
int cluster_log = 0;
|
||||
|
@ -132,6 +134,15 @@ static struct fio_option options[] = {
|
|||
.category = FIO_OPT_C_ENGINE,
|
||||
.group = FIO_OPT_G_FILENAME,
|
||||
},
|
||||
{
|
||||
.name = "mirror_file",
|
||||
.lname = "File name to mirror writes to",
|
||||
.type = FIO_OPT_STR_STORE,
|
||||
.off1 = offsetof(struct sec_options, mirror_file),
|
||||
.help = "File name to mirror writes to (for debug purpose)",
|
||||
.category = FIO_OPT_C_ENGINE,
|
||||
.group = FIO_OPT_G_FILENAME,
|
||||
},
|
||||
{
|
||||
.name = "use_rdma",
|
||||
.lname = "Use RDMA",
|
||||
|
@ -212,6 +223,16 @@ static int sec_setup(struct thread_data *td)
|
|||
td->o.open_files++;
|
||||
}
|
||||
|
||||
if (o->mirror_file)
|
||||
{
|
||||
bsd->mirror_fd = open(o->mirror_file, O_CREAT|O_RDWR, 0666);
|
||||
if (bsd->mirror_fd < 0)
|
||||
{
|
||||
td_verror(td, errno, "open mirror file");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (!o->image)
|
||||
{
|
||||
if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)))
|
||||
|
@ -265,6 +286,10 @@ static void sec_cleanup(struct thread_data *td)
|
|||
sec_data *bsd = (sec_data*)td->io_ops_data;
|
||||
if (bsd)
|
||||
{
|
||||
if (bsd->mirror_fd >= 0)
|
||||
{
|
||||
close(bsd->mirror_fd);
|
||||
}
|
||||
if (bsd->watch)
|
||||
{
|
||||
vitastor_c_close_watch(bsd->cli, bsd->watch);
|
||||
|
@ -325,6 +350,24 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
|
|||
bsd->last_sync = false;
|
||||
break;
|
||||
case DDIR_WRITE:
|
||||
if (opt->mirror_file)
|
||||
{
|
||||
size_t done = 0;
|
||||
while (done < io->xfer_buflen)
|
||||
{
|
||||
ssize_t r = pwrite(bsd->mirror_fd, io->xfer_buf+done, io->xfer_buflen-done, io->offset+done);
|
||||
if (r < 0 && errno != EAGAIN)
|
||||
{
|
||||
fprintf(stderr, "Error writing mirror file: %s\n", strerror(errno));
|
||||
io->error = errno;
|
||||
return FIO_Q_COMPLETED;
|
||||
}
|
||||
if (r > 0)
|
||||
{
|
||||
done += r;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (opt->image && vitastor_c_inode_get_readonly(bsd->watch))
|
||||
{
|
||||
io->error = EROFS;
|
||||
|
|
|
@ -4,19 +4,25 @@
|
|||
|
||||
if [ "$IMMEDIATE_COMMIT" != "" ]; then
|
||||
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1,"immediate_commit":"all"}'
|
||||
else
|
||||
NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
|
||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1}'
|
||||
fi
|
||||
|
||||
start_osd()
|
||||
{
|
||||
local i=$1
|
||||
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
|
||||
eval OSD${i}_PID=$!
|
||||
}
|
||||
|
||||
OSD_SIZE=1024
|
||||
OSD_COUNT=7
|
||||
OSD_ARGS=
|
||||
for i in $(seq 1 $OSD_COUNT); do
|
||||
dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
|
||||
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
|
||||
eval OSD${i}_PID=$!
|
||||
start_osd $i
|
||||
done
|
||||
|
||||
cd mon
|
||||
|
@ -26,11 +32,11 @@ node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1
|
|||
MON_PID=$!
|
||||
|
||||
if [ "$EC" != "" ]; then
|
||||
POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
|
||||
PG_SIZE=3
|
||||
POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
|
||||
else
|
||||
POOLCFG='"scheme":"replicated","pg_size":2,"pg_minsize":2'
|
||||
PG_SIZE=2
|
||||
PG_SIZE=${PG_SIZE:-2}
|
||||
POOLCFG='"scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":2'
|
||||
fi
|
||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}'
|
||||
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash -ex
|
||||
|
||||
# Kill OSDs while writing
|
||||
|
||||
PG_SIZE=3
|
||||
|
||||
. `dirname $0`/run_7osds.sh
|
||||
|
||||
IMG_SIZE=960
|
||||
|
||||
$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
|
||||
|
||||
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
||||
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
|
||||
-mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -cluster_log_level=10
|
||||
|
||||
kill_osds()
|
||||
{
|
||||
sleep 5
|
||||
|
||||
kill -9 $OSD1_PID
|
||||
$ETCDCTL del /vitastor/osd/state/1
|
||||
|
||||
for i in 2 3 4 5 6 7; do
|
||||
sleep 15
|
||||
echo Killing OSD $i and starting OSD $((i-1))
|
||||
p=OSD${i}_PID
|
||||
kill -9 ${!p}
|
||||
$ETCDCTL del /vitastor/osd/state/$i
|
||||
start_osd $((i-1))
|
||||
sleep 15
|
||||
done
|
||||
|
||||
sleep 5
|
||||
start_osd 7
|
||||
|
||||
sleep 5
|
||||
}
|
||||
|
||||
kill_osds &
|
||||
|
||||
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
||||
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 -fsync=256 -rw=randwrite \
|
||||
-mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120 2>/dev/null
|
||||
|
||||
qemu-img convert -S 4096 -p \
|
||||
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
|
||||
-O raw ./testdata/read.bin
|
||||
|
||||
diff ./testdata/read.bin ./testdata/mirror.bin
|
||||
|
||||
format_green OK
|
Loading…
Reference in New Issue