Release 0.6.2

- Fix a possible crash during SYNC when journal fsyncs are enabled
- Fix a memory leak in the chained read implementation

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.6.0-1) unstable; urgency=medium
+vitastor (0.6.2-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor.Dockerfile

@@ -40,10 +40,10 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.6.0; \
-    ln -s /root/packages/qemu-$REL/qemu-*/ vitastor-0.6.0/qemu; \
-    ln -s /root/fio-build/fio-*/ vitastor-0.6.0/fio; \
-    cd vitastor-0.6.0; \
+    cp -r /root/vitastor vitastor-0.6.2; \
+    ln -s /root/packages/qemu-$REL/qemu-*/ vitastor-0.6.2/qemu; \
+    ln -s /root/fio-build/fio-*/ vitastor-0.6.2/fio; \
+    cd vitastor-0.6.2; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     QEMU=$(head -n1 qemu/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     sh copy-qemu-includes.sh; \
@@ -59,8 +59,8 @@ RUN set -e -x; \
     echo "dep:fio=$FIO" > debian/substvars; \
     echo "dep:qemu=$QEMU" >> debian/substvars; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.0.orig.tar.xz vitastor-0.6.0; \
-    cd vitastor-0.6.0; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.2.orig.tar.xz vitastor-0.6.2; \
+    cd vitastor-0.6.2; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

rpm/build-tarball.sh

@@ -48,4 +48,4 @@ FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Ve
 QEMU=`rpm -qi qemu qemu-kvm | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
 perl -i -pe 's/(Requires:\s*qemu(?:-kvm)?)([^\n]+)?/$1 = '$QEMU'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.6.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.0$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.6.2/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.2$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -37,7 +37,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.0.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.2.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.0
+Version:        0.6.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.0.el7.tar.gz
+Source0:        vitastor-0.6.2.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.0.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.2.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.0
+Version:        0.6.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.0.el8.tar.gz
+Source0:        vitastor-0.6.2.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -13,7 +13,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.6.0")
+add_definitions(-DVERSION="0.6.2")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -63,14 +63,22 @@ target_link_libraries(fio_vitastor_blk
 	vitastor_blk
 )
 
+# libvitastor_common.a
+add_library(vitastor_common STATIC
+	epoll_manager.cpp etcd_state_client.cpp
+	messenger.cpp msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp ../json11/json11.cpp
+	http_client.cpp osd_ops.cpp pg_states.cpp timerfd_manager.cpp base64.cpp
+)
+target_compile_options(vitastor_common PUBLIC -fPIC)
+
 # vitastor-osd
 add_executable(vitastor-osd
-	osd_main.cpp osd.cpp osd_secondary.cpp msgr_receive.cpp msgr_send.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
+	osd_main.cpp osd.cpp osd_secondary.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
 	osd_primary.cpp osd_primary_chain.cpp osd_primary_sync.cpp osd_primary_write.cpp osd_primary_subops.cpp
-	etcd_state_client.cpp messenger.cpp msgr_stop.cpp msgr_op.cpp osd_cluster.cpp http_client.cpp osd_ops.cpp pg_states.cpp
-	osd_rmw.cpp base64.cpp timerfd_manager.cpp epoll_manager.cpp ../json11/json11.cpp
+	osd_cluster.cpp osd_rmw.cpp
 )
 target_link_libraries(vitastor-osd
+	vitastor_common
 	vitastor_blk
 	Jerasure
 )
@@ -86,11 +94,10 @@ target_link_libraries(fio_vitastor_sec
 
 # libvitastor_client.so
 add_library(vitastor_client SHARED
-	cluster_client.cpp epoll_manager.cpp etcd_state_client.cpp
-	messenger.cpp msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp ../json11/json11.cpp
-	http_client.cpp osd_ops.cpp pg_states.cpp timerfd_manager.cpp base64.cpp
+	cluster_client.cpp
 )
 target_link_libraries(vitastor_client
+	vitastor_common
 	tcmalloc_minimal
 	${LIBURING_LIBRARIES}
 )
@@ -162,10 +169,10 @@ target_link_libraries(osd_rmw_test Jerasure tcmalloc_minimal)
 
 # stub_uring_osd
 add_executable(stub_uring_osd
-	stub_uring_osd.cpp epoll_manager.cpp messenger.cpp msgr_stop.cpp msgr_op.cpp
-	msgr_send.cpp msgr_receive.cpp ringloop.cpp timerfd_manager.cpp ../json11/json11.cpp
+	stub_uring_osd.cpp
 )
 target_link_libraries(stub_uring_osd
+	vitastor_common
 	${LIBURING_LIBRARIES}
 	tcmalloc_minimal
 )

src/blockstore_sync.cpp

@@ -146,6 +146,7 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op, bool queue_has_in_prog
             my_uring_prep_fsync(sqe, journal.fd, IORING_FSYNC_DATASYNC);
             data->iov = { 0 };
             data->callback = [this, op](ring_data_t *data) { handle_sync_event(data, op); };
+            PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
             PRIV(op)->pending_ops = 1;
             PRIV(op)->op_state = SYNC_JOURNAL_SYNC_SENT;
             return 1;

src/msgr_receive.cpp

@@ -332,6 +332,7 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
         cl->read_op = op;
         cl->read_state = CL_READ_REPLY_DATA;
         cl->read_remaining = op->reply.hdr.retval;
+        free(op->buf);
         op->buf = memalign_or_die(MEM_ALIGNMENT, cl->read_remaining);
         cl->recv_list.push_back(op->buf, cl->read_remaining);
     }

src/osd_primary_chain.cpp

@@ -349,9 +349,12 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
     int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
     op_data->chain_read_count = chain_reads.size();
     op_data->chain_reads = (osd_chain_read_t*)calloc_or_die(
-        1, (sizeof(osd_chain_read_t) + sizeof(osd_rmw_stripe_t)*stripe_count) * chain_reads.size()
+        1, sizeof(osd_chain_read_t) * chain_reads.size()
+        + sizeof(osd_rmw_stripe_t) * stripe_count * op_data->chain_size
+    );
+    osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
+        ((void*)op_data->chain_reads) + sizeof(osd_chain_read_t) * op_data->chain_read_count
     );
-    osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(((void*)op_data->chain_reads) + sizeof(osd_chain_read_t) * op_data->chain_read_count);
     // Now process each subrequest as a separate read, including reconstruction if needed
     // Prepare reads
     int n_subops = 0;
@@ -361,7 +364,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         op_data->chain_reads[cri] = chain_reads[cri];
         object_id cur_oid = { .inode = chain_reads[cri].inode, .stripe = op_data->oid.stripe };
         // FIXME: maybe introduce split_read_stripes to shorten these lines and to remove read_start=req_start
-        osd_rmw_stripe_t *stripes = chain_stripes + cri*stripe_count;
+        osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
         split_stripes(pg.pg_data_size, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
         if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
         {
@@ -406,7 +409,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
     void *cur_buf = cur_op->buf;
     for (int cri = 0; cri < chain_reads.size(); cri++)
     {
-        osd_rmw_stripe_t *stripes = chain_stripes + cri*stripe_count;
+        osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
         for (int role = 0; role < stripe_count; role++)
         {
             if (stripes[role].read_end > 0)
@@ -429,7 +432,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
     int cur_subops = 0;
     for (int cri = 0; cri < chain_reads.size(); cri++)
     {
-        osd_rmw_stripe_t *stripes = chain_stripes + cri*stripe_count;
+        osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
         if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
         {
             continue;
@@ -460,7 +463,9 @@ void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
-    osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(((void*)op_data->chain_reads) + sizeof(osd_chain_read_t) * op_data->chain_read_count);
+    osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
+        ((void*)op_data->chain_reads) + sizeof(osd_chain_read_t) * op_data->chain_read_count
+    );
     // Reconstruct parts if needed
     if (op_data->degraded)
     {
@@ -468,7 +473,7 @@ void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
         for (int cri = 0; cri < op_data->chain_read_count; cri++)
         {
             // Reconstruct missing stripes
-            osd_rmw_stripe_t *stripes = chain_stripes + cri*stripe_count;
+            osd_rmw_stripe_t *stripes = chain_stripes + op_data->chain_reads[cri].chain_pos*stripe_count;
             if (op_data->scheme == POOL_SCHEME_XOR)
             {
                 reconstruct_stripes_xor(stripes, pg.pg_size, clean_entry_bitmap_size);

src/qemu_driver.c

@@ -206,7 +206,6 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
         }
         else
         {
-            assert(qemu_get_current_aio_context() == qemu_get_aio_context());
             qemu_coroutine_enter(qemu_coroutine_create((void(*)(void*))vitastor_co_get_metadata, &task));
         }
         BDRV_POLL_WHILE(bs, !task.complete);

tests/common.sh

@@ -23,8 +23,10 @@ trap 'kill -9 $(jobs -p)' EXIT
 ETCD=${ETCD:-etcd}
 ETCD_PORT=${ETCD_PORT:-12379}
 
-rm -rf ./testdata
-mkdir -p ./testdata
+if [ "$KEEP_DATA" = "" ]; then
+    rm -rf ./testdata
+    mkdir -p ./testdata
+fi
 
 $ETCD -name etcd_test --data-dir ./testdata/etcd \
     --advertise-client-urls http://127.0.0.1:$ETCD_PORT --listen-client-urls http://127.0.0.1:$ETCD_PORT \

tests/test_vm_cont.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -ex
+
+export KEEP_DATA=1
+. `dirname $0`/common.sh
+
+etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/mon/master
+etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/pg/state
+etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/osd/state
+
+build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
+OSD1_PID=$!
+build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
+OSD2_PID=$!
+build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
+OSD3_PID=$!
+
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
+MON_PID=$!
+
+sleep 3
+
+if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
+    format_error "FAILED: 1 PG NOT UP"
+fi
+
+if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then
+    sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
+    sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
+fi
+
+qemu-system-x86_64 -enable-kvm -m 1024 \
+    -drive 'file=vitastor:etcd_host=127.0.0.1\:'$ETCD_PORT'/v3:image=debian9',format=raw,if=none,id=drive-virtio-disk0,cache=none \
+    -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=off,physical_block_size=4096,logical_block_size=512 \
+    -vnc 0.0.0.0:0
+
+format_green OK

tests/test_vm_start.sh

@@ -0,0 +1,53 @@
+#!/bin/bash -ex
+
+. `dirname $0`/common.sh
+
+dd if=/dev/zero of=./testdata/test_osd1.bin bs=2048 count=1 seek=$((1024*1024-1))
+dd if=/dev/zero of=./testdata/test_osd2.bin bs=2048 count=1 seek=$((1024*1024-1))
+dd if=/dev/zero of=./testdata/test_osd3.bin bs=2048 count=1 seek=$((1024*1024-1))
+
+build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
+OSD1_PID=$!
+build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
+OSD2_PID=$!
+build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
+OSD3_PID=$!
+
+cd mon
+npm install
+cd ..
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
+MON_PID=$!
+
+$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}'
+
+sleep 2
+
+if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then
+    format_error "FAILED: 1 PG NOT CONFIGURED"
+fi
+
+if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
+    format_error "FAILED: 1 PG NOT UP"
+fi
+
+if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then
+    sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
+    sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
+fi
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"debian9","size":'$((2048*1024*1024))'}'
+
+qemu-img convert -S 4096 -p \
+    -f raw ~/debian9-kvm.raw \
+    -O raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=debian9"
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"debian9@0","size":'$((2048*1024*1024))'}'
+$ETCDCTL put /vitastor/config/inode/1/2 '{"parent_id":1,"name":"debian9","size":'$((2048*1024*1024))'}'
+
+qemu-system-x86_64 -enable-kvm -m 1024 \
+    -drive 'file=vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=debian9',format=raw,if=none,id=drive-virtio-disk0,cache=none \
+    -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=off,physical_block_size=4096,logical_block_size=512 \
+    -vnc 0.0.0.0:0
+
+format_green OK

tests/test_write.sh

@@ -39,6 +39,12 @@ if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitast
     sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
 fi
 
+# A lot of parallel syncs was crashing the primary OSD at some point
+
+LD_PRELOAD=libasan.so.5 \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -numjobs=64 -iodepth=1 -fsync=1 \
+        -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -number_ios=100
+
 LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10