Release 0.6.17

- Fix incorrect reading of extra metadata block leading to extra unknown objects in stats
- Fix CSI driver volumeMode: Block support
- Add block PVC and pod examples
- Fix build under 32 bit architectures
- Fix slow connection ramp-up caused by up_wait_retry_interval

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.6.16")
+set(VERSION "0.6.17")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.6.16
+VERSION ?= v0.6.17
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.6.16
+          image: vitalif/vitastor-csi:v0.6.17
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.6.16
+          image: vitalif/vitastor-csi:v0.6.17
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.6.16"
+    vitastorCSIDriverVersion = "0.6.17"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.6.16-1) unstable; urgency=medium
+vitastor (0.6.17-1) unstable; urgency=medium
 
   * RDMA support
   * Bugfixes

debian/vitastor.Dockerfile

@@ -33,8 +33,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.6.16; \
+    cp -r /root/vitastor vitastor-0.6.17; \
-    cd vitastor-0.6.16; \
+    cd vitastor-0.6.17; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -47,8 +47,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.16.orig.tar.xz vitastor-0.6.16; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.17.orig.tar.xz vitastor-0.6.17; \
-    cd vitastor-0.6.16; \
+    cd vitastor-0.6.17; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

mon/mon.js

@@ -224,15 +224,19 @@ const etcd_tree = {
             }, */
         },
         inodestats: {
-            /* <inode_t>: {
+            /* <pool_id>: {
-                read: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                <inode_t>: {
-                write: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                    read: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
-                delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                    write: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                    delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                },
             }, */
         },
         space: {
             /* <osd_num_t>: {
-                <inode_t>: uint64_t, // bytes
+                <pool_id>: {
+                    <inode_t>: uint64_t, // bytes
+                },
             }, */
         },
     },

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.6.16'
+VERSION = '0.6.17'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.6.16/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.16$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.6.17/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.17$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -34,7 +34,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.16.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.17.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.16
+Version:        0.6.17
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.16.el7.tar.gz
+Source0:        vitastor-0.6.17.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -33,7 +33,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.16.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.17.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.16
+Version:        0.6.17
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.16.el8.tar.gz
+Source0:        vitastor-0.6.17.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.6.16")
+add_definitions(-DVERSION="0.6.17")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/blockstore_init.cpp

@@ -131,6 +131,7 @@ resume_1:
     }
     // Skip superblock
     bs->meta_offset += bs->meta_block_size;
+    bs->meta_len -= bs->meta_block_size;
     prev_done = 0;
     done_len = 0;
     done_pos = 0;

src/cluster_client.cpp

@@ -9,6 +9,7 @@
 #define PART_SENT 1
 #define PART_DONE 2
 #define PART_ERROR 4
+#define PART_RETRY 8
 #define CACHE_DIRTY 1
 #define CACHE_FLUSHING 2
 #define CACHE_REPEATING 3
@@ -670,14 +671,17 @@ resume_2:
             if (!try_send(op, i))
             {
                 // We'll need to retry again
-                op->up_wait = true;
+                if (op->parts[i].flags & PART_RETRY)
-                if (!retry_timeout_id)
                 {
-                    retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
+                    op->up_wait = true;
+                    if (!retry_timeout_id)
                     {
-                        retry_timeout_id = 0;
+                        retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
-                        continue_ops(true);
+                        {
-                    });
+                            retry_timeout_id = 0;
+                            continue_ops(true);
+                        });
+                    }
                 }
                 op->state = 2;
             }
@@ -746,7 +750,7 @@ resume_3:
         {
             for (int i = 0; i < op->parts.size(); i++)
             {
-                op->parts[i].flags = 0;
+                op->parts[i].flags = PART_RETRY;
             }
             goto resume_2;
         }

src/osd_cluster.cpp

@@ -189,7 +189,7 @@ void osd_t::report_statistics()
     for (auto kv: bs->get_inode_space_stats())
     {
         pool_id_t pool_id = INODE_POOL(kv.first);
-        uint64_t only_inode_num = (kv.first & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1));
+        uint64_t only_inode_num = INODE_NO_POOL(kv.first);
         if (!last_pool || pool_id != last_pool)
         {
             if (last_pool)

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.6.16
+Version: 0.6.17
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/test_add_osd.sh

@@ -4,7 +4,7 @@ PG_COUNT=16
 
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -end_fsync=1 \
         -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 

tests/test_change_pg_count.sh

@@ -37,7 +37,7 @@ if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '(
     format_error "FAILED: 16 PGS NOT UP"
 fi
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -cluster_log_level=10
 

tests/test_etcd_fail.sh

@@ -5,7 +5,7 @@ ETCD_COUNT=5
 
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
         -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 
@@ -26,7 +26,7 @@ kill_etcds()
 
 kill_etcds &
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
         -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30
 

tests/test_interrupted_rebalance.sh

@@ -4,7 +4,7 @@
 
 IMG_SIZE=960
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=16 -fsync=16 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=${IMG_SIZE}M -cluster_log_level=10
 

tests/test_move_reappear.sh

@@ -21,7 +21,7 @@ if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | le
     format_error "Failed to start the PG active+degraded"
 fi
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
 

tests/test_rm.sh

@@ -3,7 +3,7 @@
 PG_COUNT=16
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 \
         -end_fsync=1 -fsync=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 

tests/test_snapshot.sh

@@ -6,7 +6,7 @@
 
 $ETCDCTL put /vitastor/config/inode/1/2 '{"name":"testimg","size":'$((32*1024*1024))'}'
 
-LD_PRELOAD="libasan.so.5 build/src/libfio_vitastor.so" \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
         -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
 
@@ -14,11 +14,11 @@ $ETCDCTL put /vitastor/config/inode/1/2 '{"name":"testimg@0","size":'$((32*1024*
 $ETCDCTL put /vitastor/config/inode/1/3 '{"parent_id":2,"name":"testimg","size":'$((32*1024*1024))'}'
 
 # Preload build/src/libfio_vitastor.so so libasan detects all symbols
-LD_PRELOAD="libasan.so.5 build/src/libfio_vitastor.so" \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=32 -buffer_pattern=0xdeadface \
         -rw=randwrite -etcd=$ETCD_URL -image=testimg -number_ios=1024
 
-LD_PRELOAD="libasan.so.5 build/src/libfio_vitastor.so" \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -rw=read -etcd=$ETCD_URL -pool=1 -inode=3 -size=32M
 
 qemu-img convert -S 4096 -p \

tests/test_splitbrain.sh

@@ -2,6 +2,7 @@
 
 OSD_COUNT=2
 PG_SIZE=2
+PG_MINSIZE=1
 SCHEME=replicated
 
 . `dirname $0`/run_3osds.sh
@@ -13,7 +14,7 @@ sleep 2
 
 # Write
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10 -number_ios=100
 

tests/test_write.sh

@@ -7,20 +7,20 @@
 
 # Random writes without immediate_commit were stalling OSDs
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=124k -direct=1 -numjobs=16 -iodepth=4 \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=68k -direct=1 -numjobs=16 -iodepth=4 \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10
 
 # A lot of parallel syncs was crashing the primary OSD at some point
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -numjobs=64 -iodepth=1 -fsync=1 \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -number_ios=100
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=32 -buffer_pattern=0xdeadface \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -number_ios=1024
 

tests/test_write_no_same.sh

@@ -11,7 +11,7 @@ GLOBAL_CONF='{"immediate_commit":"all"}'
 
 # Test basic write
 
-LD_PRELOAD=libasan.so.5 \
+LD_PRELOAD="build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=1G -cluster_log_level=10
 
 format_green OK