Release 0.6.11

- Slightly reduce journaling write amplification (requires no_same_sector_overwrites=false)
- Fix listen_backlog (it was 0) because it could more than halve OSD socket send speed
- Support IPv6 OSD addresses
- Do not try to initialize client in simple-offsets
- Fix OSDs sometimes marking PGs incomplete instead of trying to connect with peers
- Allow to configure OSD placement in node_placement
- Allow to run with 4k sector size block devices. Natural, but it was forbidden

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.6.10")
+set(VERSION "0.6.11")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.6.10
+VERSION ?= v0.6.11
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.6.10
+          image: vitalif/vitastor-csi:v0.6.11
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.6.10
+          image: vitalif/vitastor-csi:v0.6.11
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.6.10"
+    vitastorCSIDriverVersion = "0.6.11"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.6.10-1) unstable; urgency=medium
+vitastor (0.6.11-1) unstable; urgency=medium
 
   * RDMA support
   * Bugfixes

debian/vitastor.Dockerfile

@@ -33,8 +33,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.6.10; \
-    cd vitastor-0.6.10; \
+    cp -r /root/vitastor vitastor-0.6.11; \
+    cd vitastor-0.6.11; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -47,8 +47,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.10.orig.tar.xz vitastor-0.6.10; \
-    cd vitastor-0.6.10; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.11.orig.tar.xz vitastor-0.6.11; \
+    cd vitastor-0.6.11; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.6.10'
+VERSION = '0.6.11'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.6.10/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.10$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.6.11/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.11$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -34,7 +34,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.10.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.11.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.10
+Version:        0.6.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.10.el7.tar.gz
+Source0:        vitastor-0.6.11.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -33,7 +33,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.10.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.11.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.10
+Version:        0.6.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.10.el8.tar.gz
+Source0:        vitastor-0.6.11.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.6.10")
+add_definitions(-DVERSION="0.6.11")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -111,12 +111,11 @@ if (${WITH_FIO})
 	# libfio_vitastor_sec.so
 	add_library(fio_vitastor_sec SHARED
 		fio_sec_osd.cpp
+		rw_blocking.cpp
+		addr_util.cpp
 	)
 	target_link_libraries(fio_vitastor_sec
-		vitastor_common
 		tcmalloc_minimal
-		${LIBURING_LIBRARIES}
-		${IBVERBS_LIBRARIES}
 	)
 endif (${WITH_FIO})
 

src/fio_sec_osd.cpp

@@ -29,23 +29,16 @@
 #include <unordered_map>
 
 #include "addr_util.h"
-#include "epoll_manager.h"
-#include "ringloop.h"
-#include "messenger.h"
+#include "rw_blocking.h"
 #include "osd_ops.h"
 #include "fio_headers.h"
 
 struct sec_data
 {
-    epoll_manager_t *epmgr;
-    ring_loop_t *ringloop;
-    osd_messenger_t *msgr;
-    ring_consumer_t looper;
-    void *bitmap_buf;
     int connect_fd;
-    uint64_t op_id;
     /* block_size = 1 << block_order (128KB by default) */
     uint64_t block_order = 17, block_size = 1 << 17;
+    std::unordered_map<uint64_t, io_u*> queue;
     bool last_sync = false;
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
@@ -118,6 +111,9 @@ static struct fio_option options[] = {
 static int sec_setup(struct thread_data *td)
 {
     sec_data *bsd;
+    //fio_file *f;
+    //int r;
+    //int64_t size;
 
     bsd = new sec_data;
     if (!bsd)
@@ -134,6 +130,8 @@ static int sec_setup(struct thread_data *td)
         td->o.open_files++;
     }
 
+    //f = td->files[0];
+    //f->real_file_size = size;
     return 0;
 }
 
@@ -142,10 +140,6 @@ static void sec_cleanup(struct thread_data *td)
     sec_data *bsd = (sec_data*)td->io_ops_data;
     if (bsd)
     {
-        delete bsd->msgr;
-        delete bsd->epmgr;
-        delete bsd->ringloop;
-        free(bsd->bitmap_buf);
         close(bsd->connect_fd);
         delete bsd;
     }
@@ -180,45 +174,6 @@ static int sec_init(struct thread_data *td)
     int one = 1;
     setsockopt(bsd->connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
 
-    fcntl(bsd->connect_fd, F_SETFL, fcntl(bsd->connect_fd, F_GETFL, 0) | O_NONBLOCK);
-
-    json11::Json cfg = json11::Json::object{ { "use_rdma", 0 } };
-
-    bsd->bitmap_buf = malloc(4096);
-
-    bsd->ringloop = new ring_loop_t(512);
-    bsd->epmgr = new epoll_manager_t(bsd->ringloop);
-    bsd->msgr = new osd_messenger_t();
-    bsd->msgr->tfd = bsd->epmgr->tfd;
-    bsd->msgr->ringloop = bsd->ringloop;
-    bsd->msgr->repeer_pgs = [](osd_num_t){};
-    bsd->msgr->parse_config(cfg);
-    bsd->msgr->init();
-
-    bsd->looper.loop = [bsd]()
-    {
-        bsd->msgr->read_requests();
-        bsd->msgr->send_replies();
-        bsd->ringloop->submit();
-    };
-    bsd->ringloop->register_consumer(&bsd->looper);
-
-    int peer_fd = bsd->connect_fd;
-    bsd->msgr->clients[peer_fd] = new osd_client_t();
-    bsd->msgr->clients[peer_fd]->peer_addr = addr;
-    bsd->msgr->clients[peer_fd]->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
-    bsd->msgr->clients[peer_fd]->peer_fd = peer_fd;
-    bsd->msgr->clients[peer_fd]->peer_state = PEER_CONNECTED;
-    bsd->msgr->clients[peer_fd]->connect_timeout_id = -1;
-    bsd->msgr->clients[peer_fd]->osd_num = 1;
-    bsd->msgr->clients[peer_fd]->in_buf = malloc_or_die(bsd->msgr->receive_buffer_size);
-    bsd->epmgr->tfd->set_fd_handler(peer_fd, true, [msgr = bsd->msgr](int peer_fd, int epoll_events)
-    {
-        // Either OUT (connected) or HUP
-        msgr->handle_peer_epoll(peer_fd, epoll_events);
-    });
-    bsd->msgr->osd_peer_fds[1] = peer_fd;
-
     // FIXME: read config (block size) from OSD
 
     return 0;
@@ -238,12 +193,7 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
     }
 
     io->engine_data = bsd;
-
-    osd_op_t *oo = new osd_op_t();
-    oo->op_type = OSD_OP_OUT;
-    oo->peer_fd = bsd->connect_fd;
-
-    osd_any_op_t & op = oo->req;
+    osd_any_op_t op = { 0 };
 
     op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
     op.hdr.id = n;
@@ -260,9 +210,6 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.sec_rw.version = UINT64_MAX; // last unstable
             op.sec_rw.offset = io->offset % bsd->block_size;
             op.sec_rw.len = io->xfer_buflen;
-            op.sec_rw.attr_len = 4;
-            oo->bitmap = bsd->bitmap_buf;
-            oo->bitmap_len = 4;
         }
         else
         {
@@ -271,7 +218,6 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.rw.offset = io->offset;
             op.rw.len = io->xfer_buflen;
         }
-        oo->iov.push_back(io->xfer_buf, io->xfer_buflen);
         bsd->last_sync = false;
         break;
     case DDIR_WRITE:
@@ -293,7 +239,6 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.rw.offset = io->offset;
             op.rw.len = io->xfer_buflen;
         }
-        oo->iov.push_back(io->xfer_buf, io->xfer_buflen);
         bsd->last_sync = false;
         break;
     case DDIR_SYNC:
@@ -315,21 +260,6 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
         return FIO_Q_COMPLETED;
     }
 
-    oo->callback = [td, io](osd_op_t *oo)
-    {
-        sec_options *opt = (sec_options*)td->eo;
-        sec_data *bsd = (sec_data*)td->io_ops_data;
-        if (opt->trace)
-        {
-            printf("--- %s # %ld %ld\n", io->ddir == DDIR_READ ? "READ" :
-                (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), oo->reply.hdr.id, oo->reply.hdr.retval);
-        }
-        io->error = oo->reply.hdr.retval < 0 ? -oo->reply.hdr.retval : 0;
-        bsd->completed.push_back(io);
-        delete oo;
-    };
-    bsd->msgr->outbox_push(oo);
-
     if (opt->trace)
     {
         printf("+++ %s # %d\n", io->ddir == DDIR_READ ? "READ" :
@@ -339,6 +269,21 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
     io->error = 0;
     bsd->inflight++;
     bsd->op_n++;
+    bsd->queue[n] = io;
+
+    iovec iov[2] = { { .iov_base = op.buf, .iov_len = OSD_PACKET_SIZE } };
+    int iovcnt = 1, wtotal = OSD_PACKET_SIZE;
+    if (io->ddir == DDIR_WRITE)
+    {
+        iov[1] = { .iov_base = io->xfer_buf, .iov_len = io->xfer_buflen };
+        wtotal += io->xfer_buflen;
+        iovcnt++;
+    }
+    if (writev_blocking(bsd->connect_fd, iov, iovcnt) != wtotal)
+    {
+        perror("writev");
+        exit(1);
+    }
 
     if (io->error != 0)
         return FIO_Q_COMPLETED;
@@ -347,13 +292,57 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
 
 static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
 {
+    sec_options *opt = (sec_options*)td->eo;
     sec_data *bsd = (sec_data*)td->io_ops_data;
+    // FIXME timeout, at least poll. Now it's the stupidest implementation possible
+    osd_any_reply_t reply;
     while (bsd->completed.size() < min)
     {
-        bsd->ringloop->loop();
-        if (bsd->completed.size() >= min)
-            break;
-        bsd->ringloop->wait();
+        read_blocking(bsd->connect_fd, reply.buf, OSD_PACKET_SIZE);
+        if (reply.hdr.magic != SECONDARY_OSD_REPLY_MAGIC)
+        {
+            fprintf(stderr, "bad reply: magic = %lx instead of %lx\n", reply.hdr.magic, SECONDARY_OSD_REPLY_MAGIC);
+            exit(1);
+        }
+        auto it = bsd->queue.find(reply.hdr.id);
+        if (it == bsd->queue.end())
+        {
+            fprintf(stderr, "bad reply: op id %lx missing in local queue\n", reply.hdr.id);
+            exit(1);
+        }
+        io_u* io = it->second;
+        bsd->queue.erase(it);
+        if (io->ddir == DDIR_READ)
+        {
+            if (reply.hdr.retval != io->xfer_buflen)
+            {
+                fprintf(stderr, "Short read: retval = %ld instead of %llu\n", reply.hdr.retval, io->xfer_buflen);
+                exit(1);
+            }
+            read_blocking(bsd->connect_fd, io->xfer_buf, io->xfer_buflen);
+        }
+        else if (io->ddir == DDIR_WRITE)
+        {
+            if (reply.hdr.retval != io->xfer_buflen)
+            {
+                fprintf(stderr, "Short write: retval = %ld instead of %llu\n", reply.hdr.retval, io->xfer_buflen);
+                exit(1);
+            }
+        }
+        else if (io->ddir == DDIR_SYNC)
+        {
+            if (reply.hdr.retval != 0)
+            {
+                fprintf(stderr, "Sync failed: retval = %ld\n", reply.hdr.retval);
+                exit(1);
+            }
+        }
+        if (opt->trace)
+        {
+            printf("--- %s # %ld\n", io->ddir == DDIR_READ ? "READ" :
+                (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), reply.hdr.id);
+        }
+        bsd->completed.push_back(io);
     }
     return bsd->completed.size();
 }

src/messenger.h

@@ -120,6 +120,7 @@ struct osd_messenger_t
 protected:
     int keepalive_timer_id = -1;
 
+    uint32_t receive_buffer_size = 0;
     int peer_connect_interval = 0;
     int peer_connect_timeout = 0;
     int osd_idle_timeout = 0;
@@ -141,8 +142,6 @@ protected:
     std::vector<std::function<void()>> set_immediate;
 
 public:
-    uint32_t receive_buffer_size = 0;
-
     timerfd_manager_t *tfd;
     ring_loop_t *ringloop;
     // osd_num_t is only for logging and asserts
@@ -173,11 +172,10 @@ public:
     bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
 #endif
 
-    void handle_peer_epoll(int peer_fd, int epoll_events);
-
 protected:
     void try_connect_peer(uint64_t osd_num);
     void try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port);
+    void handle_peer_epoll(int peer_fd, int epoll_events);
     void handle_connect_epoll(int peer_fd);
     void on_connect_peer(osd_num_t peer_osd, int peer_fd);
     void check_peer_config(osd_client_t *cl);

src/osd.h

@@ -102,7 +102,7 @@ class osd_t
     bool no_rebalance = false;
     bool no_recovery = false;
     std::string bind_address;
-    int bind_port, listen_backlog;
+    int bind_port, listen_backlog = 128;
     // FIXME: Implement client queue depth limit
     int client_queue_depth = 128;
     bool allow_test_ops = false;

src/stub_uring_osd.cpp

@@ -110,8 +110,10 @@ void stub_exec_op(osd_messenger_t *msgr, osd_op_t *op)
     if (op->req.hdr.opcode == OSD_OP_SEC_READ)
     {
         op->reply.hdr.retval = op->req.sec_rw.len;
-        op->buf = malloc(op->req.sec_rw.len);
+        op->buf = memalign_or_die(MEM_ALIGNMENT, op->req.sec_rw.len);
         op->iov.push_back(op->buf, op->req.sec_rw.len);
+        op->reply.sec_rw.attr_len = 4;
+        op->bitmap = op->buf;
     }
     else if (op->req.hdr.opcode == OSD_OP_SEC_WRITE || op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
     {

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.6.10
+Version: 0.6.11
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}