Use real messenger in fio_sec_osd

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.6.11")
+set(VERSION "0.6.10")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.6.11
+VERSION ?= v0.6.10
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.6.11
+          image: vitalif/vitastor-csi:v0.6.10
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.6.11
+          image: vitalif/vitastor-csi:v0.6.10
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.6.11"
+    vitastorCSIDriverVersion = "0.6.10"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.6.11-1) unstable; urgency=medium
+vitastor (0.6.10-1) unstable; urgency=medium
 
   * RDMA support
   * Bugfixes

debian/vitastor.Dockerfile

@@ -33,8 +33,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.6.11; \
-    cd vitastor-0.6.11; \
+    cp -r /root/vitastor vitastor-0.6.10; \
+    cd vitastor-0.6.10; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -47,8 +47,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.11.orig.tar.xz vitastor-0.6.11; \
-    cd vitastor-0.6.11; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.10.orig.tar.xz vitastor-0.6.10; \
+    cd vitastor-0.6.10; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.6.11'
+VERSION = '0.6.10'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.6.11/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.11$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.6.10/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.10$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -34,7 +34,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.11.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.10.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.11
+Version:        0.6.10
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.11.el7.tar.gz
+Source0:        vitastor-0.6.10.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -33,7 +33,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.11.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.10.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.11
+Version:        0.6.10
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.11.el8.tar.gz
+Source0:        vitastor-0.6.10.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.6.11")
+add_definitions(-DVERSION="0.6.10")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -111,11 +111,12 @@ if (${WITH_FIO})
 	# libfio_vitastor_sec.so
 	add_library(fio_vitastor_sec SHARED
 		fio_sec_osd.cpp
-		rw_blocking.cpp
-		addr_util.cpp
 	)
 	target_link_libraries(fio_vitastor_sec
+		vitastor_common
 		tcmalloc_minimal
+		${LIBURING_LIBRARIES}
+		${IBVERBS_LIBRARIES}
 	)
 endif (${WITH_FIO})
 

src/fio_sec_osd.cpp

@@ -29,23 +29,23 @@
 #include <unordered_map>
 
 #include "addr_util.h"
-#include "rw_blocking.h"
+#include "epoll_manager.h"
+#include "ringloop.h"
+#include "messenger.h"
 #include "osd_ops.h"
 #include "fio_headers.h"
 
-struct op_buf_t
-{
-    osd_any_op_t buf;
-    io_u* fio_op;
-};
-
 struct sec_data
 {
+    epoll_manager_t *epmgr;
+    ring_loop_t *ringloop;
+    osd_messenger_t *msgr;
+    ring_consumer_t looper;
+    void *bitmap_buf;
     int connect_fd;
-    int data_fd;
+    uint64_t op_id;
     /* block_size = 1 << block_order (128KB by default) */
     uint64_t block_order = 17, block_size = 1 << 17;
-    std::unordered_map<uint64_t, op_buf_t*> queue;
     bool last_sync = false;
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
@@ -60,7 +60,6 @@ struct sec_options
     int single_primary = 0;
     int trace = 0;
     int block_order = 17;
-    int zerocopy_send = 0;
 };
 
 static struct fio_option options[] = {
@@ -111,16 +110,6 @@ static struct fio_option options[] = {
         .category = FIO_OPT_C_ENGINE,
         .group  = FIO_OPT_G_FILENAME,
     },
-    {
-        .name   = "zerocopy_send",
-        .lname  = "Use zero-copy send",
-        .type   = FIO_OPT_BOOL,
-        .off1   = offsetof(struct sec_options, zerocopy_send),
-        .help   = "Use zero-copy send (MSG_ZEROCOPY)",
-        .def    = "0",
-        .category = FIO_OPT_C_ENGINE,
-        .group  = FIO_OPT_G_FILENAME,
-    },
     {
         .name = NULL,
     },
@@ -129,9 +118,6 @@ static struct fio_option options[] = {
 static int sec_setup(struct thread_data *td)
 {
     sec_data *bsd;
-    //fio_file *f;
-    //int r;
-    //int64_t size;
 
     bsd = new sec_data;
     if (!bsd)
@@ -148,8 +134,6 @@ static int sec_setup(struct thread_data *td)
         td->o.open_files++;
     }
 
-    //f = td->files[0];
-    //f->real_file_size = size;
     return 0;
 }
 
@@ -158,7 +142,10 @@ static void sec_cleanup(struct thread_data *td)
     sec_data *bsd = (sec_data*)td->io_ops_data;
     if (bsd)
     {
-        close(bsd->data_fd);
+        delete bsd->msgr;
+        delete bsd->epmgr;
+        delete bsd->ringloop;
+        free(bsd->bitmap_buf);
         close(bsd->connect_fd);
         delete bsd;
     }
@@ -172,67 +159,65 @@ static int sec_init(struct thread_data *td)
     bsd->block_order = o->block_order == 0 ? 17 : o->block_order;
     bsd->block_size = 1 << o->block_order;
 
-    struct sockaddr_storage addr = { 0 };
-    if (!string_to_addr(o->host ? o->host : "127.0.0.1", false, o->port > 0 ? o->port : 11203, (struct sockaddr*)&addr))
+    sockaddr addr;
+    if (!string_to_addr(std::string(o->host ? o->host : "127.0.0.1"), false, o->port > 0 ? o->port : 11203, &addr))
     {
         fprintf(stderr, "server address: %s is not valid\n", o->host ? o->host : "127.0.0.1");
         return 1;
     }
 
-    bsd->connect_fd = socket(addr.ss_family, SOCK_STREAM, 0);
+    bsd->connect_fd = socket(addr.sa_family, SOCK_STREAM, 0);
     if (bsd->connect_fd < 0)
     {
         perror("socket");
         return 1;
     }
-    if (connect(bsd->connect_fd, (sockaddr*)&addr, addr.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)) < 0)
+    if (connect(bsd->connect_fd, (sockaddr*)&addr, sizeof(addr)) < 0)
     {
         perror("connect");
         return 1;
     }
     int one = 1;
     setsockopt(bsd->connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-    if (o->zerocopy_send)
-    {
-        if (setsockopt(bsd->connect_fd, SOL_SOCKET, SO_ZEROCOPY, &one, sizeof(one)) < 0)
-        {
-            perror("setsockopt zerocopy");
-            return 1;
-        }
-    }
 
-    if (!string_to_addr(o->host ? o->host : "127.0.0.1", false, 1 + (o->port > 0 ? o->port : 11203), (sockaddr*)&addr))
-    {
-        fprintf(stderr, "server address: %s is not valid\n", o->host ? o->host : "127.0.0.1");
-        return 1;
-    }
+    fcntl(bsd->connect_fd, F_SETFL, fcntl(bsd->connect_fd, F_GETFL, 0) | O_NONBLOCK);
 
-    bsd->data_fd = socket(addr.ss_family, SOCK_STREAM, 0);
-    if (bsd->data_fd < 0)
+    json11::Json cfg = json11::Json::object{ { "use_rdma", 0 } };
+
+    bsd->bitmap_buf = malloc(4096);
+
+    bsd->ringloop = new ring_loop_t(512);
+    bsd->epmgr = new epoll_manager_t(bsd->ringloop);
+    bsd->msgr = new osd_messenger_t();
+    bsd->msgr->tfd = bsd->epmgr->tfd;
+    bsd->msgr->ringloop = bsd->ringloop;
+    bsd->msgr->repeer_pgs = [](osd_num_t){};
+    bsd->msgr->parse_config(cfg);
+    bsd->msgr->init();
+
+    bsd->looper.loop = [bsd]()
     {
-        perror("socket");
-        return 1;
-    }
-/*    int mss = 4096;
-    if (setsockopt(bsd->data_fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) < 0)
+        bsd->msgr->read_requests();
+        bsd->msgr->send_replies();
+        bsd->ringloop->submit();
+    };
+    bsd->ringloop->register_consumer(&bsd->looper);
+
+    int peer_fd = bsd->connect_fd;
+    bsd->msgr->clients[peer_fd] = new osd_client_t();
+    bsd->msgr->clients[peer_fd]->peer_addr = addr;
+    bsd->msgr->clients[peer_fd]->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
+    bsd->msgr->clients[peer_fd]->peer_fd = peer_fd;
+    bsd->msgr->clients[peer_fd]->peer_state = PEER_CONNECTED;
+    bsd->msgr->clients[peer_fd]->connect_timeout_id = -1;
+    bsd->msgr->clients[peer_fd]->osd_num = 1;
+    bsd->msgr->clients[peer_fd]->in_buf = malloc_or_die(bsd->msgr->receive_buffer_size);
+    bsd->epmgr->tfd->set_fd_handler(peer_fd, true, [msgr = bsd->msgr](int peer_fd, int epoll_events)
     {
-        perror("setsockopt TCP_MAXSEG");
-        return 1;
-    }*/
-    if (connect(bsd->data_fd, (sockaddr*)&addr, addr.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)) < 0)
-    {
-        perror("connect");
-        return 1;
-    }
-    setsockopt(bsd->data_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-    if (o->zerocopy_send)
-    {
-        if (setsockopt(bsd->data_fd, SOL_SOCKET, SO_ZEROCOPY, &one, sizeof(one)) < 0)
-        {
-            perror("setsockopt zerocopy");
-            return 1;
-        }
-    }
+        // Either OUT (connected) or HUP
+        msgr->handle_peer_epoll(peer_fd, epoll_events);
+    });
+    bsd->msgr->osd_peer_fds[1] = peer_fd;
 
     // FIXME: read config (block size) from OSD
 
@@ -253,9 +238,12 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
     }
 
     io->engine_data = bsd;
-    op_buf_t *op_buf = new op_buf_t;
-    op_buf->fio_op = io;
-    osd_any_op_t &op = op_buf->buf;
+
+    osd_op_t *oo = new osd_op_t();
+    oo->op_type = OSD_OP_OUT;
+    oo->peer_fd = bsd->connect_fd;
+
+    osd_any_op_t & op = oo->req;
 
     op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
     op.hdr.id = n;
@@ -272,6 +260,9 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.sec_rw.version = UINT64_MAX; // last unstable
             op.sec_rw.offset = io->offset % bsd->block_size;
             op.sec_rw.len = io->xfer_buflen;
+            op.sec_rw.attr_len = 4;
+            oo->bitmap = bsd->bitmap_buf;
+            oo->bitmap_len = 4;
         }
         else
         {
@@ -280,6 +271,7 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.rw.offset = io->offset;
             op.rw.len = io->xfer_buflen;
         }
+        oo->iov.push_back(io->xfer_buf, io->xfer_buflen);
         bsd->last_sync = false;
         break;
     case DDIR_WRITE:
@@ -301,6 +293,7 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.rw.offset = io->offset;
             op.rw.len = io->xfer_buflen;
         }
+        oo->iov.push_back(io->xfer_buf, io->xfer_buflen);
         bsd->last_sync = false;
         break;
     case DDIR_SYNC:
@@ -322,6 +315,21 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
         return FIO_Q_COMPLETED;
     }
 
+    oo->callback = [td, io](osd_op_t *oo)
+    {
+        sec_options *opt = (sec_options*)td->eo;
+        sec_data *bsd = (sec_data*)td->io_ops_data;
+        if (opt->trace)
+        {
+            printf("--- %s # %ld %ld\n", io->ddir == DDIR_READ ? "READ" :
+                (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), oo->reply.hdr.id, oo->reply.hdr.retval);
+        }
+        io->error = oo->reply.hdr.retval < 0 ? -oo->reply.hdr.retval : 0;
+        bsd->completed.push_back(io);
+        delete oo;
+    };
+    bsd->msgr->outbox_push(oo);
+
     if (opt->trace)
     {
         printf("+++ %s # %d\n", io->ddir == DDIR_READ ? "READ" :
@@ -331,22 +339,6 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
     io->error = 0;
     bsd->inflight++;
     bsd->op_n++;
-    bsd->queue[n] = op_buf;
-
-    if (io->ddir == DDIR_WRITE)
-    {
-        // It may make you laugh but ZCTR is only stable if we write data before header :-) O_o
-        if (write_blocking(bsd->data_fd, io->xfer_buf, io->xfer_buflen) != io->xfer_buflen)
-        {
-            perror("write");
-            exit(1);
-        }
-    }
-    if (write_blocking(bsd->connect_fd, op.buf, OSD_PACKET_SIZE) != OSD_PACKET_SIZE)
-    {
-        perror("write");
-        exit(1);
-    }
 
     if (io->error != 0)
         return FIO_Q_COMPLETED;
@@ -355,72 +347,13 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
 
 static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
 {
-    sec_options *opt = (sec_options*)td->eo;
     sec_data *bsd = (sec_data*)td->io_ops_data;
-    // FIXME timeout, at least poll. Now it's the stupidest implementation possible
-    osd_any_reply_t reply;
     while (bsd->completed.size() < min)
     {
-        read_blocking(bsd->connect_fd, reply.buf, OSD_PACKET_SIZE);
-        if (reply.hdr.magic != SECONDARY_OSD_REPLY_MAGIC)
-        {
-            fprintf(stderr, "bad reply: magic = %lx instead of %lx\n", reply.hdr.magic, SECONDARY_OSD_REPLY_MAGIC);
-            exit(1);
-        }
-        auto it = bsd->queue.find(reply.hdr.id);
-        if (it == bsd->queue.end())
-        {
-            fprintf(stderr, "bad reply: op id %lx missing in local queue\n", reply.hdr.id);
-            exit(1);
-        }
-        io_u* io = it->second->fio_op;
-        delete it->second;
-        bsd->queue.erase(it);
-        if (io->ddir == DDIR_READ)
-        {
-            if (reply.hdr.retval != io->xfer_buflen)
-            {
-                fprintf(stderr, "Short read: retval = %ld instead of %llu\n", reply.hdr.retval, io->xfer_buflen);
-                exit(1);
-            }
-            if (reply.sec_rw.attr_len > 0)
-            {
-                if (reply.sec_rw.attr_len <= 8)
-                {
-                    uint64_t bitmap = 0;
-                    read_blocking(bsd->connect_fd, &bitmap, reply.sec_rw.attr_len);
-                }
-                else
-                {
-                    void *bitmap = malloc(reply.sec_rw.attr_len);
-                    read_blocking(bsd->connect_fd, bitmap, reply.sec_rw.attr_len);
-                    free(bitmap);
-                }
-            }
-            read_blocking(bsd->data_fd, io->xfer_buf, io->xfer_buflen);
-        }
-        else if (io->ddir == DDIR_WRITE)
-        {
-            if (reply.hdr.retval != io->xfer_buflen)
-            {
-                fprintf(stderr, "Short write: retval = %ld instead of %llu\n", reply.hdr.retval, io->xfer_buflen);
-                exit(1);
-            }
-        }
-        else if (io->ddir == DDIR_SYNC)
-        {
-            if (reply.hdr.retval != 0)
-            {
-                fprintf(stderr, "Sync failed: retval = %ld\n", reply.hdr.retval);
-                exit(1);
-            }
-        }
-        if (opt->trace)
-        {
-            printf("--- %s # %ld\n", io->ddir == DDIR_READ ? "READ" :
-                (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), reply.hdr.id);
-        }
-        bsd->completed.push_back(io);
+        bsd->ringloop->loop();
+        if (bsd->completed.size() >= min)
+            break;
+        bsd->ringloop->wait();
     }
     return bsd->completed.size();
 }

src/messenger.h

@@ -120,7 +120,6 @@ struct osd_messenger_t
 protected:
     int keepalive_timer_id = -1;
 
-    uint32_t receive_buffer_size = 0;
     int peer_connect_interval = 0;
     int peer_connect_timeout = 0;
     int osd_idle_timeout = 0;
@@ -142,6 +141,8 @@ protected:
     std::vector<std::function<void()>> set_immediate;
 
 public:
+    uint32_t receive_buffer_size = 0;
+
     timerfd_manager_t *tfd;
     ring_loop_t *ringloop;
     // osd_num_t is only for logging and asserts
@@ -172,10 +173,11 @@ public:
     bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
 #endif
 
+    void handle_peer_epoll(int peer_fd, int epoll_events);
+
 protected:
     void try_connect_peer(uint64_t osd_num);
     void try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port);
-    void handle_peer_epoll(int peer_fd, int epoll_events);
     void handle_connect_epoll(int peer_fd);
     void on_connect_peer(osd_num_t peer_osd, int peer_fd);
     void check_peer_config(osd_client_t *cl);

src/osd.h

@@ -102,7 +102,7 @@ class osd_t
     bool no_rebalance = false;
     bool no_recovery = false;
     std::string bind_address;
-    int bind_port, listen_backlog = 128;
+    int bind_port, listen_backlog;
     // FIXME: Implement client queue depth limit
     int client_queue_depth = 128;
     bool allow_test_ops = false;

src/rw_blocking.cpp

@@ -4,8 +4,6 @@
 #include <errno.h>
 #include <stdlib.h>
 #include <stdio.h>
-#include <sys/types.h>
-#include <sys/socket.h>
 
 #include "rw_blocking.h"
 
@@ -125,41 +123,3 @@ int writev_blocking(int fd, iovec *iov, int iovcnt)
     }
     return done;
 }
-
-int sendv_blocking(int fd, iovec *iov, int iovcnt, int flags)
-{
-    struct msghdr msg = { 0 };
-    int v = 0;
-    int done = 0;
-    while (v < iovcnt)
-    {
-        msg.msg_iov = iov+v;
-        msg.msg_iovlen = iovcnt-v;
-        ssize_t r = sendmsg(fd, &msg, flags);
-        if (r < 0)
-        {
-            if (errno != EAGAIN && errno != EPIPE)
-            {
-                perror("sendmsg");
-                exit(1);
-            }
-            continue;
-        }
-        done += r;
-        while (v < iovcnt)
-        {
-            if (iov[v].iov_len > r)
-            {
-                iov[v].iov_len -= r;
-                iov[v].iov_base += r;
-                break;
-            }
-            else
-            {
-                r -= iov[v].iov_len;
-                v++;
-            }
-        }
-    }
-    return done;
-}

src/rw_blocking.h

@@ -10,4 +10,3 @@ int read_blocking(int fd, void *read_buf, size_t remaining);
 int write_blocking(int fd, void *write_buf, size_t remaining);
 int readv_blocking(int fd, iovec *iov, int iovcnt);
 int writev_blocking(int fd, iovec *iov, int iovcnt);
-int sendv_blocking(int fd, iovec *iov, int iovcnt, int flags);

src/stub_osd.cpp

@@ -24,9 +24,7 @@
  */
 
 #include <sys/types.h>
-#include <sys/mman.h>
 #include <sys/socket.h>
-#include <poll.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <arpa/inet.h>
@@ -45,32 +43,19 @@
 
 int bind_stub(std::string bind_address, int bind_port);
 
-void run_stub(int peer_fd, int peer_data_fd);
+void run_stub(int peer_fd);
 
 int main(int narg, char *args[])
 {
     int listen_fd = bind_stub("0.0.0.0", 11203);
-    int listen_data_fd = bind_stub("0.0.0.0", 11204);
-/*    int mss = 8192;
-    if (setsockopt(listen_data_fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) < 0)
-    {
-        throw std::runtime_error(std::string("setsockopt TCP_MAXSEG: ") + strerror(errno));
-    }
-    int rcvlowat = 4096;
-    if (setsockopt(listen_data_fd, SOL_SOCKET, SO_RCVLOWAT, &rcvlowat, sizeof(rcvlowat)) < 0)
-    {
-        throw std::runtime_error(std::string("setsockopt SO_RCVLOWAT: ") + strerror(errno));
-    }*/
     // Accept new connections
     sockaddr addr;
-    socklen_t peer_addr_size;
-    int peer_fd, peer_data_fd;
-    const int one = 1;
+    socklen_t peer_addr_size = sizeof(addr);
+    int peer_fd;
     while (1)
     {
         printf("stub_osd: waiting for 1 client\n");
-        peer_addr_size = sizeof(addr);
-        peer_fd = accept(listen_fd, (sockaddr*)&addr, &peer_addr_size);
+        peer_fd = accept(listen_fd, &addr, &peer_addr_size);
         if (peer_fd == -1)
         {
             if (errno == EAGAIN)
@@ -78,27 +63,15 @@ int main(int narg, char *args[])
             else
                 throw std::runtime_error(std::string("accept: ") + strerror(errno));
         }
-        setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
         printf("stub_osd: new client %d: connection from %s\n", peer_fd,
-            addr_to_string(*((sockaddr*)&addr)).c_str());
-        printf("stub_osd: waiting for 1 data connection\n");
-        peer_addr_size = sizeof(addr);
-        peer_data_fd = accept(listen_data_fd, (sockaddr*)&addr, &peer_addr_size);
-        if (peer_data_fd == -1)
-        {
-            if (errno == EAGAIN)
-                continue;
-            else
-                throw std::runtime_error(std::string("accept: ") + strerror(errno));
-        }
-        setsockopt(peer_data_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-        printf("stub_osd: new client %d: data connection from %s\n", peer_data_fd,
-            addr_to_string(*((sockaddr*)&addr)).c_str());
-        run_stub(peer_fd, peer_data_fd);
-        close(peer_data_fd);
+            addr_to_string(addr).c_str());
+        int one = 1;
+        setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
+        run_stub(peer_fd);
         close(peer_fd);
-        printf("stub_osd: client %d / data %d disconnected\n", peer_fd, peer_data_fd);
+        printf("stub_osd: client %d disconnected\n", peer_fd);
         // Try to accept next connection
+        peer_addr_size = sizeof(addr);
     }
     return 0;
 }
@@ -107,13 +80,13 @@ int bind_stub(std::string bind_address, int bind_port)
 {
     int listen_backlog = 128;
 
-    sockaddr_storage addr = { 0 };
-    if (!string_to_addr(bind_address, 0, bind_port, (sockaddr*)&addr))
+    sockaddr addr;
+    if (!string_to_addr(bind_address, 0, bind_port, &addr))
     {
         throw std::runtime_error("bind address "+bind_address+" is not valid");
     }
 
-    int listen_fd = socket(addr.ss_family, SOCK_STREAM, 0);
+    int listen_fd = socket(addr.sa_family, SOCK_STREAM, 0);
     if (listen_fd < 0)
     {
         throw std::runtime_error(std::string("socket: ") + strerror(errno));
@@ -121,7 +94,7 @@ int bind_stub(std::string bind_address, int bind_port)
     int enable = 1;
     setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable));
 
-    if (bind(listen_fd, (sockaddr*)&addr, addr.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)) < 0)
+    if (bind(listen_fd, &addr, sizeof(addr)) < 0)
     {
         close(listen_fd);
         throw std::runtime_error(std::string("bind: ") + strerror(errno));
@@ -136,16 +109,11 @@ int bind_stub(std::string bind_address, int bind_port)
     return listen_fd;
 }
 
-void run_stub(int peer_fd, int peer_data_fd)
+void run_stub(int peer_fd)
 {
     osd_any_op_t op;
     osd_any_reply_t reply = { 0 };
-    unsigned bufsize = 4*1024*1024;
-    void *buf = mmap(NULL, bufsize, PROT_READ, MAP_SHARED, peer_data_fd, 0);
-    if (buf == MAP_FAILED)
-    {
-        throw std::runtime_error(std::string("mmap: ") + strerror(errno));
-    }
+    void *buf = NULL;
     while (1)
     {
         int r = read_blocking(peer_fd, op.buf, OSD_PACKET_SIZE);
@@ -164,36 +132,19 @@ void run_stub(int peer_fd, int peer_data_fd)
         if (op.hdr.opcode == OSD_OP_SEC_READ)
         {
             reply.hdr.retval = op.sec_rw.len;
-            void *buf = malloc(op.sec_rw.len);
+            buf = malloc(op.sec_rw.len);
             r = write_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
             if (r == OSD_PACKET_SIZE)
-                r = write_blocking(peer_data_fd, buf, op.sec_rw.len);
+                r = write_blocking(peer_fd, buf, op.sec_rw.len);
             free(buf);
             if (r < op.sec_rw.len)
                 break;
         }
         else if (op.hdr.opcode == OSD_OP_SEC_WRITE || op.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
         {
-            struct pollfd pfd = { .fd = peer_data_fd, .events = POLLIN };
-            poll(&pfd, 1, 10000);
-            struct tcp_zerocopy_receive zc = { .address = (uint64_t)buf, .length = op.sec_rw.len };
-            socklen_t zc_len = sizeof(zc);
-            r = getsockopt(peer_data_fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, &zc, &zc_len);
-            r = r == -1 ? 0 : zc.length;
-            if (r > 0)
-            {
-                uint64_t hash = 0;
-                for (int k = 0; k < r/8; k++)
-                    hash ^= ((uint64_t*)buf)[k];
-                printf("ZCTR: op=%lx r=%d len=%d skip=%d hash=%lx\n", op.hdr.id, r, zc.length, zc.recv_skip_hint, hash);
-            }
-            if (r < op.sec_rw.len)
-            {
-                int rest = op.sec_rw.len - r;
-                void *buf = malloc(rest);
-                r += read_blocking(peer_data_fd, buf, rest);
-                free(buf);
-            }
+            buf = malloc(op.sec_rw.len);
+            r = read_blocking(peer_fd, buf, op.sec_rw.len);
+            free(buf);
             reply.hdr.retval = op.sec_rw.len;
             if (r == op.sec_rw.len)
                 r = write_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
@@ -215,4 +166,5 @@ void run_stub(int peer_fd, int peer_data_fd)
             break;
         }
     }
+    free(buf);
 }

src/stub_uring_osd.cpp

@@ -110,10 +110,8 @@ void stub_exec_op(osd_messenger_t *msgr, osd_op_t *op)
     if (op->req.hdr.opcode == OSD_OP_SEC_READ)
     {
         op->reply.hdr.retval = op->req.sec_rw.len;
-        op->buf = memalign_or_die(MEM_ALIGNMENT, op->req.sec_rw.len);
+        op->buf = malloc(op->req.sec_rw.len);
         op->iov.push_back(op->buf, op->req.sec_rw.len);
-        op->reply.sec_rw.attr_len = 4;
-        op->bitmap = op->buf;
     }
     else if (op->req.hdr.opcode == OSD_OP_SEC_WRITE || op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
     {

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.6.11
+Version: 0.6.10
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/run_3osds.sh

@@ -5,7 +5,6 @@
 OSD_SIZE=${OSD_SIZE:-1024}
 PG_COUNT=${PG_COUNT:-1}
 PG_SIZE=${PG_SIZE:-3}
-PG_MINSIZE=${PG_MINSIZE:-2}
 OSD_COUNT=${OSD_COUNT:-3}
 SCHEME=${SCHEME:-ec}
 
@@ -26,9 +25,9 @@ if [ -n "$GLOBAL_CONF" ]; then
 fi
 
 if [ "$SCHEME" = "replicated" ]; then
-    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
+    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":'$((PG_SIZE-1))',"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
 else
-    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"parity_chunks":1,"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
+    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":'$PG_SIZE',"pg_minsize":'$((PG_SIZE-1))',"parity_chunks":1,"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
 fi
 
 sleep 2

tests/test_minsize_1.sh

@@ -1,17 +0,0 @@
-#!/bin/bash -ex
-
-PG_MINSIZE=1
-SCHEME=replicated
-
-. `dirname $0`/run_3osds.sh
-
-kill -INT $OSD1_PID
-kill -INT $OSD2_PID
-
-sleep 5
-
-if ! ($ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active", "degraded"]) ] | length == '$PG_COUNT); then
-    format_error "FAILED: $PG_COUNT PG(s) NOT ACTIVE+DEGRADED"
-fi
-
-format_green OK