Use io_uring SQ size for ringloop capacity - otherwise get_sqe could return NULL when space_left() was > 0 under load

Raise default io_uring size to 1024 for the same effective capacity as previously

src/cli.cpp

@@ -331,7 +331,7 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
     {
         // Create client
         json11::Json cfg_j = cfg;
-        p->ringloop = new ring_loop_t(512);
+        p->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
         p->epmgr = new epoll_manager_t(p->ringloop);
         p->cli = new cluster_client_t(p->ringloop, p->epmgr->tfd, cfg_j);
         // Smaller timeout by default for more interactiveness

src/disk_tool_resize.cpp

@@ -245,7 +245,7 @@ int disk_tool_t::resize_copy_data()
     {
         iodepth = 32;
     }
-    ringloop = new ring_loop_t(iodepth < 512 ? 512 : iodepth);
+    ringloop = new ring_loop_t(iodepth < RINGLOOP_DEFAULT_SIZE ? RINGLOOP_DEFAULT_SIZE : iodepth);
     dsk.data_fd = open(dsk.data_device.c_str(), O_DIRECT|O_RDWR);
     if (dsk.data_fd < 0)
     {

src/fio_engine.cpp

@@ -130,7 +130,7 @@ static int bs_init(struct thread_data *td)
                 config[p.first] = p.second.dump();
         }
     }
-    bsd->ringloop = new ring_loop_t(512);
+    bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     bsd->epmgr = new epoll_manager_t(bsd->ringloop);
     bsd->bs = new blockstore_t(config, bsd->ringloop, bsd->epmgr->tfd);
     while (1)

src/nbd_proxy.cpp

@@ -225,7 +225,7 @@ public:
             cfg = obj;
         }
         // Create client
-        ringloop = new ring_loop_t(512);
+        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
         epmgr = new epoll_manager_t(ringloop);
         cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
         if (!inode)

src/nfs_proxy.cpp

@@ -124,7 +124,7 @@ void nfs_proxy_t::run(json11::Json cfg)
         cfg = obj;
     }
     // Create client
-    ringloop = new ring_loop_t(512);
+    ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     epmgr = new epoll_manager_t(ringloop);
     cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
     cmd = new cli_tool_t();

src/osd_main.cpp

@@ -58,7 +58,7 @@ int main(int narg, char *args[])
     }
     signal(SIGINT, handle_sigint);
     signal(SIGTERM, handle_sigint);
-    ring_loop_t *ringloop = new ring_loop_t(512);
+    ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     osd = new osd_t(config, ringloop);
     while (1)
     {

src/ringloop.cpp

@@ -17,7 +17,7 @@ ring_loop_t::ring_loop_t(int qd)
     {
         throw std::runtime_error(std::string("io_uring_queue_init: ") + strerror(-ret));
     }
-    free_ring_data_ptr = *ring.cq.kring_entries;
+    free_ring_data_ptr = *ring.sq.kring_entries;
     ring_datas = (struct ring_data_t*)calloc(free_ring_data_ptr, sizeof(ring_data_t));
     free_ring_data = (int*)malloc(sizeof(int) * free_ring_data_ptr);
     if (!ring_datas || !free_ring_data)

src/ringloop.h

@@ -15,6 +15,8 @@
 #include <functional>
 #include <vector>
 
+#define RINGLOOP_DEFAULT_SIZE 1024
+
 static inline void my_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd, const void *addr, unsigned len, off_t offset)
 {
     // Prepare a read/write operation without clearing user_data
@@ -139,11 +141,9 @@ public:
         if (free_ring_data_ptr == 0)
             return NULL;
         struct io_uring_sqe* sqe = io_uring_get_sqe(&ring);
-        if (sqe)
-        {
-            *sqe = { 0 };
-            io_uring_sqe_set_data(sqe, ring_datas + free_ring_data[--free_ring_data_ptr]);
-        }
+        assert(sqe);
+        *sqe = { 0 };
+        io_uring_sqe_set_data(sqe, ring_datas + free_ring_data[--free_ring_data_ptr]);
         return sqe;
     }
     inline void set_immediate(const std::function<void()> cb)

src/stub_uring_osd.cpp

@@ -30,7 +30,7 @@ void stub_exec_op(osd_messenger_t *msgr, osd_op_t *op);
 int main(int narg, char *args[])
 {
     ring_consumer_t looper;
-    ring_loop_t *ringloop = new ring_loop_t(512);
+    ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     epoll_manager_t *epmgr = new epoll_manager_t(ringloop);
     osd_messenger_t *msgr = new osd_messenger_t();
     msgr->osd_num = 1351;

src/test_blockstore.cpp

@@ -11,7 +11,7 @@ int main(int narg, char *args[])
     config["meta_device"] = "./test_meta.bin";
     config["journal_device"] = "./test_journal.bin";
     config["data_device"] = "./test_data.bin";
-    ring_loop_t *ringloop = new ring_loop_t(512);
+    ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     epoll_manager_t *epmgr = new epoll_manager_t(ringloop);
     blockstore_t *bs = new blockstore_t(config, ringloop, epmgr->tfd);
 

src/test_cas.cpp

@@ -68,7 +68,7 @@ int main(int narg, char *args[])
         | cfg["inode_id"].uint64_value();
     uint64_t base_ver = 0;
     // Create client
-    auto ringloop = new ring_loop_t(512);
+    auto ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     auto epmgr = new epoll_manager_t(ringloop);
     auto cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
     cli->on_ready([&]()

src/vitastor_c.cpp

@@ -114,7 +114,7 @@ vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, v
     ring_loop_t *ringloop = NULL;
     try
     {
-        ringloop = new ring_loop_t(512);
+        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     }
     catch (std::exception & e)
     {
@@ -136,7 +136,7 @@ vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_ho
     ring_loop_t *ringloop = NULL;
     try
     {
-        ringloop = new ring_loop_t(512);
+        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     }
     catch (std::exception & e)
     {
@@ -167,7 +167,7 @@ vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
     ring_loop_t *ringloop = NULL;
     try
     {
-        ringloop = new ring_loop_t(512);
+        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     }
     catch (std::exception & e)
     {