Implement actual client operation cancel

src/CMakeLists.txt

@@ -86,11 +86,6 @@ pkg_check_modules(RDMACM librdmacm)
 if (RDMACM_LIBRARIES)
 	add_definitions(-DWITH_RDMACM)
 endif (RDMACM_LIBRARIES)
-pkg_check_modules(VFN libvfn)
-pkg_check_modules(NVME libnvme)
-if (VFN_LIBRARIES AND NVME_LIBRARIES)
-	add_definitions(-DWITH_VFN)
-endif (VFN_LIBRARIES AND NVME_LIBRARIES)
 
 find_package(OpenSSL 3.0 REQUIRED)
 add_definitions(-DWITH_OPENSSL)

src/blockstore/blockstore_disk.cpp

@@ -1,6 +1,10 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
 #include <stdexcept>
 
 #include "blockstore.h"
@@ -351,22 +355,31 @@ void blockstore_disk_t::check_lengths()
     }
 }
 
+static int bs_openmode(const std::string & mode)
+{
+    if (mode == "directsync")
+        return O_DIRECT|O_SYNC;
+    else if (mode == "cached")
+        return O_SYNC;
+    else
+        return O_DIRECT;
+}
+
 void blockstore_disk_t::open_data()
 {
-    if (data_disk)
+    if (data_fd >= 0)
     {
         throw std::runtime_error("data device is already opened");
     }
-    data_disk = mock_mode
-        ? disk_i::create_mock(MOCK_DATA_FD, data_device_size, data_device_sect)
-        : disk_i::create_and_open(data_device, data_io, disable_flock);
-    if (!data_disk)
+    data_fd = mock_mode ? MOCK_DATA_FD : open(data_device.c_str(), bs_openmode(data_io) | O_RDWR);
+    if (data_fd == -1)
     {
-        throw std::runtime_error("Failed to open data device "+data_device);
+        throw std::runtime_error("Failed to open data device "+data_device+": "+std::string(strerror(errno)));
+    }
+    if (!mock_mode)
+    {
+        check_size(data_fd, &data_device_size, &data_device_sect, "data device");
     }
-    data_off_ring = data_disk->is_off_ring();
-    data_device_size = data_disk->get_size();
-    data_device_sect = data_disk->get_block_size();
     if (disk_alignment % data_device_sect)
     {
         throw std::runtime_error(
@@ -378,33 +391,41 @@ void blockstore_disk_t::open_data()
     {
         throw std::runtime_error("data_offset exceeds device size = "+std::to_string(data_device_size));
     }
+    if (!mock_mode && !disable_flock && flock(data_fd, LOCK_EX|LOCK_NB) != 0)
+    {
+        throw std::runtime_error(std::string("Failed to lock data device: ") + strerror(errno));
+    }
 }
 
 void blockstore_disk_t::open_meta()
 {
-    if (meta_disk)
+    if (meta_fd >= 0)
     {
         throw std::runtime_error("metadata device is already opened");
     }
     if (meta_device != data_device || meta_io != data_io)
     {
-        meta_disk = mock_mode
-            ? disk_i::create_mock(MOCK_META_FD, meta_device_size, meta_device_sect)
-            : disk_i::create_and_open(meta_device, meta_io, meta_device == data_device || disable_flock);
-        if (!meta_disk)
+        meta_fd = mock_mode ? MOCK_META_FD : open(meta_device.c_str(), bs_openmode(meta_io) | O_RDWR);
+        if (meta_fd == -1)
         {
-            throw std::runtime_error("Failed to open metadata device "+meta_device);
+            throw std::runtime_error("Failed to open metadata device "+meta_device+": "+std::string(strerror(errno)));
+        }
+        if (!mock_mode)
+        {
+            check_size(meta_fd, &meta_device_size, &meta_device_sect, "metadata device");
         }
-        meta_device_size = meta_disk->get_size();
-        meta_device_sect = meta_disk->get_block_size();
         if (meta_offset >= meta_device_size)
         {
             throw std::runtime_error("meta_offset exceeds device size = "+std::to_string(meta_device_size));
         }
+        if (!mock_mode && !disable_flock && meta_device != data_device && flock(meta_fd, LOCK_EX|LOCK_NB) != 0)
+        {
+            throw std::runtime_error(std::string("Failed to lock metadata device: ") + strerror(errno));
+        }
     }
     else
     {
-        meta_disk = data_disk;
+        meta_fd = data_fd;
         meta_device_sect = data_device_sect;
         meta_device_size = 0;
         if (meta_offset >= data_device_size)
@@ -412,7 +433,6 @@ void blockstore_disk_t::open_meta()
             throw std::runtime_error("meta_offset exceeds device size = "+std::to_string(data_device_size));
         }
     }
-    meta_off_ring = meta_disk->is_off_ring();
     if (meta_block_size % meta_device_sect)
     {
         throw std::runtime_error(
@@ -424,29 +444,33 @@ void blockstore_disk_t::open_meta()
 
 void blockstore_disk_t::open_journal()
 {
-    if (journal_disk)
+    if (journal_fd >= 0)
     {
         throw std::runtime_error("journal device is already opened");
     }
     if (journal_device != meta_device || journal_io != meta_io)
     {
-        journal_disk = mock_mode
-            ? disk_i::create_mock(MOCK_JOURNAL_FD, journal_device_size, journal_device_sect)
-            : disk_i::create_and_open(journal_device, journal_io, journal_device == meta_device || disable_flock);
-        if (!journal_disk)
+        journal_fd = mock_mode ? MOCK_JOURNAL_FD : open(journal_device.c_str(), bs_openmode(journal_io) | O_RDWR);
+        if (journal_fd == -1)
         {
-            throw std::runtime_error("Failed to open journal device "+journal_device);
+            throw std::runtime_error("Failed to open journal device "+journal_device+": "+std::string(strerror(errno)));
+        }
+        if (!mock_mode)
+        {
+            check_size(journal_fd, &journal_device_size, &journal_device_sect, "journal device");
         }
-        journal_device_size = journal_disk->get_size();
-        journal_device_sect = journal_disk->get_block_size();
         if (journal_offset >= journal_device_size)
         {
             throw std::runtime_error("journal_offset exceeds device size = "+std::to_string(journal_device_size));
         }
+        if (!mock_mode && !disable_flock && journal_device != meta_device && flock(journal_fd, LOCK_EX|LOCK_NB) != 0)
+        {
+            throw std::runtime_error(std::string("Failed to lock journal device: ") + strerror(errno));
+        }
     }
     else
     {
-        journal_disk = meta_disk;
+        journal_fd = meta_fd;
         journal_device_sect = meta_device_sect;
         journal_device_size = 0;
         if (journal_offset >= (meta_device_size > 0 ? meta_device_size : data_device_size))
@@ -454,7 +478,6 @@ void blockstore_disk_t::open_journal()
             throw std::runtime_error("journal_offset exceeds device size");
         }
     }
-    journal_off_ring = journal_disk->is_off_ring();
     if (journal_block_size % journal_device_sect)
     {
         throw std::runtime_error(
@@ -466,19 +489,26 @@ void blockstore_disk_t::open_journal()
 
 void blockstore_disk_t::close_all()
 {
-    if (journal_disk && journal_disk != meta_disk && journal_disk != data_disk)
-        delete journal_disk;
-    if (meta_disk && meta_disk != data_disk)
-        delete meta_disk;
-    if (data_disk)
-        delete data_disk;
-    data_disk = meta_disk = journal_disk = NULL;
+    if (!mock_mode)
+    {
+        if (data_fd >= 0)
+            close(data_fd);
+        if (meta_fd >= 0 && meta_fd != data_fd)
+            close(meta_fd);
+        if (journal_fd >= 0 && journal_fd != meta_fd)
+            close(journal_fd);
+    }
+    data_fd = meta_fd = journal_fd = -1;
 }
 
 // Sadly DISCARD only works through ioctl(), but it seems to always block the device queue,
 // so it's not a big deal that we can only run it synchronously.
 int blockstore_disk_t::trim_data(std::function<bool(uint64_t)> is_used)
 {
+    if (mock_mode)
+    {
+        return -EINVAL;
+    }
     int r = 0;
     uint64_t j = 0, i = 0;
     uint64_t discarded = 0;
@@ -503,12 +533,12 @@ int blockstore_disk_t::trim_data(std::function<bool(uint64_t)> is_used)
                 }
                 if (range[1] > 0)
                 {
-                    r = data_disk->discard_sync(range[0], range[1]);
+                    r = ioctl(data_fd, BLKDISCARD, &range);
                     if (r != 0)
                     {
-                        fprintf(stderr, "Failed to discard %ju+%ju bytes on %s: %s (code %d)\n",
+                        fprintf(stderr, "Failed to execute BLKDISCARD %ju+%ju on %s: %s (code %d)\n",
                             range[0], range[1], data_device.c_str(), strerror(-r), r);
-                        return r;
+                        return -errno;
                     }
                     discarded += range[1];
                 }

src/blockstore/blockstore_disk.h

@@ -24,8 +24,6 @@
 
 class allocator_t;
 
-class disk_i;
-
 struct blockstore_disk_t
 {
     std::string data_device, meta_device, journal_device;
@@ -68,8 +66,6 @@ struct blockstore_disk_t
     uint64_t discard_granularity = 0;
 
     int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    disk_i *meta_disk = NULL, *data_disk = NULL, *journal_disk = NULL;
-    bool meta_off_ring = false, data_off_ring = false, journal_off_ring = false;
     uint64_t meta_offset = 0, meta_device_sect = 0, meta_device_size = 0, meta_area_size = 0, min_meta_len = 0;
     uint64_t data_offset = 0, data_device_sect = 0, data_device_size = 0, data_len = 0;
     uint64_t journal_offset = 0, journal_device_sect = 0, journal_device_size = 0, journal_len = 0;

src/blockstore/blockstore_flush.cpp

@@ -37,18 +37,18 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
 journal_flusher_co::journal_flusher_co()
 {
     wait_state = 0;
-    simple_callback_r = [this](int res)
+    simple_callback_r = [this](ring_data_t* data)
     {
         bs->live = true;
-        if (res < 0)
-            bs->disk_error_abort("read operation during flush", res);
+        if (data->res != data->iov.iov_len)
+            bs->disk_error_abort("read operation during flush", data->res, data->iov.iov_len);
         wait_count--;
     };
-    simple_callback_w = [this](int res)
+    simple_callback_w = [this](ring_data_t* data)
     {
         bs->live = true;
-        if (res < 0)
-            bs->disk_error_abort("write operation during flush", res);
+        if (data->res != data->iov.iov_len)
+            bs->disk_error_abort("write operation during flush", data->res, data->iov.iov_len);
         wait_count--;
     };
 }
@@ -161,8 +161,19 @@ void journal_flusher_t::loop()
         bs->ringloop->wakeup();
 }
 
+#define await_sqe(label) \
+    resume_##label:\
+        sqe = bs->get_sqe();\
+        if (!sqe)\
+        {\
+            wait_state = wait_base+label;\
+            return false;\
+        }\
+        data = ((ring_data_t*)sqe->user_data);
+
 bool journal_flusher_co::loop()
 {
+    int wait_base = 0;
     // This is much better than implementing the whole function as an FSM
     // Maybe I should consider a coroutine library like https://github.com/hnes/libaco ...
     // Or just C++ coroutines, but they require some wrappers
@@ -202,25 +213,18 @@ resume_0:
     res = bs->heap->get_next_compact(cur_oid);
     if ((bs->intent_write_counter >= bs->journal_trim_interval || flusher->force_fsync > 0) && co_id == 0)
     {
+        // Advance fsynced_lsn every <journal_trim_interval> intent writes
+        bs->intent_write_counter = 0;
 resume_17:
 resume_18:
 resume_19:
         if (!fsync_buffer(17))
-        {
             return false;
-        }
-        if (bs->intent_write_counter >= bs->journal_trim_interval)
-        {
-            // Advance fsynced_lsn every <journal_trim_interval> intent writes
-            bs->intent_write_counter = 0;
 resume_29:
 resume_30:
 resume_31:
-            if (!trim_lsn(29))
-            {
-                return false;
-            }
-        }
+        if (!trim_lsn(29))
+            return false;
     }
     if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
         (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))
@@ -454,22 +458,13 @@ resume_9:
         if ((read_vec[i].copy_flags & COPY_BUF_JOURNAL) &&
             !(read_vec[i].copy_flags & COPY_BUF_COALESCED))
         {
-resume_10:
             assert(read_vec[i].buf);
-            assert(clean_loc + read_vec[i].offset + read_vec[i].len <= bs->dsk.block_count*bs->dsk.data_block_size);
-            res = bs->dsk.data_disk->write_async(
-                bs->dsk.data_offset + clean_loc + read_vec[i].offset,
-                (size_t)read_vec[i].len,
-                read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
-                    ? read_vec[i].offset - read_vec[i].disk_offset : 0),
-                simple_callback_w
-            );
-            if (res == -EAGAIN)
-            {
-                wait_state = 10;
-                return false;
-            }
-            assert(res == 0);
+            await_sqe(10);
+            data->iov = (struct iovec){ read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
+                ? read_vec[i].offset - read_vec[i].disk_offset : 0), (size_t)read_vec[i].len };
+            data->callback = simple_callback_w;
+            assert(clean_loc + read_vec[i].offset + data->iov.iov_len <= bs->dsk.block_count*bs->dsk.data_block_size);
+            io_uring_prep_writev(sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + read_vec[i].offset);
             wait_count++;
         }
     }
@@ -577,7 +572,7 @@ void journal_flusher_co::fill_partial_checksum_blocks()
                 .disk_loc = clean_loc,
                 .disk_offset = blk_begin,
                 .disk_len = blk_end - blk_begin,
-                .buf = (uint8_t*)bs->dsk.data_disk->alloc_iomem(blk_end - blk_begin),
+                .buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end - blk_begin),
                 .wr = compact_info.clean_wr,
             });
         }
@@ -602,8 +597,7 @@ void journal_flusher_co::free_buffers()
         if (it->buf && !(it->copy_flags & COPY_BUF_COALESCED) &&
             (!bs->dsk.inmemory_journal || it->buf < bs->buffer_area || it->buf >= (uint8_t*)bs->buffer_area + bs->dsk.journal_len))
         {
-            disk_i *di = ((it->copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_disk : bs->dsk.data_disk);
-            di->free_iomem(it->buf);
+            free(it->buf);
         }
     }
     read_vec.clear();
@@ -754,8 +748,6 @@ resume_1:
 
 bool journal_flusher_co::read_buffered(int wait_base)
 {
-    disk_i *di = NULL;
-    copy_buffer_t *vec = NULL;
     if (wait_state == wait_base)
         goto resume_0;
     else if (wait_state == wait_base+1)
@@ -773,24 +765,19 @@ bool journal_flusher_co::read_buffered(int wait_base)
         if (!(read_vec[i].copy_flags & COPY_BUF_COALESCED) &&
             ((read_vec[i].copy_flags & COPY_BUF_JOURNAL) && !bs->dsk.inmemory_journal || (read_vec[i].copy_flags & COPY_BUF_DATA)))
         {
-resume_0:
-            vec = &read_vec[i];
-            di = ((vec->copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_disk : bs->dsk.data_disk);
-            if (!vec->buf)
-                vec->buf = (uint8_t*)di->alloc_iomem(vec->disk_len);
-            res = di->read_async(
-                ((vec->copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_offset : bs->dsk.data_offset) + vec->disk_loc + vec->disk_offset,
-                (size_t)vec->disk_len,
-                vec->buf,
-                simple_callback_r
-            );
-            if (res == -EAGAIN)
-            {
-                wait_state = wait_base+0;
-                return false;
-            }
-            assert(!res);
+            await_sqe(0);
+            auto & vec = read_vec[i];
+            if (!vec.buf)
+                vec.buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, vec.disk_len);
+            data->iov = (struct iovec){ vec.buf, (size_t)vec.disk_len };
             wait_count++;
+            io_uring_prep_readv(
+                sqe,
+                (vec.copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_fd : bs->dsk.data_fd,
+                &data->iov, 1,
+                ((vec.copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_offset : bs->dsk.data_offset) + vec.disk_loc + vec.disk_offset
+            );
+            data->callback = simple_callback_r;
         }
     }
     // Wait for reads/writes if the journal is not inmemory
@@ -833,28 +820,17 @@ resume_0:
     if (!cur_sync->sent)
     {
         // Sync batch is ready. Do it.
-resume_1:
-        res = bs->dsk.data_disk->flush_async(simple_callback_w);
-        if (res == -EAGAIN)
-        {
-            wait_state = wait_base+1;
-            return false;
-        }
-        else if (res == -EALREADY)
-        {
-            cur_sync->sent = true;
-        }
-        else
-        {
-            assert(res == 0);
-            cur_sync->sent = true;
-            wait_count++;
+        await_sqe(1);
+        data->iov = { 0 };
+        data->callback = simple_callback_w;
+        io_uring_prep_fsync(sqe, bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
+        cur_sync->sent = true;
+        wait_count++;
 resume_2:
-            if (wait_count > 0)
-            {
-                wait_state = wait_base+2;
-                return false;
-            }
+        if (wait_count > 0)
+        {
+            wait_state = wait_base+2;
+            return false;
         }
         cur_sync->done = true;
         // Wake up other flushers
@@ -892,26 +868,16 @@ resume_0:
     }
     flusher->fsyncing_meta = true;
     // Sync batch is ready. Do it.
-resume_1:
-    res = bs->dsk.meta_disk->flush_async(simple_callback_w);
-    if (res == -EAGAIN)
-    {
-        wait_state = wait_base+1;
-        return false;
-    }
-    else if (res == -EALREADY)
-    {
-    }
-    else
-    {
-        assert(res == 0);
-        wait_count++;
+    await_sqe(1);
+    data->iov = { 0 };
+    data->callback = simple_callback_w;
+    io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
+    wait_count++;
 resume_2:
-        if (wait_count > 0)
-        {
-            wait_state = wait_base+2;
-            return false;
-        }
+    if (wait_count > 0)
+    {
+        wait_state = wait_base+2;
+        return false;
     }
     // Sync completed. All previous coroutines waiting for it must be resumed
     flusher->fsyncing_meta = false;
@@ -939,7 +905,7 @@ resume_0:
 resume_1:
     assert(!wait_count);
     fsynced_lsn = bs->heap->get_completed_lsn();
-    if (bs->submit_fsyncs(wait_count) != 0)
+    if (!bs->submit_fsyncs(wait_count))
     {
         wait_state = wait_base+1;
         return false;
@@ -978,16 +944,12 @@ resume_0:
     flusher->active_flushers++;
     ((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn = fsynced_lsn;
     ((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c();
-    // Update superblock with FUA (datasync)
-resume_1:
-    res = bs->dsk.meta_disk->write_async(bs->dsk.meta_offset, bs->dsk.meta_block_size,
-        bs->meta_superblock, simple_callback_w, true);
-    if (res == -EAGAIN)
-    {
-        wait_state = wait_base+1;
-        return false;
-    }
-    assert(res == 0);
+    await_sqe(1);
+    data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
+    data->callback = simple_callback_w;
+    io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
+    // Update superblock with datasync
+    sqe->rw_flags = RWF_DSYNC;
     wait_count++;
 resume_2:
     if (wait_count > 0)

src/blockstore/blockstore_flush.h

@@ -43,12 +43,14 @@ class journal_flusher_co
     journal_flusher_t *flusher;
     int co_id;
     int wait_state, wait_count;
+    struct io_uring_sqe *sqe;
+    struct ring_data_t *data;
     uint8_t *new_csums = NULL;
     uint8_t *new_bmp = NULL;
     uint8_t *punch_bmp = NULL;
     uint8_t *new_ext_bmp = NULL;
 
-    std::function<void(int)> simple_callback_r, simple_callback_w;
+    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
 
     object_id cur_oid;
     heap_entry_t *cur_obj;

src/blockstore/blockstore_heap.cpp

@@ -2436,11 +2436,9 @@ heap_compact_t blockstore_heap_t::iterate_compaction(heap_entry_t *obj, uint64_t
             res.do_delete = false;
             res.compact_lsn = 0;
             res.compact_version = 0;
-            if ((!under_pressure || stable) && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT || wr->type() == BS_HEAP_DELETE))
+            if (!under_pressure && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_DELETE))
             {
                 // We may postpone compaction if we have an unstable overwrite when not under pressure
-                // We MUST postpone if the overwrite is STABLE (even if under pressure), 
-                // because add_compact will reject STABLE overwrites with EBUSY.
                 return res;
             }
             continue;

src/blockstore/blockstore_impl.cpp

@@ -30,15 +30,12 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
         dsk.close_all();
         throw;
     }
-    dsk.data_disk->set_ringloop(ringloop);
-    dsk.meta_disk->set_ringloop(ringloop);
-    dsk.journal_disk->set_ringloop(ringloop);
-    meta_superblock = (uint8_t*)dsk.meta_disk->alloc_iomem(dsk.meta_block_size);
+    meta_superblock = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     memset(meta_superblock, 0, dsk.meta_block_size);
     flusher = new journal_flusher_t(this);
     if (dsk.inmemory_journal)
     {
-        buffer_area = (uint8_t*)dsk.journal_disk->alloc_iomem(dsk.journal_len);
+        buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
     }
     heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
     ringloop->wakeup();
@@ -69,9 +66,9 @@ blockstore_impl_t::~blockstore_impl_t()
     if (heap)
         delete heap;
     if (buffer_area)
-        dsk.journal_disk->free_iomem(buffer_area);
+        free(buffer_area);
     if (meta_superblock)
-        dsk.meta_disk->free_iomem(meta_superblock);
+        free(meta_superblock);
     ringloop->unregister_consumer(&ring_consumer);
     dsk.close_all();
 }
@@ -130,9 +127,9 @@ void blockstore_impl_t::loop()
             if (PRIV(op)->wait_for)
             {
                 check_wait(op);
-                if (PRIV(op)->wait_for != WAIT_COMPACTION)
+                if (PRIV(op)->wait_for == WAIT_SQE)
                 {
-                    // disk queue(s) are full, stop submission
+                    // ring is full, stop submission
                     break;
                 }
                 else if (PRIV(op)->wait_for)
@@ -143,6 +140,7 @@ void blockstore_impl_t::loop()
                     continue;
                 }
             }
+            unsigned prev_sqe_pos = ringloop->save();
             // 0 = can't submit
             // 1 = in progress
             // 2 = can be removed from queue
@@ -191,7 +189,8 @@ void blockstore_impl_t::loop()
             if (wr_st == 0)
             {
                 PRIV(op)->pending_ops = 0;
-                if (PRIV(op)->wait_for != WAIT_COMPACTION)
+                ringloop->restore(prev_sqe_pos);
+                if (PRIV(op)->wait_for == WAIT_SQE)
                 {
                     // ring is full, stop submission
                     break;
@@ -262,34 +261,13 @@ bool blockstore_impl_t::is_safe_to_stop()
 
 void blockstore_impl_t::check_wait(blockstore_op_t *op)
 {
-    if (PRIV(op)->wait_for == WAIT_DATA_QUEUE)
+    if (PRIV(op)->wait_for == WAIT_SQE)
     {
-        if (dsk.data_disk->get_queue_space() < PRIV(op)->wait_detail)
+        if (ringloop->space_left() < PRIV(op)->wait_detail)
         {
+            // stop submission if there's still no free space
 #ifdef BLOCKSTORE_DEBUG
-            printf("Still waiting for %ju free data disk queue slots\n", PRIV(op)->wait_detail);
-#endif
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else if (PRIV(op)->wait_for == WAIT_META_QUEUE)
-    {
-        if (dsk.meta_disk->get_queue_space() < PRIV(op)->wait_detail)
-        {
-#ifdef BLOCKSTORE_DEBUG
-            printf("Still waiting for %ju free meta disk queue slots\n", PRIV(op)->wait_detail);
-#endif
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else if (PRIV(op)->wait_for == WAIT_JOURNAL_QUEUE)
-    {
-        if (dsk.journal_disk->get_queue_space() < PRIV(op)->wait_detail)
-        {
-#ifdef BLOCKSTORE_DEBUG
-            printf("Still waiting for %ju free journal disk queue slots\n", PRIV(op)->wait_detail);
+            printf("Still waiting for %ju SQE(s)\n", PRIV(op)->wait_detail);
 #endif
             return;
         }
@@ -426,7 +404,7 @@ void blockstore_meta_header_v3_t::set_crc32c()
     header_csum = calc;
 }
 
-void blockstore_impl_t::disk_error_abort(const char *op, int retval)
+void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expected)
 {
     if (retval == -EAGAIN)
     {
@@ -435,7 +413,7 @@ void blockstore_impl_t::disk_error_abort(const char *op, int retval)
             " Please upgrade your kernel. Aborting.\n", op);
         exit(1);
     }
-    fprintf(stderr, "Disk %s failed: %s (code %d). Can't continue, sorry :-(\n", op, strerror(-retval), retval);
+    fprintf(stderr, "Disk %s failed: result is %d, expected %d. Can't continue, sorry :-(\n", op, retval, expected);
     exit(1);
 }
 

src/blockstore/blockstore_impl.h

@@ -20,7 +20,6 @@
 #include <unordered_map>
 #include <unordered_set>
 
-#include "disk_util.h"
 #include "malloc_or_die.h"
 
 class blockstore_impl_t;
@@ -133,16 +132,21 @@ public:
 
     bool stop_sync_submitted = false;
 
+    inline struct io_uring_sqe* get_sqe()
+    {
+        return ringloop->get_sqe();
+    }
+
     void open_data();
     void open_meta();
     void open_journal();
 
-    void disk_error_abort(const char *op, int retval);
+    void disk_error_abort(const char *op, int retval, int expected);
 
     // Asynchronous init
-    int initialized = 0;
-    uint64_t metadata_buf_size = 0;
-    blockstore_init_meta* metadata_init_reader = NULL;
+    int initialized;
+    int metadata_buf_size;
+    blockstore_init_meta* metadata_init_reader;
 
     void check_wait(blockstore_op_t *op);
     void init_op(blockstore_op_t *op);
@@ -159,7 +163,7 @@ public:
     void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
         std::function<void(int&, uint32_t, uint32_t)> callback);
     void free_read_buffers(std::vector<copy_buffer_t> & rv);
-    void handle_read_event(int res, blockstore_op_t *op);
+    void handle_read_event(ring_data_t *data, blockstore_op_t *op);
     bool verify_read_checksums(blockstore_op_t *op);
 
     // Write
@@ -169,11 +173,11 @@ public:
     bool intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj);
     int dequeue_write(blockstore_op_t *op);
     int continue_write(blockstore_op_t *op);
-    void handle_write_event(int res, blockstore_op_t *op);
+    void handle_write_event(ring_data_t *data, blockstore_op_t *op);
 
     // Sync
     int continue_sync(blockstore_op_t *op);
-    int submit_fsyncs(int & wait_count);
+    bool submit_fsyncs(int & wait_count);
     int do_sync(blockstore_op_t *op, int base_state);
     bool has_unsynced();
 

src/blockstore/blockstore_init.cpp

@@ -11,6 +11,12 @@
 #define INIT_META_READING 1
 #define INIT_META_READ_DONE 2
 
+#define GET_SQE() \
+    sqe = bs->get_sqe();\
+    if (!sqe)\
+        throw std::runtime_error("io_uring is full during initialization");\
+    data = ((ring_data_t*)sqe->user_data)
+
 struct blockstore_recheck_io_t
 {
     bool is_data;
@@ -30,13 +36,14 @@ blockstore_init_meta::blockstore_init_meta(blockstore_impl_t *bs)
     this->bs = bs;
 }
 
-void blockstore_init_meta::handle_event(int res, int buf_num, const char *op)
+void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num, const char *op)
 {
-    if (res < 0)
+    if (data->res != data->iov.iov_len)
     {
         throw std::runtime_error(strprintf(
-            "%s failed at offset %ju: %s (code %d)",
-            op, (buf_num >= 0 ? bufs[buf_num].offset : last_read_offset), strerror(-res), res
+            "%s failed at offset %ju: got %s (code %d), but expected %zu",
+            op, (buf_num >= 0 ? bufs[buf_num].offset : last_read_offset), strerror(-data->res),
+            data->res, data->iov.iov_len
         ));
     }
     if (buf_num >= 0)
@@ -60,16 +67,16 @@ int blockstore_init_meta::loop()
     else if (wait_state == 7) goto resume_7;
     else if (wait_state == 8) goto resume_8;
     else if (wait_state == 9) goto resume_9;
-    if (bs->metadata_buf_size > bs->dsk.meta_disk->get_max_io_size())
-        bs->metadata_buf_size = bs->dsk.meta_disk->get_max_io_size();
-    metadata_buffer = bs->dsk.meta_disk->alloc_iomem(2*bs->metadata_buf_size);
+    metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size);
     if (!metadata_buffer)
         throw std::runtime_error("Failed to allocate metadata read buffer");
     // Read metadata superblock
+    GET_SQE();
     last_read_offset = 0;
-    res = bs->dsk.meta_disk->read_async(bs->dsk.meta_offset, bs->dsk.meta_block_size, bs->meta_superblock,
-        [this](int res) { handle_event(res, -1, "read metadata header"); });
-    assert(res == 0);
+    data->iov = { bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
+    data->callback = [this](ring_data_t *data) { handle_event(data, -1, "read metadata header"); };
+    io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
+    bs->ringloop->submit();
     submitted++;
 resume_1:
     if (submitted > 0)
@@ -158,16 +165,16 @@ resume_1:
         md_offset = 0;
         while (md_offset < bs->dsk.journal_len)
         {
-            submit_len = (bs->dsk.journal_len - md_offset < bs->metadata_buf_size ? bs->dsk.journal_len - md_offset : bs->metadata_buf_size);
-            last_read_offset = md_offset;
-            res = bs->dsk.journal_disk->read_async(
-                bs->dsk.journal_offset + md_offset, (size_t)submit_len,
+            GET_SQE();
+            data->iov = (iovec){
                 bs->buffer_area + md_offset,
-                [this](int res) { handle_event(res, -1, "read buffer area"); }
-            );
-            assert(res == 0);
-            md_offset += submit_len;
+                (size_t)(bs->dsk.journal_len - md_offset < bs->metadata_buf_size ? bs->dsk.journal_len - md_offset : bs->metadata_buf_size),
+            };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1, "read buffer area"); };
+            io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->dsk.journal_offset + md_offset);
+            md_offset += data->iov.iov_len;
             submitted++;
+            bs->ringloop->submit();
 resume_3:
             if (submitted > 0)
             {
@@ -194,31 +201,26 @@ resume_4:
                 bufs[i].size = bs->dsk.meta_area_size-next_offset > bs->metadata_buf_size
                     ? bs->metadata_buf_size : bs->dsk.meta_area_size-next_offset;
                 bufs[i].state = INIT_META_READING;
+                submitted++;
+                next_offset += bufs[i].size;
+                GET_SQE();
                 assert(bufs[i].size <= 0x7fffffff);
+                data->iov = { bufs[i].buf, (size_t)bufs[i].size };
                 if (!zero_on_init)
                 {
-                    res = bs->dsk.meta_disk->read_async(
-                        bs->dsk.meta_offset + bufs[i].offset,
-                        (size_t)bufs[i].size, bufs[i].buf,
-                        [this, i](int res) { handle_event(res, i, "read metadata"); }
-                    );
-                    assert(res == 0);
+                    data->callback = [this, i](ring_data_t *data) { handle_event(data, i, "read metadata"); };
+                    io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 }
                 else
                 {
                     // Fill metadata with empty block pattern
+                    data->callback = [this, i](ring_data_t *data) { handle_event(data, i, "clear metadata"); };
                     memset(bufs[i].buf, 0, bufs[i].size);
                     for (uint64_t o = 0; o < bufs[i].size; o += bs->dsk.meta_block_size)
                         bs->heap->fill_block_empty_space(bufs[i].buf + o, 0);
-                    res = bs->dsk.meta_disk->write_async(
-                        bs->dsk.meta_offset + bufs[i].offset,
-                        (size_t)bufs[i].size, bufs[i].buf,
-                        [this, i](int res) { handle_event(res, i, "clear metadata"); }
-                    );
-                    assert(res == 0);
+                    io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 }
-                submitted++;
-                next_offset += bufs[i].size;
+                bs->ringloop->submit();
                 break;
             }
         }
@@ -264,27 +266,26 @@ resume_6:
     while (recheck_ios.size())
     {
         blockstore_recheck_io_t* rc = recheck_ios.back();
-        disk_i *di = (rc->is_data ? bs->dsk.data_disk : bs->dsk.journal_disk);
-        if (!rc->buf)
-            rc->buf = (uint8_t*)di->alloc_iomem(rc->len);
-        int res = di->read_async(
-            (rc->is_data ? bs->dsk.data_offset : bs->dsk.journal_offset) + rc->offset,
-            rc->len, rc->buf, [this, rc](int res)
-            {
-                if (res < 0)
-                {
-                    fprintf(stderr, "Buffer area read failed at offset %ju: %d\n", rc->offset, res);
-                    exit(1);
-                }
-                rc->cb(rc->buf);
-                disk_i *di = (rc->is_data ? bs->dsk.data_disk : bs->dsk.journal_disk);
-                di->free_iomem(rc->buf);
-                delete rc;
-            }
-        );
-        if (res == -EAGAIN)
+        sqe = bs->get_sqe();
+        if (!sqe)
             break;
-        assert(res == 0);
+        if (!rc->buf)
+            rc->buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, rc->len);
+        data = ((ring_data_t*)sqe->user_data);
+        data->iov = (iovec){ rc->buf, rc->len };
+        data->callback = [this, rc](ring_data_t *data)
+        {
+            if (data->res < 0)
+            {
+                fprintf(stderr, "Buffer area read failed at offset %ju: %d\n", rc->offset, data->res);
+                exit(1);
+            }
+            rc->cb(rc->buf);
+            free(rc->buf);
+            delete rc;
+        };
+        io_uring_prep_readv(sqe, (rc->is_data ? bs->dsk.data_fd : bs->dsk.journal_fd), &data->iov, 1,
+            (rc->is_data ? bs->dsk.data_offset : bs->dsk.journal_offset) + rc->offset);
         recheck_ios.pop_back();
     }
     return 1;
@@ -306,35 +307,30 @@ resume_7:
     for (i = 0; i < recheck_mod.size(); i++)
     {
 resume_8:
-        if (wait_count >= bs->meta_write_recheck_parallelism)
+        if (wait_count >= bs->meta_write_recheck_parallelism || !(sqe = bs->get_sqe()))
         {
+            bs->ringloop->submit();
             wait_state = 8;
             return 1;
         }
         uint32_t block_num = recheck_mod[i];
         uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
-        uint8_t *buf = (uint8_t*)bs->dsk.meta_disk->alloc_iomem(bs->dsk.meta_block_size);
+        data = ((ring_data_t*)sqe->user_data);
+        uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size);
         bs->heap->get_meta_block(block_num, buf);
-        res = bs->dsk.meta_disk->write_async(block_offset, bs->dsk.meta_block_size, buf,
-            [this, buf, block_offset](int res)
-            {
-                wait_count--;
-                bs->dsk.meta_disk->free_iomem(buf);
-                if (res < 0)
-                {
-                    throw std::runtime_error(
-                        "write metadata failed at offset " + std::to_string(block_offset) + ": " + strerror(-res)
-                    );
-                }
-            }
-        );
-        if (res == -EAGAIN)
+        data->iov = { buf, bs->dsk.meta_block_size };
+        data->callback = [this, buf, block_offset](ring_data_t *data)
         {
-            bs->dsk.meta_disk->free_iomem(buf);
-            wait_state = 8;
-            return 1;
-        }
-        assert(!res);
+            wait_count--;
+            free(buf);
+            if (data->res != bs->dsk.meta_block_size)
+            {
+                throw std::runtime_error(
+                    "write metadata failed at offset " + std::to_string(block_offset) + ": " + strerror(-data->res)
+                );
+            }
+        };
+        io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, block_offset);
         wait_count++;
     }
 resume_9:
@@ -344,18 +340,18 @@ resume_9:
         wait_state = 9;
         return 1;
     }
-    bs->dsk.meta_disk->free_iomem(metadata_buffer);
+    free(metadata_buffer);
     metadata_buffer = NULL;
 do_fsync:
     if (!bs->dsk.disable_meta_fsync && !bs->readonly)
     {
+        GET_SQE();
+        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
         last_read_offset = 0;
-        res = bs->dsk.meta_disk->flush_async([this](int res) { handle_event(res, -1, "fsync metadata"); });
-        if (res != -EALREADY)
-        {
-            assert(res == 0);
-            submitted++;
-        }
+        data->iov = { 0 };
+        data->callback = [this](ring_data_t *data) { handle_event(data, -1, "fsync metadata"); };
+        submitted++;
+        bs->ringloop->submit();
     resume_5:
         if (submitted > 0)
         {
@@ -365,13 +361,13 @@ do_fsync:
     }
     if (zero_on_init && !header_written && !bs->readonly)
     {
-        last_read_offset = 0;
-        res = bs->dsk.meta_disk->write_async(
-            bs->dsk.meta_offset, (size_t)bs->dsk.meta_block_size, bs->meta_superblock,
-            [this](int res) { handle_event(res, -1, "write metadata header"); }
-        );
-        assert(res == 0);
+        GET_SQE();
         header_written = true;
+        last_read_offset = 0;
+        data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
+        data->callback = [this](ring_data_t *data) { handle_event(data, -1, "write metadata header"); };
+        io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
+        bs->ringloop->submit();
         submitted++;
     resume_2:
         if (submitted > 0)

src/blockstore/blockstore_init.h

@@ -21,19 +21,19 @@ class blockstore_init_meta
     bool zero_on_init = false;
     bool header_written = false;
     void *metadata_buffer = NULL;
-    int res = 0;
     blockstore_init_meta_buf bufs[2] = {};
     int submitted = 0;
+    struct io_uring_sqe *sqe;
+    struct ring_data_t *data;
     uint64_t md_offset = 0;
     uint64_t next_offset = 0;
     uint64_t last_read_offset = 0;
     uint64_t entries_loaded = 0;
-    uint64_t submit_len = 0;
     std::vector<uint32_t> recheck_mod;
     std::vector<blockstore_recheck_io_t*> recheck_ios;
     int i = 0, j = 0;
     bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
-    void handle_event(int res, int buf_num, const char *op);
+    void handle_event(ring_data_t *data, int buf_num, const char *op);
 public:
     blockstore_init_meta(blockstore_impl_t *bs);
     int loop();

src/blockstore/blockstore_internal.h

@@ -3,40 +3,44 @@
 
 #pragma once
 
-#define BS_SUBMIT_CHECK_JOURNAL_QUEUE(n) {\
-    if (dsk.journal_disk->get_queue_space() < (n))\
+#define BS_SUBMIT_CHECK_SQES(n) \
+    if (ringloop->space_left() < (n))\
     {\
         /* Pause until there are more requests available */\
         PRIV(op)->wait_detail = (n);\
-        PRIV(op)->wait_for = WAIT_JOURNAL_QUEUE;\
+        PRIV(op)->wait_for = WAIT_SQE;\
         return 0;\
-    } }
+    }
 
-#define BS_SUBMIT_CHECK_DATA_QUEUE(n) {\
-    if (dsk.data_disk->get_queue_space() < (n))\
-    {\
-        /* Pause until there are more requests available */\
-        PRIV(op)->wait_detail = (n);\
-        PRIV(op)->wait_for = WAIT_DATA_QUEUE;\
-        return 0;\
-    } }
+#define BS_SUBMIT_GET_SQE(sqe, data) \
+    BS_SUBMIT_GET_ONLY_SQE(sqe); \
+    struct ring_data_t *data = ((ring_data_t*)sqe->user_data)
 
-#define BS_SUBMIT_CHECK_META_QUEUE(n) {\
-    if (dsk.meta_disk->get_queue_space() < (n))\
+#define BS_SUBMIT_GET_ONLY_SQE(sqe) \
+    struct io_uring_sqe *sqe = get_sqe();\
+    if (!sqe)\
     {\
         /* Pause until there are more requests available */\
-        PRIV(op)->wait_detail = (n);\
-        PRIV(op)->wait_for = WAIT_META_QUEUE;\
+        PRIV(op)->wait_detail = 1;\
+        PRIV(op)->wait_for = WAIT_SQE;\
         return 0;\
-    } }
+    }
+
+#define BS_SUBMIT_GET_SQE_DECL(sqe) \
+    sqe = get_sqe();\
+    if (!sqe)\
+    {\
+        /* Pause until there are more requests available */\
+        PRIV(op)->wait_detail = 1;\
+        PRIV(op)->wait_for = WAIT_SQE;\
+        return 0;\
+    }
 
 #define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
 #define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
 
 // Suspend operation until there are more free SQEs
-#define WAIT_DATA_QUEUE 1
-#define WAIT_META_QUEUE 2
-#define WAIT_JOURNAL_QUEUE 3
+#define WAIT_SQE 1
 // Suspend operation until there are <wait_detail> bytes of free space in the journal on disk
 #define WAIT_COMPACTION 2
 // Suspend operation until LSN <wait_detail> becomes durable
@@ -61,3 +65,11 @@
 #define COPY_BUF_COALESCED  0x10
 #define COPY_BUF_PADDED     0x20
 #define COPY_BUF_SKIP_CSUM  0x40
+
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC 0x40
+#endif
+
+#ifndef RWF_DSYNC
+#define RWF_DSYNC 0x02
+#endif

src/blockstore/blockstore_read.cpp

@@ -88,7 +88,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     if (need_wait)
     {
 undo_wait:
-        // Unlock LSN and wait
+        // Need to wait. undo added requests, unlock lsn
         heap->unlock_entry(op->oid);
         free_read_buffers(rv);
         rv.clear();
@@ -106,13 +106,12 @@ undo_wait:
     assert(fulfilled == op->len);
     if (!fulfill_read(op))
     {
-        // Not enough SQEs. Unlock lsn and wait
         goto undo_wait;
     }
     op->version = result_version;
     if (!PRIV(op)->pending_ops)
     {
-        // Everything is fulfilled from memory
+        // everything is fulfilled from memory
         heap->unlock_entry(op->oid);
         op->retval = op->len;
         free_read_buffers(rv);
@@ -126,24 +125,6 @@ undo_wait:
 
 int blockstore_impl_t::fulfill_read(blockstore_op_t *op)
 {
-    int ndata = 0, njournal = 0;
-    uint32_t nodsk = (COPY_BUF_COALESCED|COPY_BUF_CSUM_FILL|COPY_BUF_ZERO) |
-        (dsk.inmemory_journal ? COPY_BUF_JOURNAL : 0);
-    for (auto & vec: PRIV(op)->read_vec)
-    {
-        if (!(vec.copy_flags & nodsk))
-        {
-            auto & n = (vec.copy_flags & COPY_BUF_JOURNAL) ? njournal : ndata;
-            n++;
-        }
-    }
-    if (!dsk.data_off_ring && !dsk.journal_off_ring || dsk.data_disk == dsk.journal_disk)
-    {
-        ndata += njournal;
-        njournal = 0;
-    }
-    BS_SUBMIT_CHECK_DATA_QUEUE(ndata);
-    BS_SUBMIT_CHECK_JOURNAL_QUEUE(njournal);
     for (auto & vec: PRIV(op)->read_vec)
     {
         if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_CSUM_FILL))
@@ -161,14 +142,16 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *op)
         }
         else
         {
-            disk_i *di = ((vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_disk : dsk.data_disk);
-            int res = di->read_async(
-                ((vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_offset : dsk.data_offset) + vec.disk_loc + vec.disk_offset,
-                vec.disk_len, vec.buf ? vec.buf : (op->buf + vec.offset - op->offset),
-                [this, op](int res) { handle_read_event(res, op); }
-            );
-            assert(res == 0);
+            BS_SUBMIT_GET_SQE(sqe, data);
+            data->iov = (struct iovec){ vec.buf ? vec.buf : (op->buf + vec.offset - op->offset), (size_t)vec.disk_len };
             PRIV(op)->pending_ops++;
+            io_uring_prep_readv(
+                sqe,
+                (vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_fd : dsk.data_fd,
+                &data->iov, 1,
+                ((vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_offset : dsk.data_offset) + vec.disk_loc + vec.disk_offset
+            );
+            data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); };
         }
     }
     return 1;
@@ -349,7 +332,7 @@ void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec,
         }
         else
         {
-            vec.buf = (uint8_t*)((vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_disk : dsk.data_disk)->alloc_iomem(vec.disk_len);
+            vec.buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, vec.disk_len);
         }
     }
     read_vec.insert(read_vec.begin() + pos, vec);
@@ -400,22 +383,21 @@ void blockstore_impl_t::free_read_buffers(std::vector<copy_buffer_t> & rv)
             if (!(vec.copy_flags & COPY_BUF_COALESCED) && vec.buf &&
                 (!buffer_area || vec.buf < buffer_area || vec.buf >= (uint8_t*)buffer_area + dsk.journal_len))
             {
-                disk_i *di = (vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_disk : dsk.data_disk;
-                di->free_iomem(vec.buf);
+                free(vec.buf);
                 vec.buf = NULL;
             }
         }
     }
 }
 
-void blockstore_impl_t::handle_read_event(int res, blockstore_op_t *op)
+void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op)
 {
     live = true;
     PRIV(op)->pending_ops--;
-    if (res < 0)
+    if (data->res != data->iov.iov_len)
     {
         // read error
-        op->retval = res;
+        op->retval = data->res;
     }
     if (PRIV(op)->pending_ops == 0)
     {

src/blockstore/blockstore_stable.cpp

@@ -86,7 +86,7 @@ resume_1:
         if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
         {
             priv->op_state = 1;
-            BS_SUBMIT_CHECK_META_QUEUE(1);
+            BS_SUBMIT_CHECK_SQES(1);
             prepare_meta_block_write(priv->modified_block);
 resume_2:
             if (meta_block_is_pending(priv->modified_block))

src/blockstore/blockstore_sync.cpp

@@ -27,72 +27,71 @@ bool blockstore_impl_t::has_unsynced()
     return data || buffer || meta;
 }
 
-int blockstore_impl_t::submit_fsyncs(int & wait_count)
+bool blockstore_impl_t::submit_fsyncs(int & wait_count)
 {
-    int nm = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync ? 1 : 0);
-    int nj = (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync &&
-        (!unsynced_meta_write_count || dsk.journal_disk != dsk.meta_disk) ? 1 : 0);
-    int nd = (unsynced_data_write_count > 0 && !dsk.disable_data_fsync &&
-        (!unsynced_meta_write_count || dsk.data_disk != dsk.meta_disk) &&
-        (!unsynced_buffer_write_count || dsk.data_disk != dsk.journal_disk) ? 1 : 0);
-    if (!nm && !nj && !nd)
-        return 0;
-    int rnm = nm, rnj = nj, rnd = nd;
-    if (!dsk.meta_off_ring && !dsk.journal_off_ring)
-        rnm += rnj, rnj = 0;
-    if (!dsk.meta_off_ring && !dsk.data_off_ring)
-        rnm += rnd, rnd = 0;
-    if (!dsk.journal_off_ring && !dsk.data_off_ring)
-        rnj += rnd, rnd = 0;
-    if (rnm > 0 && dsk.meta_disk->get_queue_space() < rnm)
-        return WAIT_META_QUEUE;
-    if (rnj > 0 && dsk.journal_disk->get_queue_space() < rnj)
-        return WAIT_JOURNAL_QUEUE;
-    if (rnd > 0 && dsk.data_disk->get_queue_space() < rnd)
-        return WAIT_DATA_QUEUE;
-    auto cb = [this, & wait_count](int res)
+    int n = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync ? 1 : 0) +
+        (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync &&
+            (!unsynced_meta_write_count || dsk.journal_fd != dsk.meta_fd) ? 1 : 0) +
+        (unsynced_data_write_count > 0 && !dsk.disable_data_fsync &&
+            (!unsynced_meta_write_count || dsk.data_fd != dsk.meta_fd) &&
+            (!unsynced_buffer_write_count || dsk.data_fd != dsk.journal_fd) ? 1 : 0);
+    if (ringloop->space_left() < n)
     {
-        if (res != 0)
-            disk_error_abort("sync meta", res);
+        return false;
+    }
+    if (!n)
+    {
+        return true;
+    }
+    auto cb = [this, & wait_count](ring_data_t *data)
+    {
+        if (data->res != 0)
+            disk_error_abort("sync meta", data->res, 0);
         wait_count--;
         assert(wait_count >= 0);
         if (!wait_count)
             ringloop->wakeup();
     };
-    if (nm)
+    if (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync)
     {
         // fsync meta
-        int res = dsk.meta_disk->flush_async(cb);
-        if (res != -EALREADY)
-        {
-            assert(res == 0);
-            wait_count++;
-        }
+        io_uring_sqe *sqe = get_sqe();
+        assert(sqe);
+        ring_data_t *data = ((ring_data_t*)sqe->user_data);
+        io_uring_prep_fsync(sqe, dsk.meta_fd, IORING_FSYNC_DATASYNC);
+        data->iov = { 0 };
+        data->callback = cb;
+        wait_count++;
     }
-    if (nj)
+    if (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync &&
+        (!unsynced_meta_write_count || dsk.journal_fd != dsk.meta_fd))
     {
         // fsync buffer
-        int res = dsk.journal_disk->flush_async(cb);
-        if (res != -EALREADY)
-        {
-            assert(res == 0);
-            wait_count++;
-        }
+        io_uring_sqe *sqe = get_sqe();
+        assert(sqe);
+        ring_data_t *data = ((ring_data_t*)sqe->user_data);
+        io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC);
+        data->iov = { 0 };
+        data->callback = cb;
+        wait_count++;
     }
-    if (nd)
+    if (unsynced_data_write_count > 0 && !dsk.disable_data_fsync &&
+        (!unsynced_meta_write_count || dsk.data_fd != dsk.meta_fd) &&
+        (!unsynced_buffer_write_count || dsk.data_fd != dsk.journal_fd))
     {
         // fsync data
-        int res = dsk.data_disk->flush_async(cb);
-        if (res != -EALREADY)
-        {
-            assert(res == 0);
-            wait_count++;
-        }
+        io_uring_sqe *sqe = get_sqe();
+        assert(sqe);
+        ring_data_t *data = ((ring_data_t*)sqe->user_data);
+        io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC);
+        data->iov = { 0 };
+        data->callback = cb;
+        wait_count++;
     }
     unsynced_data_write_count = 0;
     unsynced_buffer_write_count = 0;
     unsynced_meta_write_count = 0;
-    return 0;
+    return true;
 }
 
 int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
@@ -114,10 +113,11 @@ int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
     }
     assert(!PRIV(op)->lsn);
     PRIV(op)->lsn = heap->get_completed_lsn();
-    PRIV(op)->wait_for = submit_fsyncs(PRIV(op)->pending_ops);
-    if (PRIV(op)->wait_for)
+    if (!submit_fsyncs(PRIV(op)->pending_ops))
     {
         PRIV(op)->lsn = 0;
+        PRIV(op)->wait_detail = 1;
+        PRIV(op)->wait_for = WAIT_SQE;
         return 0;
     }
 resume_1:

src/blockstore/blockstore_write.cpp

@@ -16,31 +16,32 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
 {
     if (modified_blocks.find(modified_block) != modified_blocks.end())
-    {
         return;
-    }
-    uint8_t *buf = (uint8_t*)dsk.meta_disk->alloc_iomem(dsk.meta_block_size);
-    assert(((uint64_t)modified_block+2)*dsk.meta_block_size <= dsk.meta_area_size);
-    int res = dsk.meta_disk->write_async(
-        dsk.meta_offset + ((uint64_t)modified_block+1)*dsk.meta_block_size, dsk.meta_block_size, buf,
-        [this, modified_block](int res)
+    io_uring_sqe *sqe = get_sqe();
+    assert(sqe != NULL);
+    ring_data_t *data = ((ring_data_t*)sqe->user_data);
+    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
+    data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size };
+    data->callback = [this, modified_block](ring_data_t *data)
+    {
+        live = true;
+        if (data->res != data->iov.iov_len)
         {
-            live = true;
-            if (res < 0)
-            {
-                // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
-                disk_error_abort("data write", res);
-            }
-            auto it = modified_blocks.find(modified_block);
-            assert(it != modified_blocks.end());
-            dsk.meta_disk->free_iomem(it->second.buf);
-            modified_blocks.erase(it);
-            unsynced_meta_write_count++;
-            heap->complete_block_write(modified_block);
-            ringloop->wakeup();
+            // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
+            disk_error_abort("data write", data->res, data->iov.iov_len);
         }
+        auto it = modified_blocks.find(modified_block);
+        assert(it != modified_blocks.end());
+        free(it->second.buf);
+        modified_blocks.erase(it);
+        unsynced_meta_write_count++;
+        heap->complete_block_write(modified_block);
+        ringloop->wakeup();
+    };
+    assert(((uint64_t)modified_block+2)*dsk.meta_block_size <= dsk.meta_area_size);
+    io_uring_prep_writev(
+        sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + ((uint64_t)modified_block+1)*dsk.meta_block_size
     );
-    assert(res == 0);
     pending_modified_blocks.push_back(modified_block);
     modified_blocks[modified_block] = { .sent = false, .buf = buf };
 }
@@ -134,7 +135,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
             return 2;
         }
         PRIV(op)->write_type = BS_HEAP_DELETE;
-        BS_SUBMIT_CHECK_META_QUEUE(1);
+        BS_SUBMIT_CHECK_SQES(1);
         int res = heap->add_delete(obj, &PRIV(op)->modified_block);
         assert(res != EAGAIN);
         if (res == ENOSPC)
@@ -151,7 +152,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         // Big (redirect) write
         PRIV(op)->write_type = dsk.disable_data_fsync || op->opcode != BS_OP_WRITE_STABLE ? BS_HEAP_BIG_WRITE : _REDIRECT_INTENT;
-        BS_SUBMIT_CHECK_DATA_QUEUE(1);
+        BS_SUBMIT_CHECK_SQES(1);
         PRIV(op)->location = heap->find_free_data();
         if (PRIV(op)->location == UINT64_MAX)
         {
@@ -176,12 +177,12 @@ enospc:
         );
 #endif
         heap->use_data(op->oid.inode, PRIV(op)->location);
+        io_uring_sqe *sqe = get_sqe();
+        ring_data_t *data = ((ring_data_t*)sqe->user_data);
+        data->iov = (struct iovec){ op->buf, op->len };
+        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
         assert(loc+op->offset+op->len <= dsk.block_count*dsk.data_block_size);
-        int res = dsk.data_disk->write_async(
-            dsk.data_offset + loc + op->offset, op->len, op->buf,
-            [this, op](int res) { handle_write_event(res, op); }
-        );
-        assert(res == 0);
+        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + loc + op->offset);
         if (!dsk.disable_data_fsync)
         {
             // use PRIV->lsn for fsync_data_id
@@ -198,7 +199,7 @@ enospc:
     else if (intent_write_allowed(op, obj))
     {
         // Direct intent-write
-        BS_SUBMIT_CHECK_META_QUEUE(1);
+        BS_SUBMIT_CHECK_SQES(1);
         int res = 0;
         if (dsk.csum_block_size <= dsk.bitmap_granularity &&
             (obj->entry_type == (BS_HEAP_BIG_INTENT|BS_HEAP_STABLE) ||
@@ -259,15 +260,7 @@ enospc:
             return 0;
         }
         // There is sufficient space. Check SQE(s)
-        if (!op->len || dsk.journal_disk == dsk.meta_disk || !dsk.journal_off_ring && !dsk.meta_off_ring)
-        {
-            BS_SUBMIT_CHECK_META_QUEUE(1 + (op->len > 0 ? 1 : 0));
-        }
-        else
-        {
-            BS_SUBMIT_CHECK_META_QUEUE(1);
-            BS_SUBMIT_CHECK_JOURNAL_QUEUE(1);
-        }
+        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
         int res = heap->add_small_write(op->oid, &obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
             op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         BS_SUBMIT_CHECK_PLACEMENT(res, obj->lsn);
@@ -285,12 +278,11 @@ enospc:
             {
                 memcpy((uint8_t*)buffer_area + loc, op->buf, op->len);
             }
+            BS_SUBMIT_GET_SQE(sqe2, data2);
+            data2->iov = (struct iovec){ op->buf, op->len };
+            data2->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
             assert(loc+op->len <= dsk.journal_len);
-            res = dsk.journal_disk->write_async(
-                dsk.journal_offset + loc, op->len, op->buf,
-                [this, op](int res) { handle_write_event(res, op); }
-            );
-            assert(res == 0);
+            io_uring_prep_writev(sqe2, dsk.journal_fd, &data2->iov, 1, dsk.journal_offset + loc);
             PRIV(op)->pending_ops++;
         }
         else
@@ -306,7 +298,6 @@ enospc:
 
 int blockstore_impl_t::continue_write(blockstore_op_t *op)
 {
-    int res = 0;
     int op_state = PRIV(op)->op_state;
 again:
     if (op_state == 2)
@@ -359,29 +350,21 @@ resume_11:
         }
         if (PRIV(op)->lsn > data_fsync_sent)
         {
-            res = dsk.data_disk->flush_async([this, op, fs = data_fsync_cur](int res)
+            BS_SUBMIT_GET_SQE(sqe, data);
+            io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC);
+            data->iov = { 0 };
+            data->callback = [this, op, fs = data_fsync_cur](ring_data_t *data)
             {
                 live = true;
-                if (res != 0)
-                    disk_error_abort("sync data", res);
+                if (data->res != 0)
+                    disk_error_abort("sync data", data->res, 0);
                 if (fs > data_fsync_done)
                 {
                     data_fsync_done = fs;
                     ringloop->wakeup();
                 }
-            });
-            if (res == -EAGAIN)
-            {
-                return 1;
-            }
-            else
-            {
-                data_fsync_sent = data_fsync_cur;
-                if (res == -EALREADY)
-                    data_fsync_done = data_fsync_cur;
-                else
-                    assert(res == 0);
-            }
+            };
+            data_fsync_sent = data_fsync_cur;
         }
         if (PRIV(op)->lsn > data_fsync_done)
         {
@@ -391,8 +374,9 @@ resume_11:
     }
 resume_4:
     {
-        BS_SUBMIT_CHECK_META_QUEUE(1);
+        BS_SUBMIT_CHECK_SQES(1);
         auto obj = heap->read_entry(op->oid);
+        int res = 0;
         if (PRIV(op)->write_type == _REDIRECT_INTENT)
         {
             res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
@@ -492,28 +476,25 @@ resume_8:
 resume_10:
     // Direct intent-write
     // LSN is not marked as completed so big_write won't be freed
-    BS_SUBMIT_CHECK_DATA_QUEUE(1);
+    BS_SUBMIT_GET_SQE(sqe, data);
+    data->iov = (struct iovec){ op->buf, op->len };
+    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
     assert(PRIV(op)->location + op->offset <= dsk.block_count*dsk.data_block_size);
-    // FIXME: We tried to use RWF_ATOMIC here, but it's useless because Linux
-    // incorrectly enforces 2^N atomic write alignment which is a blocker for us.
-    // So now it's removed.
-    res = dsk.data_disk->write_async(
-        dsk.data_offset + PRIV(op)->location + op->offset, op->len, op->buf,
-        [this, op](int res) { handle_write_event(res, op); }
-    );
-    assert(res == 0);
+    io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
+    if (dsk.use_atomic_flag)
+        sqe->rw_flags = RWF_ATOMIC;
     PRIV(op)->pending_ops++;
     PRIV(op)->op_state = 7;
     return 1;
 }
 
-void blockstore_impl_t::handle_write_event(int res, blockstore_op_t *op)
+void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)
 {
     live = true;
-    if (res < 0)
+    if (data->res != data->iov.iov_len)
     {
         // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
-        disk_error_abort("data write", res);
+        disk_error_abort("data write", data->res, data->iov.iov_len);
     }
     PRIV(op)->pending_ops--;
     assert(PRIV(op)->pending_ops >= 0);

src/blockstore/v1/impl.cpp

@@ -1,7 +1,6 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
-#include "disk_util.h"
 #include "impl.h"
 #include "internal.h"
 
@@ -23,9 +22,6 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
         dsk.open_meta();
         dsk.open_journal();
         calc_lengths();
-        dsk.data_fd = dsk.data_disk->get_fd();
-        dsk.meta_fd = dsk.meta_disk->get_fd();
-        dsk.journal_fd = dsk.journal_disk->get_fd();
         alloc_dyn_data = dsk.clean_dyn_size > sizeof(void*) || dsk.csum_block_size > 0;
         zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
         memset(zero_object, 0, dsk.data_block_size);

src/client/cluster_client.cpp

@@ -761,6 +761,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     op->cur_inode = op->inode;
     op->retval = 0;
     op->state = 0;
+    op->canceled = false;
     op->retry_after = 0;
     op->inflight_count = 0;
     op->done_count = 0;
@@ -860,6 +861,113 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
 }
 
+// Try to cancel a previously started operation
+//
+// A canceled operation is completed with -ECANCELED, either immediately (from inside
+// this function) or later, when the messenger finishes canceling its sub-operations.
+//
+// Some operations can't be canceled at all - in that case they just complete normally.
+// The result of a canceled write is undefined - some parts of it may be written and
+// some may be not, so the write-back copy of its data is dropped, too.
+//
+// Returns true if the operation is already completed, i.e. if its callback is already called
+bool cluster_client_t::cancel(cluster_op_t *op)
+{
+    if (op->canceled || (op->flags & OP_FLUSH_BUFFER))
+    {
+        // Already canceled, or an internal write-back flush which is never canceled
+        return false;
+    }
+    // Operations which aren't started yet are just removed from their queues
+    for (auto queue: std::initializer_list<std::vector<cluster_op_t*>*>{ &offline_ops, &key_wait_ops, &wb->writeback_overflow })
+    {
+        for (auto op_it = queue->begin(); op_it != queue->end(); op_it++)
+        {
+            if (*op_it == op)
+            {
+                queue->erase(op_it);
+                op->canceled = true;
+                op->retval = -ECANCELED;
+                auto cb = std::move(op->callback);
+                cb(op);
+                return true;
+            }
+        }
+    }
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version)
+    {
+        // CAS operations aren't in the operation queue and have their own callback
+        if (!op->parts.size() || !(op->parts[0].flags & PART_SENT) ||
+            (op->parts[0].flags & (PART_DONE|PART_ERROR)))
+        {
+            // Not sent yet (waiting for a preliminary sync) => can't cancel it
+            return false;
+        }
+        op->canceled = true;
+        if (!msgr.cancel_out(&op->parts[0].op))
+        {
+            // The callback will be called with -ECANCELED by the messenger
+            return false;
+        }
+        op->retval = -ECANCELED;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return true;
+    }
+    op->canceled = true;
+    op->retry_after = 0;
+    // ERR_PRIO(-ECANCELED) is the highest, so replies to the remaining parts
+    // won't override it and won't schedule a retry in handle_op_part()
+    op->retval = -ECANCELED;
+    // msgr.cancel_out() never calls callbacks itself, so the operation
+    // can not be completed in the middle of this loop
+    for (auto & part: op->parts)
+    {
+        if ((part.flags & PART_SENT) && !(part.flags & (PART_DONE|PART_ERROR)) &&
+            msgr.cancel_out(&part.op))
+        {
+            // Canceled synchronously => the callback won't be called at all
+            part.flags |= PART_ERROR;
+            op->inflight_count--;
+        }
+    }
+    if (op->inflight_count > 0)
+    {
+        // The rest of the parts will be canceled asynchronously
+        return false;
+    }
+    finish_canceled(op);
+    return true;
+}
+
+void cluster_client_t::finish_canceled(cluster_op_t *op)
+{
+    assert(op->canceled && !op->inflight_count);
+    op->retval = -ECANCELED;
+    op->retry_after = 0;
+    if (op->opcode == OSD_OP_SYNC)
+    {
+        if (op->parts.size())
+        {
+            // Syncs were already sent - roll the write-back cache state back and
+            // mark all unsynced OSDs as dirty again so the next sync retries them
+            wb->fsync_error();
+            for (auto & part: op->parts)
+            {
+                if (!(part.flags & PART_DONE))
+                    dirty_osds.insert(part.osd_num);
+            }
+        }
+    }
+    else if (op->flush_id)
+    {
+        // The result of a canceled write is undefined, so drop the write-repeat copy
+        wb->delete_flush(op->inode, op->offset, op->len, op->flush_id);
+        op->flush_id = 0;
+    }
+    erase_op(op);
+}
+
 void cluster_client_t::execute_cas(cluster_op_t *op, bool nosync)
 {
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
@@ -1085,6 +1193,16 @@ void cluster_client_t::execute_raw(osd_num_t osd_num, osd_op_t *op)
 
 int cluster_client_t::continue_rw(cluster_op_t *op)
 {
+    if (op->canceled)
+    {
+        if (op->inflight_count > 0)
+        {
+            // Wait for the remaining parts to be canceled
+            return 0;
+        }
+        finish_canceled(op);
+        return 1;
+    }
     if (op->state == 0)
         goto resume_0;
     else if (op->state == 1)
@@ -1568,6 +1686,16 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
 
 int cluster_client_t::continue_sync(cluster_op_t *op)
 {
+    if (op->canceled)
+    {
+        if (op->inflight_count > 0)
+        {
+            // Wait for the remaining parts to be canceled
+            return 0;
+        }
+        finish_canceled(op);
+        return 1;
+    }
     if (op->state == 1)
         goto resume_1;
     if (!dirty_osds.size())
@@ -1676,7 +1804,8 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         if (ERR_PRIO(part->op.reply.hdr.retval) > ERR_PRIO(op->retval))
             op->retval = part->op.reply.hdr.retval;
         uint64_t stop_client_id = 0;
-        if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC && op->retval != -EPERM && op->retval != -EAGAIN)
+        if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC &&
+            op->retval != -EPERM && op->retval != -EAGAIN && op->retval != -ECANCELED)
         {
             stop_client_id = part->op.client_id;
             if (op->retval != -EPIPE || log_level > 0)

src/client/cluster_client.h

@@ -65,6 +65,7 @@ protected:
     uint64_t cur_inode; // for snapshot reads
     bool needs_reslice: 1;
     bool deoptimise_snapshot: 1;
+    bool canceled = false;
     int retry_after = 0;
     int inflight_count = 0, done_count = 0;
     timespec wait_up_until = {};
@@ -190,6 +191,7 @@ public:
     cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config, std::unique_ptr<etcd_state_client_t> st_cli);
     ~cluster_client_t();
     void execute(cluster_op_t *op);
+    bool cancel(cluster_op_t *op);
     void execute_raw(osd_num_t osd_num, osd_op_t *op);
     bool is_ready();
     void on_ready(std::function<void(void)> fn);
@@ -232,6 +234,7 @@ protected:
     int continue_sync(cluster_op_t *op);
     void send_sync(cluster_op_t *op, cluster_op_part_t *part);
     void handle_op_part(cluster_op_part_t *part);
+    void finish_canceled(cluster_op_t *op);
     void copy_part_bitmap(cluster_op_t *op, cluster_op_part_t *part);
     void erase_op(cluster_op_t *op);
     void calc_wait(cluster_op_t *op);

src/client/messenger.h

@@ -57,6 +57,7 @@
 #define MSGR_SENDP_IN 2
 #define MSGR_SENDP_OUT 4
 #define MSGR_SENDP_REF 8
+#define MSGR_SENDP_CANCELED 16
 
 struct osd_client_t;
 
@@ -151,6 +152,7 @@ struct osd_client_t
     // Write state
     std::deque<osd_op_t *> write_ops;
     osd_op_t *write_op = NULL;
+    bool write_op_cancel = false;
     size_t write_op_pos = 0;
     msghdr write_msg = { 0 };
     int write_state = 0;
@@ -312,6 +314,7 @@ public:
     void stop_client(uint64_t client_id, bool force_delete = false);
     void destroy_client(osd_client_t *cl);
     void outbox_push(osd_op_t *cur_op);
+    bool cancel_out(osd_op_t *cur_op);
     std::function<void(osd_op_t*)> exec_op;
     std::function<void(osd_num_t)> repeer_pgs;
     std::function<void(osd_num_t)> break_pg_locks;
@@ -352,6 +355,7 @@ protected:
     void check_peer_config(osd_client_t *cl);
     void cancel_osd_ops(osd_client_t *cl);
     void cancel_op(osd_op_t *op);
+    osd_op_t *copy_out_op_to_ignore(osd_op_t *cur_op);
 
     bool try_send(osd_client_t *cl);
     bool wakeup_send(osd_client_t *cl);

src/client/msgr_op.h

@@ -187,13 +187,17 @@ struct __attribute__((visibility("default"))) osd_op_t
     void *rmw_buf = NULL;
     std::shared_ptr<osd_op_enc_t> enc;
     uint64_t csum = 0; // network layer checksum
+    uint64_t send_free_pos = 0;
+    // Read the reply and throw it away instead of copying it anywhere.
+    // Used for stub operations left in place of canceled ones
+    bool ignore_reply = false;
     osd_primary_op_data_t* op_data = NULL;
     std::function<void(osd_op_t*)> callback;
 
     osd_op_buf_list_t iov;
 
     ~osd_op_t();
-    void cancel();
+    void cancel(int retval);
 
     bool is_recovery_related();
     uint64_t calc_data_checksum();

src/client/msgr_receive.cpp

@@ -179,14 +179,15 @@ public:
         }
         else if (flags & RDR_GCM)
         {
-            // Here, dst == NULL is not allowed
-            assert(dst != NULL);
+            // Here, dst == NULL means "decrypt in place and throw the data away".
+            // The data still has to be decrypted to keep the GCM stream and tag in sync
+            uint8_t *out = (dst ? dst+from : curbuf+done);
 #ifdef WITH_ISAL_CRYPTO
-            int r = isal_aes_gcm_dec_256_update(&cl->peer_key_isal, cl->dec_ctx, dst+from, curbuf+done, n);
+            int r = isal_aes_gcm_dec_256_update(&cl->peer_key_isal, cl->dec_ctx, out, curbuf+done, n);
             assert(!r);
 #else
             int actual_out;
-            if (EVP_DecryptUpdate(cl->dec_ctx, dst+from, &actual_out, curbuf+done, n) != 1)
+            if (EVP_DecryptUpdate(cl->dec_ctx, out, &actual_out, curbuf+done, n) != 1)
             {
                 fprintf(stderr, "DecryptUpdate error: ");
                 ERR_print_errors_fp(stderr);
@@ -196,7 +197,7 @@ public:
 #endif
             if (cl->read_csum_state && !(flags & RDR_NO_CSUM))
             {
-                XXH3_64bits_update(cl->read_csum_state, dst+from, n);
+                XXH3_64bits_update(cl->read_csum_state, out, n);
             }
             done += n;
         }
@@ -307,6 +308,8 @@ public:
     get_op_reader_t(osd_messenger_t* msgr, osd_client_t* cl):
         cl(cl), from(cl->read_op_pos), mpos(false)
     {
+        // Discarded replies are always read through in_buf - this reader needs real buffers
+        assert(!cl->read_op->ignore_reply);
         if (cl->read_op->op_type == OSD_OP_OUT &&
             cl->read_op->reply.hdr.opcode == OSD_OP_READ &&
             cl->read_op->reply.hdr.retval > 0)
@@ -397,7 +400,8 @@ void osd_messenger_t::read_requests()
             continue;
         }
         auto cl = cl_it->second;
-        if (!rxbounce && cl->read_op && cl->read_op_pos >= OSD_PACKET_SIZE && cl->read_op_size-(cl->read_op_pos-OSD_PACKET_SIZE) >= receive_buffer_size)
+        if (!rxbounce && cl->read_op && !cl->read_op->ignore_reply && cl->read_op_pos >= OSD_PACKET_SIZE &&
+            cl->read_op_size-(cl->read_op_pos-OSD_PACKET_SIZE) >= receive_buffer_size)
         {
             get_op_reader_t rdr(this, cl);
             if (!op_read_from(cl, rdr))
@@ -570,11 +574,15 @@ void osd_messenger_t::handle_immediate_ops()
             else
                 delete op;
         }
-        else
+        else if (op->callback)
         {
             // Copy lambda to be unaffected by `delete op`
             std::function<void(osd_op_t*)>(op->callback)(op);
         }
+        else
+        {
+            delete op;
+        }
     }
 }
 
@@ -859,12 +867,12 @@ bool osd_messenger_t::allocate_reply_buffers(osd_client_t *cl, osd_op_t *op)
         }
         if (op->reply.hdr.retval >= 0 && bmp_len > 0)
         {
-            assert(op->bitmap);
+            assert(op->ignore_reply || op->bitmap);
             cl->read_op_size += bmp_len;
         }
         if (op->reply.hdr.retval > 0)
         {
-            assert(op->iov.count > 0);
+            assert(op->ignore_reply || op->iov.count > 0);
             cl->read_op_size += op->reply.hdr.retval;
         }
     }
@@ -878,7 +886,8 @@ bool osd_messenger_t::allocate_reply_buffers(osd_client_t *cl, osd_op_t *op)
                 cl->client_id, op->reply.hdr.retval);
             return false;
         }
-        op->buf = malloc_or_die(cl->read_op_size);
+        if (!op->ignore_reply)
+            op->buf = malloc_or_die(cl->read_op_size);
     }
     else if (op->reply.hdr.opcode == OSD_OP_SEC_READ_BMP && op->reply.hdr.retval > 0)
     {
@@ -891,8 +900,11 @@ bool osd_messenger_t::allocate_reply_buffers(osd_client_t *cl, osd_op_t *op)
             return false;
         }
         cl->read_op_size = op->reply.hdr.retval;
-        free(op->buf);
-        op->buf = malloc_or_die(cl->read_op_size);
+        if (!op->ignore_reply)
+        {
+            free(op->buf);
+            op->buf = malloc_or_die(cl->read_op_size);
+        }
     }
     else if (op->reply.hdr.opcode == OSD_OP_SHOW_CONFIG && op->reply.hdr.retval > 0)
     {
@@ -903,14 +915,20 @@ bool osd_messenger_t::allocate_reply_buffers(osd_client_t *cl, osd_op_t *op)
                 cl->client_id, op->reply.hdr.retval, MAX_SIMPLE_PAYLOAD_SIZE);
             return false;
         }
-        free(op->buf);
-        op->buf = malloc_or_die(op->reply.hdr.retval);
+        if (!op->ignore_reply)
+        {
+            free(op->buf);
+            op->buf = malloc_or_die(op->reply.hdr.retval);
+        }
     }
     else if (op->reply.hdr.opcode == OSD_OP_DESCRIBE && op->reply.describe.result_bytes > 0)
     {
         cl->read_op_size = op->reply.describe.result_bytes;
-        free(op->buf);
-        op->buf = malloc_or_die(op->reply.describe.result_bytes);
+        if (!op->ignore_reply)
+        {
+            free(op->buf);
+            op->buf = malloc_or_die(op->reply.describe.result_bytes);
+        }
     }
     if (cl->proto_csum_status == MSGR_CSUM_FULL ||
         cl->read_op_size > 0 && cl->proto_csum_status == MSGR_CSUM_PAYLOAD)
@@ -974,7 +992,37 @@ bool osd_messenger_t::op_read_from(osd_client_t *cl, msgr_op_reader_t & rdr)
         if (!rdr.read(op->reply.buf, OSD_PACKET_SIZE, RDR_GCM | (cl->proto_csum_status == MSGR_CSUM_PAYLOAD ? RDR_NO_CSUM : 0)))
             return false;
 switched_type:
-        if (op->reply.hdr.opcode == OSD_OP_SEC_READ)
+        if (op->ignore_reply)
+        {
+            // Reply to a canceled operation - read it into nowhere and throw it away.
+            // Read flags have to exactly match the normal path below, otherwise
+            // the GCM stream and the protocol checksum would desync
+            if (op->reply.hdr.opcode == OSD_OP_SEC_READ || op->reply.hdr.opcode == OSD_OP_READ)
+            {
+                // Conditions mirror allocate_reply_buffers() which defines cl->read_op_size,
+                // so that we always consume exactly the framed length
+                uint32_t bmp_len = (op->reply.hdr.opcode == OSD_OP_SEC_READ
+                    ? op->reply.sec_rw.attr_len : op->reply.rw.bitmap_len);
+                if (op->reply.hdr.retval >= 0 && bmp_len > 0 && !rdr.read(NULL, bmp_len, RDR_GCM))
+                    return false;
+                // Note that RDR_XTS is intentionally not passed - the stub has no keys and
+                // XTS is length-preserving, so both the GCM stream and the checksum match
+                if (op->reply.hdr.retval > 0 && !rdr.read(NULL, op->reply.hdr.retval,
+                    (cl->proto_csum_status == MSGR_CSUM_GCM ? RDR_GCM : 0)))
+                    return false;
+            }
+            else
+            {
+                // All other replies have a single payload region, always read with RDR_GCM
+                size_t payload = cl->read_op_size;
+                if (cl->proto_csum_status == MSGR_CSUM_FULL ||
+                    payload > 0 && cl->proto_csum_status == MSGR_CSUM_PAYLOAD)
+                    payload -= 8;
+                if (payload > 0 && !rdr.read(NULL, payload, RDR_GCM))
+                    return false;
+            }
+        }
+        else if (op->reply.hdr.opcode == OSD_OP_SEC_READ)
         {
             if (op->reply.sec_rw.attr_len > 0)
             {

src/client/msgr_send.cpp

@@ -436,7 +436,7 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
     auto cl_it = clients.find(cur_op->client_id);
     if (cl_it == clients.end() || cl_it->second->peer_state == PEER_STOPPED)
     {
-        cur_op->cancel();
+        cur_op->cancel(-EPIPE);
         return;
     }
     osd_client_t *cl = cl_it->second;
@@ -494,6 +494,81 @@ bool osd_messenger_t::wakeup_send(osd_client_t *cl)
     return true;
 }
 
+// Try to cancel an outbound operation
+// Returns true when canceled synchronously, false if canceling is impossible or will be
+// performed asynchronously and the caller has to wait for the callback to be called
+bool osd_messenger_t::cancel_out(osd_op_t *cur_op)
+{
+    assert(cur_op->client_id);
+    assert(cur_op->op_type == OSD_OP_OUT);
+    auto cl_it = clients.find(cur_op->client_id);
+    if (cl_it == clients.end() || cl_it->second->peer_state == PEER_STOPPED)
+    {
+        // Client is stopped => messenger will call the callback anyway
+        return false;
+    }
+    osd_client_t *cl = cl_it->second;
+    auto sent_it = cl->sent_ops.find(cur_op->req.hdr.id);
+    if (sent_it == cl->sent_ops.end())
+    {
+        // Reply header is already received => callback is pending
+        return false;
+    }
+    if (sent_it->second != cur_op)
+    {
+        // Already canceled
+        return false;
+    }
+    // Make a copy to have some space to place the reply to and forget the original object
+    if (cl->write_op == cur_op)
+    {
+        // Operation is being sent => wait for full send, then cancel it
+        sent_it->second = copy_out_op_to_ignore(cur_op);
+        cl->write_op_cancel = true;
+        return false;
+    }
+    for (auto wr_it = cl->write_ops.begin(); wr_it != cl->write_ops.end(); wr_it++)
+    {
+        if (*wr_it == cur_op)
+        {
+            // Operation is only queued => just remove it from the send queue
+            cl->sent_ops.erase(sent_it);
+            cl->write_ops.erase(wr_it);
+            cur_op->callback = NULL;
+            return true;
+        }
+    }
+    sent_it->second = copy_out_op_to_ignore(cur_op);
+    if (cur_op->send_free_pos >= cl->next_send_free)
+    {
+        // In progress of sending => free it when the send_free reference is cleared
+        assert(cur_op->send_free_pos < cl->next_send_free + cl->send_free.size());
+        auto & sf = cl->send_free[cur_op->send_free_pos - cl->next_send_free];
+        assert((osd_op_t*)sf.buf == cur_op);
+        sf.flags |= MSGR_SENDP_CANCELED;
+        return false;
+    }
+    // Operation is fully sent and not referenced by the send_free queue
+    cur_op->callback = NULL;
+    return true;
+}
+
+// Make a stub operation to swallow the reply to a canceled operation.
+// The reply body is read into nowhere, so no data buffers are needed here -
+// only the request header, which is required to validate the reply
+osd_op_t *osd_messenger_t::copy_out_op_to_ignore(osd_op_t *cur_op)
+{
+    osd_op_t *op_copy = new osd_op_t;
+    op_copy->op_type = OSD_OP_OUT;
+    op_copy->ignore_reply = true;
+    op_copy->tv_begin = cur_op->tv_begin;
+    op_copy->client_id = cur_op->client_id;
+    op_copy->osd_num = cur_op->osd_num;
+    op_copy->bitmap_len = cur_op->bitmap_len;
+    memcpy(op_copy->req.buf, cur_op->req.buf, OSD_PACKET_SIZE);
+    return op_copy;
+}
+
 bool osd_messenger_t::try_send(osd_client_t *cl)
 {
     assert(cl->peer_state != PEER_RDMA);
@@ -686,7 +761,11 @@ size_t osd_messenger_t::copy_ops_to_with(osd_client_t *cl, uint8_t *dst, size_t
             break;
         }
         assert(cl->send_free.size() < UINT64_MAX - cl->next_send_free);
-        cl->send_free.push_back({ .buf = op, .flags = (uint32_t)((op->op_type == OSD_OP_IN ? MSGR_SENDP_IN : MSGR_SENDP_OUT) | MSGR_SENDP_REF) });
+        uint32_t flags = ((op->op_type == OSD_OP_IN ? MSGR_SENDP_IN : MSGR_SENDP_OUT) | MSGR_SENDP_REF);
+        if (cl->write_op_cancel)
+            flags |= MSGR_SENDP_CANCELED;
+        op->send_free_pos = cl->next_send_free + cl->send_free.size();
+        cl->send_free.push_back({ .buf = op, .flags = flags });
     }
     return wr.get_done();
 }
@@ -694,6 +773,7 @@ size_t osd_messenger_t::copy_ops_to_with(osd_client_t *cl, uint8_t *dst, size_t
 void osd_messenger_t::next_write_op(osd_client_t *cl)
 {
     cl->write_op = cl->write_ops.front();
+    cl->write_op_cancel = false;
     cl->write_ops.pop_front();
     if (cl->proto_csum_status == MSGR_CSUM_FULL || cl->proto_csum_status == MSGR_CSUM_PAYLOAD)
     {

src/client/msgr_stop.cpp

@@ -22,24 +22,24 @@ void osd_client_t::cancel_ops()
     sent_ops.clear();
     for (auto op: cancel_ops)
     {
-        op->cancel();
+        op->cancel(-EPIPE);
     }
 }
 
-void osd_op_t::cancel()
+void osd_op_t::cancel(int retval)
 {
     if (op_type == OSD_OP_OUT && callback)
     {
         reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
         reply.hdr.id = req.hdr.id;
         reply.hdr.opcode = req.hdr.opcode;
-        reply.hdr.retval = -EPIPE;
+        reply.hdr.retval = retval;
         // Copy lambda to be unaffected by `delete this`
         (std::function<void(osd_op_t*)>(callback))(this);
     }
     else
     {
-        // This function is only called in stop_client(), so it's fine to destroy the operation
+        // It's fine to just free responses to incoming ops
         delete this;
     }
 }
@@ -57,6 +57,8 @@ void osd_client_t::unref_send_free(uint64_t from, uint64_t to)
         {
             if (it->flags & MSGR_SENDP_IN)
                 delete (osd_op_t*)it->buf;
+            else if (it->flags & MSGR_SENDP_CANCELED)
+                ((osd_op_t*)it->buf)->cancel(-ECANCELED);
             else if (it->flags & MSGR_SENDP_BUF)
                 free(it->buf);
         }
@@ -67,6 +69,8 @@ void osd_client_t::unref_send_free(uint64_t from, uint64_t to)
         {
             if (it->flags & MSGR_SENDP_IN)
                 delete (osd_op_t*)it->buf;
+            else if (it->flags & MSGR_SENDP_CANCELED)
+                ((osd_op_t*)it->buf)->cancel(-ECANCELED);
             else if (it->flags & MSGR_SENDP_BUF)
                 free(it->buf);
             to++;
@@ -204,14 +208,21 @@ osd_client_t::~osd_client_t()
     if (read_op)
     {
         // read_op may be an incoming op or a continued response for an outbound op
-        read_op->cancel();
+        read_op->cancel(-EPIPE);
         read_op = NULL;
     }
     // Delete unsent replies - before cancel_ops() because it may delete outbound operations.
     if (write_op)
     {
         if (write_op->op_type == OSD_OP_IN)
+        {
             delete write_op;
+        }
+        else if (write_op_cancel)
+        {
+            // not in sent_ops anymore - should be canceled here
+            write_op->cancel(-ECANCELED);
+        }
         write_op = NULL;
     }
     for (auto op: write_ops)

src/disk_tool/CMakeLists.txt

@@ -10,7 +10,6 @@ add_executable(vitastor-disk
 	disk_tool.h
 	disk_tool_journal.cpp
 	disk_tool_meta.cpp
-	disk_tool_nvme.cpp
 	disk_tool_prepare.cpp
 	disk_tool_resize_auto.cpp
 	disk_tool_resize.cpp

src/disk_tool/disk_simple_offsets.cpp

@@ -1,11 +1,16 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
 #include "json11/json11.hpp"
 #include "str_util.h"
 #include "blockstore.h"
 #include "blockstore_disk.h"
-#include "disk_util.h"
 #include "blockstore_heap.h"
 #include "ondisk_formats.h"
 
@@ -57,23 +62,39 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
             fprintf(stderr, "Device path is missing\n");
             exit(1);
         }
-        disk_i *disk = disk_i::create_and_open(device, "cached,ro", true);
-        if (!disk)
+        struct stat st;
+        if (stat(device.c_str(), &st) < 0)
         {
-            fprintf(stderr, "Failed to open %s\n", device.c_str());
+            fprintf(stderr, "Can't stat %s: %s\n", device.c_str(), strerror(errno));
             exit(1);
         }
-        device_size = disk->get_size();
-        uint64_t blksize = disk->get_block_size();
-        if (blksize < device_block_size)
+        if (S_ISBLK(st.st_mode))
         {
-            fprintf(
-                stderr, "Warning: %s reports %ju byte blocks, but we use %ju."
-                " Set --device_block_size=%ju if you're sure it works well with %ju byte blocks.\n",
-                device.c_str(), blksize, device_block_size, blksize, blksize
-            );
+            int fd = open(device.c_str(), O_DIRECT|O_RDONLY);
+            if (fd < 0 || ioctl(fd, BLKGETSIZE64, &device_size) < 0)
+            {
+                fprintf(stderr, "Failed to get device size for %s: %s\n", device.c_str(), strerror(errno));
+                exit(1);
+            }
+            close(fd);
+            if (st.st_blksize < device_block_size)
+            {
+                fprintf(
+                    stderr, "Warning: %s reports %ju byte blocks, but we use %ju."
+                    " Set --device_block_size=%ju if you're sure it works well with %ju byte blocks.\n",
+                    device.c_str(), (uint64_t)st.st_blksize, device_block_size, (uint64_t)st.st_blksize, (uint64_t)st.st_blksize
+                );
+            }
+        }
+        else if (S_ISREG(st.st_mode))
+        {
+            device_size = st.st_size;
+        }
+        else
+        {
+            fprintf(stderr, "%s is neither a block device nor a regular file\n", device.c_str());
+            exit(1);
         }
-        delete disk;
     }
     if (!device_size)
     {

src/disk_tool/disk_tool.cpp

@@ -207,9 +207,6 @@ static const char *help_text =
     "  Write metadata from JSON taken from standard input in the same format as produced by `dump-meta`.\n"
     "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
     "\n"
-    "vitastor-disk find-pci <serial> [nsid]\n"
-    "  Find PCI ID and attributes of an NVMe device with a specified serial number.\n"
-    "\n"
     "vitastor-disk simple-offsets <device>\n"
     "  Calculate offsets for old simple&stupid (no superblock) OSD deployment. Options:\n"
     "    --object_size 128k       Set blockstore block size\n"
@@ -476,16 +473,6 @@ int main(int argc, char *argv[])
     {
         return self.raw_resize();
     }
-    else if (!strcmp(cmd[0], "find-pci"))
-    {
-        if (cmd.size() != 2 && cmd.size() != 3)
-        {
-            fprintf(stderr, "S/N and optionally NS number is required\n");
-            return 1;
-        }
-        self.find_pci_id(cmd[1], cmd.size() == 3 ? stoull_full(cmd[2]) : 0);
-        return 0;
-    }
     else if (!strcmp(cmd[0], "simple-offsets"))
     {
         // Calculate offsets for simple & stupid OSD deployment without superblock

src/disk_tool/disk_tool.h

@@ -17,7 +17,6 @@
 #include "blockstore_disk.h"
 #include "blockstore.h"
 #include "blockstore_heap.h"
-#include "disk_util.h"
 #include "ondisk_formats.h"
 #include "crc32c.h"
 #include "allocator.h"
@@ -86,7 +85,7 @@ struct disk_tool_t
     uint64_t total_blocks = 0, free_first = 0, free_last = 0;
     uint64_t new_clean_entry_bitmap_size = 0, new_data_csum_size = 0, new_clean_entry_size = 0, new_entries_per_block = 0;
     uint32_t new_meta_format = 0;
-    disk_i *new_journal_disk = NULL, *new_meta_disk = NULL;
+    int new_journal_fd = -1, new_meta_fd = -1;
     resizer_data_moving_t *moving_blocks = NULL;
     bool atomic_warned = false;
 
@@ -153,8 +152,6 @@ struct disk_tool_t
     int clear_osd_superblock(const std::string & dev);
     int trim_data(std::string device);
 
-    std::string find_pci_id(const std::string & nvme_serial, uint32_t nvme_nsid);
-
     json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
     uint32_t write_osd_superblock(std::string device, json11::Json params);
 
@@ -180,6 +177,7 @@ uint64_t get_atomic_write_size(const std::string & dev);
 uint64_t get_device_size(const std::string & dev, bool should_exist = false);
 std::string get_parent_device(std::string dev);
 int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std::string *out, std::string *err);
+int write_zero(int fd, uint64_t offset, uint64_t size);
 json11::Json read_parttable(std::string dev);
 uint64_t dev_size_from_parttable(json11::Json pt);
 uint64_t free_from_parttable(json11::Json pt);

src/disk_tool/disk_tool_journal.cpp

@@ -22,22 +22,18 @@ int disk_tool_t::dump_journal()
         printf("[\n");
     if (all)
     {
-        dsk.journal_disk = disk_i::create_and_open(dsk.journal_device, (options["io"] == "cached" ? "cached,ro" : "ro"), true);
-        if (!dsk.journal_disk)
+        dsk.journal_fd = open(dsk.journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
+        if (dsk.journal_fd < 0)
         {
-            fprintf(stderr, "Failed to open journal device %s\n", dsk.journal_device.c_str());
+            fprintf(stderr, "Failed to open journal device %s: %s\n", dsk.journal_device.c_str(), strerror(errno));
             return 1;
         }
-        void *journal_buf = dsk.journal_disk->alloc_iomem(dsk.journal_block_size);
+        void *journal_buf = memalign_or_die(MEM_ALIGNMENT, dsk.journal_block_size);
         journal_pos = 0;
         while (journal_pos < dsk.journal_len)
         {
-            int r = dsk.journal_disk->read_sync(dsk.journal_offset+journal_pos, dsk.journal_block_size, journal_buf);
-            if (r != 0)
-            {
-                fprintf(stderr, "Failed to read journal block at %ju: %s (code %u)\n", dsk.journal_offset+journal_pos, strerror(-r), -r);
-                break;
-            }
+            int r = pread(dsk.journal_fd, journal_buf, dsk.journal_block_size, dsk.journal_offset+journal_pos);
+            assert(r == dsk.journal_block_size);
             uint64_t s;
             for (s = 0; s < dsk.journal_block_size; s += 8)
             {
@@ -94,9 +90,9 @@ int disk_tool_t::dump_journal()
                 journal_pos += dsk.journal_block_size;
             }
         }
-        dsk.journal_disk->free_iomem(journal_buf);
-        delete dsk.journal_disk;
-        dsk.journal_disk = NULL;
+        free(journal_buf);
+        close(dsk.journal_fd);
+        dsk.journal_fd = -1;
     }
     else
     {
@@ -131,28 +127,23 @@ int disk_tool_t::process_journal(std::function<int(void*)> block_fn, bool do_ope
 {
     if (do_open)
     {
-        if (dsk.journal_disk)
+        if (dsk.journal_fd >= 0)
         {
             fprintf(stderr, "Bug: journal device is already opened\n");
             return 1;
         }
-        dsk.journal_disk = disk_i::create_and_open(dsk.journal_device, (options["io"] == "cached" ? "cached,ro" : "ro"), true);
-        if (!dsk.journal_disk)
+        dsk.journal_fd = open(dsk.journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
+        if (dsk.journal_fd < 0)
         {
-            fprintf(stderr, "Failed to open journal device %s\n", dsk.journal_device.c_str());
+            fprintf(stderr, "Failed to open journal device %s: %s\n", dsk.journal_device.c_str(), strerror(errno));
             return 1;
         }
     }
-    void *data = dsk.journal_disk->alloc_iomem(dsk.journal_block_size);
+    void *data = memalign_or_die(MEM_ALIGNMENT, dsk.journal_block_size);
     journal_pos = 0;
-    journal_entry *je = NULL;
-    int r = dsk.journal_disk->read_sync(dsk.journal_offset+journal_pos, dsk.journal_block_size, data);
-    if (r != 0)
-    {
-        fprintf(stderr, "Failed to read journal header at %ju: %s (code %u)\n", dsk.journal_offset+journal_pos, strerror(-r), -r);
-        goto ret;
-    }
-    je = (journal_entry*)(data);
+    int r = pread(dsk.journal_fd, data, dsk.journal_block_size, dsk.journal_offset+journal_pos);
+    assert(r == dsk.journal_block_size);
+    journal_entry *je = (journal_entry*)(data);
     if (je->magic != JOURNAL_MAGIC || je->type != JE_START || je_crc32(je) != je->crc32)
     {
         fprintf(stderr, "offset %08jx: journal superblock is invalid\n", journal_pos);
@@ -184,28 +175,19 @@ int disk_tool_t::process_journal(std::function<int(void*)> block_fn, bool do_ope
         {
             if (journal_pos >= dsk.journal_len)
                 journal_pos = dsk.journal_block_size;
-            r = dsk.journal_disk->read_sync(dsk.journal_offset+journal_pos, dsk.journal_block_size, data);
-            if (r != 0)
-            {
-                fprintf(stderr, "Failed to read journal block at %ju: %s (code %u)\n", dsk.journal_offset+journal_pos, strerror(-r), -r);
-                break;
-            }
+            r = pread(dsk.journal_fd, data, dsk.journal_block_size, dsk.journal_offset+journal_pos);
+            assert(r == dsk.journal_block_size);
             r = block_fn(data);
             if (r <= 0)
                 break;
         }
     }
-ret:
-    if (data)
-    {
-        dsk.journal_disk->free_iomem(data);
-        data = NULL;
-    }
     if (do_open)
     {
-        delete dsk.journal_disk;
-        dsk.journal_disk = NULL;
+        close(dsk.journal_fd);
+        dsk.journal_fd = -1;
     }
+    free(data);
     return r;
 }
 
@@ -245,14 +227,8 @@ int disk_tool_t::process_journal_block(void *buf, std::function<void(int, journa
                 journal_pos = dsk.journal_block_size;
                 wrapped = true;
             }
-            small_write_data = dsk.journal_disk->alloc_iomem(je->small_write.len);
-            int r = dsk.journal_disk->read_sync(dsk.journal_offset+je->small_write.data_offset, je->small_write.len, small_write_data);
-            if (r != 0)
-            {
-                fprintf(stderr, "Failed to read %u journal bytes at %ju: %s (code %u)\n", je->small_write.len,
-                    dsk.journal_offset+je->small_write.data_offset, strerror(-r), -r);
-                break;
-            }
+            small_write_data = memalign_or_die(MEM_ALIGNMENT, je->small_write.len);
+            assert(pread(dsk.journal_fd, small_write_data, je->small_write.len, dsk.journal_offset+je->small_write.data_offset) == je->small_write.len);
             data_crc32 = je_start.csum_block_size ? 0 : crc32c(0, small_write_data, je->small_write.len);
             data_csum_valid = (data_crc32 == je->small_write.crc32_data);
             if (je_start.csum_block_size && je->small_write.len > 0)
@@ -300,7 +276,7 @@ int disk_tool_t::process_journal_block(void *buf, std::function<void(int, journa
         iter_fn(entry, je);
         if (je->type == JE_SMALL_WRITE || je->type == JE_SMALL_WRITE_INSTANT)
         {
-            dsk.journal_disk->free_iomem(small_write_data);
+            free(small_write_data);
             small_write_data = NULL;
         }
         pos += je->size;
@@ -462,7 +438,7 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
 
 int disk_tool_t::write_json_journal(json11::Json entries)
 {
-    new_journal_buf = (uint8_t*)dsk.journal_disk->alloc_iomem(new_journal_len);
+    new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len);
     new_journal_ptr = new_journal_buf;
     new_journal_data = new_journal_ptr + dsk.journal_block_size;
     new_journal_in_pos = 0;
@@ -521,7 +497,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
             if (new_journal_ptr-new_journal_buf >= new_journal_len)
             {
                 fprintf(stderr, "Error: entries don't fit to the new journal\n");
-                dsk.journal_disk->free_iomem(new_journal_buf);
+                free(new_journal_buf);
                 return 1;
             }
             new_journal_data = new_journal_ptr+dsk.journal_block_size;
@@ -529,7 +505,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
             if (dsk.journal_block_size < entry_size)
             {
                 fprintf(stderr, "Error: journal entry too large (%u bytes)\n", entry_size);
-                dsk.journal_disk->free_iomem(new_journal_buf);
+                free(new_journal_buf);
                 return 1;
             }
         }
@@ -539,7 +515,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
             if (new_journal_data - new_journal_buf + len > new_journal_len)
             {
                 fprintf(stderr, "Error: entries don't fit to the new journal\n");
-                dsk.journal_disk->free_iomem(new_journal_buf);
+                free(new_journal_buf);
                 return 1;
             }
             *((journal_entry_small_write*)ne) = (journal_entry_small_write){
@@ -562,7 +538,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
             if (ne->small_write.len > 0 && !rec["data"].is_string())
             {
                 fprintf(stderr, "Error: entry data is missing, please generate the dump with --json --format data\n");
-                dsk.journal_disk->free_iomem(new_journal_buf);
+                free(new_journal_buf);
                 return 1;
             }
             if (ne->small_write.len > 0)
@@ -623,6 +599,6 @@ int disk_tool_t::write_json_journal(json11::Json entries)
         new_journal_in_pos += ne->size;
     }
     int r = resize_write_new_journal();
-    dsk.journal_disk->free_iomem(new_journal_buf);
+    free(new_journal_buf);
     return r;
 }

src/disk_tool/disk_tool_meta.cpp

@@ -22,36 +22,31 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v3_t *)>
     if (buf_size % dsk.meta_block_size)
         buf_size = 8*dsk.meta_block_size;
     uint8_t *data = NULL;
-    blockstore_meta_header_v3_t *hdr = NULL;
+    data = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, buf_size);
+    blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     if (do_open)
     {
-        if (dsk.meta_disk)
+        if (dsk.meta_fd >= 0)
         {
             fprintf(stderr, "Bug: Metadata device is already opened\n");
 close_error:
             r = 1;
             goto close_free;
         }
-        dsk.meta_disk = disk_i::create_and_open(dsk.meta_device, (options["io"] == "cached" ? "cached,ro" : "ro"), true);
-        if (!dsk.meta_disk)
+        dsk.meta_fd = open(dsk.meta_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
+        if (dsk.meta_fd < 0)
         {
-            fprintf(stderr, "Failed to open metadata device %s\n", dsk.meta_device.c_str());
+            fprintf(stderr, "Failed to open metadata device %s: %s\n", dsk.meta_device.c_str(), strerror(errno));
             goto close_error;
         }
     }
-    else if (!dsk.meta_disk)
+    else if (dsk.meta_fd < 0)
     {
         fprintf(stderr, "Bug: Metadata device is not opened\n");
         goto close_error;
     }
-    data = (uint8_t*)dsk.meta_disk->alloc_iomem(buf_size);
-    hdr = (blockstore_meta_header_v3_t *)dsk.meta_disk->alloc_iomem(dsk.meta_block_size);
     // Check superblock
-    if ((r = dsk.meta_disk->read_sync(dsk.meta_offset, dsk.meta_block_size, hdr)) != 0)
-    {
-        fprintf(stderr, "Failed to read metadata header: %s (code %d)\n", strerror(-r), -r);
-        goto close_error;
-    }
+    pread_blocking(dsk.meta_fd, hdr, dsk.meta_block_size, dsk.meta_offset);
     if (hdr->zero == 0 && hdr->magic == BLOCKSTORE_META_MAGIC_V1 && hdr->version == BLOCKSTORE_META_FORMAT_HEAP)
     {
         if (hdr->data_csum_type != 0 &&
@@ -67,40 +62,35 @@ close_error:
         // Load buffer_area
         if (with_data)
         {
+            buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
             if (dsk.journal_device == dsk.meta_device || dsk.journal_device == "")
             {
-                dsk.journal_disk = dsk.meta_disk;
+                dsk.journal_fd = dsk.meta_fd;
             }
             else if (do_open)
             {
-                if (dsk.journal_disk)
+                if (dsk.journal_fd >= 0)
                 {
                     fprintf(stderr, "Bug: Metadata device is already opened\n");
                     goto close_error;
                 }
-                dsk.journal_disk = disk_i::create_and_open(dsk.journal_device, (options["io"] == "cached" ? "cached,ro" : "ro"), true);
-                if (!dsk.journal_disk)
+                dsk.journal_fd = open(dsk.journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
+                if (dsk.journal_fd < 0)
                 {
-                    fprintf(stderr, "Failed to open journal device %s\n", dsk.journal_device.c_str());
+                    fprintf(stderr, "Failed to open journal device %s: %s\n", dsk.journal_device.c_str(), strerror(errno));
                     goto close_error;
                 }
             }
-            else if (!dsk.journal_disk)
+            else if (dsk.journal_fd < 0)
             {
                 fprintf(stderr, "Bug: journal device is not opened\n");
                 goto close_error;
             }
-            buffer_area = (uint8_t*)dsk.journal_disk->alloc_iomem(dsk.journal_len);
             uint64_t journal_pos = 0;
             while (journal_pos < dsk.journal_len)
             {
                 uint64_t read_len = buf_size < dsk.journal_len-journal_pos ? buf_size : dsk.journal_len-journal_pos;
-                if ((r = dsk.journal_disk->read_sync(dsk.journal_offset+journal_pos, read_len, buffer_area+journal_pos)) != 0)
-                {
-                    fprintf(stderr, "Failed to read %ju bytes at %ju from journal device %s: %s (code %d)\n",
-                        read_len, dsk.journal_offset+journal_pos, dsk.journal_device.c_str(), strerror(-r), -r);
-                    goto close_error;
-                }
+                pread_blocking(dsk.journal_fd, buffer_area+journal_pos, read_len, dsk.journal_offset+journal_pos);
                 journal_pos += read_len;
             }
         }
@@ -118,12 +108,7 @@ close_error:
         while (meta_pos < hdr->meta_area_size)
         {
             uint64_t read_len = buf_size < hdr->meta_area_size-meta_pos ? buf_size : hdr->meta_area_size-meta_pos;
-            if ((r = dsk.meta_disk->read_sync(dsk.meta_offset+meta_pos, read_len, data)) != 0)
-            {
-                fprintf(stderr, "Failed to read %ju bytes at %ju from metadata device %s: %s (code %d)\n",
-                    read_len, dsk.meta_offset+meta_pos, dsk.meta_device.c_str(), strerror(-r), -r);
-                goto close_error;
-            }
+            pread_blocking(dsk.meta_fd, data, read_len, dsk.meta_offset+meta_pos);
             r = heap->load_blocks(meta_pos-dsk.meta_block_size, read_len, data, true, entries_loaded);
             meta_pos += read_len;
         }
@@ -185,12 +170,7 @@ csum_unknown:
         while (meta_pos < dsk.min_meta_len)
         {
             uint64_t read_len = buf_size < dsk.min_meta_len-meta_pos ? buf_size : dsk.min_meta_len-meta_pos;
-            if ((r = dsk.meta_disk->read_sync(dsk.meta_offset+meta_pos, read_len, data)) != 0)
-            {
-                fprintf(stderr, "Failed to read %ju bytes at %ju from metadata device %s: %s (code %d)\n",
-                    read_len, dsk.meta_offset+meta_pos, dsk.meta_device.c_str(), strerror(-r), -r);
-                goto close_error;
-            }
+            pread_blocking(dsk.meta_fd, data, read_len, dsk.meta_offset+meta_pos);
             meta_pos += read_len;
             for (uint64_t blk = 0; blk < read_len; blk += dsk.meta_block_size)
             {
@@ -225,12 +205,7 @@ csum_unknown:
         while (meta_pos < dsk.min_meta_len)
         {
             uint64_t read_len = buf_size < dsk.min_meta_len-meta_pos ? buf_size : dsk.min_meta_len-meta_pos;
-            if ((r = dsk.meta_disk->read_sync(dsk.meta_offset+meta_pos, read_len, data)) != 0)
-            {
-                fprintf(stderr, "Failed to read %ju bytes at %ju from metadata device %s: %s (code %d)\n",
-                    read_len, dsk.meta_offset+meta_pos, dsk.meta_device.c_str(), strerror(-r), -r);
-                goto close_error;
-            }
+            pread_blocking(dsk.meta_fd, data, read_len, dsk.meta_offset+meta_pos);
             meta_pos += read_len;
             for (uint64_t blk = 0; blk < read_len; blk += dsk.meta_block_size)
             {
@@ -246,29 +221,23 @@ csum_unknown:
         }
     }
 close_free:
-    if (data)
-    {
-        dsk.meta_disk->free_iomem(data);
-        data = NULL;
-    }
-    if (hdr)
-    {
-        dsk.meta_disk->free_iomem(hdr);
-        hdr = NULL;
-    }
+    free(data);
+    free(hdr);
     if (buffer_area)
     {
-        dsk.journal_disk->free_iomem(buffer_area);
+        free(buffer_area);
         buffer_area = NULL;
     }
     if (do_open)
     {
-        if (dsk.journal_disk && dsk.journal_disk != dsk.meta_disk)
-            delete dsk.journal_disk;
-        if (dsk.meta_disk)
-            delete dsk.meta_disk;
-        dsk.meta_disk = NULL;
-        dsk.journal_disk = NULL;
+        close(dsk.meta_fd);
+        dsk.meta_fd = -1;
+        if (dsk.journal_fd >= 0)
+        {
+            if (dsk.journal_fd != dsk.meta_fd)
+                close(dsk.journal_fd);
+            dsk.journal_fd = -1;
+        }
     }
     return r;
 }
@@ -551,7 +520,7 @@ int disk_tool_t::write_json_meta(json11::Json meta)
     {
         return write_json_heap(meta, meta["journal"]);
     }
-    new_meta_buf = (uint8_t*)dsk.meta_disk->alloc_iomem(new_meta_len);
+    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
     memset(new_meta_buf, 0, new_meta_len);
     blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
     new_hdr->zero = 0;
@@ -589,7 +558,7 @@ int disk_tool_t::write_json_meta(json11::Json meta)
         uint64_t mb = 1 + data_block/new_entries_per_block;
         if (mb >= new_meta_len/new_hdr->meta_block_size)
         {
-            dsk.meta_disk->free_iomem(new_meta_buf);
+            free(new_meta_buf);
             new_meta_buf = NULL;
             fprintf(stderr, "Metadata (data block %ju) doesn't fit into the new area\n", data_block);
             return 1;
@@ -623,7 +592,7 @@ int disk_tool_t::write_json_meta(json11::Json meta)
 int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
 {
     assert(new_meta_len >= sizeof(blockstore_meta_header_v3_t));
-    new_meta_buf = (uint8_t*)dsk.meta_disk->alloc_iomem(new_meta_len);
+    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
     memset(new_meta_buf, 0, new_meta_len);
     new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
     new_meta_hdr->zero = 0;
@@ -658,7 +627,7 @@ int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
     new_journal_buf = NULL;
     if (new_journal_len)
     {
-        new_journal_buf = (uint8_t*)dsk.journal_disk->alloc_iomem(new_journal_len);
+        new_journal_buf = (uint8_t*)memalign(MEM_ALIGNMENT, new_journal_len);
         memset(new_journal_buf, 0, new_journal_len);
     }
     uint64_t total_used_space = 0;
@@ -715,7 +684,8 @@ int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
                     fprintf(stderr, "Write entry %s has invalid type: %s, aborting\n",
                         write_entry.dump().c_str(), write_entry["type"].dump().c_str());
 close_err0:
-                    free_new_meta();
+                    free(new_meta_buf);
+                    new_meta_buf = NULL;
                     return 1;
                 }
                 uint64_t wr_offset = write_entry["offset"].uint64_value();
@@ -802,7 +772,8 @@ close_err0:
         {
             fprintf(stderr, "Metadata should include journal in you want to convert it to the \"heap\" format\n");
 close_err:
-            free_new_meta();
+            free(new_meta_buf);
+            new_meta_buf = NULL;
             return 1;
         }
         std::map<object_id, std::vector<json11::Json::object>> journal_by_object;

src/disk_tool/disk_tool_nvme.cpp

@@ -1,158 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-
-#ifdef WITH_VFN
-
-#include "disk_tool.h"
-#include "str_util.h"
-#include "json_util.h"
-#include "disk_util.h"
-#include "malloc_or_die.h"
-
-#define NVME_CLASS 0x010800
-#define NVME_CLASS_MASK 0xffff00
-
-const char *oncs_bit_names[] = {
-    "compare",
-    "write uncorrectable",
-    "dsm (deallocate)",
-    "write zeroes",
-    "save features",
-    "reservations",
-    "timestamp",
-    "verify",
-    "copy",
-    "copies are atomic",
-    "copies are fast",
-    "write zeroes with deallocate",
-    "namespaces zeroes"
-};
-
-bool bind_to_vfio(const std::string & dev)
-{
-    int r;
-    if (access("/sys/bus/pci/drivers/vfio-pci", F_OK) != 0 && errno == ENOENT)
-    {
-        r = system("modprobe vfio-pci");
-        if (r != 0)
-        {
-            fprintf(stderr, "Failed to load vfio-pci kernel module: %s (code %u)\n", strerror(errno), errno);
-            return false;
-        }
-    }
-    r = write_file("/sys/bus/pci/drivers/vfio-pci/bind", dev);
-    if (r == -ENODEV)
-    {
-        auto vid = str_replace(trim(read_file("/sys/bus/pci/devices/"+dev+"/vendor")), "0x", "");
-        auto pid = str_replace(trim(read_file("/sys/bus/pci/devices/"+dev+"/device")), "0x", "");
-        r = write_file("/sys/bus/pci/drivers/vfio-pci/new_id", vid+" "+pid);
-        if (r != 0)
-        {
-            fprintf(stderr, "Failed to register VID:PID %s:%s in vfio-pci: %s (code %u)\n",
-                vid.c_str(), pid.c_str(), strerror(-r), -r);
-            return false;
-        }
-        r = write_file("/sys/bus/pci/drivers/vfio-pci/bind", dev);
-    }
-    if (r != 0)
-    {
-        fprintf(stderr, "Failed to bind %s to vfio-pci driver: %s (code %u)\n",
-            dev.c_str(), strerror(-r), -r);
-        return false;
-    }
-    return true;
-}
-
-std::string disk_tool_t::find_pci_id(const std::string & nvme_serial, uint32_t nvme_nsid)
-{
-    auto dev = vfn_disk_i::find_pci_id(nvme_serial, nvme_nsid);
-    if (dev == "")
-    {
-        return dev;
-    }
-    std::unique_ptr<vfn_disk_i> dsk(vfn_disk_i::create());
-    if (!dsk->open_dev(dev))
-    {
-        continue;
-    }
-    const nvme_id_ctrl* id_ctrl = dsk->get_id_ctrl();
-    std::string serial = trim(std::string(id_ctrl->sn, 20));
-    fprintf(stderr, "NVMe device %s found - PCI ID %s\n", serial.c_str(), dev.c_str());
-    // Interesting fields: mdts, tnvmcap, nn, vwc, awupf, sgls, msdbd, cap.cmbs, hmpre, hmmin, oncs
-    fprintf(stderr, "Maximum transfer size: %ju bytes\n", dsk->get_mdts());
-    fprintf(stderr, "Total capacity: %ju bytes\n", *(uint64_t*)id_ctrl->tnvmcap);
-    fprintf(stderr, "Max number of namespaces: %u\n", id_ctrl->nn);
-    fprintf(stderr, "Volatile cache: %s\n", (id_ctrl->vwc & NVME_CTRL_VWC_PRESENT) ? "yes" : "no");
-    fprintf(stderr, "Atomic write unit power fail: %u blocks (probably 4k)\n", id_ctrl->awupf+1);
-    fprintf(stderr, "SGL support: %s",
-        (id_ctrl->sgls & NVME_CTRL_SGLS_SUPPORTED) == 1 ? "unaligned" :
-        ((id_ctrl->sgls & NVME_CTRL_SGLS_SUPPORTED) == 2 ? "8 byte aligned" : "no support"));
-    if (id_ctrl->sgls & NVME_CTRL_SGLS_KEYED)
-        fprintf(stderr, ", keyed (RDMA)");
-    if (id_ctrl->sgls & NVME_CTRL_SGLS_BIT_BUCKET)
-        fprintf(stderr, ", bit bucket");
-    if (id_ctrl->sgls & NVME_CTRL_SGLS_MPTR_BYTE_ALIGNED)
-        fprintf(stderr, ", byte-aligned mptr");
-    if (id_ctrl->sgls & NVME_CTRL_SGLS_OVERSIZE)
-        fprintf(stderr, ", oversized");
-    if (id_ctrl->sgls & NVME_CTRL_SGLS_MPTR_SGL)
-        fprintf(stderr, ", SGL mptr");
-    if (id_ctrl->sgls & NVME_CTRL_SGLS_OFFSET)
-        fprintf(stderr, ", offset");
-    if (id_ctrl->sgls & NVME_CTRL_SGLS_TPORT)
-        fprintf(stderr, ", transport data block");
-    fprintf(stderr, "\nMax SGL data block descriptors: %u\n", id_ctrl->msdbd);
-    fprintf(stderr, "Optional command support:");
-    int oncs_n = 0;
-    for (int i = 0; i < sizeof(oncs_bit_names)/sizeof(oncs_bit_names[0]); i++)
-    {
-        if ((id_ctrl->oncs >> i) & 1)
-            fprintf(stderr, "%s%s", oncs_n++ > 0 ? ", " : " ", oncs_bit_names[i]);
-    }
-    fprintf(stderr, "%s\n", oncs_n > 0 ? "" : "none");
-    fprintf(stderr, "CMB: %s\n", NVME_CAP_CMBS(dsk->get_cap()) ? "supported" : "not supported");
-    fprintf(stderr, "HMB: min %u bytes, preferred %u bytes\n", id_ctrl->hmmin*4096, id_ctrl->hmpre*4096);
-    if (nvme_nsid > 0)
-    {
-        if (!dsk->open_ns(nvme_nsid))
-        {
-            return "";
-        }
-        // Interesting fields: nsze, nsfeat (bit 3 - nguid, bit 1 - nawupf), nlbaf,
-        // flbas, mc (bit 1 - OK, bit 0 - not OK), nawupf, nabspf, nabo, nguid, lbaf0-lbaf15
-        const nvme_id_ns *id_ns = dsk->get_id_ns();
-        uint8_t fmt = NVME_FLBAS_LOWER(id_ns->flbas);
-        uint64_t blksize = 1 << id_ns->lbaf[fmt].ds;
-        uint64_t ms = id_ns->lbaf[fmt].ms;
-        fprintf(stderr, "Namespace size: %ju bytes\n", (uint64_t)id_ns->nsze * blksize);
-        fprintf(stderr, "Format %u/%u: %ju byte sectors + %ju byte metadata\n", id_ns->flbas, id_ns->nlbaf+1, blksize, ms);
-        if (id_ns->nsfeat & NVME_NS_FEAT_ID_REUSE)
-        {
-            fprintf(stderr, "Namespace GUID: ");
-            for (int i = 0; i < sizeof(id_ns->nguid); i++)
-                fprintf(stderr, "%02x", id_ns->nguid[i]);
-            fprintf(stderr, "\n");
-        }
-        if (id_ns->nsfeat & NVME_NS_FEAT_NATOMIC)
-        {
-            fprintf(stderr, "Namespace atomic write unit power fail: %u blocks\n", id_ns->nawupf+1);
-            fprintf(stderr, "Namespace atomic boundary power fail: %u blocks\n", id_ns->nabspf);
-            fprintf(stderr, "Namespace atomic boundary offset: %u blocks\n", id_ns->nabo);
-        }
-    }
-    return dev;
-}
-
-#else
-
-#include "disk_tool.h"
-#include <string>
-
-std::string disk_tool_t::find_pci_id(const std::string & nvme_serial, uint32_t nvme_nsid)
-{
-    fprintf(stderr, "Vitastor is built without userspace NVMe (libvfn) support\n");
-    exit(1);
-    return "";
-}
-
-#endif

src/disk_tool/disk_tool_prepare.cpp

@@ -4,7 +4,6 @@
 #include "disk_tool.h"
 #include "str_util.h"
 #include "json_util.h"
-#include "malloc_or_die.h"
 
 void disk_tool_t::parse_meta_reserve()
 {
@@ -273,24 +272,10 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         }
     }
     // Zero out the first block of metadata and journal - OSD will zero the rest on the first run
-    int rz = 0;
-    if (dsk.meta_disk)
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), 4096) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), 4096) != 0)
     {
-        void *zero_buf = dsk.meta_disk->alloc_iomem(4096);
-        memset(zero_buf, 0, 4096);
-        rz = dsk.meta_disk->write_sync(sb["meta_offset"].uint64_value(), 4096, zero_buf);
-        dsk.meta_disk->free_iomem(zero_buf);
-    }
-    if (!rz && dsk.journal_disk)
-    {
-        void *zero_buf = dsk.journal_disk->alloc_iomem(4096);
-        memset(zero_buf, 0, 4096);
-        rz = dsk.journal_disk->write_sync(sb["journal_offset"].uint64_value(), 4096, zero_buf);
-        dsk.journal_disk->free_iomem(zero_buf);
-    }
-    if (rz != 0)
-    {
-        fprintf(stderr, "Failed to zero out metadata or journal header: %s (code %d)\n", strerror(-rz), -rz);
+        fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();
         return 1;
     }

src/disk_tool/disk_tool_resize.cpp

@@ -303,17 +303,16 @@ int disk_tool_t::resize_copy_data()
         iodepth = 32;
     }
     ringloop = new ring_loop_t(iodepth < RINGLOOP_DEFAULT_SIZE ? RINGLOOP_DEFAULT_SIZE : iodepth);
-    dsk.data_disk = disk_i::create_and_open(dsk.data_device, (options["io"] == "cached" ? "cached" : ""), true);
-    if (!dsk.data_disk)
+    dsk.data_fd = open(dsk.data_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
+    if (dsk.data_fd < 0)
     {
-        fprintf(stderr, "Failed to open data device %s\n", dsk.data_device.c_str());
+        fprintf(stderr, "Failed to open data device %s: %s\n", dsk.data_device.c_str(), strerror(errno));
         delete ringloop;
         ringloop = NULL;
         return 1;
     }
-    dsk.data_disk->set_ringloop(ringloop);
     moving_blocks = new resizer_data_moving_t[iodepth];
-    moving_blocks[0].buf = dsk.data_disk->alloc_iomem(iodepth*dsk.data_block_size);
+    moving_blocks[0].buf = memalign_or_die(MEM_ALIGNMENT, iodepth*dsk.data_block_size);
     for (int i = 1; i < iodepth; i++)
     {
         moving_blocks[i].buf = (uint8_t*)moving_blocks[0].buf + i*dsk.data_block_size;
@@ -335,76 +334,52 @@ int disk_tool_t::resize_copy_data()
             }
             if (moving_blocks[i].state == DM_ST_TO_READ)
             {
-                int res = dsk.data_disk->read_async(
-                    dsk.data_offset + moving_blocks[i].old_loc*dsk.data_block_size,
-                    dsk.data_block_size, moving_blocks[i].buf, [this, i](int res)
+                struct io_uring_sqe *sqe = ringloop->get_sqe();
+                if (sqe)
+                {
+                    moving_blocks[i].state = DM_ST_READING;
+                    struct ring_data_t *data = ((ring_data_t*)sqe->user_data);
+                    data->iov = (struct iovec){ moving_blocks[i].buf, dsk.data_block_size };
+                    io_uring_prep_readv(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + moving_blocks[i].old_loc*dsk.data_block_size);
+                    data->callback = [this, i](ring_data_t *data)
                     {
-                        if (res != 0)
+                        if (data->res != dsk.data_block_size)
                         {
                             fprintf(
                                 stderr, "Failed to read %ju bytes at %ju from %s: %s\n", dsk.data_block_size,
                                 dsk.data_offset + moving_blocks[i].old_loc*dsk.data_block_size, dsk.data_device.c_str(),
-                                strerror(-res)
+                                data->res < 0 ? strerror(-data->res) : "short read"
                             );
                             exit(1);
                         }
                         moving_blocks[i].state = DM_ST_TO_WRITE;
                         ringloop->wakeup();
-                    }
-                );
-                if (res == 0)
-                {
-                    moving_blocks[i].state = DM_ST_READING;
-                }
-                else if (res == -EAGAIN)
-                {
-                    break;
-                }
-                else
-                {
-                    fprintf(
-                        stderr, "Failed to submit read of %ju bytes at %ju from %s: %s\n", dsk.data_block_size,
-                        dsk.data_offset + moving_blocks[i].old_loc*dsk.data_block_size, dsk.data_device.c_str(),
-                        strerror(-res)
-                    );
-                    exit(1);
-                }
+                    };
+                 }
             }
             if (moving_blocks[i].state == DM_ST_TO_WRITE)
             {
-                int res = dsk.data_disk->write_async(
-                    dsk.data_offset + moving_blocks[i].new_loc*dsk.data_block_size,
-                    dsk.data_block_size, moving_blocks[i].buf, [this, i](int res)
+                struct io_uring_sqe *sqe = ringloop->get_sqe();
+                if (sqe)
+                {
+                    moving_blocks[i].state = DM_ST_WRITING;
+                    struct ring_data_t *data = ((ring_data_t*)sqe->user_data);
+                    data->iov = (struct iovec){ moving_blocks[i].buf, dsk.data_block_size };
+                    io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + moving_blocks[i].new_loc*dsk.data_block_size);
+                    data->callback = [this, i](ring_data_t *data)
                     {
-                        if (res != 0)
+                        if (data->res != dsk.data_block_size)
                         {
                             fprintf(
                                 stderr, "Failed to write %ju bytes at %ju to %s: %s\n", dsk.data_block_size,
                                 dsk.data_offset + moving_blocks[i].new_loc*dsk.data_block_size, dsk.data_device.c_str(),
-                                strerror(-res)
+                                data->res < 0 ? strerror(-data->res) : "short write"
                             );
                             exit(1);
                         }
                         moving_blocks[i].state = DM_ST_EMPTY;
                         ringloop->wakeup();
-                    }
-                );
-                if (res == 0)
-                {
-                    moving_blocks[i].state = DM_ST_WRITING;
-                }
-                else if (res == -EAGAIN)
-                {
-                    break;
-                }
-                else
-                {
-                    fprintf(
-                        stderr, "Failed to submit write of %ju bytes at %ju to %s: %s\n", dsk.data_block_size,
-                        dsk.data_offset + moving_blocks[i].new_loc*dsk.data_block_size, dsk.data_device.c_str(),
-                        strerror(-res)
-                    );
-                    exit(1);
+                    };
                 }
             }
             remap_active += moving_blocks[i].state != DM_ST_EMPTY ? 1 : 0;
@@ -420,11 +395,11 @@ int disk_tool_t::resize_copy_data()
         ringloop->wait();
     }
     ringloop->unregister_consumer(&ring_consumer);
-    dsk.data_disk->free_iomem(moving_blocks[0].buf);
+    free(moving_blocks[0].buf);
     delete[] moving_blocks;
     moving_blocks = NULL;
-    delete dsk.data_disk;
-    dsk.data_disk = NULL;
+    close(dsk.data_fd);
+    dsk.data_fd = -1;
     delete ringloop;
     ringloop = NULL;
     return 0;
@@ -432,7 +407,7 @@ int disk_tool_t::resize_copy_data()
 
 void disk_tool_t::resize_alloc_journal()
 {
-    new_journal_buf = (uint8_t*)dsk.journal_disk->alloc_iomem(new_journal_len);
+    new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len);
     memset(new_journal_buf, 0, new_journal_len);
     new_journal_ptr = new_journal_buf;
     new_journal_data = new_journal_ptr + dsk.journal_block_size;
@@ -537,24 +512,17 @@ int disk_tool_t::resize_rebuild_journal()
 
 int disk_tool_t::resize_write_new_journal()
 {
-    new_journal_disk = disk_i::create_and_open(new_journal_device, (options["io"] == "cached" ? "cached" : ""), true);
-    if (!new_journal_disk)
+    new_journal_fd = open(new_journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
+    if (new_journal_fd < 0)
     {
-        fprintf(stderr, "Failed to open new journal device %s\n", new_journal_device.c_str());
+        fprintf(stderr, "Failed to open new journal device %s: %s\n", new_journal_device.c_str(), strerror(errno));
         return 1;
     }
-    int r = new_journal_disk->write_sync(new_journal_offset, new_journal_len, new_journal_buf);
-    if (r == 0)
-    {
-        r = new_journal_disk->flush_sync();
-    }
-    if (r != 0)
-    {
-        fprintf(stderr, "Failed to write new journal to %s: %s (code %d)\n", new_journal_device.c_str(), strerror(-r), -r);
-    }
-    delete new_journal_disk;
-    new_journal_disk = NULL;
-    return r;
+    pwrite_blocking(new_journal_fd, new_journal_buf, new_journal_len, new_journal_offset);
+    fsync(new_journal_fd);
+    close(new_journal_fd);
+    new_journal_fd = -1;
+    return 0;
 }
 
 void disk_tool_t::remap_big_write(blockstore_heap_t *heap, heap_entry_t *wr)
@@ -591,7 +559,7 @@ void disk_tool_t::remap_small_write(blockstore_heap_t *heap, heap_entry_t *wr)
 
 int disk_tool_t::resize_rebuild_meta()
 {
-    new_meta_buf = (uint8_t*)dsk.meta_disk->alloc_iomem(new_meta_len);
+    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
     memset(new_meta_buf, 0, new_meta_len);
     new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
     uint64_t new_meta_pos = dsk.meta_block_size;
@@ -723,41 +691,34 @@ int disk_tool_t::resize_rebuild_meta()
 
 int disk_tool_t::resize_write_new_meta()
 {
-    new_meta_disk = disk_i::create_and_open(new_meta_device, (options["io"] == "cached" ? "cached" : ""), true);
-    if (!new_meta_disk)
+    new_meta_fd = open(new_meta_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
+    if (new_meta_fd < 0)
     {
-        fprintf(stderr, "Failed to open new metadata device %s\n", new_meta_device.c_str());
+        fprintf(stderr, "Failed to open new metadata device %s: %s\n", new_meta_device.c_str(), strerror(errno));
         return 1;
     }
-    int r = new_meta_disk->write_sync(new_meta_offset, new_meta_len, new_meta_buf);
-    if (r == 0)
-    {
-        r = new_meta_disk->flush_sync();
-    }
-    if (r != 0)
-    {
-        fprintf(stderr, "Failed to write new metadata to %s: %s (code %d)\n", new_meta_device.c_str(), strerror(-r), -r);
-    }
-    delete new_meta_disk;
-    new_meta_disk = NULL;
-    return r;
+    pwrite_blocking(new_meta_fd, new_meta_buf, new_meta_len, new_meta_offset);
+    fsync(new_meta_fd);
+    close(new_meta_fd);
+    new_meta_fd = -1;
+    return 0;
 }
 
 void disk_tool_t::free_new_meta()
 {
     if ((uint8_t*)new_meta_hdr != new_meta_buf)
     {
-        dsk.meta_disk->free_iomem(new_meta_hdr);
+        free(new_meta_hdr);
         new_meta_hdr = NULL;
     }
     if (new_meta_buf)
     {
-        dsk.meta_disk->free_iomem(new_meta_buf);
+        free(new_meta_buf);
         new_meta_buf = NULL;
     }
     if (new_journal_buf)
     {
-        dsk.journal_disk->free_iomem(new_journal_buf);
+        free(new_journal_buf);
         new_journal_buf = NULL;
     }
 }

src/disk_tool/disk_tool_resize_auto.cpp

@@ -37,6 +37,8 @@ int disk_tool_t::resize_data(std::string device)
         fprintf(stderr, "%s\n", e.what());
         return 1;
     }
+    // Save FD numbers because calc_lengths() relies on them
+    int old_journal_fd = dsk.journal_fd, old_meta_fd = dsk.meta_fd, old_data_fd = dsk.data_fd;
     dsk.close_all();
     bool dry_run = options.find("dry_run") != options.end();
     auto old_journal_device = dsk.journal_device;
@@ -56,7 +58,13 @@ int disk_tool_t::resize_data(std::string device)
             ? dsk.data_device_size : new_data_dev_size;
         dsk.data_device_size = new_data_dev_size;
         dsk.cfg_data_size = 0;
+        dsk.journal_fd = old_journal_fd;
+        dsk.meta_fd = old_meta_fd;
+        dsk.data_fd = old_data_fd;
         dsk.calc_lengths(true);
+        dsk.journal_fd = -1;
+        dsk.meta_fd = -1;
+        dsk.data_fd = -1;
     }
     std::map<std::string, std::string> move_options;
     if (options.find("move_journal") != options.end())

src/disk_tool/disk_tool_udev.cpp

@@ -155,16 +155,15 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
     json11::Json osd_params;
     std::string json_err;
     std::string real_device, device_type, real_data, real_meta, real_journal;
-    int r = 0;
-    auto di = disk_i::create_and_open(device, (options["io"] == "cached" ? "cached,ro" : "ro"), true);
-    if (!di)
+    int r, fd = open(device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
+    if (fd < 0)
     {
-        fprintf(stderr, "Failed to open device %s\n", device.c_str());
+        fprintf(stderr, "Failed to open device %s: %s\n", device.c_str(), strerror(errno));
         return osd_params;
     }
-    buf = (uint8_t*)di->alloc_iomem(4096);
-    r = di->read_sync(0, 4096, buf);
-    if (r != 0)
+    buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096);
+    r = pread_blocking(fd, buf, 4096, 0);
+    if (r != 4096)
     {
         fprintf(stderr, "Failed to read OSD superblock from %s: %s\n", device.c_str(), strerror(errno));
         goto ex;
@@ -187,10 +186,10 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
     if (sb->size > 4096)
     {
         uint64_t sb_size = ((sb->size+4095)/4096)*4096;
-        di->free_iomem(buf);
-        buf = (uint8_t*)di->alloc_iomem(sb_size);
-        r = di->read_sync(0, sb_size, buf);
-        if (r != 0)
+        free(buf);
+        buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, sb_size);
+        r = pread_blocking(fd, buf, sb_size, 0);
+        if (r != sb_size)
         {
             fprintf(stderr, "Failed to read OSD superblock from %s: %s\n", device.c_str(), strerror(errno));
             goto ex;
@@ -225,12 +224,12 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         osd_params = json11::Json();
         goto ex;
     }
-    real_device = disk_i::realpath(device);
-    real_data = disk_i::realpath(osd_params["data_device"].string_value());
+    real_device = realpath_str(device);
+    real_data = realpath_str(osd_params["data_device"].string_value());
     real_meta = osd_params["meta_device"].string_value() != "" && osd_params["meta_device"] != osd_params["data_device"]
-        ? disk_i::realpath(osd_params["meta_device"].string_value()) : "";
+        ? realpath_str(osd_params["meta_device"].string_value()) : "";
     real_journal = osd_params["journal_device"].string_value() != "" && osd_params["journal_device"] != osd_params["meta_device"]
-        ? disk_i::realpath(osd_params["journal_device"].string_value()) : "";
+        ? realpath_str(osd_params["journal_device"].string_value()) : "";
     if (real_journal == real_meta)
     {
         real_journal = "";
@@ -266,12 +265,8 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
         { "real_journal_device", real_journal },
     };
 ex:
-    if (buf)
-    {
-        di->free_iomem(buf);
-        buf = NULL;
-    }
-    delete di;
+    free(buf);
+    close(fd);
     return osd_params;
 }
 

src/disk_tool/disk_tool_utils.cpp

@@ -3,6 +3,7 @@
 
 #include <sys/wait.h>
 #include <sys/ioctl.h>
+#include <dirent.h>
 
 #include "disk_tool.h"
 #include "rw_blocking.h"
@@ -233,6 +234,30 @@ err_pipe1:
     return 255;
 }
 
+int write_zero(int fd, uint64_t offset, uint64_t size)
+{
+    uint64_t buf_len = 1024*1024;
+    void *zero_buf = memalign_or_die(MEM_ALIGNMENT, buf_len);
+    memset(zero_buf, 0, buf_len);
+    ssize_t r;
+    while (size > 0)
+    {
+        r = pwrite(fd, zero_buf, size > buf_len ? buf_len : size, offset);
+        if (r > 0)
+        {
+            size -= r;
+            offset += r;
+        }
+        else if (errno != EAGAIN && errno != EINTR)
+        {
+            free(zero_buf);
+            return -1;
+        }
+    }
+    free(zero_buf);
+    return 0;
+}
+
 // Returns false in case of an error
 // Returns null if there is no partition table
 json11::Json read_parttable(std::string dev)

src/test/CMakeLists.txt

@@ -27,6 +27,17 @@ target_link_libraries(stub_uring_osd
 	${RDMACM_LIBRARIES}
 )
 
+# test_messenger
+add_executable(test_messenger EXCLUDE_FROM_ALL test_messenger.cpp)
+target_link_libraries(test_messenger
+	vitastor_net
+	${LIBURING_LIBRARIES}
+	${IBVERBS_LIBRARIES}
+	${RDMACM_LIBRARIES}
+)
+add_dependencies(build_tests test_messenger)
+add_test(NAME test_messenger COMMAND test_messenger)
+
 # test_allocator
 add_executable(test_allocator EXCLUDE_FROM_ALL test_allocator.cpp)
 target_link_libraries(test_allocator util)

src/test/mock/messenger.cpp

@@ -25,7 +25,7 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
     auto cl_it = clients.find(cur_op->client_id);
     if (cl_it == clients.end() || cl_it->second->peer_state == PEER_STOPPED)
     {
-        cur_op->cancel();
+        cur_op->cancel(-EPIPE);
         return;
     }
     auto cl = cl_it->second;
@@ -34,6 +34,35 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
     cl->sent_ops[cur_op->req.hdr.id] = cur_op;
 }
 
+// Set to true to make cancel_out() pretend that operations are already sent
+// and have to be canceled asynchronously
+bool mock_msgr_async_cancel = false;
+
+bool osd_messenger_t::cancel_out(osd_op_t *cur_op)
+{
+    assert(cur_op->client_id);
+    assert(cur_op->op_type == OSD_OP_OUT);
+    auto cl_it = clients.find(cur_op->client_id);
+    if (cl_it == clients.end() || cl_it->second->peer_state == PEER_STOPPED)
+    {
+        return false;
+    }
+    auto cl = cl_it->second;
+    auto sent_it = cl->sent_ops.find(cur_op->req.hdr.id);
+    if (sent_it == cl->sent_ops.end() || sent_it->second != cur_op)
+    {
+        return false;
+    }
+    if (mock_msgr_async_cancel)
+    {
+        // The caller is expected to "complete" the operation with -ECANCELED itself
+        return false;
+    }
+    cl->sent_ops.erase(sent_it);
+    cur_op->callback = NULL;
+    return true;
+}
+
 void osd_messenger_t::parse_config(const json11::Json & config, bool init)
 {
 }

src/test/test_cluster_client.cpp

@@ -12,6 +12,9 @@
 #include "cluster_client_impl.h"
 #include "msgr_encrypt.h"
 
+// Defined in test/mock/messenger.cpp
+extern bool mock_msgr_async_cancel;
+
 class cluster_client_test_t
 {
 public:
@@ -53,6 +56,9 @@ void configure_single_pg_pool(etcd_state_client_mock_t *mock)
     });
 }
 
+// Last operation posted by test_write()/test_read()/test_sync(), for cancellation tests
+cluster_op_t *last_posted_op = NULL;
+
 int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c, std::function<void()> cb = NULL, bool instant = false)
 {
     printf("Post write %jx+%jx\n", offset, len);
@@ -78,6 +84,7 @@ int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c,
         if (cb != NULL)
             cb();
     };
+    last_posted_op = op;
     cli->execute(op);
     if (instant)
     {
@@ -113,6 +120,7 @@ int *test_read(cluster_client_t *cli, uint64_t offset, uint64_t len, std::functi
         if (cb != NULL)
             cb();
     };
+    last_posted_op = op;
     cli->execute(op);
     return r;
 }
@@ -129,10 +137,11 @@ int *test_sync(cluster_client_t *cli)
         if (*r == -1000)
             printf("Error: Not allowed to complete yet\n");
         assert(*r != -1000);
-        *r = op->retval == 0 ? 1 : 0;
+        *r = op->retval == 0 ? 1 : op->retval;
         printf("Done sync r=%d\n", op->retval);
         delete op;
     };
+    last_posted_op = op;
     cli->execute(op);
     return r;
 }
@@ -1122,6 +1131,127 @@ void test_snapshot_over_nonexistent_pool()
     printf("[ok] test_snapshot_over_nonexistent_pool\n");
 }
 
+void test_cancel()
+{
+    json11::Json config;
+    timerfd_manager_t *tfd = new timerfd_manager_t([](int fd, bool wr, std::function<void(int, int)> callback){});
+    etcd_state_client_mock_t *mock = new etcd_state_client_mock_t();
+    mock->pause();
+    cluster_client_t *cli = new cluster_client_t(NULL, tfd, config, std::unique_ptr<etcd_state_client_t>(mock));
+
+    // Cancel an operation posted before the configuration is loaded
+    int *r1 = test_write(cli, 0, 4096, 0x55);
+    cluster_op_t *op1 = last_posted_op;
+    can_complete(r1);
+    assert(cli->cancel(op1));
+    assert(*r1 == -ECANCELED);
+    delete r1;
+
+    configure_single_pg_pool(mock);
+    mock->resume();
+    pretend_connected(cli, 1);
+
+    // Cancel an operation which is queued, but not sent because the OSD is down
+    pretend_disconnected(cli, 1);
+    r1 = test_write(cli, 0, 4096, 0x56);
+    op1 = last_posted_op;
+    can_complete(r1);
+    assert(cli->cancel(op1));
+    assert(*r1 == -ECANCELED);
+    delete r1;
+    // The canceled write must not be repeated when the OSD comes back
+    pretend_connected(cli, 1);
+    check_op_count(cli, 1, 0);
+
+    // Cancel an already sent operation - the messenger cancels it synchronously
+    r1 = test_write(cli, 0, 4096, 0x57);
+    op1 = last_posted_op;
+    check_op_count(cli, 1, 1);
+    can_complete(r1);
+    assert(cli->cancel(op1));
+    assert(*r1 == -ECANCELED);
+    delete r1;
+    check_op_count(cli, 1, 0);
+    // The connection must NOT be dropped because of a cancellation
+    assert(cli->msgr.osd_peers.find(1) != cli->msgr.osd_peers.end());
+
+    // Cancel an already sent operation - the messenger cancels it asynchronously
+    mock_msgr_async_cancel = true;
+    r1 = test_write(cli, 0, 4096, 0x58);
+    op1 = last_posted_op;
+    check_op_count(cli, 1, 1);
+    osd_op_t *part = find_op(cli, 1, OSD_OP_WRITE, 0, 4096);
+    assert(!cli->cancel(op1));
+    can_complete(r1);
+    pretend_op_completed(cli, part, -ECANCELED);
+    assert(*r1 == -ECANCELED);
+    delete r1;
+    assert(cli->msgr.osd_peers.find(1) != cli->msgr.osd_peers.end());
+    mock_msgr_async_cancel = false;
+
+    // Cancel a sync which waits for a previous write
+    r1 = test_write(cli, 0, 4096, 0x59);
+    check_op_count(cli, 1, 1);
+    int *r2 = test_sync(cli);
+    cluster_op_t *op2 = last_posted_op;
+    check_op_count(cli, 1, 1);
+    can_complete(r2);
+    assert(cli->cancel(op2));
+    assert(*r2 == -ECANCELED);
+    delete r2;
+    // The write must complete as usual
+    can_complete(r1);
+    pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 4096), 0);
+    check_completed(r1);
+
+    // Cancel an already sent sync - the next sync must still sync the same OSD
+    r2 = test_sync(cli);
+    op2 = last_posted_op;
+    check_op_count(cli, 1, 1);
+    can_complete(r2);
+    assert(cli->cancel(op2));
+    assert(*r2 == -ECANCELED);
+    delete r2;
+    check_op_count(cli, 1, 0);
+    r2 = test_sync(cli);
+    check_op_count(cli, 1, 1);
+    can_complete(r2);
+    pretend_op_completed(cli, find_op(cli, 1, OSD_OP_SYNC, 0, 0), 0);
+    check_completed(r2);
+
+    // Cancel a CAS write
+    int *r3 = new int;
+    *r3 = -1000;
+    cluster_op_t *op3 = new cluster_op_t();
+    op3->opcode = OSD_OP_WRITE;
+    op3->inode = 0x1000000000001;
+    op3->offset = 0;
+    op3->len = 4096;
+    op3->version = 1;
+    op3->iov.push_back(malloc_or_die(4096), 4096);
+    memset(op3->iov.buf[0].iov_base, 0x5a, 4096);
+    op3->callback = [r3](cluster_op_t *op)
+    {
+        assert(*r3 != -1000);
+        *r3 = op->retval;
+        printf("Done CAS write %jx+%jx r=%d\n", op->offset, op->len, op->retval);
+        free(op->iov.buf[0].iov_base);
+        delete op;
+    };
+    printf("Post CAS write 0+1000\n");
+    cli->execute(op3);
+    check_op_count(cli, 1, 1);
+    can_complete(r3);
+    assert(cli->cancel(op3));
+    assert(*r3 == -ECANCELED);
+    delete r3;
+    check_op_count(cli, 1, 0);
+
+    delete cli;
+    delete tfd;
+    printf("[ok] test_cancel\n");
+}
+
 int main(int narg, char *args[])
 {
     test1();
@@ -1135,5 +1265,6 @@ int main(int narg, char *args[])
     cluster_client_test_t::test_vault();
     test_pg_config_disappears();
     test_snapshot_over_nonexistent_pool();
+    test_cancel();
     return 0;
 }

src/test/test_heap.cpp

@@ -1010,211 +1010,6 @@ void test_iterate_compaction()
         assert(small_writes == 1);
     }
 
-    {
-        blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_recheck();
-
-        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) BIG_STABLE(v3 l3) un-fsynced
-        // -> skip compaction of l2 into l1 EVEN WHEN under pressure,
-        // because add_compact rejects stable overwrites with EBUSY
-        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
-        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
-
-        heap.use_data(INODE_WITH_POOL(1, 1), 128*1024);
-        uint32_t mblock = 999999;
-        res = _test_do_big_write(heap, dsk, 1, 0, 3, 128*1024, true, 0, 0, buffer_area.data(), &mblock);
-        assert(res == 0);
-        heap.start_block_write(mblock);
-
-        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
-        auto obj = heap.read_entry(oid);
-        assert(obj->lsn == 3);
-        assert(obj->is_overwrite());
-        assert(heap.get_fsynced_lsn() == 2);
-
-        // When not under pressure: postpone compaction
-        int small_writes = 0;
-        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.compact_version == 0);
-        assert(!compact_info.clean_wr);
-        assert(!compact_info.do_delete);
-        assert(small_writes == 0);
-
-        // When under pressure: MUST ALSO postpone compaction because obj is STABLE!
-        // Before the fix, this would incorrectly compact l2 (compact_lsn == 2)
-        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.compact_version == 0);
-        assert(!compact_info.clean_wr);
-        assert(!compact_info.do_delete);
-        assert(small_writes == 0);
-
-        // Verify that add_compact() rejects stable overwrites with EBUSY
-        uint32_t compact_mblock = 999999;
-        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
-        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
-        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
-        assert(res == EBUSY);
-
-        heap.complete_block_write(mblock);
-        assert(heap.get_fsynced_lsn() == 3);
-
-        // Once fsynced, l3 is durable and becomes the clean_wr base
-        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.clean_wr && compact_info.clean_wr->lsn == 3);
-        assert(small_writes == 0);
-    }
-
-    {
-        blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_recheck();
-
-        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) BIG_INTENT(v3 l3) un-fsynced
-        // -> skip compaction of l2 into l1 both when not under pressure and under pressure
-        // Before the fix, BIG_INTENT was missing from the check and was never skipped.
-        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
-        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
-
-        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
-        auto obj = heap.read_entry(oid);
-        uint8_t ext_bitmap[dsk.clean_entry_bitmap_size];
-        memset(ext_bitmap, 0x8e, dsk.clean_entry_bitmap_size);
-        heap.use_data(INODE_WITH_POOL(1, 1), 128*1024);
-        uint32_t mblock = 999999;
-        res = heap.add_redirect_intent(oid, &obj, 3, 0, dsk.data_block_size, 128*1024, ext_bitmap, buffer_area.data(), &mblock);
-        assert(res == 0);
-        heap.start_block_write(mblock);
-
-        obj = heap.read_entry(oid);
-        assert(obj->lsn == 3);
-        assert(obj->type() == BS_HEAP_BIG_INTENT);
-        assert(obj->is_overwrite());
-        assert(heap.get_fsynced_lsn() == 2);
-
-        // When not under pressure: MUST postpone compaction!
-        // Before the fix, BIG_INTENT was not checked, so it incorrectly compacted l2 (compact_lsn == 2)
-        int small_writes = 0;
-        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.compact_version == 0);
-        assert(!compact_info.clean_wr);
-        assert(!compact_info.do_delete);
-        assert(small_writes == 0);
-
-        // When under pressure: MUST ALSO postpone compaction!
-        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.compact_version == 0);
-        assert(!compact_info.clean_wr);
-        assert(!compact_info.do_delete);
-        assert(small_writes == 0);
-
-        // Verify add_compact() rejects stable overwrite with EBUSY
-        uint32_t compact_mblock = 999999;
-        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
-        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
-        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
-        assert(res == EBUSY);
-
-        // Complete block write
-        heap.complete_block_write(mblock);
-        assert(heap.get_fsynced_lsn() == 3);
-
-        // Once fsynced, BIG_INTENT is clean_wr base
-        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.clean_wr && compact_info.clean_wr->lsn == 3);
-        assert(small_writes == 0);
-    }
-
-    {
-        blockstore_heap_t heap(&dsk, buffer_area.data());
-        heap.finish_recheck();
-
-        // Case: BIG_STABLE(v1 l1) SMALL_STABLE(v2 l2) DELETE_STABLE(l3) un-fsynced
-        // -> skip compaction of l2 into l1 EVEN WHEN under pressure,
-        // because add_compact rejects stable overwrites with EBUSY
-        _test_big_write(heap, dsk, 1, 0, 1, 0, true, 0, 4096, buffer_area.data());
-        _test_small_write(heap, dsk, 1, 0, 2, 0, 4096, 0, true, buffer_area.data(), false);
-
-        object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
-        auto obj = heap.read_entry(oid);
-        uint32_t mblock = 999999;
-        res = heap.add_delete(obj, &mblock);
-        assert(res == 0);
-        heap.start_block_write(mblock);
-
-        obj = heap.read_entry(oid);
-        assert(obj->lsn == 3);
-        assert(obj->type() == BS_HEAP_DELETE);
-        assert(obj->is_overwrite());
-        assert(heap.get_fsynced_lsn() == 2);
-
-        // When not under pressure: postpone compaction
-        int small_writes = 0;
-        auto compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.compact_version == 0);
-        assert(!compact_info.clean_wr);
-        assert(!compact_info.do_delete);
-        assert(small_writes == 0);
-
-        // When under pressure: MUST ALSO postpone compaction because DELETE is STABLE!
-        // Before the fix, this would incorrectly compact l2 (compact_lsn == 2)
-        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), true, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.compact_lsn == 0);
-        assert(compact_info.compact_version == 0);
-        assert(!compact_info.clean_wr);
-        assert(!compact_info.do_delete);
-        assert(small_writes == 0);
-
-        // Verify add_compact() rejects stable overwrite with EBUSY
-        uint32_t compact_mblock = 999999;
-        uint8_t ref_int_bitmap[dsk.clean_entry_bitmap_size];
-        memset(ref_int_bitmap, 0, dsk.clean_entry_bitmap_size);
-        res = heap.add_compact(obj, 2, 2, 0, false, &compact_mblock, ref_int_bitmap, ref_int_bitmap, NULL);
-        assert(res == EBUSY);
-
-        // Complete block write
-        heap.complete_block_write(mblock);
-        assert(heap.get_fsynced_lsn() == 3);
-
-        // Once fsynced, DELETE is processed
-        compact_info = heap.iterate_compaction(obj, heap.get_fsynced_lsn(), false, [&](heap_entry_t *wr)
-        {
-            small_writes++;
-        });
-        assert(compact_info.do_delete);
-        assert(!compact_info.clean_wr);
-        assert(small_writes == 0);
-    }
-
     {
         blockstore_heap_t heap(&dsk, buffer_area.data());
         heap.finish_recheck();

src/test/test_messenger.cpp

@@ -0,0 +1,302 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+//
+// Tests for the messenger receive path, in particular for reading and
+// discarding replies to canceled operations (osd_op_t::ignore_reply)
+
+#undef NDEBUG
+#include <assert.h>
+#include <stdio.h>
+#include <vector>
+
+#include "messenger.h"
+#include "util/xxh_x86dispatch.h"
+
+#define BMP_LEN 8
+#define DATA_LEN 4096
+
+// The receive path is protected, so reach it through a subclass
+struct test_msgr_t: osd_messenger_t
+{
+    bool feed(osd_client_t *cl, uint8_t *buf, size_t len) { return handle_read_buffer(cl, buf, len); }
+    void drain() { handle_immediate_ops(); }
+    osd_op_t *make_stub(osd_op_t *op) { return copy_out_op_to_ignore(op); }
+    size_t serialize(osd_client_t *cl, uint8_t *dst, size_t len) { return copy_ops_to(cl, dst, len); }
+    size_t immediate_count() { return set_immediate_ops.size(); }
+};
+
+static void append_csum(std::vector<uint8_t> & buf, int csum_status, size_t payload)
+{
+    if (csum_status == MSGR_CSUM_FULL)
+    {
+        // The header is covered by the checksum too
+        uint64_t csum = XXH3_64bits(buf.data(), buf.size());
+        buf.resize(buf.size()+8);
+        memcpy(buf.data()+buf.size()-8, &csum, 8);
+    }
+    else if (csum_status == MSGR_CSUM_PAYLOAD)
+    {
+        uint64_t csum = XXH3_64bits(buf.data()+OSD_PACKET_SIZE, payload);
+        buf.resize(buf.size()+8);
+        memcpy(buf.data()+buf.size()-8, &csum, 8);
+    }
+}
+
+// SEC_READ reply: header + bitmap + data
+static std::vector<uint8_t> make_read_reply(uint64_t id, uint8_t fill, int csum_status = 0)
+{
+    std::vector<uint8_t> buf(OSD_PACKET_SIZE + BMP_LEN + DATA_LEN);
+    osd_any_reply_t *r = (osd_any_reply_t*)buf.data();
+    memset(buf.data(), 0, OSD_PACKET_SIZE);
+    r->hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
+    r->hdr.id = id;
+    r->hdr.opcode = OSD_OP_SEC_READ;
+    r->hdr.retval = DATA_LEN;
+    r->sec_rw.attr_len = BMP_LEN;
+    memset(buf.data()+OSD_PACKET_SIZE, 0xB0|fill, BMP_LEN);
+    memset(buf.data()+OSD_PACKET_SIZE+BMP_LEN, fill, DATA_LEN);
+    append_csum(buf, csum_status, BMP_LEN+DATA_LEN);
+    return buf;
+}
+
+// SEC_LIST reply - a single payload region, unlike reads
+static std::vector<uint8_t> make_list_reply(uint64_t id, uint64_t count, int csum_status = 0)
+{
+    size_t payload = sizeof(obj_ver_id)*count;
+    std::vector<uint8_t> buf(OSD_PACKET_SIZE + payload);
+    osd_any_reply_t *r = (osd_any_reply_t*)buf.data();
+    memset(buf.data(), 0, OSD_PACKET_SIZE);
+    r->hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
+    r->hdr.id = id;
+    r->hdr.opcode = OSD_OP_SEC_LIST;
+    r->hdr.retval = count;
+    memset(buf.data()+OSD_PACKET_SIZE, 0x5A, payload);
+    append_csum(buf, csum_status, payload);
+    return buf;
+}
+
+static osd_op_t *make_read_op(uint64_t id)
+{
+    osd_op_t *op = new osd_op_t;
+    op->op_type = OSD_OP_OUT;
+    op->req.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
+    op->req.hdr.id = id;
+    op->req.hdr.opcode = OSD_OP_SEC_READ;
+    op->req.sec_rw.len = DATA_LEN;
+    op->bitmap_len = BMP_LEN;
+    return op;
+}
+
+// A read operation with preallocated buffers, like cluster_client and the OSD make them
+static osd_op_t *make_real_read_op(uint64_t id, int *done, uint8_t **out_data)
+{
+    osd_op_t *op = make_read_op(id);
+    op->buf = malloc_or_die(BMP_LEN + DATA_LEN);
+    op->iov.push_back(op->buf, DATA_LEN);
+    op->bitmap = (uint8_t*)op->buf + DATA_LEN;
+    *out_data = (uint8_t*)op->buf;
+    op->callback = [done](osd_op_t *op) { *done = op->reply.hdr.retval; };
+    return op;
+}
+
+// Put a stub in place of a canceled operation, like cancel_out() does
+static void cancel_op(test_msgr_t *msgr, osd_client_t *cl, osd_op_t *op)
+{
+    osd_op_t *stub = msgr->make_stub(op);
+    assert(stub->ignore_reply);
+    assert(stub->op_type == OSD_OP_OUT);
+    // No data buffers are allocated for a discarded reply
+    assert(!stub->buf && !stub->iov.count && !stub->bitmap);
+    cl->sent_ops[op->req.hdr.id] = stub;
+    delete op;
+}
+
+static osd_client_t *add_client(test_msgr_t *msgr, int csum_status)
+{
+    auto cl = new osd_client_t();
+    cl->client_id = msgr->next_client_id++;
+    cl->peer_fd = -1;
+    cl->peer_state = PEER_CONNECTED;
+    cl->proto_csum_status = csum_status;
+    msgr->clients[cl->client_id] = cl;
+    return cl;
+}
+
+// A canceled reply is read and thrown away, and the stub is freed
+static void test_discard_one_chunk(test_msgr_t *msgr)
+{
+    auto cl = add_client(msgr, 0);
+    cancel_op(msgr, cl, make_read_op(1));
+    auto buf = make_read_reply(1, 0x11);
+    assert(msgr->feed(cl, buf.data(), buf.size()));
+    assert(cl->sent_ops.size() == 0);
+    assert(msgr->immediate_count() == 1);
+    msgr->drain();
+    msgr->stop_client(cl->client_id);
+    printf("[ok] discarded reply, single chunk\n");
+}
+
+// The same, but split into small buffers - checks that the skip resumes correctly
+static void test_discard_chunked(test_msgr_t *msgr)
+{
+    auto cl = add_client(msgr, 0);
+    cancel_op(msgr, cl, make_read_op(1));
+    auto buf = make_read_reply(1, 0x22);
+    size_t chunk = 100;
+    for (size_t pos = 0; pos < buf.size(); pos += chunk)
+    {
+        size_t n = buf.size()-pos < chunk ? buf.size()-pos : chunk;
+        assert(msgr->feed(cl, buf.data()+pos, n));
+    }
+    assert(cl->sent_ops.size() == 0);
+    msgr->drain();
+    msgr->stop_client(cl->client_id);
+    printf("[ok] discarded reply, %zu-byte chunks\n", chunk);
+}
+
+// The important one: if the discard consumes the wrong number of bytes,
+// the next reply in the same buffer is garbage
+static void test_stream_stays_in_sync(test_msgr_t *msgr, int csum_status)
+{
+    auto cl = add_client(msgr, csum_status);
+    cancel_op(msgr, cl, make_read_op(1));
+    int done = 0;
+    uint8_t *data = NULL;
+    osd_op_t *real = make_real_read_op(2, &done, &data);
+    cl->sent_ops[2] = real;
+    auto b1 = make_read_reply(1, 0x33, csum_status);
+    auto b2 = make_read_reply(2, 0x44, csum_status);
+    std::vector<uint8_t> both(b1);
+    both.insert(both.end(), b2.begin(), b2.end());
+    assert(msgr->feed(cl, both.data(), both.size()));
+    msgr->drain();
+    assert(!cl->io_error);
+    assert(cl->sent_ops.size() == 0);
+    assert(done == DATA_LEN);
+    for (int i = 0; i < DATA_LEN; i++)
+        assert(data[i] == 0x44);
+    assert(((uint8_t*)real->bitmap)[0] == (0xB0|0x44));
+    delete real;
+    msgr->stop_client(cl->client_id);
+    printf("[ok] stream stays in sync after a discarded reply, csum_status=%d\n", csum_status);
+}
+
+// SEC_LIST goes through the single-payload-region branch of the discard path
+static void test_discard_list(test_msgr_t *msgr, int csum_status)
+{
+    auto cl = add_client(msgr, csum_status);
+    osd_op_t *op = new osd_op_t;
+    op->op_type = OSD_OP_OUT;
+    op->req.hdr.id = 1;
+    op->req.hdr.opcode = OSD_OP_SEC_LIST;
+    cancel_op(msgr, cl, op);
+    int done = 0;
+    uint8_t *data = NULL;
+    osd_op_t *real = make_real_read_op(2, &done, &data);
+    cl->sent_ops[2] = real;
+    auto b1 = make_list_reply(1, 37, csum_status);
+    auto b2 = make_read_reply(2, 0x77, csum_status);
+    std::vector<uint8_t> both(b1);
+    both.insert(both.end(), b2.begin(), b2.end());
+    assert(msgr->feed(cl, both.data(), both.size()));
+    msgr->drain();
+    assert(!cl->io_error);
+    assert(cl->sent_ops.size() == 0);
+    assert(done == DATA_LEN);
+    for (int i = 0; i < DATA_LEN; i++)
+        assert(data[i] == 0x77);
+    delete real;
+    msgr->stop_client(cl->client_id);
+    printf("[ok] discarded SEC_LIST + real reply, csum_status=%d\n", csum_status);
+}
+
+// The data is thrown away, but it's still checksummed on the way
+static void test_discard_checks_csum(test_msgr_t *msgr, int csum_status)
+{
+    auto cl = add_client(msgr, csum_status);
+    uint64_t client_id = cl->client_id;
+    cancel_op(msgr, cl, make_read_op(1));
+    auto buf = make_read_reply(1, 0x88, csum_status);
+    buf[OSD_PACKET_SIZE+BMP_LEN+10] ^= 0xff;
+    printf("-- a checksum mismatch message is expected below:\n");
+    msgr->feed(cl, buf.data(), buf.size());
+    // A checksum mismatch stops the client, so it has to be gone by now
+    assert(msgr->clients.find(client_id) == msgr->clients.end());
+    printf("[ok] corruption inside a discarded reply is still detected, csum_status=%d\n", csum_status);
+}
+
+// Build a SEC_READ reply the way an OSD does, for the real send path
+static osd_op_t *make_reply_op(uint64_t id, uint8_t fill)
+{
+    osd_op_t *op = new osd_op_t;
+    op->op_type = OSD_OP_IN;
+    op->req.hdr.id = id;
+    op->req.hdr.opcode = OSD_OP_SEC_READ;
+    op->req.sec_rw.len = DATA_LEN;
+    op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
+    op->reply.hdr.id = id;
+    op->reply.hdr.opcode = OSD_OP_SEC_READ;
+    op->reply.hdr.retval = DATA_LEN;
+    op->reply.sec_rw.attr_len = BMP_LEN;
+    op->buf = malloc_or_die(BMP_LEN + DATA_LEN);
+    memset(op->buf, fill, DATA_LEN);
+    op->bitmap = (uint8_t*)op->buf + DATA_LEN;
+    memset(op->bitmap, 0xB0|fill, BMP_LEN);
+    op->iov.push_back(op->buf, DATA_LEN);
+    return op;
+}
+
+// Serialize two replies through the real send path and feed them into the real receive
+// path, discarding the first one. Unlike the tests above, the checksum is produced by
+// the messenger itself, so this also verifies that the sender and the receiver agree
+static void test_roundtrip(test_msgr_t *msgr, int csum_status)
+{
+    auto scl = add_client(msgr, csum_status);
+    scl->write_ops.push_back(make_reply_op(1, 0x11));
+    scl->write_ops.push_back(make_reply_op(2, 0x22));
+    std::vector<uint8_t> wire(2*(OSD_PACKET_SIZE + BMP_LEN + DATA_LEN + 8));
+    size_t n = msgr->serialize(scl, wire.data(), wire.size());
+    assert(!scl->write_op && !scl->write_ops.size());
+    wire.resize(n);
+
+    auto rcl = add_client(msgr, csum_status);
+    cancel_op(msgr, rcl, make_read_op(1));
+    int done = 0;
+    uint8_t *data = NULL;
+    osd_op_t *real = make_real_read_op(2, &done, &data);
+    rcl->sent_ops[2] = real;
+    assert(msgr->feed(rcl, wire.data(), wire.size()));
+    msgr->drain();
+    assert(!rcl->io_error);
+    assert(rcl->sent_ops.size() == 0);
+    assert(done == DATA_LEN);
+    for (int i = 0; i < DATA_LEN; i++)
+        assert(data[i] == 0x22);
+    assert(((uint8_t*)real->bitmap)[0] == (0xB0|0x22));
+    delete real;
+    msgr->stop_client(rcl->client_id);
+    msgr->stop_client(scl->client_id);
+    printf("[ok] send->receive round trip with a discarded reply, csum_status=%d\n", csum_status);
+}
+
+int main()
+{
+    setvbuf(stdout, NULL, _IONBF, 0);
+    auto msgr = new test_msgr_t();
+    test_discard_one_chunk(msgr);
+    test_discard_chunked(msgr);
+    test_stream_stays_in_sync(msgr, 0);
+    test_stream_stays_in_sync(msgr, MSGR_CSUM_PAYLOAD);
+    test_stream_stays_in_sync(msgr, MSGR_CSUM_FULL);
+    test_discard_list(msgr, 0);
+    test_discard_list(msgr, MSGR_CSUM_PAYLOAD);
+    test_discard_list(msgr, MSGR_CSUM_FULL);
+    test_discard_checks_csum(msgr, MSGR_CSUM_PAYLOAD);
+    test_discard_checks_csum(msgr, MSGR_CSUM_FULL);
+    test_roundtrip(msgr, 0);
+    test_roundtrip(msgr, MSGR_CSUM_PAYLOAD);
+    test_roundtrip(msgr, MSGR_CSUM_FULL);
+    delete msgr;
+    printf("All messenger tests passed\n");
+    return 0;
+}

src/util/CMakeLists.txt

@@ -19,27 +19,20 @@ add_library(util STATIC
 	../../json11/json11.cpp
 	xxh_x86dispatch.c
 )
-if (RDMACM_LIBRARIES OR (VFN_LIBRARIES AND NVME_LIBRARIES))
+if (RDMACM_LIBRARIES)
 	target_sources(util PRIVATE
 		dma_alloc_impl.cpp
 	)
-endif (RDMACM_LIBRARIES OR (VFN_LIBRARIES AND NVME_LIBRARIES))
+endif (RDMACM_LIBRARIES)
 if (RDMACM_LIBRARIES)
 	target_sources(util PRIVATE
 		rdma_alloc.cpp
 	)
 endif (RDMACM_LIBRARIES)
-if (VFN_LIBRARIES AND NVME_LIBRARIES)
-	target_sources(util PRIVATE
-		vfn_alloc.cpp
-	)
-endif (VFN_LIBRARIES AND NVME_LIBRARIES)
 target_compile_options(util PUBLIC -fPIC)
 target_link_libraries(util
 	${LIBURING_LIBRARIES}
 	${OPENSSL_LIBRARIES}
-	${VFN_LIBRARIES}
-	${NVME_LIBRARIES}
 )
 
 # libcrc32c.a to isolate the isa-l dependency

src/util/disk_util.h

@@ -4,76 +4,6 @@
 #pragma once
 
 #include <string>
-#include <map>
-#include <functional>
 #include <stdint.h>
-#include <time.h>
-
-#ifdef WITH_VFN
-#include <nvme/types.h>
-#endif
-
-class ring_loop_i;
-
-class disk_i
-{
-public:
-    static disk_i *create_and_open(const std::string & path, const std::string & mode, bool disable_flock = false);
-    static disk_i *create_mock(int mock_fd, uint64_t mock_size, uint64_t mock_sectsize);
-    static std::string realpath(const std::string & path);
-    virtual ~disk_i() = default;
-    virtual void set_ringloop(ring_loop_i *ringloop) = 0;
-    virtual bool is_off_ring() const = 0;
-    virtual int get_fd() const = 0;
-    virtual void close() = 0;
-    virtual uint64_t get_size() const = 0;
-    virtual uint64_t get_block_size() const = 0;
-    virtual uint64_t get_max_io_size() const = 0;
-    virtual void *alloc_iomem(size_t size) = 0;
-    virtual void free_iomem(void *buf) = 0;
-    virtual uint32_t get_queue_space() const = 0;
-    // all errors are passed as negative errno's
-    virtual int rw_sync(bool is_wr, uint64_t offset, size_t len, void *buf) = 0;
-    virtual int flush_sync() = 0;
-    virtual int discard_sync(uint64_t offset, uint64_t len) = 0;
-    virtual int rw_async(bool is_wr, uint64_t offset, size_t len, void *buf, bool fua, std::function<void(int res)> cb) = 0;
-    virtual int flush_async(std::function<void(int res)> cb) = 0;
-    virtual int discard_async(uint64_t offset, uint64_t len, std::function<void(int res)> cb) = 0;
-    inline int read_sync(uint64_t offset, size_t len, void *buf)
-    {
-        return rw_sync(false, offset, len, buf);
-    }
-    inline int write_sync(uint64_t offset, size_t len, void *buf)
-    {
-        return rw_sync(true, offset, len, buf);
-    }
-    inline int read_async(uint64_t offset, size_t len, void *buf, std::function<void(int res)> cb)
-    {
-        return rw_async(false, offset, len, buf, false, cb);
-    }
-    inline int write_async(uint64_t offset, size_t len, void *buf, std::function<void(int res)> cb, bool fua = false)
-    {
-        return rw_async(true, offset, len, buf, fua, cb);
-    }
-};
 
 void check_size(int fd, uint64_t *size, uint64_t *sectsize, const std::string & name);
-
-#ifdef WITH_VFN
-class vfn_disk_i: public disk_i
-{
-public:
-    static vfn_disk_i *create();
-    virtual ~vfn_disk_i() = default;
-    virtual bool open_dev(const std::string & pci_id) = 0;
-    virtual bool open_ns(uint32_t ns_num) = 0;
-    virtual const nvme_id_ctrl* get_id_ctrl() const = 0;
-    virtual const nvme_id_ns* get_id_ns() const = 0;
-    virtual uint64_t get_mdts() const = 0;
-    virtual uint64_t get_cap() const = 0;
-
-    static bool is_vfn_path(const std::string & path);
-    static bool bind_to_vfio(const std::string & dev);
-    static std::string find_pci_id(const std::string & nvme_serial, uint32_t nvme_nsid);
-};
-#endif

src/util/ringloop.cpp

@@ -52,8 +52,6 @@ void ring_loop_t::register_consumer(ring_consumer_t *consumer)
 {
     unregister_consumer(consumer);
     consumers.push_back(consumer);
-    if (consumer->is_idle)
-        idle_consumers++;
 }
 
 void ring_loop_t::wakeup()
@@ -67,8 +65,6 @@ void ring_loop_t::unregister_consumer(ring_consumer_t *consumer)
     {
         if (consumers[i] == consumer)
         {
-            if (consumer->is_idle)
-                idle_consumers--;
             consumers.erase(consumers.begin()+i, consumers.begin()+i+1);
             break;
         }
@@ -158,30 +154,10 @@ void ring_loop_t::loop()
     }
     do
     {
-        do
+        loop_again = false;
+        for (int i = 0; i < consumers.size(); i++)
         {
-            loop_again = false;
-            for (int i = 0; i < consumers.size(); i++)
-            {
-                if (!consumers[i]->is_idle)
-                    consumers[i]->loop();
-            }
-            if (immediate_queue.size())
-            {
-                immediate_queue2.swap(immediate_queue);
-                for (auto & cb: immediate_queue2)
-                    cb();
-                immediate_queue2.clear();
-            }
-        } while (loop_again);
-        if (idle_consumers > 0)
-        {
-            // idle consumers are those who poll devices when everything is idle
-            for (int i = 0; i < consumers.size(); i++)
-            {
-                if (consumers[i]->is_idle)
-                    consumers[i]->loop();
-            }
+            consumers[i]->loop();
             if (immediate_queue.size())
             {
                 immediate_queue2.swap(immediate_queue);

src/util/ringloop.h

@@ -32,7 +32,6 @@ struct ring_data_t
 struct ring_consumer_t
 {
     std::function<void(void)> loop;
-    bool is_idle = false;
 };
 
 class __attribute__((visibility("default"))) ring_loop_i
@@ -61,7 +60,6 @@ class __attribute__((visibility("default"))) ring_loop_t: public ring_loop_i
     std::vector<ring_consumer_t*> consumers;
     std::vector<ring_data_t> ring_datas;
     std::vector<int> free_ring_data;
-    int idle_consumers = 0;
     std::mutex mu;
     bool mt;
     bool in_loop;

src/util/vfn_alloc.cpp

@@ -1,56 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-//
-// libvfn (IOMMU) variant of our simple allocator
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
-#include <vfn/iommu.h>
-
-#include "dma_alloc_impl.h"
-
-static_assert(sizeof(iova_t) == sizeof(void*));
-
-class vfn_allocator_t: public dma_allocator_t
-{
-protected:
-    iommu_ctx *ctx = NULL;
-
-    void* reg_mr(void *buf, size_t len) override
-    {
-        iova_t iova = 0;
-        if (iommu_map_vaddr(ctx, buf, len, &iova, 0))
-        {
-            fprintf(stderr, "iommu_map_vaddr %zu bytes failed: %s\n", len, strerror(errno));
-            exit(1);
-        }
-        return (void*)iova;
-    }
-
-    void dereg_mr(void *buf, size_t len, void *handle) override
-    {
-        if (iommu_unmap_vaddr(ctx, buf, &len))
-        {
-            fprintf(stderr, "iommu_unmap_vaddr failed: %s\n", strerror(errno));
-            exit(1);
-        }
-    }
-public:
-    vfn_allocator_t(iommu_ctx *ctx, size_t region_size, size_t max_unused)
-        : dma_allocator_t(region_size, max_unused, 1), ctx(ctx)
-    {
-        alignment = page_size;
-    }
-
-    ~vfn_allocator_t()
-    {
-        free_unused_buffers(0, true);
-    }
-};
-
-dma_allocator_i *vfn_malloc_create(iommu_ctx *ctx, size_t region_size, size_t max_unused)
-{
-    return new vfn_allocator_t(ctx, region_size, max_unused);
-}

src/util/vfn_alloc.h

@@ -1,11 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-//
-// libvfn (IOMMU) variant of our simple allocator
-
-#pragma once
-
-#include "rdma_alloc.h"
-
-struct iommu_ctx;
-dma_allocator_i *vfn_malloc_create(iommu_ctx *ctx, size_t rdma_alloc_size, size_t rdma_max_unused);