WIP: Introduce meson build files

meson.build

@@ -0,0 +1,625 @@
+project(
+  'vitastor',
+  ['cpp', 'c'],
+  version: '3.0.0',
+  license: 'VNPL-1.1',
+  default_options: [
+    'buildtype=debugoptimized',
+    'cpp_std=c++17',
+    'b_pie=true',
+    'warning_level=1',
+  ],
+)
+
+cc = meson.get_compiler('c')
+cpp = meson.get_compiler('cpp')
+pkgconfig = import('pkgconfig')
+
+with_qemu = get_option('with_qemu')
+with_fio = get_option('with_fio')
+with_asan = get_option('with_asan')
+with_system_liburing = get_option('with_system_liburing')
+enable_coverage = get_option('enable_coverage')
+build_tests_opt = get_option('build_tests')
+qemu_plugindir = get_option('qemu_plugindir')
+
+common_args = [
+  '-DVITASTOR_VERSION="@0@"'.format(meson.project_version()),
+  '-D_GNU_SOURCE',
+  '-D_LARGEFILE64_SOURCE',
+  '-D_FILE_OFFSET_BITS=64',
+  '-Wno-sign-compare',
+  '-Wno-comment',
+  '-Wno-parentheses',
+  '-Wno-pointer-arith',
+  '-fdiagnostics-color=always',
+  '-fvisibility=hidden',
+  '-fno-omit-frame-pointer',
+]
+add_project_arguments(common_args, language: ['c', 'cpp'])
+add_project_link_arguments(['-fno-omit-frame-pointer'], language: ['c', 'cpp'])
+
+if with_asan
+  add_project_arguments(['-fsanitize=address'], language: ['c', 'cpp'])
+  add_project_link_arguments(['-fsanitize=address', '-fno-omit-frame-pointer'], language: ['c', 'cpp'])
+endif
+
+if enable_coverage
+  add_project_arguments(['-coverage'], language: ['c', 'cpp'])
+  add_project_link_arguments(['-coverage'], language: ['c', 'cpp'])
+endif
+
+have_nbd_netlink = cc.has_header('linux/nbd-netlink.h')
+
+threads_dep = dependency('threads')
+ibverbs_dep = dependency('libibverbs', required: false)
+if ibverbs_dep.found()
+  add_project_arguments(['-DWITH_RDMA'], language: ['c', 'cpp'])
+endif
+
+isal_dep = dependency('libisal', required: false)
+if isal_dep.found()
+  add_project_arguments(['-DWITH_ISAL'], language: ['c', 'cpp'])
+endif
+
+rdmacm_dep = dependency('librdmacm', required: false)
+if rdmacm_dep.found()
+  add_project_arguments(['-DWITH_RDMACM'], language: ['c', 'cpp'])
+endif
+
+glib_dep = dependency('glib-2.0', required: with_qemu)
+jerasure_dep = dependency('Jerasure', required: false)
+if not jerasure_dep.found()
+  jerasure_dep = cpp.find_library('Jerasure', required: true)
+endif
+
+tcmalloc_dep = dependency('tcmalloc_minimal', required: false)
+if not tcmalloc_dep.found()
+  tcmalloc_dep = cpp.find_library('tcmalloc_minimal', required: true)
+endif
+
+nl3_dep = dependency(['libnl-3.0', 'libnl-genl-3.0'], required : false)
+
+liburing_dep = dependency('liburing', required: with_system_liburing, version: '>=2.10')
+liburing_inc = []
+if not liburing_dep.found()
+  liburing_inc = include_directories('src/liburing/include')
+  liburing_lib = static_library(
+    'uring',
+    [
+      'src/liburing/queue.c',
+      'src/liburing/register.c',
+      'src/liburing/setup.c',
+      'src/liburing/syscall.c',
+      'src/liburing/version.c',
+    ],
+    include_directories: liburing_inc,
+    c_args: ['-D_GNU_SOURCE', '-D_LARGEFILE_SOURCE', '-D_FILE_OFFSET_BITS=64'],
+    pic: true,
+    install: false,
+  )
+  liburing_dep = declare_dependency(
+    link_with: liburing_lib,
+    include_directories: liburing_inc,
+  )
+endif
+
+inc_common = include_directories(
+  'src',
+  'src/blockstore',
+  'src/cmd',
+  'src/client',
+  'src/disk_tool',
+  'src/kv',
+  'src/nfs',
+  'src/osd',
+  'src/test',
+  'src/util',
+  '/usr/include/jerasure',
+)
+
+inc = [inc_common, liburing_inc]
+vitastor_common_sources = [
+  'src/util/epoll_manager.cpp',
+  'src/client/etcd_state_client.cpp',
+  'src/client/messenger.cpp',
+  'src/util/addr_util.cpp',
+  'src/client/msgr_stop.cpp',
+  'src/client/msgr_op.cpp',
+  'src/client/msgr_send.cpp',
+  'src/client/msgr_receive.cpp',
+  'src/util/ringloop.cpp',
+  'json11/json11.cpp',
+  'src/client/http_client.cpp',
+  'src/client/osd_ops.cpp',
+  'src/client/pg_states.cpp',
+  'src/util/timerfd_manager.cpp',
+  'src/util/str_util.cpp',
+  'src/util/json_util.cpp',
+]
+if ibverbs_dep.found()
+  vitastor_common_sources += 'src/client/msgr_rdma.cpp'
+endif
+if rdmacm_dep.found()
+  vitastor_common_sources += 'src/client/msgr_rdmacm.cpp'
+endif
+
+vitastor_common = static_library(
+  'vitastor_common',
+  vitastor_common_sources,
+  include_directories: inc,
+  dependencies: [threads_dep, ibverbs_dep, rdmacm_dep],
+  pic: true,
+  install: false,
+)
+vitastor_common_dep = declare_dependency(
+  link_with: vitastor_common,
+  include_directories: inc,
+  dependencies: [threads_dep, ibverbs_dep, rdmacm_dep],
+)
+
+vitastor_blk = static_library(
+  'vitastor_blk',
+  [
+    'src/util/allocator.cpp',
+    'src/util/crc32c.c',
+    'src/util/ringloop.cpp',
+    'src/blockstore/multilist.cpp',
+    'src/blockstore/blockstore_heap.cpp',
+    'src/blockstore/blockstore_disk.cpp',
+    'src/blockstore/blockstore.cpp',
+    'src/blockstore/blockstore_impl.cpp',
+    'src/blockstore/blockstore_init.cpp',
+    'src/blockstore/blockstore_open.cpp',
+    'src/blockstore/blockstore_flush.cpp',
+    'src/blockstore/blockstore_read.cpp',
+    'src/blockstore/blockstore_stable.cpp',
+    'src/blockstore/blockstore_sync.cpp',
+    'src/blockstore/blockstore_write.cpp',
+    'src/blockstore/v1/flush.cpp',
+    'src/blockstore/v1/impl.cpp',
+    'src/blockstore/v1/init.cpp',
+    'src/blockstore/v1/journal.cpp',
+    'src/blockstore/v1/open.cpp',
+    'src/blockstore/v1/read.cpp',
+    'src/blockstore/v1/rollback.cpp',
+    'src/blockstore/v1/stable.cpp',
+    'src/blockstore/v1/sync.cpp',
+    'src/blockstore/v1/write.cpp',
+  ],
+  include_directories: inc,
+  dependencies: [liburing_dep, isal_dep],
+  link_with: vitastor_common,
+  pic: true,
+  install: false,
+)
+
+fio_vitastor_blk = []
+if with_fio
+  fio_vitastor_blk = shared_library(
+    'fio_vitastor_blk',
+    [
+      'src/blockstore/fio_engine.cpp',
+      'json11/json11.cpp',
+    ],
+    include_directories: inc,
+    link_with: vitastor_blk,
+    install: true,
+  )
+endif
+
+vitastor_cli = static_library(
+  'vitastor_cli',
+  [
+    'src/cmd/cli_common.cpp',
+    'src/cmd/cli_alloc_osd.cpp',
+    'src/cmd/cli_status.cpp',
+    'src/cmd/cli_describe.cpp',
+    'src/cmd/cli_fix.cpp',
+    'src/cmd/cli_ls.cpp',
+    'src/cmd/cli_create.cpp',
+    'src/cmd/cli_dd.cpp',
+    'src/cmd/cli_modify.cpp',
+    'src/cmd/cli_modify_osd.cpp',
+    'src/cmd/cli_osd_tree.cpp',
+    'src/cmd/cli_pg_ls.cpp',
+    'src/cmd/cli_flatten.cpp',
+    'src/cmd/cli_merge.cpp',
+    'src/cmd/cli_rm_data.cpp',
+    'src/cmd/cli_rm_wildcard.cpp',
+    'src/cmd/cli_rm.cpp',
+    'src/cmd/cli_rm_osd.cpp',
+    'src/cmd/cli_pool_cfg.cpp',
+    'src/cmd/cli_pool_create.cpp',
+    'src/cmd/cli_pool_ls.cpp',
+    'src/cmd/cli_pool_modify.cpp',
+    'src/cmd/cli_pool_rm.cpp',
+  ],
+  include_directories: inc,
+  pic: true,
+  install: false,
+)
+
+vitastor_client = shared_library(
+  'vitastor_client',
+  [
+    'src/client/cluster_client.cpp',
+    'src/client/cluster_client_list.cpp',
+    'src/client/cluster_client_wb.cpp',
+    'src/client/vitastor_c.cpp',
+  ],
+  include_directories: inc,
+  link_with: [vitastor_common, vitastor_cli],
+  dependencies: [liburing_dep, ibverbs_dep, rdmacm_dep],
+  version: meson.project_version(),
+  soversion: '0',
+  install: true,
+)
+
+pkgconfig.generate(
+  name: 'vitastor',
+  description: 'Vitastor client library',
+  version: meson.project_version(),
+  libraries: vitastor_client,
+  subdirs: ['client'],
+  filebase: 'vitastor',
+)
+
+if with_fio
+  fio_vitastor = shared_library(
+    'fio_vitastor',
+    ['src/client/fio_cluster.cpp'],
+    include_directories: inc,
+    link_with: vitastor_client,
+    install: true,
+  )
+
+  fio_vitastor_sec = shared_library(
+    'fio_vitastor_sec',
+    [
+      'src/client/fio_sec_osd.cpp',
+      'src/util/rw_blocking.cpp',
+      'src/util/addr_util.cpp',
+    ],
+    include_directories: inc,
+    dependencies: [tcmalloc_dep],
+    install: true,
+  )
+endif
+
+if nl3_dep.found()
+  vitastor_nbd_args = have_nbd_netlink ? ['-DHAVE_NBD_NETLINK_H'] : []
+vitastor_nbd = executable(
+  'vitastor-nbd',
+  ['src/client/nbd_proxy.cpp'],
+  include_directories: inc,
+  link_with: vitastor_client,
+  dependencies: [nl3_dep],
+  c_args: vitastor_nbd_args,
+  cpp_args: vitastor_nbd_args,
+  install: true,
+)
+endif
+
+vitastor_ublk = executable(
+  'vitastor-ublk',
+  ['src/client/ublk_server.cpp'],
+  include_directories: inc,
+  link_with: vitastor_client,
+  install: true,
+)
+
+if with_qemu
+  qemu_vitastor_args = -DVITASTOR_SOURCE_TREE
+  qemu_vitastor = shared_library(
+    'block-vitastor',
+    ['src/client/qemu_driver.c'],
+    include_directories: [
+      inc,
+      include_directories('qemu/b/qemu', 'qemu/include'),
+    ],
+    link_with: vitastor_client,
+    dependencies: [glib_dep],
+    c_args: qemu_vitastor_args,
+    cpp_args: qemu_vitastor_args,
+    name_prefix: '',
+    install: true,
+    install_dir: join_paths(get_option('libdir'), qemu_plugindir),
+  )
+endif
+
+test_cluster_client = executable(
+  'test_cluster_client',
+  [
+    'src/test/test_cluster_client.cpp',
+    'src/client/pg_states.cpp',
+    'src/client/osd_ops.cpp',
+    'src/client/cluster_client.cpp',
+    'src/client/cluster_client_list.cpp',
+    'src/client/cluster_client_wb.cpp',
+    'src/client/msgr_op.cpp',
+    'src/test/mock/messenger.cpp',
+    'src/client/etcd_state_client.cpp',
+    'src/util/timerfd_manager.cpp',
+    'src/util/addr_util.cpp',
+    'src/util/str_util.cpp',
+    'src/util/json_util.cpp',
+    'json11/json11.cpp',
+  ],
+  include_directories: [inc, include_directories('src/test/mock')],
+  dependencies: threads_dep,
+  build_by_default: false,
+  override_options: ['cpp_args=-D__MOCK__'],
+)
+
+vitastor_disk = executable(
+  'vitastor-disk',
+  [
+    'src/disk_tool/disk_tool.cpp',
+    'src/disk_tool/disk_simple_offsets.cpp',
+    'src/disk_tool/disk_tool_discard.cpp',
+    'src/disk_tool/disk_tool_journal.cpp',
+    'src/disk_tool/disk_tool_meta.cpp',
+    'src/disk_tool/disk_tool_prepare.cpp',
+    'src/disk_tool/disk_tool_resize.cpp',
+    'src/disk_tool/disk_tool_resize_auto.cpp',
+    'src/disk_tool/disk_tool_udev.cpp',
+    'src/disk_tool/disk_tool_utils.cpp',
+    'src/disk_tool/disk_tool_upgrade.cpp',
+    'src/util/crc32c.c',
+    'src/util/str_util.cpp',
+    'src/util/json_util.cpp',
+    'json11/json11.cpp',
+    'src/util/rw_blocking.cpp',
+    'src/util/allocator.cpp',
+    'src/util/ringloop.cpp',
+    'src/blockstore/blockstore_disk.cpp',
+    'src/blockstore/blockstore_heap.cpp',
+    'src/blockstore/multilist.cpp',
+  ],
+  include_directories: inc,
+  dependencies: [tcmalloc_dep, liburing_dep, isal_dep],
+  install: true,
+)
+
+vitastor_cli_exe = executable(
+  'vitastor-cli',
+  ['src/cmd/cli.cpp'],
+  include_directories: inc,
+  link_with: vitastor_client,
+  install: true,
+)
+
+vitastor_kv = shared_library(
+  'vitastor_kv',
+  ['src/kv/kv_db.cpp'],
+  include_directories: inc,
+  link_with: vitastor_client,
+  version: meson.project_version(),
+  soversion: '0',
+  install: true,
+)
+
+vitastor_kv_cli = executable(
+  'vitastor-kv',
+  ['src/kv/kv_cli.cpp'],
+  include_directories: inc,
+  link_with: vitastor_kv,
+  install: true,
+)
+
+vitastor_kv_stress = executable(
+  'vitastor-kv-stress',
+  ['src/kv/kv_stress.cpp'],
+  include_directories: inc,
+  link_with: vitastor_kv,
+  install: true,
+)
+
+vitastor_osd = executable(
+  'vitastor-osd',
+  [
+    'src/osd/osd_main.cpp',
+    'src/osd/osd.cpp',
+    'src/osd/osd_secondary.cpp',
+    'src/osd/osd_peering.cpp',
+    'src/osd/osd_flush.cpp',
+    'src/osd/osd_peering_pg.cpp',
+    'src/osd/osd_primary.cpp',
+    'src/osd/osd_primary_chain.cpp',
+    'src/osd/osd_primary_sync.cpp',
+    'src/osd/osd_primary_write.cpp',
+    'src/osd/osd_primary_subops.cpp',
+    'src/osd/osd_cluster.cpp',
+    'src/osd/osd_rmw.cpp',
+    'src/osd/osd_scrub.cpp',
+    'src/osd/osd_primary_describe.cpp',
+  ],
+  include_directories: inc,
+  link_with: [vitastor_common, vitastor_blk],
+  dependencies: [jerasure_dep, isal_dep, ibverbs_dep, rdmacm_dep],
+  install: true,
+)
+
+osd_rmw_test = executable(
+  'osd_rmw_test',
+  ['src/osd/osd_rmw_test.cpp', 'src/util/allocator.cpp'],
+  dependencies: [jerasure_dep, isal_dep, tcmalloc_dep],
+  build_by_default: false,
+)
+osd_rmw_test_je = []
+if isal_dep.found()
+  osd_rmw_test_je = executable(
+    'osd_rmw_test_je',
+    ['src/osd/osd_rmw_test.cpp', 'src/util/allocator.cpp'],
+    dependencies: [jerasure_dep, tcmalloc_dep],
+    build_by_default: false,
+    override_options: ['cpp_args=-DNO_ISAL'],
+  )
+endif
+
+osd_peering_pg_test = executable(
+  'osd_peering_pg_test',
+  ['src/osd/osd_peering_pg_test.cpp', 'src/osd/osd_peering_pg.cpp'],
+  dependencies: tcmalloc_dep,
+  build_by_default: false,
+)
+
+vitastor_nfs = executable(
+  'vitastor-nfs',
+  [
+    'src/nfs/nfs_proxy.cpp',
+    'src/nfs/nfs_proxy_rdma.cpp',
+    'src/nfs/nfs_block.cpp',
+    'src/nfs/nfs_kv.cpp',
+    'src/nfs/nfs_kv_create.cpp',
+    'src/nfs/nfs_kv_defrag.cpp',
+    'src/nfs/nfs_kv_getattr.cpp',
+    'src/nfs/nfs_kv_link.cpp',
+    'src/nfs/nfs_kv_lookup.cpp',
+    'src/nfs/nfs_kv_read.cpp',
+    'src/nfs/nfs_kv_readdir.cpp',
+    'src/nfs/nfs_kv_remove.cpp',
+    'src/nfs/nfs_kv_rename.cpp',
+    'src/nfs/nfs_kv_setattr.cpp',
+    'src/nfs/nfs_kv_write.cpp',
+    'src/nfs/nfs_fsstat.cpp',
+    'src/nfs/nfs_mount.cpp',
+    'src/nfs/nfs_portmap.cpp',
+    'src/nfs/rdma_alloc.cpp',
+    'src/util/sha256.c',
+    'src/nfs/proto/xdr_impl.cpp',
+    'src/nfs/proto/rpc_rdma_xdr.cpp',
+    'src/nfs/proto/rpc_xdr.cpp',
+    'src/nfs/proto/portmap_xdr.cpp',
+    'src/nfs/proto/nfs_xdr.cpp',
+  ],
+  include_directories: inc,
+  link_with: [vitastor_client, vitastor_kv],
+  dependencies: rdmacm_dep,
+  install: true,
+)
+
+stub_osd = executable(
+  'stub_osd',
+  ['src/test/stub_osd.cpp', 'src/util/rw_blocking.cpp', 'src/util/addr_util.cpp'],
+  dependencies: tcmalloc_dep,
+  build_by_default: false,
+)
+stub_bench = executable(
+  'stub_bench',
+  ['src/test/stub_bench.cpp', 'src/util/rw_blocking.cpp', 'src/util/addr_util.cpp'],
+  dependencies: tcmalloc_dep,
+  build_by_default: false,
+)
+osd_test = executable(
+  'osd_test',
+  ['src/test/osd_test.cpp', 'src/util/rw_blocking.cpp', 'src/util/addr_util.cpp'],
+  dependencies: tcmalloc_dep,
+  build_by_default: false,
+)
+
+bindiff = executable(
+  'bindiff',
+  ['src/test/bindiff.c'],
+  build_by_default: false,
+)
+
+stub_uring_osd = executable(
+  'stub_uring_osd',
+  ['src/test/stub_uring_osd.cpp'],
+  link_with: vitastor_common,
+  dependencies: [liburing_dep, ibverbs_dep, rdmacm_dep, tcmalloc_dep],
+  build_by_default: false,
+)
+
+test_allocator = executable(
+  'test_allocator',
+  ['src/test/test_allocator.cpp', 'src/util/allocator.cpp'],
+  dependencies: [tcmalloc_dep],
+  build_by_default: false,
+)
+
+test_heap_args = []
+if enable_coverage
+  test_heap_args += '-coverage'
+endif
+
+test_heap = executable(
+  'test_heap',
+  [
+    'src/test/test_heap.cpp',
+    'src/blockstore/multilist.cpp',
+    'src/blockstore/blockstore_heap.cpp',
+    'src/util/crc32c.c',
+    'src/util/allocator.cpp',
+    'src/blockstore/blockstore_disk.cpp',
+    'src/util/str_util.cpp',
+  ],
+  dependencies: isal_dep,
+  build_by_default: false,
+  cpp_args: test_heap_args,
+  link_args: test_heap_args,
+)
+
+test_cas = executable(
+  'test_cas',
+  ['src/test/test_cas.cpp'],
+  link_with: vitastor_client,
+  build_by_default: false,
+)
+
+test_crc32 = executable(
+  'test_crc32',
+  ['src/test/test_crc32.cpp'],
+  link_with: vitastor_blk,
+  dependencies: isal_dep,
+  build_by_default: false,
+)
+
+test_blockstore = executable(
+  'test_blockstore',
+  ['src/test/test_blockstore.cpp', 'src/test/ringloop_mock.cpp'],
+  link_with: [vitastor_blk, vitastor_common],
+  dependencies: isal_dep,
+  build_by_default: false,
+)
+
+test_atomic = executable(
+  'test_atomic',
+  ['src/test/test_atomic.cpp', 'src/util/ringloop.cpp'],
+  dependencies: liburing_dep,
+  build_by_default: false,
+)
+
+if build_tests_opt
+  test('test_cluster_client', test_cluster_client)
+  test('osd_rmw_test', osd_rmw_test)
+  if isal_dep.found()
+    test('osd_rmw_test_jerasure', osd_rmw_test_je)
+  endif
+  test('osd_peering_pg_test', osd_peering_pg_test)
+  test('test_allocator', test_allocator)
+  test('test_heap', test_heap)
+  test('test_blockstore', test_blockstore)
+endif
+
+install_symlink(
+  'vitastor-dump-journal',
+  install_dir: get_option('bindir'),
+  pointing_to: 'vitastor-disk',
+)
+install_symlink(
+  'vitastor-rm',
+  install_dir: get_option('bindir'),
+  pointing_to: 'vitastor-cli',
+)
+install_symlink(
+  'vita',
+  install_dir: get_option('bindir'),
+  pointing_to: 'vitastor-cli',
+)
+
+install_headers('src/client/vitastor_c.h', subdir: 'client')
+install_headers('src/kv/vitastor_kv.h', subdir: 'kv')
+

meson_options.txt

@@ -0,0 +1,7 @@
+option('with_qemu', type: 'boolean', value: false, description: 'Build QEMU driver inside source tree')
+option('with_fio', type: 'boolean', value: true, description: 'Build FIO driver')
+option('qemu_plugindir', type: 'string', value: 'qemu', description: 'QEMU plugin directory suffix')
+option('with_asan', type: 'boolean', value: false, description: 'Enable AddressSanitizer flags')
+option('with_system_liburing', type: 'boolean', value: false, description: 'Use system-provided liburing instead of bundled sources')
+option('enable_coverage', type: 'boolean', value: false, description: 'Enable coverage flags for builds')
+option('build_tests', type: 'boolean', value: true, description: 'Register unit/integration tests')

src/client/msgr_rdma.cpp

@@ -67,6 +67,20 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_close_device(context);
 }
 
+msgr_rdma_buf_t::~msgr_rdma_buf_t()
+{
+    if (buf)
+    {
+        free(buf);
+        buf = NULL;
+    }
+    if (mr)
+    {
+        ibv_dereg_mr(mr);
+        mr = NULL;
+    }
+}
+
 msgr_rdma_connection_t::~msgr_rdma_connection_t()
 {
     ctx->reserve_cqe(-max_send-max_recv);
@@ -84,26 +98,6 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     if (qp)
         ibv_destroy_qp(qp);
 #endif
-    if (recv_buffers.size())
-    {
-        for (auto b: recv_buffers)
-        {
-            if (b.mr)
-                ibv_dereg_mr(b.mr);
-            free(b.buf);
-        }
-        recv_buffers.clear();
-    }
-    if (send_out.mr)
-    {
-        ibv_dereg_mr(send_out.mr);
-        send_out.mr = NULL;
-    }
-    if (send_out.buf)
-    {
-        free(send_out.buf);
-        send_out.buf = NULL;
-    }
     send_out_size = 0;
 }
 
@@ -493,7 +487,7 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
             .grh        = {
                 .dgid = dest->gid,
                 .sgid_index = conn->ctx->gid_index,
-                .hop_limit = 1, // FIXME can it vary?
+                .hop_limit = 64, // FIXME can it vary?
             },
             .dlid       = dest->lid,
             .sl         = 0, // service level
@@ -663,7 +657,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
     {
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
-        rc->send_out.buf = malloc_or_die(rc->send_out_size);
+        rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
         if (!rc->ctx->odp)
         {
             rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
@@ -712,12 +706,12 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
         try_send_rdma_nodp(cl);
 }
 
-static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
+static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 {
     ibv_sge sge = {
-        .addr = (uintptr_t)b.buf,
+        .addr = (uintptr_t)buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : b.mr->lkey,
+        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : cl->rdma_conn->recv_buf.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
@@ -737,19 +731,20 @@ static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
 bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
-    while (rc->cur_recv < rc->max_recv)
+    assert(!rc->recv_buf.buf);
+    rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
+    if (!rc->ctx->odp)
     {
-        msgr_rdma_buf_t b;
-        b.buf = malloc_or_die(rc->max_msg);
-        if (!rc->ctx->odp)
+        rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
+        if (!rc->recv_buf.mr)
         {
-            b.mr = ibv_reg_mr(rc->ctx->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
-            if (!b.mr)
-            {
-                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-                exit(1);
-            }
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
         }
+    }
+    for (uint32_t i = 0; i < rc->max_recv; i++)
+    {
+        uint8_t *b = rc->recv_buf.buf + i*rc->max_recv;
         rc->recv_buffers.push_back(b);
         try_recv_rdma_wr(cl, b);
     }
@@ -807,7 +802,7 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 cl->ping_time_remaining = 0;
                 cl->idle_time_remaining = osd_idle_timeout;
                 rc->cur_recv--;
-                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
+                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
                     clear_immediate_ops(client_id);

src/client/msgr_rdma.h

@@ -54,8 +54,10 @@ struct msgr_rdma_context_t
 
 struct msgr_rdma_buf_t
 {
-    void *buf = NULL;
+    uint8_t *buf = NULL;
     ibv_mr *mr = NULL;
+
+    ~msgr_rdma_buf_t();
 };
 
 struct msgr_rdma_connection_t
@@ -72,7 +74,8 @@ struct msgr_rdma_connection_t
     int cur_send = 0, cur_recv = 0;
     int send_pos = 0, send_buf_pos = 0;
     int next_recv_buf = 0;
-    std::vector<msgr_rdma_buf_t> recv_buffers;
+    std::vector<void*> recv_buffers;
+    msgr_rdma_buf_t recv_buf;
     std::vector<uint64_t> send_sizes;
     msgr_rdma_buf_t send_out;
     int send_out_pos = 0, send_done_pos = 0, send_out_size = 0;