From cc798bd412e816bc1c1da4c91b904d2c7ff987e9 Mon Sep 17 00:00:00 2001 From: Xianqiang Lei Date: Tue, 20 Oct 2026 18:09:48 +0800 Subject: [PATCH 1/2] test: add a test tool for testing the send/reply of a single message --- src/test/CMakeLists.txt | 11 + src/test/test_msgr_send_reply.cpp | 714 ++++++++++++++++++++++++++++++ 2 files changed, 725 insertions(+) create mode 100644 src/test/test_msgr_send_reply.cpp diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 61737e85..6337e643 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -27,6 +27,17 @@ target_link_libraries(stub_uring_osd ${RDMACM_LIBRARIES} ) +add_executable(test_msgr_send_reply + EXCLUDE_FROM_ALL + test_msgr_send_reply.cpp +) +target_link_libraries(test_msgr_send_reply + vitastor_net + ${LIBURING_LIBRARIES} + ${IBVERBS_LIBRARIES} + ${RDMACM_LIBRARIES} +) + # test_msgr_stop add_executable(test_msgr_stop EXCLUDE_FROM_ALL diff --git a/src/test/test_msgr_send_reply.cpp b/src/test/test_msgr_send_reply.cpp new file mode 100644 index 00000000..f198df94 --- /dev/null +++ b/src/test/test_msgr_send_reply.cpp @@ -0,0 +1,714 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) + +// One-shot transfer through osd_messenger_t over TCP or RDMA. +// +// test_msgr_send_reply server --ip ADDR [--port N] [--net tcp|rdma] [--zerocopy] +// test_msgr_send_reply client --ip ADDR [--port N] [--net tcp|rdma] +// [--send_size SIZE] [--reply_size SIZE] [--zerocopy] +// +// --ip ADDR required. Server bind address, or the address the client +// connects to. --net rdma selects a RoCEv2 GID in the /24 +// that contains this address. +// --port N TCP port. Default 11223. +// --net tcp|rdma transport. Default tcp. +// --send_size SIZE client only. OSD_OP_SEC_WRITE payload. Default 1 MiB. +// --reply_size SIZE client only. OSD_OP_SEC_READ reply payload. Default 1 MiB. +// --zerocopy send with io_uring sendmsg_zc. The default is a copying send. +// +// SIZE is a number of MiB, with an optional m/MiB suffix. +// The server takes the lengths from the requests. It checks the write pattern and +// fills the read reply with the same pattern. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "json11/json11.hpp" +#include "addr_util.h" +#include "ringloop.h" +#include "epoll_manager.h" +#include "messenger.h" + +static const uint32_t DEFAULT_PAYLOAD_MIB = 1; +static const uint32_t MAX_PAYLOAD_MIB = UINT32_MAX / (1024u * 1024u); +static uint32_t g_send_mib = DEFAULT_PAYLOAD_MIB; +static uint32_t g_send_bytes = DEFAULT_PAYLOAD_MIB * 1024u * 1024u; +static uint32_t g_reply_mib = DEFAULT_PAYLOAD_MIB; +static uint32_t g_reply_bytes = DEFAULT_PAYLOAD_MIB * 1024u * 1024u; +static const int DEFAULT_PORT = 11223; +static const osd_num_t SERVER_OSD = 2; +static const osd_num_t CLIENT_OSD = 1; +static const int TEST_TIMEOUT_SEC = 180; +// Copying sendmsg is the default. --zerocopy opts into io_uring sendmsg_zc. +static bool g_force_copy = true; +static bool g_use_rdma = false; + +enum +{ + PHASE_WAIT = 0, + PHASE_WRITE = 1, + PHASE_READ = 2, + PHASE_DONE = 3, + PHASE_FAIL = 4, +}; + +static osd_messenger_t *g_msgr = NULL; +static int g_phase = PHASE_WAIT; +static int g_exit = -1; +static bool g_server_role = false; +static bool g_seen_client = false; +static bool g_write_ok = false; +static bool g_read_ok = false; +static time_t g_start = 0; +static timespec g_t0; +static uint8_t *g_write_buf = NULL; + +static uint64_t pattern_word(size_t i) +{ + return 0xC0FFEE0000000000ull + (uint64_t)i; +} + +static void fill_pattern(uint8_t *buf, size_t len) +{ + uint64_t *p = (uint64_t*)buf; + size_t n = len / sizeof(uint64_t); + for (size_t i = 0; i < n; i++) + p[i] = pattern_word(i); +} + +// Returns the byte offset of the first mismatch, or len if the buffer matches. +static size_t check_pattern(const uint8_t *buf, size_t len) +{ + const uint64_t *p = (const uint64_t*)buf; + size_t n = len / sizeof(uint64_t); + for (size_t i = 0; i < n; i++) + { + if (p[i] != pattern_word(i)) + return i * sizeof(uint64_t); + } + return len; +} + +static double elapsed_ms(const timespec & t0) +{ + timespec t1; + clock_gettime(CLOCK_MONOTONIC, &t1); + return (t1.tv_sec - t0.tv_sec) * 1000.0 + (t1.tv_nsec - t0.tv_nsec) / 1e6; +} + +static const char *peer_mode(osd_client_t *cl) +{ + if (cl->peer_state == PEER_RDMA) + return "rdma"; + if (cl->peer_state == PEER_CONNECTED) + return "tcp"; + return "other"; +} + +static void reply_op(osd_messenger_t *msgr, osd_op_t *op, int64_t retval) +{ + op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC; + op->reply.hdr.id = op->req.hdr.id; + op->reply.hdr.opcode = op->req.hdr.opcode; + op->reply.hdr.retval = retval; + msgr->outbox_push(op); +} + +static void fail_test(const char *who, const char *why) +{ + fprintf(stderr, "[%s] FAIL: %s\n", who, why); + g_phase = PHASE_FAIL; + g_exit = 1; +} + +static void start_read(); + +static void on_write_reply(osd_op_t *op) +{ + double ms = elapsed_ms(g_t0); + int64_t retval = op->reply.hdr.retval; + delete op; + if (g_phase != PHASE_WRITE) + return; + if (retval != (int64_t)g_send_bytes) + { + fprintf(stderr, "[client] SEC_WRITE retval=%jd (%s), %.1f ms\n", + retval, retval < 0 ? strerror(-(int)retval) : "unexpected", ms); + fail_test("client", "SEC_WRITE was not accepted"); + return; + } + double mib_s = (g_send_bytes / 1024.0 / 1024.0) / (ms / 1000.0); + fprintf(stderr, "[client] SEC_WRITE %u bytes ok, %.1f ms, %.1f MiB/s\n", + g_send_bytes, ms, mib_s); + start_read(); +} + +static void on_read_reply(osd_op_t *op) +{ + double ms = elapsed_ms(g_t0); + int64_t retval = op->reply.hdr.retval; + if (g_phase != PHASE_READ) + { + delete op; + return; + } + if (retval != (int64_t)g_reply_bytes) + { + fprintf(stderr, "[client] SEC_READ retval=%jd (%s), %.1f ms\n", + retval, retval < 0 ? strerror(-(int)retval) : "unexpected", ms); + delete op; + fail_test("client", "SEC_READ was not accepted"); + return; + } + size_t bad = check_pattern((uint8_t*)op->buf, g_reply_bytes); + delete op; + if (bad != g_reply_bytes) + { + fprintf(stderr, "[client] SEC_READ pattern mismatch at offset %zu\n", bad); + fail_test("client", "SEC_READ payload mismatch"); + return; + } + double mib_s = (g_reply_bytes / 1024.0 / 1024.0) / (ms / 1000.0); + fprintf(stderr, "[client] SEC_READ %u bytes ok, %.1f ms, %.1f MiB/s\n", + g_reply_bytes, ms, mib_s); + fprintf(stderr, "[client] PASS\n"); + g_phase = PHASE_DONE; + g_exit = 0; +} + +static void start_read() +{ + auto cl_it = g_msgr->osd_peers.find(SERVER_OSD); + if (cl_it == g_msgr->osd_peers.end()) + { + fail_test("client", "peer gone before SEC_READ"); + return; + } + osd_op_t *op = new osd_op_t(); + op->op_type = OSD_OP_OUT; + op->client_id = cl_it->second->client_id; + op->req.hdr.magic = SECONDARY_OSD_OP_MAGIC; + op->req.hdr.opcode = OSD_OP_SEC_READ; + op->req.sec_rw.oid = { .inode = 1, .stripe = 0 }; + op->req.sec_rw.version = 1; + op->req.sec_rw.len = g_reply_bytes; + op->buf = memalign_or_die(MEM_ALIGNMENT, g_reply_bytes); + op->iov.push_back(op->buf, g_reply_bytes); + op->callback = on_read_reply; + g_phase = PHASE_READ; + clock_gettime(CLOCK_MONOTONIC, &g_t0); + fprintf(stderr, "[client] sending one SEC_READ of %u bytes\n", g_reply_bytes); + g_msgr->outbox_push(op); +} + +static void start_write() +{ + auto cl_it = g_msgr->osd_peers.find(SERVER_OSD); + if (cl_it == g_msgr->osd_peers.end()) + { + fail_test("client", "connected but peer is missing"); + return; + } + osd_client_t *cl = cl_it->second; + fprintf(stderr, "[client] connected to OSD %ju client_id=%ju mode=%s\n", + SERVER_OSD, cl->client_id, peer_mode(cl)); + if (g_use_rdma && cl->peer_state != PEER_RDMA) + { + fail_test("client", "RDMA requested but the peer stayed on TCP"); + return; + } + if (!g_use_rdma && cl->peer_state == PEER_RDMA) + { + fail_test("client", "TCP requested but the peer switched to RDMA"); + return; + } + osd_op_t *op = new osd_op_t(); + op->op_type = OSD_OP_OUT; + op->client_id = cl->client_id; + op->req.hdr.magic = SECONDARY_OSD_OP_MAGIC; + op->req.hdr.opcode = OSD_OP_SEC_WRITE; + op->req.sec_rw.oid = { .inode = 1, .stripe = 0 }; + op->req.sec_rw.version = 1; + op->req.sec_rw.len = g_send_bytes; + op->buf = g_write_buf; + g_write_buf = NULL; + op->iov.push_back(op->buf, g_send_bytes); + op->callback = on_write_reply; + g_phase = PHASE_WRITE; + clock_gettime(CLOCK_MONOTONIC, &g_t0); + fprintf(stderr, "[client] sending one SEC_WRITE of %u bytes\n", g_send_bytes); + g_msgr->outbox_push(op); +} + +static void exec_show_config(osd_op_t *op) +{ + std::string json_err; + json11::Json req = op->req.show_conf.json_len + ? json11::Json::parse(std::string((char*)op->buf), json_err) + : json11::Json(); + auto cl = g_msgr->clients.at(op->client_id); + cl->in_osd_num = req["osd_num"].uint64_value(); + if (req["features"]["check_sequencing"].bool_value()) + cl->check_sequencing = true; + json11::Json::object features; + int peer_csums = (int)req["features"]["proto_checksums"].uint64_value(); + if (g_msgr->use_proto_checksums && peer_csums) + { + if (g_msgr->use_proto_checksums == MSGR_CSUM_FULL && peer_csums == MSGR_CSUM_FULL) + cl->proto_csum_status = MSGR_CSUM_FULL | MSGR_CSUM_NEG; + else + cl->proto_csum_status = MSGR_CSUM_PAYLOAD | MSGR_CSUM_NEG; + features["proto_checksums"] = (uint64_t)g_msgr->use_proto_checksums; + } + json11::Json::object wire = { + { "osd_num", (uint64_t)SERVER_OSD }, + { "protocol_version", (uint64_t)OSD_PROTOCOL_VERSION }, + { "features", features }, + }; +#ifdef WITH_RDMA + if (g_msgr->is_rdma_enabled()) + { + wire["rdma_enabled"] = true; + if (req["connect_rdma"].is_string()) + { + json11::Json data = g_msgr->connect_rdma( + op->client_id, req["connect_rdma"].string_value(), req["rdma_max_msg"].uint64_value()); + if (!data.is_null()) + { + for (auto & kv: data.object_items()) + wire[kv.first] = kv.second; + } + } + } +#endif + if (op->buf) + free(op->buf); + std::string cfg = json11::Json(wire).dump(); + op->buf = malloc_or_die(cfg.size() + 1); + memcpy(op->buf, cfg.c_str(), cfg.size() + 1); + op->iov.push_back(op->buf, cfg.size() + 1); + fprintf(stderr, "[server] SHOW_CONFIG from OSD %ju, checksum=%d, reply %s\n", + cl->in_osd_num, cl->proto_csum_status, cfg.c_str()); + reply_op(g_msgr, op, cfg.size() + 1); +} + +static void exec_sec_write(osd_op_t *op) +{ + auto cl = g_msgr->clients.at(op->client_id); + uint32_t len = op->req.sec_rw.len; + if (!len || !op->buf) + { + fprintf(stderr, "[server] SEC_WRITE empty payload\n"); + g_write_ok = false; + reply_op(g_msgr, op, -EINVAL); + return; + } + g_send_bytes = len; + bool mode_ok = g_use_rdma ? cl->peer_state == PEER_RDMA : cl->peer_state == PEER_CONNECTED; + size_t bad = check_pattern((uint8_t*)op->buf, len); + if (!mode_ok || bad != len) + { + fprintf(stderr, "[server] SEC_WRITE %s at mode=%s (want %s), mismatch_off=%zu\n", + mode_ok ? "pattern mismatch" : "transport mismatch", + peer_mode(cl), g_use_rdma ? "rdma" : "tcp", bad); + g_write_ok = false; + reply_op(g_msgr, op, -EIO); + return; + } + g_write_ok = true; + fprintf(stderr, "[server] SEC_WRITE %u bytes pattern ok (mode=%s)\n", + len, peer_mode(cl)); + reply_op(g_msgr, op, len); +} + +static void exec_sec_read(osd_op_t *op) +{ + auto cl = g_msgr->clients.at(op->client_id); + uint32_t len = op->req.sec_rw.len; + if (!len) + { + fprintf(stderr, "[server] SEC_READ empty payload\n"); + reply_op(g_msgr, op, -EINVAL); + return; + } + g_reply_bytes = len; + op->buf = memalign_or_die(MEM_ALIGNMENT, len); + fprintf(stderr, "[server] filling %u byte SEC_READ reply\n", len); + fill_pattern((uint8_t*)op->buf, len); + op->iov.push_back(op->buf, len); + op->reply.sec_rw.attr_len = 0; + g_read_ok = true; + fprintf(stderr, "[server] sending one SEC_READ reply of %u bytes (mode=%s)\n", + len, peer_mode(cl)); + reply_op(g_msgr, op, len); +} + +static void server_exec_op(osd_op_t *op) +{ + g_seen_client = true; + if (op->req.hdr.opcode == OSD_OP_SHOW_CONFIG) + exec_show_config(op); + else if (op->req.hdr.opcode == OSD_OP_PING) + reply_op(g_msgr, op, 0); + else if (op->req.hdr.opcode == OSD_OP_SEC_WRITE || op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) + exec_sec_write(op); + else if (op->req.hdr.opcode == OSD_OP_SEC_READ) + exec_sec_read(op); + else + { + fprintf(stderr, "[server] unsupported opcode %ju\n", op->req.hdr.opcode); + reply_op(g_msgr, op, -EINVAL); + } +} + +static json11::Json msgr_config(const std::string & ip) +{ + json11::Json::object cfg = { + { "log_level", 3 }, + { "use_rdma", g_use_rdma }, + { "osd_idle_timeout", TEST_TIMEOUT_SEC }, + { "osd_ping_timeout", TEST_TIMEOUT_SEC }, + { "peer_connect_timeout", 10 }, + { "peer_connect_interval", 1 }, + { "proto_checksums", "payload" }, + { "use_sync_send_recv", false }, + }; + + if (g_use_rdma) + cfg["osd_network"] = ip + "/24"; + cfg["min_zerocopy_send_size"] = g_force_copy ? -1 : 0; + return cfg; +} + +static int setup_msgr(ring_loop_t *ringloop, epoll_manager_t *epmgr, osd_num_t osd_num, const std::string & ip) +{ + g_msgr = new osd_messenger_t(); + g_msgr->osd_num = osd_num; + g_msgr->tfd = epmgr->tfd; + g_msgr->ringloop = ringloop; + g_msgr->repeer_pgs = [](osd_num_t peer) + { + if (!g_server_role && peer == SERVER_OSD && g_phase == PHASE_WAIT) + start_write(); + }; + g_msgr->break_pg_locks = [](osd_num_t) {}; + if (g_server_role) + g_msgr->exec_op = [](osd_op_t *op) { server_exec_op(op); }; + g_msgr->parse_config(msgr_config(ip), true); + uint32_t op_bytes = g_server_role + ? MAX_PAYLOAD_MIB * 1024u * 1024u + : (g_send_bytes > g_reply_bytes ? g_send_bytes : g_reply_bytes); + g_msgr->bs_block_size = op_bytes; + g_msgr->max_write_request_size = op_bytes; + g_msgr->init(); +#ifdef WITH_RDMA + bool rdma_on = g_msgr->is_rdma_enabled(); +#else + bool rdma_on = false; +#endif + if (g_server_role) + { + fprintf(stderr, "[server] osd=%ju transport=%s zerocopy=%d sendmsg_zc=%d rdma_ready=%d\n", + osd_num, g_use_rdma ? "rdma" : "tcp", (int)!g_force_copy, (int)g_msgr->has_sendmsg_zc, (int)rdma_on); + } + else + { + fprintf(stderr, "[client] osd=%ju send=%u MiB (%u bytes) reply=%u MiB (%u bytes) transport=%s zerocopy=%d sendmsg_zc=%d rdma_ready=%d\n", + osd_num, g_send_mib, g_send_bytes, g_reply_mib, g_reply_bytes, + g_use_rdma ? "rdma" : "tcp", (int)!g_force_copy, (int)g_msgr->has_sendmsg_zc, (int)rdma_on); + } + if (g_use_rdma && !rdma_on) + { + fprintf(stderr, "[%s] FAIL: RDMA requested but no device matched %s\n", + g_server_role ? "server" : "client", ip.c_str()); + g_exit = 1; + } + return 0; +} + +static void arm_timeout(timerfd_manager_t *tfd) +{ + tfd->set_timer(1000, true, [](int) + { + if (g_exit >= 0) + return; + if (g_server_role && g_seen_client && g_msgr->clients.empty()) + { + if (g_write_ok && g_read_ok) + { + fprintf(stderr, "[server] PASS write %u bytes + read %u bytes, peer disconnected\n", + g_send_bytes, g_reply_bytes); + g_exit = 0; + } + else + { + fprintf(stderr, "[server] FAIL peer disconnected write_ok=%d read_ok=%d\n", + (int)g_write_ok, (int)g_read_ok); + g_exit = 1; + } + return; + } + if (time(NULL) - g_start >= TEST_TIMEOUT_SEC) + { + fprintf(stderr, "[%s] FAIL timeout after %d s phase=%d write_ok=%d read_ok=%d\n", + g_server_role ? "server" : "client", TEST_TIMEOUT_SEC, g_phase, + (int)g_write_ok, (int)g_read_ok); + g_exit = 1; + } + }); +} + +static int run_loop(ring_loop_t *ringloop) +{ + while (g_exit < 0) + { + ringloop->loop(); + if (g_exit >= 0) + break; + ringloop->wait(); + } + return g_exit; +} + +static int run_server(const std::string & bind_ip, int port) +{ + g_server_role = true; + ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE); + epoll_manager_t *epmgr = new epoll_manager_t(ringloop); + setup_msgr(ringloop, epmgr, SERVER_OSD, bind_ip); + if (g_exit > 0) + return g_exit; + int listen_fd = create_and_bind_socket(bind_ip, port, 128, NULL); + if (listen_fd < 0) + { + fprintf(stderr, "listen failed\n"); + return 1; + } + fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK); + epmgr->set_fd_handler(listen_fd, false, [listen_fd](int, int) + { + g_msgr->accept_connections(listen_fd); + }); + ring_consumer_t looper; + looper.loop = [ringloop]() + { + g_msgr->read_requests(); + g_msgr->send_replies(); + ringloop->submit(); + }; + ringloop->register_consumer(&looper); + arm_timeout(epmgr->tfd); + fprintf(stderr, "[server] listening on %s:%d\n", bind_ip.c_str(), port); + int rc = run_loop(ringloop); + delete g_msgr; + delete epmgr; + delete ringloop; + return rc < 0 ? 1 : rc; +} + +static int run_client(const std::string & host, int port) +{ + g_server_role = false; + fprintf(stderr, "[client] filling %u byte pattern\n", g_send_bytes); + g_write_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, g_send_bytes); + fill_pattern(g_write_buf, g_send_bytes); + ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE); + epoll_manager_t *epmgr = new epoll_manager_t(ringloop); + setup_msgr(ringloop, epmgr, CLIENT_OSD, host); + if (g_exit > 0) + return g_exit; + ring_consumer_t looper; + looper.loop = [ringloop]() + { + g_msgr->read_requests(); + g_msgr->send_replies(); + ringloop->submit(); + }; + ringloop->register_consumer(&looper); + arm_timeout(epmgr->tfd); + g_msgr->connect_peer(SERVER_OSD, json11::Json::object { + { "addresses", json11::Json::array { host } }, + { "port", port }, + }); + fprintf(stderr, "[client] connecting to %s:%d\n", host.c_str(), port); + int rc = run_loop(ringloop); + delete g_msgr; + delete epmgr; + delete ringloop; + return rc < 0 ? 1 : rc; +} + +static void usage(const char *argv0) +{ + fprintf(stderr, + "USAGE:\n" + " %s server --ip ADDR [--port N] [--net tcp|rdma] [--zerocopy]\n" + " %s client --ip ADDR [--port N] [--net tcp|rdma]\n" + " [--send_size SIZE] [--reply_size SIZE] [--zerocopy]\n" + "The client sends one SEC_WRITE of --send_size, then one SEC_READ of --reply_size.\n" + "The server uses those request lengths. SIZE is MiB, with an optional m/MiB\n" + "suffix (default %u, max %u).\n" + "--ip is required. Defaults: --port %d --net tcp, copying send.\n" + "--zerocopy uses io_uring sendmsg_zc. --net rdma selects a RoCEv2 GID in the /24 of --ip.\n", + argv0, argv0, DEFAULT_PAYLOAD_MIB, MAX_PAYLOAD_MIB, DEFAULT_PORT); +} + +static bool parse_u32(const char *begin, const char *end, uint32_t *out) +{ + if (begin >= end) + return false; + uint64_t v = 0; + for (const char *p = begin; p < end; p++) + { + if (*p < '0' || *p > '9') + return false; + v = v * 10u + (uint32_t)(*p - '0'); + if (v > UINT32_MAX) + return false; + } + *out = (uint32_t)v; + return true; +} + +static bool suffix_is_mib(const char *s) +{ + char buf[8]; + size_t n = strlen(s); + if (n == 0 || n >= sizeof(buf)) + return false; + for (size_t i = 0; i < n; i++) + buf[i] = (s[i] >= 'A' && s[i] <= 'Z') ? (char)(s[i] - 'A' + 'a') : s[i]; + buf[n] = 0; + return !strcmp(buf, "m") || !strcmp(buf, "mib"); +} + +static uint32_t parse_mib_size(const char *arg, const char *what) +{ + const char *end = arg; + while (*end >= '0' && *end <= '9') + end++; + uint32_t mib = 0; + if (!parse_u32(arg, end, &mib) || !(end[0] == 0 || suffix_is_mib(end)) || + mib < 1 || mib > MAX_PAYLOAD_MIB) + { + char msg[96]; + snprintf(msg, sizeof(msg), "bad %s (1..%u MiB): %s", what, MAX_PAYLOAD_MIB, arg); + throw std::runtime_error(msg); + } + return mib; +} + +static void set_send_mib(uint32_t mib) +{ + g_send_mib = mib; + g_send_bytes = mib * 1024u * 1024u; +} + +static void set_reply_mib(uint32_t mib) +{ + g_reply_mib = mib; + g_reply_bytes = mib * 1024u * 1024u; +} + +static int parse_port(const char *arg) +{ + uint32_t p = 0; + if (!parse_u32(arg, arg + strlen(arg), &p) || p < 1 || p > 65535) + throw std::runtime_error(std::string("bad port: ") + arg); + return (int)p; +} + +static void set_transport(const char *value) +{ + if (!strcmp(value, "rdma")) + g_use_rdma = true; + else if (!strcmp(value, "tcp")) + g_use_rdma = false; + else + throw std::runtime_error(std::string("bad --net (tcp or rdma): ") + value); +} + +// Returns the value of --name VALUE or --name=VALUE, or NULL if arg is another flag. +static const char *flag_value(int argc, char **argv, int & i, const char *name) +{ + const char *arg = argv[i]; + size_t n = strlen(name); + if (!strncmp(arg, name, n) && arg[n] == '=') + return arg + n + 1; + if (strcmp(arg, name)) + return NULL; + if (i + 1 >= argc) + throw std::runtime_error(std::string(name) + " needs a value"); + return argv[++i]; +} + +static void apply_arg(int argc, char **argv, int & i, bool server, std::string & host, int & port) +{ + const char *value; + if ((value = flag_value(argc, argv, i, "--ip"))) + host = value; + else if ((value = flag_value(argc, argv, i, "--net"))) + set_transport(value); + else if ((value = flag_value(argc, argv, i, "--send_size"))) + { + if (server) + throw std::runtime_error("server does not take --send_size; the client request sets it"); + set_send_mib(parse_mib_size(value, "send_size")); + } + else if ((value = flag_value(argc, argv, i, "--reply_size"))) + { + if (server) + throw std::runtime_error("server does not take --reply_size; the client request sets it"); + set_reply_mib(parse_mib_size(value, "reply_size")); + } + else if ((value = flag_value(argc, argv, i, "--port"))) + port = parse_port(value); + else if (!strcmp(argv[i], "--zerocopy")) + g_force_copy = false; + else + throw std::runtime_error(std::string("unknown argument: ") + argv[i]); +} + +int main(int argc, char *argv[]) +{ + signal(SIGPIPE, SIG_IGN); + g_start = time(NULL); + if (argc < 2) + { + usage(argv[0]); + return 2; + } + std::string mode = argv[1]; + std::string host; + int port = DEFAULT_PORT; + try + { + if (mode == "server" || mode == "client") + { + for (int i = 2; i < argc; i++) + apply_arg(argc, argv, i, mode == "server", host, port); + if (host.empty()) + throw std::runtime_error("--ip is required"); + if (mode == "server") + return run_server(host, port); + return run_client(host, port); + } + } + catch (std::exception & e) + { + fprintf(stderr, "test_msgr_send_reply: %s\n", e.what()); + return 1; + } + usage(argv[0]); + return 2; +} -- 2.47.3 From 38d25bf8cfc4d33e299317042636432dfa138289 Mon Sep 17 00:00:00 2001 From: Xianqiang Lei Date: Mon, 28 Sep 2026 15:10:52 +0800 Subject: [PATCH 2/2] client: resend only the unsent tail after a short io_uring send A short io_uring send (sendmsg/sendmsg_zc with MSG_WAITALL) used to disconnect the peer. With large replies such as SEC_LIST filling the socket buffer this happened repeatedly and caused endless re-peering. Now the already accepted prefix stays on the zerocopy path and waits for its release notification as usual, while only the unsent tail is resent with a copying sendmsg before any new ops are appended. -EAGAIN/-EINTR completions are handled the same way, as a zero-byte short send, instead of leaving the batch stuck in send_list. The send_free range of the short-written batch is recorded once, on its first short send, and released only after both the tail is fully sent and, for a zerocopy send, its release notification arrives. Notifications of other batches are still released immediately, so none of them can be lost and pin send_free. --- src/client/messenger.cpp | 35 ++- src/client/messenger.h | 11 +- src/client/msgr_send.cpp | 96 ++++++-- src/test/CMakeLists.txt | 11 + src/test/test_msgr_send.cpp | 472 ++++++++++++++++++++++++++++++++++++ 5 files changed, 600 insertions(+), 25 deletions(-) create mode 100644 src/test/test_msgr_send.cpp diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index ea93c59c..36336a01 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -550,8 +550,8 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd) int one = 1; setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one)); cl->peer_state = PEER_CONNECTED; - // EPOLLOUT is only required without io_uring, to continue partially sent messages - tfd->set_fd_handler(peer_fd, use_sync_send_recv, [this](int peer_fd, int epoll_events) + // EPOLLOUT is registered on demand when a send returns EAGAIN + tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events) { handle_peer_epoll(peer_fd, epoll_events); }); @@ -591,7 +591,7 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events) } if (epoll_events & EPOLLOUT) { - // The socket became writable again - continue the interrupted use_sync_send_recv + // The socket became writable again - continue the interrupted send // read_requests() above may have stopped the client, so look it up again auto cl_it = clients.find(cl->client_id); if (cl_it == clients.end()) @@ -599,11 +599,28 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events) return; } cl = cl_it->second; - cl->write_ready = true; - // send_list is only left non-empty when we're waiting for EPOLLOUT - if (cl->send_list.size() && cl->peer_state != PEER_RDMA) + if (!cl->write_ready) { - try_send(cl); + cl->write_ready = true; + tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events) + { + handle_peer_epoll(peer_fd, epoll_events); + }); + } + if (cl->peer_state != PEER_RDMA && + (use_sync_send_recv ? cl->send_list.size() > 0 : + (cl->send_plain_tail && !cl->write_msg.msg_iovlen || + cl->write_op || cl->write_ops.size() || cl->hs && cl->hs->out_size()))) + { + if (use_sync_send_recv) + try_send(cl); + else if (cl->write_state == 0) + { + cl->write_state = CL_WRITE_READY; + write_ready_clients.push_back(cl->client_id); + if (ringloop) + ringloop->wakeup(); + } } } } @@ -841,8 +858,8 @@ void osd_messenger_t::accept_connections(int listen_fd) { continue; } - // Add FD to epoll. EPOLLOUT is only required without io_uring - tfd->set_fd_handler(peer_fd, use_sync_send_recv, [this](int peer_fd, int epoll_events) + // EPOLLOUT is registered on demand when a send returns EAGAIN + tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events) { handle_peer_epoll(peer_fd, epoll_events); }); diff --git a/src/client/messenger.h b/src/client/messenger.h index 2d964779..5ed2b2ad 100644 --- a/src/client/messenger.h +++ b/src/client/messenger.h @@ -154,10 +154,19 @@ struct osd_client_t size_t write_op_pos = 0; msghdr write_msg = { 0 }; int write_state = 0; - // Socket is writable. Only tracked with use_sync_send_recv, io_uring doesn't need it + // Socket is writable. Cleared when a send returns EAGAIN; EPOLLOUT sets it again. bool write_ready = true; std::vector send_list; size_t send_list_size = 0; + // Remainder of a short io_uring send. Resent without zerocopy so the + // already accepted prefix can stay on the zerocopy path. + bool send_plain_tail = false; + // The tail points into send_free[send_tail_from..send_tail_to) of the batch + // that was short-written. That range is released only after both the tail + // is fully sent and, if it was a zerocopy send, its release notification arrives. + bool send_tail_zc_pending = false; + uint64_t send_tail_from = 0; + uint64_t send_tail_to = 0; uint64_t next_send_free = 0; std::deque send_free; op_aes_xts_encrypt_t *xts_enc_ctx = NULL; diff --git a/src/client/msgr_send.cpp b/src/client/msgr_send.cpp index fbc9ef10..f6414949 100644 --- a/src/client/msgr_send.cpp +++ b/src/client/msgr_send.cpp @@ -17,8 +17,8 @@ #define GCM_TMP_BUF_SIZE 4096 -// How many times to retry sendmsg() in place before falling back to EPOLLOUT. -// Only used with use_sync_send_recv - io_uring waits for the socket by itself +// How many times to retry a blocking-style sendmsg() before waiting for EPOLLOUT. +// The io_uring path submits once, then waits for EPOLLOUT after EAGAIN. #define MSGR_SEND_RETRIES 4 class msgr_op_writer_t @@ -508,6 +508,12 @@ restart: { return true; } + if (!cl->write_ready) + { + // Socket buffer is full, the unsent tail is still in send_list. + // Do not copy more data into it, just wait for EPOLLOUT + return true; + } if (!use_sync_send_recv) { if (!ringloop->space_left()) @@ -521,14 +527,11 @@ restart: return true; } } - else if (!cl->write_ready) - { - // Socket buffer is full, the unsent tail is still in send_list. - // Do not copy more data into it, just wait for EPOLLOUT - return true; - } + // A short send left its tail in send_list. Finish that tail before + // appending new ops, and do not zerocopy it (see handle_send). + bool resume_plain_tail = cl->send_plain_tail && cl->send_list.size(); uint64_t send_free_from = cl->next_send_free + cl->send_free.size(); - if (cl->hs) + if (!resume_plain_tail && cl->hs) { // Send handshake message if (cl->hs->out_size()) @@ -546,12 +549,14 @@ restart: goto copy_ops; } } - else + else if (!resume_plain_tail) { copy_ops: copy_ops_to_with(cl, NULL, 0); } uint64_t send_free_to = cl->next_send_free + cl->send_free.size(); + // The tail resend owns no send_free entries, handle_send relies on it + assert(!resume_plain_tail || send_free_from == send_free_to); if (cl->io_error) { stop_client(cl->client_id); @@ -588,7 +593,7 @@ copy_ops: { handle_send(data->res, data->prev, data->more, cl, send_free_from, send_free_to); }; - bool use_zc = has_sendmsg_zc && min_zerocopy_send_size >= 0; + bool use_zc = !cl->send_plain_tail && has_sendmsg_zc && min_zerocopy_send_size >= 0; if (use_zc && min_zerocopy_send_size > 0 && cl->send_list_size/cl->write_msg.msg_iovlen < min_zerocopy_send_size) { @@ -628,6 +633,10 @@ copy_ops: if (cl->send_list.size() && --send_retries <= 0) { cl->write_ready = false; + tfd->set_fd_handler(cl->peer_fd, true, [this](int peer_fd, int epoll_events) + { + handle_peer_epoll(peer_fd, epoll_events); + }); return true; } } @@ -771,11 +780,25 @@ bool osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t stop_client(cl->client_id); return false; } + // EAGAIN queued nothing. EINTR is resent immediately; EAGAIN waits for EPOLLOUT + // so a full send buffer cannot spin the ring thread. + bool uring_eagain = !use_sync_send_recv && !prev && result == -EAGAIN; + bool uring_retry = uring_eagain || (!use_sync_send_recv && !prev && result == -EINTR); + if (uring_retry) + { + result = 0; + } if (result >= 0) { if (prev) { // Second (zero-copy release) notification + if (cl->send_plain_tail && free_from == cl->send_tail_from && free_to == cl->send_tail_to) + { + // The unsent tail still points into these buffers, release them after it's sent + cl->send_tail_zc_pending = false; + return true; + } cl->unref_send_free(free_from, free_to); return true; } @@ -783,13 +806,56 @@ bool osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t { cl->send_list.clear(); cl->send_list_size = 0; + if (cl->send_plain_tail) + { + cl->send_plain_tail = false; + // Otherwise the zerocopy notification releases it through the common path + if (!cl->send_tail_zc_pending) + cl->unref_send_free(cl->send_tail_from, cl->send_tail_to); + } } else if (!use_sync_send_recv) { - fprintf(stderr, "Client %ju socket write error: expected to send " - "%zu bytes with MSG_WAITALL but sent %u. Disconnecting client\n", cl->client_id, cl->send_list_size, result); - stop_client(cl->client_id); - return false; + // io_uring may still return a short write for MSG_WAITALL (depending on the + // kernel version or when interrupted), for example with large SEC_LIST replies + // filling the socket buffer. Keep the zerocopy prefix and resend only the tail + // with a copying sendmsg. + if (result <= 0 && !uring_retry) + { + fprintf(stderr, "Client %ju socket write error: expected to send " + "%zu bytes with MSG_WAITALL but sent %u. Disconnecting client\n", cl->client_id, cl->send_list_size, result); + stop_client(cl->client_id); + return false; + } + if (!cl->send_plain_tail) + { + // First short write of the original batch. Repeated short writes of + // the tail itself own no send_free entries and must not overwrite it. + cl->send_tail_from = free_from; + cl->send_tail_to = free_to; + cl->send_tail_zc_pending = more; + } + eat_send_list(cl, result); + cl->send_list_size -= result; + cl->send_plain_tail = true; + if (uring_eagain) + { + cl->write_ready = false; + tfd->set_fd_handler(cl->peer_fd, true, [this](int peer_fd, int epoll_events) + { + handle_peer_epoll(peer_fd, epoll_events); + }); + return true; + } + // A positive short write made progress. Submit the tail once; + // if that also returns EAGAIN, the branch above waits for EPOLLOUT. + if (cl->write_state == 0) + { + cl->write_state = CL_WRITE_READY; + write_ready_clients.push_back(cl->client_id); + } + ringloop->wakeup(); + return true; } else { diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 6337e643..f58e167e 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -38,6 +38,17 @@ target_link_libraries(test_msgr_send_reply ${RDMACM_LIBRARIES} ) +# test_msgr_send +add_executable(test_msgr_send EXCLUDE_FROM_ALL test_msgr_send.cpp) +target_link_libraries(test_msgr_send + vitastor_net + ${LIBURING_LIBRARIES} + ${IBVERBS_LIBRARIES} + ${RDMACM_LIBRARIES} +) +add_dependencies(build_tests test_msgr_send) +add_test(NAME test_msgr_send COMMAND test_msgr_send) + # test_msgr_stop add_executable(test_msgr_stop EXCLUDE_FROM_ALL diff --git a/src/test/test_msgr_send.cpp b/src/test/test_msgr_send.cpp new file mode 100644 index 00000000..03b932b0 --- /dev/null +++ b/src/test/test_msgr_send.cpp @@ -0,0 +1,472 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) + +// Short io_uring send handling in osd_messenger_t: the unsent tail is resent +// without zerocopy and every send_free range is released exactly once, after +// both the tail is sent and the zerocopy release notification arrives. + +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "messenger.h" + +#define TEST_DATA_LEN 4096 +#define TEST_BATCH_LEN (OSD_PACKET_SIZE + TEST_DATA_LEN) + +// Records SQEs and delivers completions by hand, with the same prev/more +// semantics as ring_loop_t::loop() +class send_ring_t: public ring_loop_i +{ +public: + std::deque sqes; + std::deque datas; + + void register_consumer(ring_consumer_t *consumer) override {} + void unregister_consumer(ring_consumer_t *consumer) override {} + int register_eventfd() override { return -1; } + io_uring_sqe* get_sqe() override + { + sqes.emplace_back(); + datas.emplace_back(); + io_uring_sqe *sqe = &sqes.back(); + ring_data_t *d = &datas.back(); + *sqe = {}; + d->res = 0; + d->prev = d->more = false; + io_uring_sqe_set_data(sqe, d); + return sqe; + } + void set_immediate(const std::function & cb) override { cb(); } + int submit() override { return 0; } + int wait() override { return 0; } + unsigned space_left() override { return 1024; } + bool has_work() override { return false; } + bool has_sendmsg_zc() override { return true; } + void loop() override {} + void wakeup() override {} + unsigned save() override { return 0; } + void restore(unsigned sqe_tail) override {} + + // more = true emulates a CQE with IORING_CQE_F_MORE (first zerocopy completion) + void complete(size_t i, int res, bool more) + { + assert(i < datas.size()); + ring_data_t *d = &datas[i]; + assert(d->callback); + if (more) + { + d->res = res; + d->more = true; + d->callback(d); + d->prev = true; + d->more = false; + } + else + { + ring_data_t dl; + dl.iov = d->iov; + dl.res = res; + dl.more = false; + dl.prev = d->prev; + dl.callback.swap(d->callback); + d->prev = false; + dl.callback(&dl); + } + } + + bool is_zc(size_t i) + { + assert(i < sqes.size()); + assert(sqes[i].opcode == IORING_OP_SENDMSG_ZC || sqes[i].opcode == IORING_OP_SENDMSG); + return sqes[i].opcode == IORING_OP_SENDMSG_ZC; + } + + msghdr *msg(size_t i) + { + return (msghdr*)sqes[i].addr; + } +}; + +class test_msgr_t: public osd_messenger_t +{ +public: + void set_zerocopy(bool enabled) + { + min_zerocopy_send_size = enabled ? 0 : -1; + } + + osd_client_t *add_client() + { + int fds[2]; + int r = socketpair(AF_UNIX, SOCK_STREAM, 0, fds); + assert(r == 0); + close(fds[1]); + osd_client_t *cl = new osd_client_t(); + cl->client_id = next_client_id++; + cl->peer_fd = fds[0]; + cl->peer_state = PEER_CONNECTED; + clients[cl->client_id] = cl; + clients_by_fd[cl->peer_fd] = cl; + return cl; + } + + // Reply to a SEC_READ with TEST_DATA_LEN bytes of data, sent from op->buf + osd_op_t *push_reply(osd_client_t *cl, uint8_t fill) + { + osd_op_t *op = new osd_op_t(); + op->op_type = OSD_OP_IN; + op->client_id = cl->client_id; + op->req.hdr.opcode = OSD_OP_SEC_READ; + op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC; + op->reply.hdr.opcode = OSD_OP_SEC_READ; + op->reply.hdr.retval = TEST_DATA_LEN; + op->buf = malloc_or_die(TEST_DATA_LEN); + memset(op->buf, fill, TEST_DATA_LEN); + op->iov.push_back(op->buf, TEST_DATA_LEN); + cl->write_ops.push_back(op); + return op; + } + + bool send(osd_client_t *cl) + { + return try_send(cl); + } + + bool is_write_ready(osd_client_t *cl) + { + for (auto id: write_ready_clients) + if (id == cl->client_id) + return true; + return false; + } + + void socket_writable(osd_client_t *cl) + { + handle_peer_epoll(cl->peer_fd, EPOLLOUT); + } +}; + +struct test_env_t +{ + timerfd_manager_t *tfd; + send_ring_t *ring; + test_msgr_t *msgr; + std::map fd_write_watched; + + test_env_t(bool zerocopy) + { + tfd = new timerfd_manager_t([this](int fd, bool wr, std::function callback) + { + if (callback) + fd_write_watched[fd] = wr; + else + fd_write_watched.erase(fd); + }); + ring = new send_ring_t(); + msgr = new test_msgr_t(); + msgr->tfd = tfd; + msgr->ringloop = ring; + msgr->has_sendmsg_zc = true; + msgr->set_zerocopy(zerocopy); + } + + ~test_env_t() + { + delete msgr; + delete ring; + delete tfd; + } +}; + +static void check_released(osd_client_t *cl, uint64_t next_send_free, size_t send_free_size) +{ + if (cl->next_send_free != next_send_free || cl->send_free.size() != send_free_size) + { + fprintf(stderr, "send_free: expected next=%ju size=%zu, got next=%ju size=%zu\n", + next_send_free, send_free_size, cl->next_send_free, cl->send_free.size()); + abort(); + } +} + +static void check_tail(send_ring_t *ring, size_t i, osd_op_t *op, size_t sent) +{ + assert(!ring->is_zc(i)); + assert(ring->sqes[i].msg_flags & MSG_WAITALL); + msghdr *msg = ring->msg(i); + size_t total = 0; + for (size_t j = 0; j < msg->msg_iovlen; j++) + total += msg->msg_iov[j].iov_len; + assert(total == TEST_BATCH_LEN - sent); + assert(sent >= OSD_PACKET_SIZE); + assert(msg->msg_iov[0].iov_base == (uint8_t*)op->buf + (sent - OSD_PACKET_SIZE)); +} + +// Zerocopy short write, the tail is sent before the release notification arrives +static void test_zc_tail_before_notify() +{ + printf("test_zc_tail_before_notify\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + assert(env.ring->is_zc(0)); + env.ring->complete(0, 1000, true); + assert(cl->send_plain_tail); + assert(cl->send_tail_zc_pending); + assert(cl->send_list_size == TEST_BATCH_LEN - 1000); + assert(env.msgr->is_write_ready(cl)); + env.msgr->send_replies(); + check_tail(env.ring, 1, a, 1000); + env.ring->complete(1, TEST_BATCH_LEN - 1000, false); + assert(!cl->send_plain_tail); + // The zerocopy prefix is still owned by the kernel + check_released(cl, 0, 1); + env.ring->complete(0, 0, false); + check_released(cl, 1, 0); +} + +// Zerocopy short write, the release notification arrives before the tail is sent +static void test_zc_notify_before_tail() +{ + printf("test_zc_notify_before_tail\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + env.ring->complete(0, 1000, true); + // Notification before the tail is even submitted + env.ring->complete(0, 0, false); + assert(!cl->send_tail_zc_pending); + check_released(cl, 0, 1); + env.msgr->send_replies(); + check_tail(env.ring, 1, a, 1000); + check_released(cl, 0, 1); + env.ring->complete(1, TEST_BATCH_LEN - 1000, false); + check_released(cl, 1, 0); +} + +// A notification of an older zerocopy batch arrives while the tail is pending. +// It must be released at once and must not swallow the notification of the short batch. +static void test_zc_other_notify_during_tail() +{ + printf("test_zc_other_notify_during_tail\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + env.msgr->push_reply(cl, 0x11); + env.msgr->send(cl); + env.ring->complete(0, TEST_BATCH_LEN, true); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + assert(env.ring->is_zc(1)); + env.ring->complete(1, 1000, true); + assert(cl->send_tail_from == 1 && cl->send_tail_to == 2); + env.msgr->send_replies(); + check_tail(env.ring, 2, a, 1000); + env.ring->complete(0, 0, false); + check_released(cl, 1, 1); + env.ring->complete(1, 0, false); + check_released(cl, 1, 1); + env.ring->complete(2, TEST_BATCH_LEN - 1000, false); + check_released(cl, 2, 0); +} + +// Copying short write while an older zerocopy notification is still in flight +static void test_plain_short_with_zc_notify() +{ + printf("test_plain_short_with_zc_notify\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + env.msgr->push_reply(cl, 0x11); + env.msgr->send(cl); + assert(env.ring->is_zc(0)); + env.ring->complete(0, TEST_BATCH_LEN, true); + env.msgr->set_zerocopy(false); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + assert(!env.ring->is_zc(1)); + env.ring->complete(1, 1000, false); + assert(cl->send_plain_tail); + assert(!cl->send_tail_zc_pending); + env.msgr->send_replies(); + check_tail(env.ring, 2, a, 1000); + env.ring->complete(0, 0, false); + check_released(cl, 1, 1); + env.ring->complete(2, TEST_BATCH_LEN - 1000, false); + check_released(cl, 2, 0); +} + +// The tail itself is short-written again: the owner range must be kept +static void test_tail_short_again() +{ + printf("test_tail_short_again\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + env.ring->complete(0, 1000, true); + env.msgr->send_replies(); + check_tail(env.ring, 1, a, 1000); + env.ring->complete(1, 1000, false); + assert(cl->send_plain_tail); + assert(cl->send_tail_from == 0 && cl->send_tail_to == 1); + assert(cl->send_tail_zc_pending); + env.msgr->send_replies(); + check_tail(env.ring, 2, a, 2000); + env.ring->complete(0, 0, false); + check_released(cl, 0, 1); + env.ring->complete(2, TEST_BATCH_LEN - 2000, false); + check_released(cl, 1, 0); +} + +// io_uring returns -EAGAIN: nothing was queued, so the batch waits for EPOLLOUT +// and is then resent through the copying tail path. A second EAGAIN does not +// submit another SQE until the socket is writable again. +static void test_eagain_resend() +{ + printf("test_eagain_resend\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + env.ring->complete(0, -EAGAIN, false); + assert(env.msgr->clients.find(cl->client_id) != env.msgr->clients.end()); + assert(cl->send_plain_tail); + assert(!cl->send_tail_zc_pending); + assert(cl->send_list_size == TEST_BATCH_LEN); + assert(!cl->write_ready); + assert(env.fd_write_watched[cl->peer_fd]); + assert(!env.msgr->is_write_ready(cl)); + env.msgr->send_replies(); + assert(env.ring->sqes.size() == 1); + env.msgr->socket_writable(cl); + assert(cl->write_ready); + assert(!env.fd_write_watched[cl->peer_fd]); + assert(env.msgr->is_write_ready(cl)); + env.msgr->send_replies(); + assert(env.ring->sqes.size() == 2); + assert(!env.ring->is_zc(1)); + assert(env.ring->msg(1)->msg_iov[0].iov_base == &a->reply); + env.ring->complete(1, -EAGAIN, false); + assert(!cl->write_ready); + assert(env.fd_write_watched[cl->peer_fd]); + assert(!env.msgr->is_write_ready(cl)); + env.msgr->send_replies(); + assert(env.ring->sqes.size() == 2); + env.msgr->socket_writable(cl); + assert(!env.fd_write_watched[cl->peer_fd]); + env.msgr->send_replies(); + assert(env.ring->sqes.size() == 3); + env.ring->complete(2, TEST_BATCH_LEN, false); + assert(!cl->send_plain_tail); + check_released(cl, 1, 0); +} + +// Failed zerocopy send: -EAGAIN carries IORING_CQE_F_MORE. The release +// notification must not free the buffer, and the resend still waits for EPOLLOUT. +static void test_eagain_zc_more() +{ + printf("test_eagain_zc_more\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + env.ring->complete(0, -EAGAIN, true); + assert(cl->send_plain_tail); + assert(cl->send_tail_zc_pending); + assert(!cl->write_ready); + assert(cl->send_list_size == TEST_BATCH_LEN); + env.ring->complete(0, 0, false); + assert(!cl->send_tail_zc_pending); + check_released(cl, 0, 1); + assert(env.ring->sqes.size() == 1); + env.msgr->socket_writable(cl); + env.msgr->send_replies(); + assert(!env.ring->is_zc(1)); + assert(env.ring->msg(1)->msg_iov[0].iov_base == &a->reply); + env.ring->complete(1, TEST_BATCH_LEN, false); + assert(!cl->send_plain_tail); + check_released(cl, 1, 0); +} + +// A zero-byte send without an error is still a broken socket +static void test_zero_bytes_disconnects() +{ + printf("test_zero_bytes_disconnects\n"); + test_env_t env(false); + osd_client_t *cl = env.msgr->add_client(); + uint64_t client_id = cl->client_id; + env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + env.ring->complete(0, 0, false); + assert(env.msgr->clients.find(client_id) == env.msgr->clients.end()); +} + +// New replies queued while the tail is pending must go after it, in a separate send +static void test_new_ops_wait_for_tail() +{ + printf("test_new_ops_wait_for_tail\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + osd_op_t *a = env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + env.ring->complete(0, 1000, true); + osd_op_t *b = env.msgr->push_reply(cl, 0xbb); + env.msgr->send(cl); + check_tail(env.ring, 1, a, 1000); + assert(cl->write_ops.size() == 1); + assert(env.ring->sqes.size() == 2); + env.ring->complete(1, TEST_BATCH_LEN - 1000, false); + assert(env.msgr->is_write_ready(cl)); + env.msgr->send_replies(); + assert(env.ring->sqes.size() == 3); + assert(env.ring->is_zc(2)); + assert(env.ring->msg(2)->msg_iov[0].iov_base == &b->reply); + env.ring->complete(2, TEST_BATCH_LEN, true); + env.ring->complete(2, 0, false); + // The first batch still waits for its notification and blocks the head + check_released(cl, 0, 2); + env.ring->complete(0, 0, false); + check_released(cl, 2, 0); +} + +// A spurious EPOLLOUT while a normal io_uring send is in flight must not +// enqueue the client into write_ready_clients +static void test_inflight_epollout_ignored() +{ + printf("test_inflight_epollout_ignored\n"); + test_env_t env(true); + osd_client_t *cl = env.msgr->add_client(); + env.msgr->push_reply(cl, 0xaa); + env.msgr->send(cl); + assert(cl->write_msg.msg_iovlen > 0); + assert(cl->send_list.size() > 0); + env.msgr->socket_writable(cl); + assert(!env.msgr->is_write_ready(cl)); + env.ring->complete(0, TEST_BATCH_LEN, true); + env.ring->complete(0, 0, false); + check_released(cl, 1, 0); +} + +int main(int narg, char *args[]) +{ + test_zc_tail_before_notify(); + test_zc_notify_before_tail(); + test_zc_other_notify_during_tail(); + test_plain_short_with_zc_notify(); + test_tail_short_again(); + test_eagain_resend(); + test_eagain_zc_more(); + test_zero_bytes_disconnects(); + test_new_ops_wait_for_tail(); + test_inflight_epollout_ignored(); + printf("OK\n"); + return 0; +} -- 2.47.3