WIP Replace exit/abort in client with on_fatal_error_hook

- Check the result of EVP_KDF_CTX_new() in msgr_handshake_ctx_t::init().
- Check the created cipher context instead of msgr_handshake_t::ctx
  after EVP_CIPHER_CTX_new() in encrypt() and decrypt().
- Set the error message and the error state when the handshake signature
  is invalid, and clear the OpenSSL error queue so that its errors are not
  reported by the next on_error() call.

src/client/cluster_client.cpp

@@ -14,6 +14,11 @@
 
 cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config, std::unique_ptr<etcd_state_client_t> st_cli_ptr)
 {
+    auto on_fatal_error_hook = [](const std::string& error)
+    {
+        throw std::runtime_error(error);
+    };
+
     wb = new writeback_cache_t();
 
     cli_config = config.object_items();
@@ -23,12 +28,13 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     this->ringloop = ringloop;
     this->tfd = tfd;
 
-    msgr.osd_num = 0;
-    msgr.tfd = tfd;
-    msgr.ringloop = ringloop;
-    msgr.repeer_pgs = [this](osd_num_t peer_osd)
+    msgr = new osd_messenger_t();
+    msgr->osd_num = 0;
+    msgr->tfd = tfd;
+    msgr->ringloop = ringloop;
+    msgr->repeer_pgs = [this](osd_num_t peer_osd)
     {
-        if (msgr.osd_peers.find(peer_osd) != msgr.osd_peers.end())
+        if (msgr->osd_peers.find(peer_osd) != msgr->osd_peers.end())
         {
             // peer_osd just connected
             continue_ops();
@@ -45,14 +51,16 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
             }
         }
     };
-    msgr.exec_op = [this](osd_op_t *op)
+    msgr->exec_op = [this](osd_op_t *op)
     {
         // Garbage in
         fprintf(stderr, "Can't handle incoming operation from client %ju\n", op->client_id);
-        msgr.stop_client(op->client_id);
+        msgr->stop_client(op->client_id);
         delete op;
     };
-    msgr.parse_config(config, true);
+    auto exit_hook = msgr->on_fatal_error_hook;
+    msgr->on_fatal_error_hook = on_fatal_error_hook;
+    msgr->parse_config(config, true);
 
     st_cli = std::move(st_cli_ptr);
     st_cli->on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
@@ -64,6 +72,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     st_cli->on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
     st_cli->on_reload_hook = [this]() { this->st_cli->load_global_config(); };
     st_cli->on_inode_change_hook = [this](uint64_t inode, bool removed) { on_change_inode_hook(inode, removed); };
+    st_cli->on_fatal_error_hook = on_fatal_error_hook;
 
     st_cli->parse_config(config);
     st_cli->infinite_start = false;
@@ -72,11 +81,30 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
         st_cli->infinite_start = config["client_infinite_start"].bool_value();
     }
     st_cli->load_global_config();
+
+    if (ringloop)
+    {
+        ringloop->loop();
+    }
+
+    msgr->on_fatal_error_hook = exit_hook;
+    st_cli->on_fatal_error_hook = exit_hook;
+}
+
+void cluster_client_t::set_on_fatal_error_hook(std::function<void(const std::string&)> cb)
+{
+    st_cli->on_fatal_error_hook = cb;
+    msgr->on_fatal_error_hook = cb;
+}
+
+void cluster_client_t::fatal_error(const std::string & error)
+{
+    (std::function<void(const std::string&)>(msgr->on_fatal_error_hook))(error);
 }
 
 cluster_client_t::~cluster_client_t()
 {
-    vault_destroy();
+    aborted = true;
     if (retry_timeout_id >= 0)
     {
         tfd->clear_timer(retry_timeout_id);
@@ -89,13 +117,21 @@ cluster_client_t::~cluster_client_t()
         list_retry_timeout_id = -1;
         list_retry_time = {};
     }
-    msgr.repeer_pgs = [](osd_num_t){};
+    msgr->repeer_pgs = [](osd_num_t){};
     if (ringloop)
     {
         ringloop->unregister_consumer(&consumer);
     }
     delete wb;
     wb = NULL;
+    delete msgr;
+    msgr = NULL;
+    while (op_queue_head)
+    {
+        op_queue_head->retval = -ECANCELED;
+        erase_op(op_queue_head);
+    }
+    vault_destroy();
 }
 
 cluster_op_t::~cluster_op_t()
@@ -150,13 +186,15 @@ std::vector<osd_num_t> cluster_op_t::get_left_on_dead()
 
 void cluster_client_t::continue_raw_ops(osd_num_t peer_osd)
 {
+    if (aborted)
+        return;
     auto it = raw_ops.find(peer_osd);
     while (it != raw_ops.end() && it->first == peer_osd)
     {
         auto op = it->second;
         op->op_type = OSD_OP_OUT;
-        op->client_id = msgr.osd_peers.at(peer_osd)->client_id;
-        msgr.outbox_push(op);
+        op->client_id = msgr->osd_peers.at(peer_osd)->client_id;
+        msgr->outbox_push(op);
         raw_ops.erase(it++);
     }
 }
@@ -165,14 +203,14 @@ void cluster_client_t::init_msgr()
 {
     if (msgr_initialized)
         return;
-    msgr.init();
+    msgr->init();
     msgr_initialized = true;
     if (ringloop)
     {
         consumer.loop = [this]()
         {
-            msgr.read_requests();
-            msgr.send_replies();
+            msgr->read_requests();
+            msgr->send_replies();
             this->ringloop->submit();
         };
         ringloop->register_consumer(&consumer);
@@ -482,7 +520,7 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
     }
     // vault
     vault_parse_config();
-    msgr.parse_config(config, false);
+    msgr->parse_config(config, false);
     st_cli->parse_config(config);
     st_cli->load_pgs();
 }
@@ -643,9 +681,9 @@ bool cluster_client_t::get_immediate_commit(uint64_t inode)
 void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
 {
     osd_tree_metrics.erase(peer_osd);
-    if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
+    if (msgr->wanted_peers.find(peer_osd) != msgr->wanted_peers.end())
     {
-        msgr.connect_peer(peer_osd, st_cli->peer_states[peer_osd]);
+        msgr->connect_peer(peer_osd, st_cli->peer_states[peer_osd]);
         continue_lists();
     }
 }
@@ -862,6 +900,13 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
 
 void cluster_client_t::execute_cas(cluster_op_t *op, bool nosync)
 {
+    if (aborted)
+    {
+        op->retval = -ECANCELED;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return;
+    }
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
     {
         // Atomic writes to multiple stripes are unsupported
@@ -878,7 +923,14 @@ void cluster_client_t::execute_cas(cluster_op_t *op, bool nosync)
         sync_op->flags = 0;
         sync_op->callback = [this, op](cluster_op_t* sync_op)
         {
-            execute_cas(op, true);
+            if (sync_op->retval != 0)
+            {
+                op->retval = sync_op->retval;
+                auto cb = std::move(op->callback);
+                cb(op);
+            }
+            else
+                execute_cas(op, true);
             delete sync_op;
         };
         execute_internal(sync_op);
@@ -894,18 +946,25 @@ void cluster_client_t::execute_cas(cluster_op_t *op, bool nosync)
     op->needs_reslice = false;
     int res = try_send(op, 0, [this, op](osd_op_t *part)
     {
+        if (aborted)
+        {
+            op->retval = -ECANCELED;
+            auto cb = std::move(op->callback);
+            cb(op);
+            return;
+        }
         int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
         op->retval = part->reply.hdr.retval;
         if (op->retval != expected && op->retval >= 0)
             op->retval = -EIO;
         op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
-        auto peer_it = msgr.osd_peers.find(op->parts[0].osd_num);
+        auto peer_it = msgr->osd_peers.find(op->parts[0].osd_num);
         if (op->retval != expected || (op->flags & OP_IMMEDIATE_COMMIT))
         {
             auto cb = std::move(op->callback);
             cb(op);
         }
-        else if (peer_it == msgr.osd_peers.end())
+        else if (peer_it == msgr->osd_peers.end())
         {
             // Care must be taken to make sure that the client doesn't reconnect to the OSD
             // before executing the previously completed operation callback (!)
@@ -937,7 +996,7 @@ void cluster_client_t::execute_cas(cluster_op_t *op, bool nosync)
                     cb(op);
                 },
             };
-            msgr.outbox_push(part);
+            msgr->outbox_push(part);
         }
     });
     if (res == TRY_SEND_CONNECTING || res == TRY_SEND_OFFLINE)
@@ -1068,23 +1127,25 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
 
 void cluster_client_t::execute_raw(osd_num_t osd_num, osd_op_t *op)
 {
-    auto peer_it = msgr.osd_peers.find(osd_num);
-    if (peer_it != msgr.osd_peers.end())
+    auto peer_it = msgr->osd_peers.find(osd_num);
+    if (peer_it != msgr->osd_peers.end())
     {
         op->op_type = OSD_OP_OUT;
         op->client_id = peer_it->second->client_id;
-        msgr.outbox_push(op);
+        msgr->outbox_push(op);
     }
     else
     {
-        if (msgr.wanted_peers.find(osd_num) == msgr.wanted_peers.end())
-            msgr.connect_peer(osd_num, st_cli->peer_states[osd_num]);
+        if (msgr->wanted_peers.find(osd_num) == msgr->wanted_peers.end())
+            msgr->connect_peer(osd_num, st_cli->peer_states[osd_num]);
         raw_ops.emplace(osd_num, op);
     }
 }
 
 int cluster_client_t::continue_rw(cluster_op_t *op)
 {
+    if (aborted)
+        return 0;
     if (op->state == 0)
         goto resume_0;
     else if (op->state == 1)
@@ -1509,8 +1570,8 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                 primary_osd = nearest_osd;
         }
         part->osd_num = primary_osd;
-        auto peer_it = msgr.osd_peers.find(primary_osd);
-        if (peer_it != msgr.osd_peers.end())
+        auto peer_it = msgr->osd_peers.find(primary_osd);
+        if (peer_it != msgr->osd_peers.end())
         {
             osd_client_t *cl = peer_it->second;
             part->flags |= PART_SENT|PART_VALID;
@@ -1554,12 +1615,12 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                 },
             };
             part->op.iov = part->iov;
-            msgr.outbox_push(&part->op);
+            msgr->outbox_push(&part->op);
             return TRY_SEND_OK;
         }
-        else if (msgr.wanted_peers.find(primary_osd) == msgr.wanted_peers.end())
+        else if (msgr->wanted_peers.find(primary_osd) == msgr->wanted_peers.end())
         {
-            msgr.connect_peer(primary_osd, st_cli->peer_states[primary_osd]);
+            msgr->connect_peer(primary_osd, st_cli->peer_states[primary_osd]);
             return TRY_SEND_CONNECTING;
         }
     }
@@ -1568,6 +1629,8 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
 
 int cluster_client_t::continue_sync(cluster_op_t *op)
 {
+    if (aborted)
+        return 0;
     if (op->state == 1)
         goto resume_1;
     if (!dirty_osds.size())
@@ -1581,8 +1644,8 @@ int cluster_client_t::continue_sync(cluster_op_t *op)
     for (auto do_it = dirty_osds.begin(); do_it != dirty_osds.end(); )
     {
         osd_num_t sync_osd = *do_it;
-        auto peer_it = msgr.osd_peers.find(sync_osd);
-        if (peer_it == msgr.osd_peers.end())
+        auto peer_it = msgr->osd_peers.find(sync_osd);
+        if (peer_it == msgr->osd_peers.end())
             dirty_osds.erase(do_it++);
         else
             do_it++;
@@ -1635,7 +1698,7 @@ resume_1:
 
 void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
 {
-    osd_client_t *cl = msgr.osd_peers.at(part->osd_num);
+    osd_client_t *cl = msgr->osd_peers.at(part->osd_num);
     part->flags |= PART_SENT;
     op->inflight_count++;
     part->op = (osd_op_t){
@@ -1652,7 +1715,7 @@ void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
             handle_op_part(part);
         },
     };
-    msgr.outbox_push(&part->op);
+    msgr->outbox_push(&part->op);
 }
 
 static inline void mem_or(void *res, const void *r2, unsigned int len)
@@ -1707,7 +1770,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         reset_retry_timer(op->retry_after);
         if (stop_client_id)
         {
-            msgr.stop_client(stop_client_id);
+            msgr->stop_client(stop_client_id);
         }
         op->inflight_count--;
         if (op->inflight_count == 0 && !op->retry_after)

src/client/cluster_client.h

@@ -176,11 +176,12 @@ class __attribute__((visibility("default"))) cluster_client_t
     std::multimap<osd_num_t, osd_op_t*> raw_ops;
     int continuing_ops = 0;
     bool msgr_initialized = false;
+    bool aborted = false;
 
 public:
     std::unique_ptr<etcd_state_client_t> st_cli;
+    osd_messenger_t* msgr = NULL;
 
-    osd_messenger_t msgr;
     void init_msgr();
 
     json11::Json::object cli_config, file_config, etcd_global_config;
@@ -194,6 +195,7 @@ public:
     bool is_ready();
     void on_ready(std::function<void(void)> fn);
     bool flush();
+    void set_on_fatal_error_hook(std::function<void(const std::string&)>);
 
     bool get_immediate_commit(uint64_t inode);
 
@@ -201,6 +203,7 @@ public:
         int status, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)> pg_callback);
 
 protected:
+    void fatal_error(const std::string& error);
     void continue_ops(int time_passed = 0);
 
     std::shared_ptr<inode_cache_t> inode_cache_get(inode_t ino);

src/client/cluster_client_list.cpp

@@ -301,14 +301,14 @@ int cluster_client_t::start_pg_listing(inode_list_pg_t *pg)
     bool conn = true;
     for (osd_num_t peer_osd: all_peers)
     {
-        if (msgr.osd_peers.find(peer_osd) == msgr.osd_peers.end())
+        if (msgr->osd_peers.find(peer_osd) == msgr->osd_peers.end())
         {
             // Initiate connection
             if (st_cli->peer_states[peer_osd].is_null())
             {
                 return LIST_PG_WAIT_ACTIVE;
             }
-            msgr.connect_peer(peer_osd, st_cli->peer_states[peer_osd]);
+            msgr->connect_peer(peer_osd, st_cli->peer_states[peer_osd]);
             conn = false;
         }
     }
@@ -346,7 +346,7 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
     osd_op_t *op = new osd_op_t();
     op->op_type = OSD_OP_OUT;
     // Already checked that it exists above, but anyway
-    op->client_id = msgr.osd_peers.at(cur_list->osd_num)->client_id;
+    op->client_id = msgr->osd_peers.at(cur_list->osd_num)->client_id;
     op->req = (osd_any_op_t){
         .sec_list = {
             .header = {
@@ -425,7 +425,7 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
         finish_list_pg(cur_list->pg, true);
         continue_listing(cur_list->pg->lst);
     };
-    msgr.outbox_push(op);
+    msgr->outbox_push(op);
 }
 
 void cluster_client_t::finish_list_pg(inode_list_pg_t *pg, bool retry_epipe)

src/client/cluster_client_real.cpp

@@ -98,8 +98,8 @@ void cluster_client_t::vault_load_keys()
         vault_http_ctx = http_context_init(tfd, vault_client_cert, vault_client_key, vault_ca, true, error);
         if (!vault_http_ctx)
         {
-            fprintf(stderr, "Failed to initialize HTTP context for Vault: %s\n", error.c_str());
-            exit(1);
+            fatal_error("Failed to initialize HTTP context for Vault: "+error);
+            return;
         }
         vault_http_cli = http_init(vault_http_ctx);
     }

src/client/etcd_state_client.cpp

@@ -10,6 +10,20 @@
 #include "str_util.h"
 #include "json_util.h"
 
+etcd_state_client_t::etcd_state_client_t()
+{
+    this->on_fatal_error_hook = [](const std::string & error)
+    {
+        fprintf(stderr, "%s\n", error.c_str());
+        exit(1);
+    };
+}
+
+void etcd_state_client_t::fatal_error(const std::string & error)
+{
+    (std::function<void(const std::string&)>(on_fatal_error_hook))(error);
+}
+
 etcd_state_client_t::~etcd_state_client_t()
 {
     for (auto watch: watches)

src/client/etcd_state_client.h

@@ -144,6 +144,7 @@ protected:
     std::set<osd_num_t> seen_peers;
     bool new_pg_config = false;
 
+    void fatal_error(const std::string & error);
     void add_etcd_url(std::string);
     void reset_pg_exists();
     void clean_nonexistent_pgs();
@@ -192,6 +193,7 @@ public:
     std::function<void()> on_reload_hook;
     std::function<void(inode_t, bool)> on_inode_change_hook;
     std::function<void(http_co_t *)> on_start_watcher_hook;
+    std::function<void(const std::string&)> on_fatal_error_hook;
 
     json11::Json::object serialize_inode_cfg(inode_config_t *cfg);
     inode_config_t deserialize_inode_cfg(uint64_t inode_num, json11::Json value, uint64_t mod_revision);
@@ -215,6 +217,7 @@ public:
     inode_watch_t* watch_inode(std::string name);
     void close_watch(inode_watch_t* watch);
     int address_count();
+    etcd_state_client_t();
     virtual ~etcd_state_client_t();
 
     static uint32_t parse_immediate_commit(const std::string & immediate_commit_str, uint32_t default_value);

src/client/etcd_state_client_http.cpp

@@ -49,7 +49,11 @@ void etcd_state_client_http_t::etcd_add_watch(json11::Json watch)
 
 std::string etcd_state_client_http_t::get_username()
 {
-    return http_context_get_ssl_cn(get_http_ctx());
+    if (!get_http_ctx())
+    {
+        return "";
+    }
+    return http_context_get_ssl_cn(http_ctx);
 }
 
 http_context_t *etcd_state_client_http_t::get_http_ctx()
@@ -60,8 +64,7 @@ http_context_t *etcd_state_client_http_t::get_http_ctx()
         http_ctx = http_context_init(tfd, etcd_client_cert, etcd_client_key, etcd_ca, true, error);
         if (!http_ctx)
         {
-            fprintf(stderr, "Failed to initialize HTTP context: %s\n", error.c_str());
-            exit(1);
+            fatal_error("Failed to initialize HTTP context: "+error);
         }
     }
     return http_ctx;
@@ -70,7 +73,11 @@ http_context_t *etcd_state_client_http_t::get_http_ctx()
 void etcd_state_client_http_t::etcd_call_oneshot(const std::string & etcd_url, const std::string & api, json11::Json payload,
     int timeout, std::function<void(std::string, json11::Json)> callback)
 {
-    auto http_cli = http_init(get_http_ctx());
+    if (!get_http_ctx())
+    {
+        return;
+    }
+    auto http_cli = http_init(http_ctx);
     http_json_post(http_cli, etcd_url+api, payload, "", { .timeout = timeout }, [http_cli, callback](http_message_t *response)
     {
         std::string err;
@@ -140,7 +147,11 @@ void etcd_state_client_http_t::etcd_call_selected(const std::string & api, json1
             callback(err, data);
     };
     if (!keepalive_client)
-        keepalive_client = http_init(get_http_ctx());
+    {
+        if (!get_http_ctx())
+            return;
+        keepalive_client = http_init(http_ctx);
+    }
     http_request(keepalive_client, url.addr, req, { .timeout = timeout, .keepalive = true, .ssl = url.ssl }, cb);
 }
 
@@ -159,8 +170,12 @@ void etcd_state_client_http_t::pick_next_etcd(std::function<void()> cb)
 {
     if (!etcd_addresses.size() && !etcd_local.size())
     {
-        fprintf(stderr, "etcd_address is missing in Vitastor configuration\n");
-        exit(1);
+        fatal_error("etcd_address is missing in Vitastor configuration");
+        return;
+    }
+    if (!get_http_ctx())
+    {
+        return;
     }
     if (selected_etcd_url.addr != "")
     {
@@ -196,7 +211,7 @@ void etcd_state_client_http_t::pick_next_etcd(std::function<void()> cb)
     for (auto & url: etcd_name_urls)
     {
         resolve_count++;
-        http_resolve(get_http_ctx(), url.ssl, url.addr, [this, url](const std::string & error, const std::vector<std::string>& addresses)
+        http_resolve(http_ctx, url.ssl, url.addr, [this, url](const std::string & error, const std::vector<std::string>& addresses)
         {
             if (error != "")
                 fprintf(stderr, "Error resolving %s: %s\n", url.addr.c_str(), error.c_str());
@@ -228,8 +243,8 @@ void etcd_state_client_http_t::pick_next_etcd_on_resolve()
 {
     if (!etcd_urls_to_try.size())
     {
-        fprintf(stderr, "None of etcd_address could be resolved\n");
-        exit(1);
+        fatal_error("None of etcd_address could be resolved");
+        return;
     }
     if (!rand_initialized)
     {
@@ -274,7 +289,11 @@ void etcd_state_client_http_t::start_etcd_watcher_selected()
             etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
     }
     if (!etcd_watch_ws)
-        etcd_watch_ws = http_init(get_http_ctx());
+    {
+        if (!get_http_ctx())
+            return;
+        etcd_watch_ws = http_init(http_ctx);
+    }
     else
         http_close(etcd_watch_ws);
     open_websocket(etcd_watch_ws, url.addr, url.hostname, url.path+"/watch", { .timeout = etcd_slow_timeout, .ssl = url.ssl },
@@ -324,20 +343,17 @@ void etcd_state_client_http_t::start_etcd_watcher_selected()
                                 etcd_watch_revision_config = etcd_watch_revision_osd = etcd_watch_revision_pg = 0;
                                 on_reload_hook();
                             }
-                            return;
                         }
                         else
                         {
-                            fprintf(stderr, "Revisions before %ju were compacted by etcd, exiting\n",
-                                data["result"]["compact_revision"].uint64_value());
-                            exit(1);
+                            fatal_error("Revisions before "+data["result"]["compact_revision"].as_string()+" were compacted by etcd, exiting");
                         }
                     }
                     else
                     {
-                        fprintf(stderr, "Watch canceled by etcd, reason: %s, exiting\n", data["result"]["cancel_reason"].string_value().c_str());
-                        exit(1);
+                        fatal_error("Watch canceled by etcd, reason: "+data["result"]["cancel_reason"].string_value()+", exiting");
                     }
+                    return;
                 }
                 // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
                 // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
@@ -492,9 +508,9 @@ void etcd_state_client_http_t::load_global_config()
     {
         if (err != "")
         {
-            fprintf(stderr, "Error reading configuration from etcd: %s\n", err.c_str());
             if (infinite_start)
             {
+                fprintf(stderr, "Error reading configuration from etcd: %s\n", err.c_str());
                 tfd->set_timer(etcd_slow_timeout, false, [this](int timer_id)
                 {
                     load_global_config();
@@ -502,7 +518,7 @@ void etcd_state_client_http_t::load_global_config()
             }
             else
             {
-                exit(1);
+                fatal_error("Error reading configuration from etcd: "+err);
             }
         }
     });

src/client/messenger.cpp

@@ -17,6 +17,20 @@
 #include "msgr_rdma.h"
 #endif
 
+osd_messenger_t::osd_messenger_t()
+{
+    this->on_fatal_error_hook = [](const std::string & error)
+    {
+        fprintf(stderr, "%s\n", error.c_str());
+        exit(1);
+    };
+}
+
+void osd_messenger_t::fatal_error(const std::string & error)
+{
+    (std::function<void(const std::string&)>(on_fatal_error_hook))(error);
+}
+
 void osd_messenger_t::init()
 {
     init_tls();
@@ -198,7 +212,7 @@ osd_messenger_t::~osd_messenger_t()
     destroy_tls();
 }
 
-static int parse_proto_checksums(const json11::Json & val, int default_value)
+int osd_messenger_t::parse_proto_checksums(const json11::Json & val, int default_value)
 {
     if (val.is_string())
     {
@@ -216,9 +230,9 @@ static int parse_proto_checksums(const json11::Json & val, int default_value)
     }
     else if (val.is_null())
         return default_value;
-    fprintf(stderr, "proto_checksums should be \"full\", \"payload\", \"gcm\", \"none\""
-        ", \"\" or null (default), but it is: %s\n", val.dump().c_str());
-    exit(1);
+    fatal_error("proto_checksums should be \"full\", \"payload\", \"gcm\", \"none\""
+        ", \"\" or null (default), but it is: "+val.dump());
+    return default_value;
 }
 
 void osd_messenger_t::parse_config(const json11::Json & config, bool init)

src/client/messenger.h

@@ -257,6 +257,7 @@ protected:
     bool gcm_enabled = false;
     msgr_handshake_ctx_i *hs_ctx = NULL;
 
+    void fatal_error(const std::string & error);
     void init_tls();
     void destroy_tls();
     bool init_tls_client(osd_client_t *cl);
@@ -317,10 +318,12 @@ public:
     std::function<void(osd_num_t)> break_pg_locks;
     std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
     std::function<void(osd_client_t*)> handshake_hook;
+    std::function<void(const std::string&)> on_fatal_error_hook;
     void read_requests();
     void send_replies();
     void accept_connections(int listen_fd);
     void destroy_iothreads();
+    osd_messenger_t();
     ~osd_messenger_t();
 
     static json11::Json::object read_config(const json11::Json & config);
@@ -344,6 +347,7 @@ public:
     void measure_exec(osd_op_t *cur_op);
 
 protected:
+    int parse_proto_checksums(const json11::Json & val, int default_value);
     void try_connect_peer(uint64_t osd_num);
     void try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_host, int peer_port);
     void handle_peer_epoll(int peer_fd, int epoll_events);

src/client/msgr_encrypt.cpp

@@ -597,10 +597,9 @@ void osd_messenger_t::init_tls()
         if (client_cert.empty() || client_key.empty() || osd_ca.empty() || osd_num && client_ca.empty())
         {
             if (osd_num)
-                fprintf(stderr, "Vitastor transport encryption requires osd_cert, osd_pkey, osd_ca, client_ca options for OSDs\n");
+                fatal_error("Vitastor transport encryption requires osd_cert, osd_pkey, osd_ca, client_ca options for OSDs");
             else
-                fprintf(stderr, "Vitastor transport encryption requires cert, pkey and osd_ca options\n");
-            exit(1);
+                fatal_error("Vitastor transport encryption requires cert, pkey and osd_ca options\n");
         }
         else
         {
@@ -609,8 +608,8 @@ void osd_messenger_t::init_tls()
             hs_ctx = msgr_handshake_ctx_i::create_ctx();
             if (!hs_ctx->init(client_cert, client_key, osd_ca, client_ca, admin_ca))
             {
-                fprintf(stderr, "Error: %s\n", hs_ctx->get_error().c_str());
-                exit(1);
+                gcm_enabled = false;
+                fatal_error("Error: "+hs_ctx->get_error());
             }
 #endif
         }

src/client/msgr_handshake.cpp

@@ -205,6 +205,8 @@ bool msgr_handshake_ctx_t::init(const std::string & pem_cert, const std::string
         return on_error("EVP_KDF_fetch: ");
     kdf_ctx = EVP_KDF_CTX_new(kdf);
     EVP_KDF_free(kdf);
+    if (!kdf_ctx)
+        return on_error("EVP_KDF_CTX_new: ");
     return true;
 }
 
@@ -245,6 +247,9 @@ bool msgr_handshake_ctx_t::on_error(const std::string & prefix)
 bool msgr_handshake_ctx_t::derive_kdf(const uint8_t* insecret, size_t insecret_len,
     const uint8_t* salt, size_t salt_len, const char *label, uint8_t *outsecret, size_t outsize)
 {
+    // Some OpenSSL versions (for example 3.0.13, 3.3.3, 3.5.0) ignore empty salt,
+    // so without reset the salt from the previous call (key rotation) would be used
+    EVP_KDF_CTX_reset(kdf_ctx);
     OSSL_PARAM params[5];
     int n = 0;
     params[n++] = OSSL_PARAM_construct_utf8_string("digest", (char*)"sha384", (size_t)7);
@@ -405,14 +410,20 @@ bool msgr_handshake_t::verify(const uint8_t *signature, size_t signature_len)
     if (EVP_DigestVerifyUpdate(md_ctx.get(), full_handshake.data(), full_handshake.size()) <= 0)
         return on_error("EVP_DigestVerifyUpdate: ");
     if (EVP_DigestVerifyFinal(md_ctx.get(), signature, signature_len) <= 0)
+    {
+        // Clear errors of the invalid signature, otherwise the next on_error() would report them
+        ERR_clear_error();
+        error = "Handshake signature verification failed";
+        state = MSGR_HS_ERROR;
         return false;
+    }
     return true;
 }
 
 bool msgr_handshake_t::encrypt(const uint8_t* src, size_t len, uint8_t* dest)
 {
     std::unique_ptr<EVP_CIPHER_CTX, decltype(&EVP_CIPHER_CTX_free)> enc_ctx(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
-    if (!ctx)
+    if (!enc_ctx)
         return on_error("EVP_CIPHER_CTX_new: ");
     if (EVP_EncryptInit_ex(enc_ctx.get(), EVP_aes_256_gcm(), NULL, hs_key.data(), hs_key.data() + AES_256_GCM_KEY_SIZE) <= 0)
         return on_error("EVP_EncryptInit AES-256-GCM: ");
@@ -438,7 +449,7 @@ bool msgr_handshake_t::decrypt(const uint8_t *src, size_t & len, uint8_t* dest)
         return false;
     }
     std::unique_ptr<EVP_CIPHER_CTX, decltype(&EVP_CIPHER_CTX_free)> dec_ctx(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
-    if (!ctx)
+    if (!dec_ctx)
         return on_error("EVP_CIPHER_CTX_new: ");
     if (EVP_DecryptInit_ex(dec_ctx.get(), EVP_aes_256_gcm(), NULL, peer_hs_key.data(), peer_hs_key.data() + AES_256_GCM_KEY_SIZE) <= 0)
         return on_error("EVP_DecryptInit AES-256-GCM: ");

src/client/qemu_driver.c

@@ -1160,7 +1160,7 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
 }
 
 static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
-#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1 || defined BDRV_WANT_PRECISE
     unsigned int mode,
 #else
     bool want_zero,
@@ -1171,7 +1171,7 @@ static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
     // Not allocated => return 0
     // Error => return -errno
     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
-#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1 || defined BDRV_WANT_PRECISE
     int want_zero = (mode == BDRV_WANT_PRECISE);
 #endif
     VitastorRPC task;

src/client/vitastor_c.cpp

@@ -30,6 +30,18 @@ struct vitastor_c
 
     QEMUSetFDHandler *aio_set_fd_handler = NULL;
     void *aio_ctx = NULL;
+
+    ~vitastor_c()
+    {
+        if (ringloop)
+            delete ringloop;
+        if (epmgr)
+            delete epmgr;
+        else if (tfd)
+            delete tfd;
+        if (cli)
+            delete cli;
+    }
 };
 
 extern "C" {
@@ -103,7 +115,16 @@ vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *a
         rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
     );
     auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
-    self->cli = cluster_client_t::create(NULL, self->tfd, cfg_json);
+    try
+    {
+        self->cli = cluster_client_t::create(NULL, self->tfd, cfg_json);
+    }
+    catch (std::exception & e)
+    {
+        fprintf(stderr, "Failed to create Vitastor client: %s\n", e.what());
+        delete self;
+        return NULL;
+    }
     return self;
 }
 
@@ -111,47 +132,45 @@ vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, v
     const char *config_path, const char *etcd_host, const char *etcd_prefix,
     int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
 {
-    ring_loop_t *ringloop = NULL;
-    try
-    {
-        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
-    }
-    catch (std::exception & e)
-    {
-        return NULL;
-    }
     json11::Json cfg_json = vitastor_c_common_config(
         config_path, etcd_host, etcd_prefix, use_rdma,
         rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
     );
     auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
-    self->ringloop = ringloop;
-    self->cli = cluster_client_t::create(self->ringloop, self->tfd, cfg_json);
-    ringloop->loop();
+    try
+    {
+        self->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
+        self->cli = cluster_client_t::create(self->ringloop, self->tfd, cfg_json);
+    }
+    catch (std::exception & e)
+    {
+        fprintf(stderr, "Failed to create Vitastor client: %s\n", e.what());
+        delete self;
+        return NULL;
+    }
     return self;
 }
 
 vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix,
     int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
 {
-    ring_loop_t *ringloop = NULL;
-    try
-    {
-        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
-    }
-    catch (std::exception & e)
-    {
-        return NULL;
-    }
     json11::Json cfg_json = vitastor_c_common_config(
         config_path, etcd_host, etcd_prefix, use_rdma,
         rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
     );
     vitastor_c *self = new vitastor_c;
-    self->ringloop = ringloop;
-    self->epmgr = new epoll_manager_t(self->ringloop);
-    self->cli = cluster_client_t::create(self->ringloop, self->epmgr->tfd, cfg_json);
-    ringloop->loop();
+    try
+    {
+        self->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
+        self->epmgr = new epoll_manager_t(self->ringloop);
+        self->cli = cluster_client_t::create(self->ringloop, self->epmgr->tfd, cfg_json);
+    }
+    catch (std::exception & e)
+    {
+        fprintf(stderr, "Failed to create Vitastor client: %s\n", e.what());
+        delete self;
+        return NULL;
+    }
     return self;
 }
 
@@ -166,15 +185,6 @@ int vitastor_c_uring_register_eventfd(vitastor_c *client)
 
 vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
 {
-    ring_loop_t *ringloop = NULL;
-    try
-    {
-        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
-    }
-    catch (std::exception & e)
-    {
-        return NULL;
-    }
     json11::Json::object cfg;
     for (int i = 0; i < options_len-1; i += 2)
     {
@@ -189,10 +199,18 @@ vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
     }
     json11::Json cfg_json(cfg);
     vitastor_c *self = new vitastor_c;
-    self->ringloop = ringloop;
-    self->epmgr = new epoll_manager_t(self->ringloop);
-    self->cli = cluster_client_t::create(self->ringloop, self->epmgr->tfd, cfg_json);
-    ringloop->loop();
+    try
+    {
+        self->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
+        self->epmgr = new epoll_manager_t(self->ringloop);
+        self->cli = cluster_client_t::create(self->ringloop, self->epmgr->tfd, cfg_json);
+    }
+    catch (std::exception & e)
+    {
+        fprintf(stderr, "Failed to create Vitastor client: %s\n", e.what());
+        delete self;
+        return NULL;
+    }
     return self;
 }
 
@@ -205,8 +223,17 @@ vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len)
     }
     json11::Json cfg_json(cfg);
     vitastor_c *self = new vitastor_c;
-    self->epmgr = new epoll_manager_t(NULL);
-    self->cli = cluster_client_t::create(NULL, self->epmgr->tfd, cfg_json);
+    try
+    {
+        self->epmgr = new epoll_manager_t(NULL);
+        self->cli = cluster_client_t::create(NULL, self->epmgr->tfd, cfg_json);
+    }
+    catch (std::exception & e)
+    {
+        fprintf(stderr, "Failed to create Vitastor client: %s\n", e.what());
+        delete self;
+        return NULL;
+    }
     return self;
 }
 
@@ -217,13 +244,6 @@ void* vitastor_c_get_internal_client(vitastor_c *client)
 
 void vitastor_c_destroy(vitastor_c *client)
 {
-    delete client->cli;
-    if (client->epmgr)
-        delete client->epmgr;
-    else if (client->tfd)
-        delete client->tfd;
-    if (client->ringloop)
-        delete client->ringloop;
     delete client;
 }
 

src/test/CMakeLists.txt

@@ -88,3 +88,9 @@ add_test(NAME test_blockstore_chaos COMMAND test_blockstore_chaos)
 # test_atomic
 add_executable(test_atomic test_atomic.cpp)
 target_link_libraries(test_atomic util)
+
+# test_msgr_kdf
+add_executable(test_msgr_kdf EXCLUDE_FROM_ALL test_msgr_kdf.cpp)
+target_link_libraries(test_msgr_kdf vitastor_common)
+add_dependencies(build_tests test_msgr_kdf)
+add_test(NAME test_msgr_kdf COMMAND test_msgr_kdf)

src/test/test_cluster_client.cpp

@@ -177,7 +177,7 @@ void pretend_disconnected(cluster_client_t *cli, osd_num_t osd_num)
 }
 
 #define check_disconnected(cli, osd_num) {\
-    if (cli->msgr.osd_peers.find(osd_num) != cli->msgr.osd_peers.end())\
+    if (cli->msgr->osd_peers.find(osd_num) != cli->msgr->osd_peers.end())\
     {\
         printf("OSD %ju not disconnected as it ought to be\n", (uint64_t)osd_num);\
         assert(0);\
@@ -185,7 +185,7 @@ void pretend_disconnected(cluster_client_t *cli, osd_num_t osd_num)
 }
 
 #define check_op_count(cli, osd_num, ops) {\
-    osd_client_t *cl = cli->msgr.osd_peers.at(osd_num);\
+    osd_client_t *cl = cli->msgr->osd_peers.at(osd_num);\
     int real_ops = cl->sent_ops.size();\
     if (real_ops != ops)\
     {\
@@ -196,7 +196,7 @@ void pretend_disconnected(cluster_client_t *cli, osd_num_t osd_num)
 
 osd_op_t *find_op(cluster_client_t *cli, osd_num_t osd_num, uint64_t opcode, uint64_t offset, uint64_t len)
 {
-    osd_client_t *cl = cli->msgr.osd_peers.at(osd_num);
+    osd_client_t *cl = cli->msgr->osd_peers.at(osd_num);
     auto op_it = cl->sent_ops.begin();
     while (op_it != cl->sent_ops.end())
     {
@@ -225,7 +225,7 @@ void pretend_op_completed(cluster_client_t *cli, osd_op_t *op, int64_t retval)
         ? "sync" : (op->req.hdr.opcode == OSD_OP_WRITE ? "write" : "read"), op->req.rw.offset, op->req.rw.len);
     uint64_t op_id = op->req.hdr.id;
     uint64_t client_id = op->client_id;
-    cli->msgr.clients[client_id]->sent_ops.erase(op_id);
+    cli->msgr->clients[client_id]->sent_ops.erase(op_id);
     op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
     op->reply.hdr.id = op->req.hdr.id;
     op->reply.hdr.opcode = op->req.hdr.opcode;
@@ -308,7 +308,7 @@ void test1()
         uint64_t replay_start = UINT64_MAX;
         uint64_t replay_end = 0;
         std::vector<osd_op_t*> replay_ops;
-        auto osd_cl = cli->msgr.osd_peers.at(1);
+        auto osd_cl = cli->msgr->osd_peers.at(1);
         for (auto & op_p: osd_cl->sent_ops)
         {
             auto op = op_p.second;
@@ -739,7 +739,7 @@ void test_deoptimize_snapshot_read()
         };
         cli->execute(op);
         check_op_count(cli, 1, 1);
-        auto cl = cli->msgr.osd_peers.at(1);
+        auto cl = cli->msgr->osd_peers.at(1);
         osd_op_t *read_op = cl->sent_ops.begin()->second;
         assert(read_op->req.hdr.opcode == OSD_OP_READ);
         assert(read_op->req.rw.inode == child_inode);
@@ -781,7 +781,7 @@ void test_deoptimize_snapshot_read()
         };
         cli->execute(op);
         check_op_count(cli, 1, 1);
-        auto cl = cli->msgr.osd_peers.at(1);
+        auto cl = cli->msgr->osd_peers.at(1);
         osd_op_t *read_op = cl->sent_ops.begin()->second;
         assert(read_op->req.hdr.opcode == OSD_OP_READ);
         assert(read_op->req.rw.inode == child_inode);
@@ -815,7 +815,7 @@ void test_deoptimize_snapshot_read()
         };
         cli->execute(op);
         check_op_count(cli, 1, 1);
-        auto cl = cli->msgr.osd_peers.at(1);
+        auto cl = cli->msgr->osd_peers.at(1);
         osd_op_t *child_read = cl->sent_ops.begin()->second;
         assert(child_read->req.hdr.opcode == OSD_OP_READ);
         assert(child_read->req.rw.inode == child_inode);

src/test/test_msgr_kdf.cpp

@@ -0,0 +1,122 @@
+// Copyright (c) Alexey Shabalin, 2026+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+// Known-answer test for msgr_handshake_ctx_i::derive_kdf (HKDF-SHA384).
+// Keys must be the same with any OpenSSL version, and the salt used for key rotation
+// must not affect the next derive_kdf call without salt.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include <string>
+#include <vector>
+
+#include "msgr_handshake.h"
+
+// Self-signed EC P-384 certificate, only used to initialize the handshake context
+static const char *test_key =
+    "-----BEGIN PRIVATE KEY-----\n"
+    "MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDA0CIkBTlNDaxTDbpc3\n"
+    "2JBY4ofK7PfLDw4D2jEEQ/kTQRAXOobqFCjSqefm2Bbmz9yhZANiAAQZEaeZPB3z\n"
+    "60xhhxxGYn2/qUgRXcoyPkGBzBXT54esK/qcuumFo1/uoB2+PTNT9lRqYO9s4xhC\n"
+    "naDV653Ynp6J8O0AnyhzqDaxHIjG84yWlslTWZMyDvqAtp1ZsIPprSk=\n"
+    "-----END PRIVATE KEY-----\n";
+
+static const char *test_cert =
+    "-----BEGIN CERTIFICATE-----\n"
+    "MIIBwjCCAUegAwIBAgIUMTD1mUZ7KJZYC2m28V+S3Y3M2wkwCgYIKoZIzj0EAwIw\n"
+    "GDEWMBQGA1UEAwwNdml0YXN0b3ItdGVzdDAgFw0yNjA5MzAxODM3NDJaGA8yMTI2\n"
+    "MDkwNjE4Mzc0MlowGDEWMBQGA1UEAwwNdml0YXN0b3ItdGVzdDB2MBAGByqGSM49\n"
+    "AgEGBSuBBAAiA2IABBkRp5k8HfPrTGGHHEZifb+pSBFdyjI+QYHMFdPnh6wr+py6\n"
+    "6YWjX+6gHb49M1P2VGpg72zjGEKdoNXrndienonw7QCfKHOoNrEciMbzjJaWyVNZ\n"
+    "kzIO+oC2nVmwg+mtKaNQME4wHQYDVR0OBBYEFLkMxzaAAbRTajF+q6HbCWjJtDkP\n"
+    "MB8GA1UdIwQYMBaAFLkMxzaAAbRTajF+q6HbCWjJtDkPMAwGA1UdEwQFMAMBAf8w\n"
+    "CgYIKoZIzj0EAwIDaQAwZgIxAJ0uH7Y2LmXKnRHSgYLqmpeKFqsvmbph3oWEOErX\n"
+    "+NJK56FzMceivmjvMkt4WMa69wIxAO14TLCFZeESLltGwIzdM0hv/Iysiec88ip+\n"
+    "OlW2PuLyhw8f/4HHp7Ooyu0ki3AHEA==\n"
+    "-----END CERTIFICATE-----\n";
+
+static const char *hs_key_hex =
+    "f5ed89f2161b25f4ac4a097179c12008804f2c8cfe757a8d810e166720dc6f67"
+    "86f2852a19697bd869551b10";
+
+static const char *key_hex =
+    "2b4047bb889aa0e63c5ca73e2edfc8b29dca0887bed9058a390c341bfb8a677c"
+    "d632115d5342470a48c76c6094a1e27945dffead049d308c53e3bbaf9a4e71f1"
+    "5758e3b3414416f2c9f0ea59235f2a022ad836ff8d8b91ca926f5d5c07ac5dc8"
+    "57f3ed36b1cd085b26c2471aa55c314804efeb9da022fd7a25c035cfdb773eb2"
+    "75f10d6f2f1551bc53dca1d34b44ba602c3b2e975ddd068732fa5d40ac0302f9"
+    "408f357361dd8ba6de9cfb42114663ae3bec06b94fce87227283e28464f27468"
+    "1677d78682b142e17fcdfaf6ac24eca4c31b11f3fde6186d5e1850fda7ec6747"
+    "7fb2d1e6b1d3998b38373904";
+
+static const char *rotated_key_hex =
+    "33b3f43c75660d051ef7465b4e9f61e74c3715af5bb3730a0093931874b0c2f2"
+    "73363984f3ae5e7cc6023200bfd3bdff0ee55d6ee10feb78396b0f74663c59f6"
+    "394a861c9f1f86b29b5d5fd491e196f05f18576592c86c7c0713ec9fad9e8053"
+    "a9113ba72ad56b725e57bf887891abb56fcac346b8f5eb30cc58c255b904ee88"
+    "2e7db934efadd32a76e235be842c4a647bcfe4d96a1d11b68d793d7356261bc0"
+    "0b10f345a456674855cc6621440a94201dfef0ee724d0913c24f74c3e4f21faf"
+    "a263f16f512a84103aeb50507466a2df51c2f35f9c49a27c6697d031a80dcd27"
+    "78959c188de53c8a4d757260";
+
+static void fill(uint8_t *buf, size_t len, uint8_t start)
+{
+    for (size_t i = 0; i < len; i++)
+        buf[i] = (uint8_t)(start + i);
+}
+
+static std::string to_hex(const uint8_t *buf, size_t len)
+{
+    std::string r;
+    char tmp[3];
+    for (size_t i = 0; i < len; i++)
+    {
+        snprintf(tmp, sizeof(tmp), "%02x", buf[i]);
+        r += tmp;
+    }
+    return r;
+}
+
+static void check(msgr_handshake_ctx_i *ctx, const char *name, const uint8_t *secret, size_t secret_len,
+    const uint8_t *salt, size_t salt_len, const char *label, size_t out_len, const char *expected)
+{
+    std::vector<uint8_t> out(out_len);
+    if (!ctx->derive_kdf(secret, secret_len, salt, salt_len, label, out.data(), out_len))
+    {
+        printf("%s: derive_kdf failed\n", name);
+        exit(1);
+    }
+    std::string actual = to_hex(out.data(), out_len);
+    if (actual != expected)
+    {
+        printf("%s: key mismatch\nexpected: %s\nactual:   %s\n", name, expected, actual.c_str());
+        exit(1);
+    }
+}
+
+int main(int narg, char *args[])
+{
+    // Same sizes as in msgr_handshake_t::derive_hs_keys() and osd_messenger_t::derive_aes_keys()
+    const size_t hs_key_len = AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE;
+    const size_t key_len = AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE + 192 /* XXH_SECRET_DEFAULT_SIZE */;
+    uint8_t secret[48], old_key[key_len];
+    fill(secret, sizeof(secret), 0x00);
+    fill(old_key, sizeof(old_key), 0x80);
+    msgr_handshake_ctx_i *ctx = msgr_handshake_ctx_i::create_ctx();
+    if (!ctx->init(test_cert, test_key, test_cert, "", ""))
+    {
+        printf("handshake context init failed: %s\n", ctx->get_error().c_str());
+        exit(1);
+    }
+    check(ctx, "handshake key", secret, sizeof(secret), NULL, 0, "server hs key", hs_key_len, hs_key_hex);
+    check(ctx, "initial key", secret, sizeof(secret), NULL, 0, "client key", key_len, key_hex);
+    // Key rotation uses the old key as salt
+    check(ctx, "rotated key", secret, sizeof(secret), old_key, sizeof(old_key), "client key", key_len, rotated_key_hex);
+    // Salt of the previous call must not affect the next call without salt
+    check(ctx, "handshake key after rotation", secret, sizeof(secret), NULL, 0, "server hs key", hs_key_len, hs_key_hex);
+    delete ctx;
+    printf("OK\n");
+    return 0;
+}