Add an alternative RDMA implementation via RDMA-CM

Required for non-RoCE cards: iWARP and, possibly, Infiniband

src/client/messenger.cpp

@@ -149,11 +149,6 @@ void osd_messenger_t::init()
         );
         if (!rdma_contexts.size())
         {
-            if (force_rdma)
-            {
-                fprintf(stderr, "[OSD %ju] Couldn't initialize RDMA, force_rdma is enabled, exiting\n", osd_num);
-                exit(1);
-            }
             if (log_level > 0)
                 fprintf(stderr, "[OSD %ju] Couldn't initialize RDMA, proceeding with TCP only\n", osd_num);
         }
@@ -306,10 +301,6 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->use_rdmacm = config["use_rdmacm"].bool_value() || config["use_rdmacm"].uint64_value() != 0;
     this->disable_tcp = this->use_rdmacm && (config["disable_tcp"].bool_value() || config["disable_tcp"].uint64_value() != 0);
 #endif
-    if (!config["force_rdma"].is_null())
-    {
-        this->force_rdma = config["force_rdma"].bool_value() || config["force_rdma"].uint64_value() != 0;
-    }
     this->rdma_device = config["rdma_device"].string_value();
     this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
     if (!config["rdma_gid_index"].is_null())

src/client/messenger.h

@@ -179,7 +179,6 @@ protected:
     bool use_rdma = true;
     bool use_rdmacm = false;
     bool disable_tcp = false;
-    bool force_rdma = false;
     std::string rdma_device;
     uint64_t rdma_port_num = 1;
     int rdma_mtu = 0;

src/client/msgr_rdma.cpp

@@ -328,6 +328,11 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
     ctx->gid_index = gid_index;
+    if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+    {
+        fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(dev), gid_index);
+        goto cleanup;
+    }
 
     ctx->pd = ibv_alloc_pd(ctx->context);
     if (!ctx->pd)
@@ -336,14 +341,15 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
 
+    if (ibv_query_device_ex(ctx->context, NULL, &ctx->attrx))
+    {
+        fprintf(stderr, "Couldn't query RDMA device for its features\n");
+        goto cleanup;
+    }
+
     ctx->odp = odp;
     if (ctx->odp)
     {
-        if (ibv_query_device_ex(ctx->context, NULL, &ctx->attrx))
-        {
-            fprintf(stderr, "Couldn't query RDMA device for its features\n");
-            goto cleanup;
-        }
         if (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
             !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
             !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
@@ -572,7 +578,7 @@ static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
     int err = ibv_post_send(cl->rdma_conn->qp, &wr, &bad_wr);
     if (err || bad_wr)
     {
-        fprintf(stderr, "RDMA send failed: %s\n", strerror(err));
+        fprintf(stderr, "RDMA send failed: %s (code %d)\n", strerror(err), err);
         exit(1);
     }
     cl->rdma_conn->cur_send++;