Add an alternative RDMA implementation via RDMA-CM

Required for non-RoCE cards: iWARP and, possibly, Infiniband

src/client/messenger.cpp

@@ -149,6 +149,11 @@ void osd_messenger_t::init()
         );
         if (!rdma_contexts.size())
         {
+            if (force_rdma)
+            {
+                fprintf(stderr, "[OSD %ju] Couldn't initialize RDMA, force_rdma is enabled, exiting\n", osd_num);
+                exit(1);
+            }
             if (log_level > 0)
                 fprintf(stderr, "[OSD %ju] Couldn't initialize RDMA, proceeding with TCP only\n", osd_num);
         }
@@ -301,6 +306,10 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->use_rdmacm = config["use_rdmacm"].bool_value() || config["use_rdmacm"].uint64_value() != 0;
     this->disable_tcp = this->use_rdmacm && (config["disable_tcp"].bool_value() || config["disable_tcp"].uint64_value() != 0);
 #endif
+    if (!config["force_rdma"].is_null())
+    {
+        this->force_rdma = config["force_rdma"].bool_value() || config["force_rdma"].uint64_value() != 0;
+    }
     this->rdma_device = config["rdma_device"].string_value();
     this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
     if (!config["rdma_gid_index"].is_null())

src/client/messenger.h

@@ -179,6 +179,7 @@ protected:
     bool use_rdma = true;
     bool use_rdmacm = false;
     bool disable_tcp = false;
+    bool force_rdma = false;
     std::string rdma_device;
     uint64_t rdma_port_num = 1;
     int rdma_mtu = 0;

src/client/msgr_rdma.cpp

@@ -328,11 +328,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
     ctx->gid_index = gid_index;
-    if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
-    {
-        fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(dev), gid_index);
-        goto cleanup;
-    }
 
     ctx->pd = ibv_alloc_pd(ctx->context);
     if (!ctx->pd)
@@ -341,15 +336,14 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
 
-    if (ibv_query_device_ex(ctx->context, NULL, &ctx->attrx))
-    {
-        fprintf(stderr, "Couldn't query RDMA device for its features\n");
-        goto cleanup;
-    }
-
     ctx->odp = odp;
     if (ctx->odp)
     {
+        if (ibv_query_device_ex(ctx->context, NULL, &ctx->attrx))
+        {
+            fprintf(stderr, "Couldn't query RDMA device for its features\n");
+            goto cleanup;
+        }
         if (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
             !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
             !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
@@ -578,7 +572,7 @@ static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
     int err = ibv_post_send(cl->rdma_conn->qp, &wr, &bad_wr);
     if (err || bad_wr)
     {
-        fprintf(stderr, "RDMA send failed: %s (code %d)\n", strerror(err), err);
+        fprintf(stderr, "RDMA send failed: %s\n", strerror(err));
         exit(1);
     }
     cl->rdma_conn->cur_send++;