Add an alternative RDMA implementation via RDMA-CM

Required for non-RoCE cards: iWARP and, possibly, Infiniband

src/client/messenger.cpp

@@ -344,9 +344,7 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->log_level = config["log_level"].uint64_value();
     // OSD public & cluster networks
     this->osd_networks.clear();
-    if (config["bind_address"].is_string())
-        this->osd_networks.push_back(config["bind_address"].string_value());
-    else if (config["osd_network"].is_string())
+    if (config["osd_network"].is_string())
         this->osd_networks.push_back(config["osd_network"].string_value());
     else
         for (auto v: config["osd_network"].array_items())
@@ -386,33 +384,35 @@ void osd_messenger_t::connect_peer(uint64_t peer_osd, json11::Json peer_state)
     if (wanted_peers[peer_osd].raw_address_list != peer_state["addresses"])
     {
         wanted_peers[peer_osd].raw_address_list = peer_state["addresses"];
-        if (osd_cluster_network_masks.size())
+        // We are an OSD -> try to select a cluster address
+        // We are a client -> try to select a public address
+        // OSD only has 1 address -> don't try anything, it's pointless
+        // FIXME: Maybe support optional fallback from cluster to public network?
+        auto & match_masks = (this->osd_num ? osd_cluster_network_masks : osd_network_masks);
+        if (peer_state["addresses"].array_items().size() > 1 && match_masks.size())
         {
-            json11::Json::array address_list, cluster_address_list;
+            json11::Json::array address_list;
             for (auto json_addr: peer_state["addresses"].array_items())
             {
                 struct sockaddr_storage addr;
                 auto ok = string_to_addr(json_addr.string_value(), false, 0, &addr);
                 if (ok)
                 {
-                    bool is_cluster = false;
-                    for (auto & mask: osd_cluster_network_masks)
+                    bool matches = false;
+                    for (auto & mask: match_masks)
                     {
                         if (cidr_sockaddr_match(addr, mask))
                         {
-                            is_cluster = true;
+                            matches = true;
                             break;
                         }
                     }
-                    if (is_cluster)
-                        cluster_address_list.push_back(json_addr);
-                    else
+                    if (matches)
                         address_list.push_back(json_addr);
                 }
             }
-            // FIXME: Maybe support optional fallback from cluster to public network?
-            if (this->osd_num && cluster_address_list.size() > 0)
-                address_list = cluster_address_list;
+            if (!address_list.size())
+                address_list = peer_state["addresses"].array_items();
             wanted_peers[peer_osd].address_list = address_list;
         }
         else

src/osd/osd.cpp

@@ -169,6 +169,7 @@ void osd_t::parse_config(bool init)
         else
             immediate_commit = IMMEDIATE_NONE;
         // Bind address
+        bind_address = config["bind_address"].string_value();
         bind_port = config["bind_port"].uint64_value();
         if (bind_port <= 0 || bind_port > 65535)
             bind_port = 0;
@@ -331,7 +332,11 @@ void osd_t::parse_config(bool init)
 
 void osd_t::bind_socket()
 {
-    if (msgr.all_osd_network_masks.size())
+    if (bind_address != "")
+    {
+        bind_addresses.push_back(bind_address);
+    }
+    else if (msgr.all_osd_network_masks.size())
     {
         bind_addresses = getifaddr_list(msgr.all_osd_network_masks);
         if (!bind_addresses.size())

src/osd/osd.h

@@ -107,6 +107,7 @@ class osd_t
     bool no_recovery = false;
     bool no_scrub = false;
     bool allow_net_split = false;
+    std::string bind_address;
     int bind_port, listen_backlog = 128;
     bool use_rdmacm = false;
     bool disable_tcp = false;