test: add a test to enable and test rdma connect

Signed-off-by: Diwakar Sharma <[email protected]>

io-engine-tests/src/compose/rpc/v0.rs

@@ -7,7 +7,7 @@ use mayastor::{
 };
 
 use std::{
-    net::{SocketAddr, TcpStream},
+    net::{Ipv4Addr, SocketAddr, TcpStream},
     thread,
     time::Duration,
 };
@@ -86,7 +86,7 @@ impl<'a> GrpcConnect<'a> {
             handles.push(
                 RpcHandle::connect(
                     v.0.clone(),
-                    format!("{}:10124", v.1 .1)
+                    format!("{}:10124", v.1 .1.unwrap_or(Ipv4Addr::new(127, 0, 0, 1)))
                         .parse::<std::net::SocketAddr>()
                         .unwrap(),
                 )
@@ -103,7 +103,7 @@ impl<'a> GrpcConnect<'a> {
         match self.ct.containers().iter().find(|&c| c.0 == name) {
             Some(container) => Ok(RpcHandle::connect(
                 container.0.clone(),
-                format!("{}:10124", container.1 .1)
+                format!("{}:10124", container.1 .1.unwrap_or(Ipv4Addr::new(127, 0, 0, 1)))
                     .parse::<std::net::SocketAddr>()
                     .unwrap(),
             )

io-engine-tests/src/compose/rpc/v1.rs

@@ -1,7 +1,7 @@
 use composer::ComposeTest;
 
 use std::{
-    net::{SocketAddr, TcpStream},
+    net::{Ipv4Addr, SocketAddr, TcpStream},
     sync::Arc,
     thread,
     time::Duration,
@@ -151,7 +151,7 @@ impl<'a> GrpcConnect<'a> {
             handles.push(
                 RpcHandle::connect(
                     v.0.clone(),
-                    format!("{}:10124", v.1 .1)
+                    format!("{}:10124", v.1 .1.unwrap_or(Ipv4Addr::new(127, 0, 0, 1)))
                         .parse::<std::net::SocketAddr>()
                         .unwrap(),
                 )
@@ -167,7 +167,7 @@ impl<'a> GrpcConnect<'a> {
         match self.ct.containers().iter().find(|&c| c.0 == name) {
             Some(container) => Ok(RpcHandle::connect(
                 container.0.clone(),
-                format!("{}:10124", container.1 .1)
+                format!("{}:10124", container.1 .1.unwrap_or(Ipv4Addr::new(127, 0, 0, 1)))
                     .parse::<std::net::SocketAddr>()
                     .unwrap(),
             )

io-engine-tests/src/lib.rs

@@ -8,6 +8,7 @@ use std::{io, io::Write, process::Command, time::Duration};
 
 use crossbeam::channel::{after, select, unbounded};
 use once_cell::sync::OnceCell;
+use regex::Regex;
 use run_script::{self, ScriptOptions};
 use url::{ParseError, Url};
 
@@ -36,6 +37,8 @@ pub mod snapshot;
 pub mod test;
 pub mod test_task;
 
+use cli_tools::run_command_args;
+
 pub use compose::MayastorTest;
 
 /// call F cnt times, and sleep for a duration between each invocation
@@ -550,6 +553,84 @@ pub fn reactor_run_millis(milliseconds: u64) {
     reactor_poll!(r);
 }
 
+// We try to figure out the net interface that is being used to connect
+// to internet(public IP). Then use that interface name to setup rdma rxe device.
+//
+// # host we want to "reach"
+// host=google.com
+//
+// # get the ip of that host (works with dns and /etc/hosts. In case we get
+// # multiple IP addresses, we just want one of them
+// host_ip=$(getent ahosts "$host" | awk '{print $1; exit}')
+//
+// # only list the interface used to reach a specific host/IP. We only want the part
+// # between dev and the remainder (use grep for that)
+// ip route get "$host_ip" | grep -Po '(?<=(dev ))(\S+)'
+pub fn setup_rdma_rxe_device() -> String {
+        let test_host = "google.com";
+        let ns_entries = run_command_args(
+            "getent",
+            vec!["ahosts", test_host],
+            Some("get ip of test host"),
+        )
+        .unwrap();
+
+        let test_host_ip = ns_entries
+            .1
+            .first()
+            .unwrap()
+            .to_string_lossy()
+            .split_whitespace()
+            .next()
+            .unwrap()
+            .to_owned();
+
+        let ent = &run_command_args(
+            "ip",
+            vec!["route", "get", test_host_ip.as_str()],
+            Some("get routing entry"),
+        )
+        .unwrap()
+        .1[0];
+
+        // match/grep the interface name
+        let pattern = r"dev (\S+)";
+        let re = Regex::new(pattern).unwrap();
+
+        let iface = re
+            .captures(ent.to_str().unwrap())
+            .unwrap()
+            .get(1)
+            .expect("interface not found")
+            .as_str();
+        println!("Using interface {} to create rdma rxe device", iface);
+
+        // now try to install rdma_rxe module and create rxe device.
+        // todo: More dynamic checks for module may be needed depending on platform.
+        // Assume linux-modules-extra-$(uname -r) is installed.
+        let _ = run_command_args(
+            "modprobe",
+            vec!["rdma_rxe"],
+            Some("install rdma_rxe kernel module"),
+        );
+        let _ = run_command_args(
+            "rdma",
+            vec!["link", "add", "rxe0", "type", "rxe", "netdev", iface],
+            Some("Create rxe device"),
+        );
+
+        iface.to_string()
+    }
+
+    pub fn delete_rdma_rxe_device() {
+        let _ = run_command_args(
+            "rdma",
+            vec!["link", "delete", "rxe0"],
+            Some("Delete rxe device"),
+        )
+        .expect("rxe device delete");
+    }    
+
 pub fn composer_init() {
     std::fs::create_dir_all("/var/run/dpdk").ok();
     let path = std::path::PathBuf::from(std::env!("CARGO_MANIFEST_DIR"));

io-engine-tests/src/nvme.rs

@@ -15,7 +15,7 @@ pub struct NmveConnectGuard {
 
 impl NmveConnectGuard {
     pub fn connect(target_addr: &str, nqn: &str) -> Self {
-        nvme_connect(target_addr, nqn, true);
+        nvme_connect(target_addr, nqn, "tcp", true);
 
         Self {
             nqn: nqn.to_string(),
@@ -85,11 +85,12 @@ pub fn nvme_discover(target_addr: &str) -> Vec<BTreeMap<String, String>> {
 pub fn nvme_connect(
     target_addr: &str,
     nqn: &str,
+    transport: &str,
     must_succeed: bool,
 ) -> ExitStatus {
     let status = Command::new("nvme")
         .args(["connect"])
-        .args(["-t", "tcp"])
+        .args(["-t", transport])
         .args(["-a", target_addr])
         .args(["-s", "8420"])
         .args(["-c", "1"])

io-engine/tests/nexus_io.rs

@@ -200,12 +200,12 @@ async fn nexus_io_multipath() {
         .unwrap();
 
     let nqn = format!("{HOSTNQN}:nexus-{NEXUS_UUID}");
-    nvme_connect("127.0.0.1", &nqn, true);
+    nvme_connect("127.0.0.1", &nqn, "tcp", true);
 
     // The first attempt will fail with "Duplicate cntlid x with y" error from
     // kernel
     for i in 0 .. 2 {
-        let status_c0 = nvme_connect(&ip0.to_string(), &nqn, false);
+        let status_c0 = nvme_connect(&ip0.to_string(), &nqn, "tcp", false);
         if i == 0 && status_c0.success() {
             break;
         }
@@ -270,7 +270,7 @@ async fn nexus_io_multipath() {
 
     // Connect to remote replica to check key registered
     let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
     let rep_dev = get_mayastor_nvme_device();
 
@@ -404,7 +404,7 @@ async fn nexus_io_resv_acquire() {
 
     // Connect to remote replica to check key registered
     let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
     let rep_dev = get_mayastor_nvme_device();
 
@@ -601,7 +601,7 @@ async fn nexus_io_resv_preempt() {
     // Connect to remote replica to check key registered
     let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
 
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
     let rep_dev = get_mayastor_nvme_device();
 
@@ -748,7 +748,7 @@ async fn nexus_io_resv_preempt() {
         .await
         .unwrap();
 
-    nvme_connect(&ip0.to_string(), &rep_nqn, true);
+    nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
     let rep_dev = get_mayastor_nvme_device();
 
     // After restart the reservations should still be in place!
@@ -899,7 +899,7 @@ async fn nexus_io_resv_preempt_tabled() {
         // Connect to remote replica to check key registered
         let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}");
 
-        nvme_connect(&ip0.to_string(), &rep_nqn, true);
+        nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true);
 
         let rep_dev = get_mayastor_nvme_device();