server/
server.rs

1//! This demo shows how to establish a connection between server and client
2//! and send msg to the other end.
3//!
4//! You can try this example by running:
5//!
6//!     cargo run --example server
7//!
8//! And then start client in another terminal by running:
9//!
10//!     cargo run --example client <server_ip> <port>
11//!
12//! The default port is 7471.
13
14use rdma_sys::*;
15use std::ptr::null_mut;
16
17static SERVER: &str = "0.0.0.0\0";
18static PORT: &str = "7471\0";
19
20fn run() -> i32 {
21    let mut send_msg = vec![1_u8; 16];
22    let mut recv_msg = vec![0_u8; 16];
23    let mut hints = unsafe { std::mem::zeroed::<rdma_addrinfo>() };
24    let mut res: *mut rdma_addrinfo = null_mut();
25    hints.ai_flags = RAI_PASSIVE.try_into().unwrap();
26    hints.ai_port_space = rdma_port_space::RDMA_PS_TCP.try_into().unwrap();
27    let mut ret = unsafe {
28        rdma_getaddrinfo(
29            SERVER.as_ptr().cast(),
30            PORT.as_ptr().cast(),
31            &hints,
32            &mut res,
33        )
34    };
35
36    if ret != 0 {
37        println!("rdma_getaddrinfo");
38        return ret;
39    }
40
41    let mut listen_id = null_mut();
42    let mut id = null_mut();
43
44    let mut init_attr = unsafe { std::mem::zeroed::<ibv_qp_init_attr>() };
45    init_attr.cap.max_send_wr = 1;
46    init_attr.cap.max_recv_wr = 1;
47    init_attr.cap.max_send_sge = 1;
48    init_attr.cap.max_recv_sge = 1;
49    init_attr.cap.max_inline_data = 16;
50    init_attr.sq_sig_all = 1;
51    ret = unsafe { rdma_create_ep(&mut listen_id, res, null_mut(), &mut init_attr) };
52    // Check to see if we got inline data allowed or not
53    if ret != 0 {
54        println!("rdma_create_ep");
55        unsafe {
56            rdma_freeaddrinfo(res);
57        }
58        return ret;
59    }
60    ret = unsafe { rdma_listen(listen_id, 0) };
61    if ret != 0 {
62        println!("rdma_listen");
63        unsafe {
64            rdma_destroy_ep(listen_id);
65        }
66        return ret;
67    }
68
69    ret = unsafe { rdma_get_request(listen_id, &mut id) };
70    if ret != 0 {
71        println!("rdma_get_request");
72        unsafe {
73            rdma_destroy_ep(listen_id);
74        }
75        return ret;
76    }
77
78    let mut qp_attr = unsafe { std::mem::zeroed::<ibv_qp_attr>() };
79    ret = unsafe {
80        ibv_query_qp(
81            (*id).qp,
82            &mut qp_attr,
83            ibv_qp_attr_mask::IBV_QP_CAP.0.try_into().unwrap(),
84            &mut init_attr,
85        )
86    };
87
88    if ret != 0 {
89        println!("ibv_query_qp");
90        unsafe {
91            rdma_destroy_ep(id);
92        }
93        return ret;
94    }
95
96    let mut send_flags = 0_u32;
97    if init_attr.cap.max_inline_data >= 16 {
98        send_flags = ibv_send_flags::IBV_SEND_INLINE.0;
99    } else {
100        println!("rdma_server: device doesn't support IBV_SEND_INLINE, using sge sends");
101    }
102
103    let recv_mr = unsafe { rdma_reg_msgs(id, recv_msg.as_mut_ptr().cast(), 16) };
104    if recv_mr.is_null() {
105        ret = -1;
106        println!("rdma_reg_msgs for recv_msg");
107        unsafe {
108            rdma_dereg_mr(recv_mr);
109        }
110        return ret;
111    }
112
113    let mut send_mr = null_mut();
114    if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) == 0 {
115        send_mr = unsafe { rdma_reg_msgs(id, send_msg.as_mut_ptr().cast(), 16) };
116        if send_mr.is_null() {
117            ret = -1;
118            println!("rdma_reg_msgs for send_msg");
119            unsafe {
120                rdma_dereg_mr(recv_mr);
121            }
122            return ret;
123        }
124    }
125    ret = unsafe { rdma_post_recv(id, null_mut(), recv_msg.as_mut_ptr().cast(), 16, recv_mr) };
126
127    if ret != 0 {
128        println!("rdma_post_recv");
129        unsafe {
130            rdma_dereg_mr(recv_mr);
131        }
132        return ret;
133    }
134
135    ret = unsafe { rdma_accept(id, null_mut()) };
136    if ret != 0 {
137        println!("rdma_accept");
138        if (send_flags & ibv_send_flags::IBV_SEND_INLINE.0) == 0 {
139            unsafe { rdma_dereg_mr(send_mr) };
140        }
141        return ret;
142    }
143
144    let mut wc = unsafe { std::mem::zeroed::<ibv_wc>() };
145    while ret == 0 {
146        ret = unsafe { rdma_get_recv_comp(id, &mut wc) };
147    }
148    if ret < 0 {
149        println!("rdma_get_recv_comp");
150        unsafe {
151            rdma_disconnect(id);
152        }
153        return ret;
154    }
155    println!("rdma_server: recv msg : {:?}", recv_msg);
156    ret = unsafe {
157        rdma_post_send(
158            id,
159            null_mut(),
160            send_msg.as_mut_ptr().cast(),
161            16,
162            send_mr,
163            send_flags.try_into().unwrap(),
164        )
165    };
166    if ret != 0 {
167        println!("rdma_post_send");
168        unsafe {
169            rdma_disconnect(id);
170        }
171        return ret;
172    }
173
174    while ret == 0 {
175        ret = unsafe { rdma_get_send_comp(id, &mut wc) };
176    }
177    if ret < 0 {
178        println!("rdma_get_send_comp");
179    } else {
180        ret = 0;
181    }
182    ret
183}
184
185fn main() {
186    println!("rdma_server: start");
187    let ret = run();
188    if ret != 0 {
189        println!(
190            "rdma_server: ret error {:?}",
191            std::io::Error::from_raw_os_error(-ret)
192        );
193        if ret == -1 {
194            println!(
195                "rdma_server: last os error {:?}",
196                std::io::Error::last_os_error()
197            );
198        }
199    }
200    println!("rdma_server: end");
201}
server/server.rs

server/
server.rs
