rrddmma 0.7.3

A Rust library for RDMA.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

#[cfg(mlnx5)]
fn main() {
    eprintln!("DC is not yet implemented on MLNX v5.x");
}

#[cfg(mlnx4)]
fn main() -> anyhow::Result<()> {
    use rrddmma::{prelude::*, wrap::RegisteredMem};
    use std::{thread, time::Duration};

    fn client(ep: QpEndpoint) -> anyhow::Result<()> {
        fn make_dci(dev: &str) -> anyhow::Result<Qp> {
            let Nic { context, ports } = Nic::finder().dev_name(dev).probe()?;
            let pd = Pd::new(&context)?;
            let cq = Cq::new(&context, Cq::DEFAULT_CQ_DEPTH)?;
            let mut qp = Qp::builder()
                .qp_type(QpType::DcIni)
                .caps(QpCaps::for_dc_ini())
                .send_cq(&cq)
                .recv_cq(&cq)
                .sq_sig_all(true)
                .build(&pd)?;
            qp.bind_local_port(&ports[0], None)?;
            Ok(qp)
        }

        let qp = make_dci("mlx5_0").inspect_err(|e| eprintln!("Err: {:?}", e))?;
        let peer = qp.make_peer(ep)?;

        // Send the message to the server.
        let mem = RegisteredMem::new_with_content(qp.pd(), "Hello, rrddmma!".as_bytes())?;
        for _ in 0..100000 {
            qp.send(&[mem.as_slice()], Some(&peer), None, 0, true, true)?;
            qp.scq().poll_one_blocking()?;
        }
        Ok(())
    }

    fn make_dct(dev: &str) -> anyhow::Result<Dct> {
        let Nic { context, ports } = Nic::finder().dev_name(dev).probe()?;
        let pd = Pd::new(&context)?;
        let cq = Cq::new(&context, Cq::DEFAULT_CQ_DEPTH)?;
        let dct = Dct::builder()
            .pd(&pd)
            .cq(&cq)
            .port(&ports[0], None)
            .inline_size(64)
            .build(&context)?;
        Ok(dct)
    }

    let dct = make_dct("mlx5_0")?;
    let ep = dct.endpoint();
    let cli = thread::spawn(move || client(ep));

    // Receive a message from the client.
    let mem = RegisteredMem::new(dct.pd(), 4096)?;
    let mut total_recv_time = Duration::ZERO;
    for i in 0..100000 {
        total_recv_time += {
            let start = std::time::Instant::now();
            dct.srq().recv(&[mem.as_slice()], 0)?;
            start.elapsed()
        };

        let wc = dct.cq().poll_one_blocking()?;
        if (i + 1) % 10000 == 0 {
            println!("{}", String::from_utf8_lossy(&mem[..wc.ok()?]));
        }
    }
    cli.join().unwrap()?;

    println!("Average recv time: {:?}", total_recv_time / 100000);
    Ok(())
}