async-rdma 0.5.0

A rust async wrapper for RDMA ibvers lib
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

//! This demo shows how to establish a connection between server and client
//! and the usage of rdma `read`, `write` and `send&recv` APIs.
//!
//! You can try this example by running:
//!
//!     cargo run --example server
//!
//! And start client in another terminal by running:
//!
//!     cargo run --example client

use async_rdma::{LocalMrReadAccess, Rdma, RdmaBuilder};
use clippy_utilities::Cast;
use std::{alloc::Layout, io};

/// receive data from client
async fn receive_data_from_client(rdma: &Rdma) -> io::Result<()> {
    // receive data
    let lmr = rdma.receive().await?;
    let data = *lmr.as_slice();
    println!("{:?}", data);
    assert_eq!(data, [1_u8; 8]);
    Ok(())
}

/// receive data and immdiate number from client
async fn receive_data_with_imm_from_client(rdma: &Rdma) -> io::Result<()> {
    // receive data with imm
    let (lmr, imm) = rdma.receive_with_imm().await?;
    let data = *lmr.as_slice();
    println!("{:?}", data);
    assert_eq!(data, [imm.unwrap().cast(); 8]);
    Ok(())
}

/// read data from rmr by RDMA READ
async fn read_rmr_from_client(rdma: &Rdma) -> io::Result<()> {
    // alloc 8 bytes local memory
    let mut lmr = rdma.alloc_local_mr(Layout::new::<[u8; 8]>())?;
    // receive the mr's meta data from client
    let rmr = rdma.receive_remote_mr().await?;
    // get data in rmr through RDMA READ
    rdma.read(&mut lmr, &rmr).await?;
    let data = *lmr.as_slice();
    println!("{:?}", data);
    assert_eq!(data, [[1_u8; 4], [2_u8; 4]].concat());
    Ok(())
}

/// receive lmr which was written by client through RDMA WRITE
async fn receive_mr_after_being_written(rdma: &Rdma) -> io::Result<()> {
    // receive mr's meta data from client
    let lmr = rdma.receive_local_mr().await?;
    let data = *lmr.as_slice();
    println!("{:?}", data);
    assert_eq!(data, [[0_u8; 4], [1_u8; 4]].concat());
    Ok(())
}

/// receive lmr which was written by client through RDMA WRITE with an immediate number
async fn receive_mr_after_being_written_with_imm(rdma: &Rdma) -> io::Result<()> {
    // receive the immediate data sent by `write_with_imm`
    let imm = rdma.receive_write_imm().await?;
    // receive mr's meta data from client
    let lmr = rdma.receive_local_mr().await?;
    // assert the content of lmr, which was `write` by client
    let data = *lmr.as_slice();
    println!("{:?}", data);
    assert_eq!(data, [[0_u8; 4], [1_u8; 4]].concat());
    // RC supports a message size of zero to 2^31 bytes so we can use u32 as usize
    assert_ne!(data[imm.wrapping_sub(1) as usize], data[imm as usize]);
    Ok(())
}

/// receive lmr which was written by client through RDMA ATOMIC_CAS
async fn receive_mr_after_being_written_by_cas(rdma: &Rdma) -> io::Result<()> {
    // receive mr's meta data from client
    let lmr = rdma.receive_local_mr().await?;
    // assert the content of lmr, which was write by cas
    let data = *lmr.as_slice();
    println!("{:?}", data);
    assert_eq!(data, [1_u8; 8]);
    Ok(())
}

#[tokio::main]
async fn main() {
    println!("server start");
    let mut rdma = RdmaBuilder::default()
        .listen("localhost:5555")
        .await
        .unwrap();
    println!("accepted");
    receive_data_from_client(&rdma).await.unwrap();
    receive_data_with_imm_from_client(&rdma).await.unwrap();
    read_rmr_from_client(&rdma).await.unwrap();
    receive_mr_after_being_written(&rdma).await.unwrap();
    receive_mr_after_being_written_with_imm(&rdma)
        .await
        .unwrap();
    receive_mr_after_being_written_by_cas(&rdma).await.unwrap();
    println!("server done");

    // create new `Rdma`s (connections) that has the same `mr_allocator` and `event_listener` as parent
    for _ in 0..3 {
        let rdma = rdma.listen().await.unwrap();
        println!("accepted");
        receive_data_from_client(&rdma).await.unwrap();
        receive_data_with_imm_from_client(&rdma).await.unwrap();
        read_rmr_from_client(&rdma).await.unwrap();
        receive_mr_after_being_written(&rdma).await.unwrap();
        receive_mr_after_being_written_with_imm(&rdma)
            .await
            .unwrap();
        receive_mr_after_being_written_by_cas(&rdma).await.unwrap();
    }
    println!("server done");
}