async-rdma 0.5.0

A rust async wrapper for RDMA ibvers lib
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

//! This demo shows how to establish a connection between server and client
//! and the usage of rdma `read`, `write` and `send&recv` APIs.
//!
//! You can try this example by running:
//!
//!     cargo run --example server
//!
//! And then start client in another terminal by running:
//!
//!     cargo run --example client

use async_rdma::{LocalMrReadAccess, LocalMrWriteAccess, Rdma, RdmaBuilder};
use std::{
    alloc::Layout,
    io::{self, Write},
};

/// send data to serer
async fn send_data_to_server(rdma: &Rdma) -> io::Result<()> {
    // alloc 8 bytes local memory
    let mut lmr = rdma.alloc_local_mr(Layout::new::<[u8; 8]>())?;
    // write data into lmr
    let _num = lmr.as_mut_slice().write(&[1_u8; 8])?;
    // send data in mr to the remote end
    rdma.send(&lmr).await?;
    Ok(())
}

/// send data and immdiate number to server
async fn send_data_with_imm_to_server(rdma: &Rdma) -> io::Result<()> {
    // alloc 8 bytes local memory
    let mut lmr = rdma.alloc_local_mr(Layout::new::<[u8; 8]>())?;
    // write data into lmr
    let _num = lmr.as_mut_slice().write(&[1_u8; 8])?;
    // send data and imm to the remote end
    rdma.send_with_imm(&lmr, 1_u32).await?;
    Ok(())
}

/// write data into local mr and send mr's meta data to server and wait to be read
async fn send_lmr_to_server(rdma: &Rdma) -> io::Result<()> {
    // alloc 8 bytes local memory
    let mut lmr = rdma.alloc_local_mr(Layout::new::<[u8; 8]>())?;
    println!("{:?}", *lmr.as_slice());
    assert_eq!(*lmr.as_slice(), [0_u8; 8]);

    // write data into lmr
    let _num = lmr.as_mut_slice().write(&[1_u8; 8])?;
    println!("{:?}", *lmr.as_slice());
    assert_eq!(*lmr.as_slice(), [1_u8; 8]);

    // write data to a part of it
    let _num = lmr
        .get_mut(4..8)
        .unwrap()
        .as_mut_slice()
        .write(&[2_u8; 4])?;
    println!("{:?}", *lmr.as_slice());
    assert_eq!(*lmr.as_slice(), [[1_u8; 4], [2_u8; 4]].concat());

    // send lmr's meta data to the remote
    rdma.send_local_mr(lmr).await?;
    Ok(())
}

/// request remote memory region and write data to it by RDMA WRITE
async fn request_then_write(rdma: &Rdma) -> io::Result<()> {
    // alloc 8 bytes remote memory
    let mut rmr = rdma.request_remote_mr(Layout::new::<[u8; 8]>()).await?;
    // alloc 8 bytes local memory
    let mut lmr = rdma.alloc_local_mr(Layout::new::<[u8; 8]>())?;
    // write data into lmr
    let _num = lmr.as_mut_slice().write(&[1_u8; 8])?;
    // write the second half of the data in lmr to the rmr
    rdma.write(&lmr.get(4..8).unwrap(), &mut rmr.get_mut(4..8).unwrap())
        .await?;
    // send rmr's meta data to the remote end
    rdma.send_remote_mr(rmr).await?;
    Ok(())
}

/// request remote memory region and write data to it with immdiate number by RDMA WRITE
async fn request_then_write_with_imm(rdma: &Rdma) -> io::Result<()> {
    // alloc 8 bytes remote memory
    let mut rmr = rdma.request_remote_mr(Layout::new::<[u8; 8]>()).await?;
    // alloc 8 bytes local memory
    let mut lmr = rdma.alloc_local_mr(Layout::new::<[u8; 8]>())?;
    // write data into lmr
    let _num = lmr.as_mut_slice().write(&[1_u8; 8])?;
    // write the second half of the data in lmr to the rmr
    rdma.write_with_imm(
        &lmr.get(4..8).unwrap(),
        &mut rmr.get_mut(4..8).unwrap(),
        4_u32,
    )
    .await?;
    // send rmr's meta data to the remote end
    rdma.send_remote_mr(rmr).await?;
    Ok(())
}

/// request remote memory region and write data to it by RDMA ATOMIC_CAS
async fn request_then_write_cas(rdma: &Rdma) -> io::Result<()> {
    // alloc 8 bytes remote memory
    let mut rmr = rdma.request_remote_mr(Layout::new::<[u8; 8]>()).await?;
    let new_value = u64::from_le_bytes([1_u8; 8]);
    // read, compare with rmr and swap `old_value` with `new_value`
    rdma.atomic_cas(0, new_value, &mut rmr).await?;
    // send rmr's meta data to the remote end
    rdma.send_remote_mr(rmr).await?;
    Ok(())
}

#[tokio::main]
async fn main() {
    println!("client start");
    let addr = "localhost:5555";
    let mut rdma = RdmaBuilder::default().connect(addr).await.unwrap();
    println!("connected");
    send_data_to_server(&rdma).await.unwrap();
    send_data_with_imm_to_server(&rdma).await.unwrap();
    send_lmr_to_server(&rdma).await.unwrap();
    request_then_write(&rdma).await.unwrap();
    request_then_write_with_imm(&rdma).await.unwrap();
    request_then_write_cas(&rdma).await.unwrap();
    println!("client done");

    // create new `Rdma`s (connections) that has the same `mr_allocator` and `event_listener` as parent
    for _ in 0..3 {
        let rdma = rdma.new_connect(addr).await.unwrap();
        println!("connected");
        send_data_to_server(&rdma).await.unwrap();
        send_data_with_imm_to_server(&rdma).await.unwrap();
        send_lmr_to_server(&rdma).await.unwrap();
        request_then_write(&rdma).await.unwrap();
        request_then_write_with_imm(&rdma).await.unwrap();
        request_then_write_cas(&rdma).await.unwrap();
    }
    println!("client done");
}