1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
use std::mem;
use std::ptr::NonNull;
use std::{ptr, slice};
use crate::pkthdr::PacketHeader;
use crate::transport::{ControlMsg, LKey, RKey, UdTransport};
use crate::util::{buddy::BuddyAllocator, buffer::*, likely::*};
/// Message buffer that can contain application data for requests and responses.
#[derive(Debug)]
pub struct MsgBuf {
/// Pointer to the first *application data* byte.
data: NonNull<u8>,
/// Max data bytes in the MsgBuf.
/// This number must be a multiple of 8.
max_len: usize,
/// Valid data bytes in the MsgBuf.
len: usize,
/// Backing buffer.
buffer: Buffer,
/// Auxiliary data, also for padding.
pub aux_data: u64,
}
impl MsgBuf {
/// Maximum application data bytes in a single `MsgBuf`.
pub const MAX_DATA_LEN: usize = BuddyAllocator::MAX_ALLOC_SIZE - 64;
/// Return the needed buffer size for the given data length.
#[inline(always)]
pub(crate) fn buf_len(data_len: usize) -> usize {
// Roundup data length to multiplicity of 8.
let max_len = data_len.next_multiple_of(8);
// For long messages, there is an extra control message at the end.
if likely(max_len <= UdTransport::max_data_in_pkt()) {
max_len
} else {
max_len + mem::size_of::<ControlMsg>()
}
}
/// Create a new `MsgBuf` on owned buffer.
///
/// This static method should be used to create a `MsgBuf` that is mutable
/// by the user. If the `MsgBuf` will never be mutated (e.g., a large request
/// for the request handler), use [`owned_immutable()`] instead.
#[inline]
pub(crate) fn owned(buf: Buffer, data_len: usize) -> Self {
// Roundup data length to multiplicity of 8.
assert!(data_len < Self::MAX_DATA_LEN);
let max_len = data_len.next_multiple_of(8);
// Ensure that the buffer has enough space.
let buf_avail_len = if unlikely(buf.len() % 8 != 0) {
buf.len() - (buf.len() % 8)
} else {
buf.len()
};
assert!(
Self::buf_len(data_len) <= buf_avail_len,
"buffer too small: {} (avail {}) < {}",
buf.len(),
buf_avail_len,
Self::buf_len(data_len)
);
let rkey = buf.rkey();
// The return value.
let ret = Self {
// SAFETY: guaranteed not null.
data: unsafe { NonNull::new_unchecked(buf.as_ptr()) },
max_len,
len: data_len,
buffer: buf,
aux_data: 0,
};
// If short message, return it directly ...
if likely(max_len <= UdTransport::max_data_in_pkt()) {
return ret;
}
// ... otherwise, we need to fill in the control message.
let ctrl = ret.ctrl_msg();
// SAFETY: `MsgBuf::ctrl_msg()` ensures buffer validity.
unsafe {
ptr::write_volatile(
ctrl,
ControlMsg {
addr: ret.as_ptr() as _,
rkey,
},
)
};
ret
}
/// Create a new `MsgBuf` on owned buffer, but guarantee that the buffer
/// will never be mutated. Therefore, we do not need to store the header
/// or the control message, and do not need to roundup data length.
/// Only store application data, no metadata at all.
#[inline]
pub(crate) fn owned_immutable(buf: Buffer, data_len: usize) -> Self {
// Roundup data length to multiplicity of 8.
assert!(data_len < Self::MAX_DATA_LEN);
Self {
// SAFETY: guaranteed not null.
data: unsafe { NonNull::new_unchecked(buf.as_ptr()) },
max_len: data_len,
len: data_len,
buffer: buf,
aux_data: 0,
}
}
/// Create a new `MsgBuf` on not-owned buffer. It must not be remote-accessible.
#[inline]
pub(crate) fn borrowed(data: NonNull<u8>, data_len: usize) -> Self {
Self {
data,
max_len: data_len,
len: data_len,
buffer: Buffer::fake(0, 0),
aux_data: 0,
}
}
/// Create a placeholder `MsgBuf`.
/// This should only be useful when initializing `SSlot`s.
#[inline]
pub(crate) fn dummy() -> Self {
Self {
data: NonNull::dangling(),
max_len: 0,
len: 0,
buffer: Buffer::fake(0, 0),
aux_data: 0,
}
}
/// Clone a `MsgBuf` as borrowed.
/// The resulting `MsgBuf` will not do anything when dropped.
#[inline]
pub(crate) fn clone_borrowed(&self) -> Self {
Self {
data: self.data,
max_len: self.max_len,
len: self.len,
buffer: Buffer::fake(self.lkey(), self.rkey()),
aux_data: self.aux_data,
}
}
/// Get a pointer to the packet header right before this MsgBuf.
///
/// # Safety
///
/// For only receive buffers returned by the transport, there is a packet header
/// before application data. Therefore, it is only safe to call this method on
/// receive buffers returned by [`UdTransport::rx_next()`]. For other `MsgBuf`s,
/// calling this method results in instant undefined behavior.
#[inline]
pub(crate) unsafe fn pkt_hdr(&self) -> *mut PacketHeader {
// SAFETY: the entire MsgBuf must be within the same allocated buffer.
let hdr = self.data.as_ptr().sub(mem::size_of::<PacketHeader>());
debug_assert!(!hdr.is_null());
debug_assert!(
(hdr as usize) % mem::align_of::<PacketHeader>() == 0,
"misaligned header"
);
hdr as _
}
/// Get a pointer to the control message.
///
/// # Panics
///
/// Panic (in debug mode) if the message buffer is not large enough to hold the control message.
#[inline]
pub(crate) fn ctrl_msg(&self) -> *mut ControlMsg {
debug_assert!(
self.max_len > UdTransport::max_data_in_pkt(),
"MsgBuf size {} too small to hold control message",
self.max_len
);
// SAFETY: the entire MsgBuf must be within the same allocated buffer.
let ctrl = unsafe { self.data.as_ptr().add(self.max_len) };
debug_assert!(!ctrl.is_null());
debug_assert!(
(ctrl as usize) % mem::align_of::<ControlMsg>() == 0,
"misaligned control message"
);
ctrl as _
}
/// Get the local key of the backing buffer.
#[inline(always)]
pub(crate) fn lkey(&self) -> LKey {
self.buffer.lkey()
}
/// Get the remote key of the backing buffer.
#[inline(always)]
pub(crate) fn rkey(&self) -> RKey {
self.buffer.rkey()
}
/// Return `true` if the packet header and the application data can fit in one packet.
#[inline(always)]
pub(crate) fn is_small(&self) -> bool {
self.len <= UdTransport::max_data_in_pkt()
}
}
impl MsgBuf {
/// Return a pointer to the first application data byte.
#[inline(always)]
pub fn as_ptr(&self) -> *const u8 {
self.data.as_ptr()
}
/// Return a mutable pointer to the first application data byte.
#[inline(always)]
pub fn as_mut_ptr(&mut self) -> *mut u8 {
self.data.as_ptr()
}
/// Return the length of application data.
#[allow(clippy::len_without_is_empty)]
#[inline(always)]
pub fn len(&self) -> usize {
self.len
}
/// Return the capacity of application data.
/// This is the largest value that can be passed into [`set_len()`](`Self::set_len()`).
#[inline(always)]
pub fn capacity(&self) -> usize {
self.max_len
}
/// Set the application data length message buffer.
///
/// # Panics
///
/// Panic if `len` is larger than the capacity.
#[inline(always)]
pub fn set_len(&mut self, len: usize) {
assert!(
len <= self.max_len,
"desired MsgBuf len {} > capacity {}",
len,
self.max_len
);
self.len = len;
}
/// View the application data as a `[u8]` slice.
///
/// # Safety
///
/// This method has the same safety requirements as [`std::slice::from_raw_parts()`].
#[inline(always)]
pub unsafe fn as_slice(&self) -> &[u8] {
slice::from_raw_parts(self.data.as_ptr(), self.len)
}
/// View the application data as a mutable `[u8]` slice.
///
/// # Safety
///
/// This method has the same safety requirements as [`std::slice::from_raw_parts_mut()`].
#[inline(always)]
pub unsafe fn as_mut_slice(&mut self) -> &mut [u8] {
slice::from_raw_parts_mut(self.data.as_ptr(), self.len)
}
}