1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
//! Endpoint kernel split by layer.
mod authority;
mod branch_recv;
mod core;
pub(crate) mod endpoint_init;
mod evidence;
mod evidence_store;
mod frontier;
mod frontier_state;
mod lane_port;
mod session;
mod lane_slots {
pub(super) struct LaneSlotArray<T> {
ptr: *mut Option<T>,
len: u16,
}
impl<T> LaneSlotArray<T> {
pub(super) unsafe fn init_from_parts(dst: *mut Self, ptr: *mut Option<T>, len: usize) {
if len > u16::MAX as usize {
crate::invariant();
}
/* SAFETY: endpoint initialization passes an unpublished
`LaneSlotArray` field. The backing pointer and checked u16 length
are written before any lane slot accessor can observe the array. */
unsafe {
core::ptr::addr_of_mut!((*dst).ptr).write(ptr);
core::ptr::addr_of_mut!((*dst).len).write(len as u16);
}
let mut idx = 0usize;
while idx < len {
/* SAFETY: `idx < len` selects one slot in the endpoint-owned
lane slot backing slice, and every slot is initialized to
`None` before the endpoint is published. */
unsafe {
ptr.add(idx).write(None);
}
idx += 1;
}
}
#[inline]
pub(super) fn len(&self) -> usize {
self.len as usize
}
#[inline]
fn slot_ptr(&self, lane_idx: usize) -> Option<*mut Option<T>> {
if lane_idx >= self.len() {
return None;
}
/* SAFETY: `lane_idx < self.len` bounds this endpoint lane-slot
array; the pointer was installed from the endpoint arena during
initialization. */
Some(unsafe { self.ptr.add(lane_idx) })
}
#[inline]
pub(super) fn get(&self, lane_idx: usize) -> Option<&Option<T>> {
self.slot_ptr(lane_idx).map(|ptr| {
/* SAFETY: `slot_ptr` returned a lane slot owned by this array;
shared access is tied to `&self` and cannot mutate the option. */
unsafe { &*ptr }
})
}
#[inline]
pub(super) fn get_mut(&mut self, lane_idx: usize) -> Option<&mut Option<T>> {
self.slot_ptr(lane_idx).map(|ptr| {
/* SAFETY: `&mut self` is the lane-slot mutation token, so the
returned mutable option is the only live borrow of this slot. */
unsafe { &mut *ptr }
})
}
#[inline]
pub(super) fn iter_mut(&mut self) -> core::slice::IterMut<'_, Option<T>> {
let len = self.len();
let ptr = if len == 0 {
core::ptr::NonNull::<Option<T>>::dangling().as_ptr()
} else {
self.ptr
};
/* SAFETY: `ptr,len` describe the initialized endpoint lane-slot
slice installed by `init_from_parts`; zero-length arrays use a
dangling pointer accepted by `from_raw_parts_mut`. */
unsafe { core::slice::from_raw_parts_mut(ptr, len).iter_mut() }
}
}
impl<T> core::ops::Index<usize> for LaneSlotArray<T> {
type Output = Option<T>;
#[inline]
fn index(&self, index: usize) -> &Self::Output {
crate::invariant_some(self.get(index))
}
}
impl<T> core::ops::IndexMut<usize> for LaneSlotArray<T> {
#[inline]
fn index_mut(&mut self, index: usize) -> &mut Self::Output {
crate::invariant_some(self.get_mut(index))
}
}
impl<T> Drop for LaneSlotArray<T> {
fn drop(&mut self) {
let mut idx = 0usize;
while idx < self.len() {
/* SAFETY: `idx < self.len` selects an initialized lane slot in
this array, and Drop owns the array so no slot borrow remains. */
unsafe {
core::ptr::drop_in_place(self.ptr.add(idx));
}
idx += 1;
}
}
}
#[cfg(test)]
mod tests {
use super::LaneSlotArray;
use core::mem::MaybeUninit;
#[test]
fn lane_slot_array_accepts_full_u8_lane_domain() {
let mut storage = std::vec::Vec::with_capacity(256);
storage.resize_with(256, MaybeUninit::<Option<u16>>::uninit);
let mut array = MaybeUninit::<LaneSlotArray<u16>>::uninit();
/* SAFETY: `array` is this test's uninitialized `LaneSlotArray`
storage and `storage` owns 256 uninitialized option slots until the
initialized array takes responsibility for dropping them. */
unsafe {
LaneSlotArray::init_from_parts(
array.as_mut_ptr(),
storage.as_mut_ptr().cast::<Option<u16>>(),
256,
);
}
let mut array =
/* SAFETY: `LaneSlotArray::init_from_parts` returned after writing
both fields and every `Option<u16>` slot in `storage`. */
unsafe { array.assume_init() };
assert_eq!(array.len(), 256);
*array.get_mut(255).expect("lane 255 slot") = Some(7);
assert_eq!(*array.get(255).expect("lane 255 slot"), Some(7));
}
}
}
mod decision_state;
pub(crate) mod layout;
mod observe;
mod offer;
mod public_ops;
mod public_poll;
mod recv;
mod recv_commit_plan;
pub(crate) use self::core::cursor_endpoint_storage_layout;
pub(super) use self::core::*;
pub(crate) use self::core::{
CursorEndpoint, PublicSlotOwnership, SendInit, SendPreview, SendRuntimeDesc,
};
pub(crate) use self::frontier::FrontierScratchLayout;
pub(crate) use self::lane_port::RawSendPayload;
pub(crate) use self::layout::EndpointArenaLayout;