1use std::mem::{align_of, size_of};
4use std::sync::Arc;
5use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
6
7use orbit_core::process::ProcessOwner;
8use orbit_core::shm::ShmRegion;
9
10use crate::{Error, Joined, LinkSegment, Result};
11
12const MAGIC: u64 = u64::from_le_bytes(*b"OLCTRL01");
13const MASK: u64 = 3;
14const FREE: u64 = 0;
15const PENDING: u64 = 1;
16const CANCELLED: u64 = 2;
17const FAILED: u64 = 3;
18
19#[repr(C)]
20struct Header {
21 magic: AtomicU64,
22 lanes: AtomicU64,
23 slots_per_lane: AtomicU64,
24}
25
26#[repr(C)]
27struct Slot {
28 state: AtomicU64,
29 origin_epoch: AtomicU64,
30 runner: AtomicU64,
31 runner_epoch: AtomicU64,
32}
33
34const _: () = assert!(size_of::<Slot>() == 32);
35const _: () =
36 assert!(size_of::<Header>().is_multiple_of(align_of::<AtomicU64>()));
37
38#[derive(Clone, Copy, Debug, Eq, PartialEq)]
41pub struct ExchangeControlTicket {
42 pub origin: u16,
43 pub origin_epoch: u64,
44 pub runner: u16,
45 pub runner_epoch: u64,
46 pub slot: u32,
47 pub generation: u64,
48}
49
50#[derive(Clone, Copy, Debug, Eq, PartialEq)]
51pub enum ExchangeControlState {
52 Pending,
53 Cancelled,
54 Failed,
55}
56
57pub struct ExchangeControls {
60 segment: Arc<LinkSegment>,
61 region: ShmRegion,
62 kind: u8,
63 slots_per_lane: usize,
64 next: AtomicUsize,
65 process: ProcessOwner,
66}
67
68impl ExchangeControls {
69 pub fn open(
72 segment: Arc<LinkSegment>,
73 kind: u8,
74 slots_per_lane: u32,
75 ) -> Result<Self> {
76 let spec = segment.spec();
77 if kind == spec.inbox_kind
78 || kind == spec.streams.kind
79 || slots_per_lane == 0
80 {
81 return Err(Error::Malformed(
82 "invalid Link exchange control geometry",
83 ));
84 }
85 let lanes = segment.inbox().lanes();
86 let slots_per_lane = slots_per_lane as usize;
87 let size = lanes
88 .checked_mul(slots_per_lane)
89 .and_then(|slots| slots.checked_mul(size_of::<Slot>()))
90 .and_then(|bytes| {
91 bytes.checked_add(
92 size_of::<Header>() + lanes * size_of::<AtomicU64>(),
93 )
94 })
95 .ok_or(Error::Malformed(
96 "Link exchange control geometry overflow",
97 ))?;
98 let process = ProcessOwner::current()?;
99 let (region, lock) = segment.open_control_region(kind, size)?;
100 let table = Self {
101 segment,
102 region,
103 kind,
104 slots_per_lane,
105 next: AtomicUsize::new(0),
106 process,
107 };
108 let header = unsafe { &*table.region.as_ptr().cast::<Header>() };
111 if table.region.created() {
112 header.lanes.store(lanes as u64, Ordering::Relaxed);
113 header
114 .slots_per_lane
115 .store(slots_per_lane as u64, Ordering::Relaxed);
116 header.magic.store(MAGIC, Ordering::Release);
117 }
118 if header.magic.load(Ordering::Acquire) != MAGIC
119 || header.lanes.load(Ordering::Relaxed) != lanes as u64
120 || header.slots_per_lane.load(Ordering::Relaxed)
121 != slots_per_lane as u64
122 {
123 return Err(Error::Malformed(
124 "Link exchange control layout mismatch",
125 ));
126 }
127 drop(lock);
128 Ok(table)
129 }
130
131 pub fn reserve(
135 self: &Arc<Self>,
136 caller: &Joined,
137 runner: usize,
138 runner_epoch: u64,
139 ) -> Result<Option<ExchangeControl>> {
140 self.check_owner(caller)?;
141 let inbox = self.segment.inbox();
142 let origin_epoch = caller.epoch();
143 if inbox.lane_epoch(runner)? != runner_epoch
144 || inbox.consumers(runner)? == 0
145 || inbox.consumers(caller.lane)? == 0
146 {
147 return Err(Error::NoConsumer);
148 }
149
150 self.prepare_lane(caller.lane, origin_epoch)?;
151 let first =
152 self.next.fetch_add(1, Ordering::Relaxed) % self.slots_per_lane;
153 for offset in 0..self.slots_per_lane {
154 let index = (first + offset) % self.slots_per_lane;
155 let slot = self.slot(caller.lane, index);
156 let old = slot.state.load(Ordering::Acquire);
157 if old & MASK != FREE {
158 continue;
159 }
160 let Some(generation) = (old & !MASK).checked_add(MASK + 1) else {
162 continue;
163 };
164 if slot
165 .state
166 .compare_exchange(
167 old,
168 generation | PENDING,
169 Ordering::AcqRel,
170 Ordering::Acquire,
171 )
172 .is_err()
173 {
174 continue;
175 }
176 slot.origin_epoch.store(origin_epoch, Ordering::Relaxed);
177 slot.runner.store(runner as u64, Ordering::Relaxed);
178 slot.runner_epoch.store(runner_epoch, Ordering::Release);
179 return Ok(Some(ExchangeControl {
180 table: Arc::clone(self),
181 ticket: ExchangeControlTicket {
182 origin: caller.lane as u16,
183 origin_epoch,
184 runner: runner as u16,
185 runner_epoch,
186 slot: index as u32,
187 generation,
188 },
189 }));
190 }
191 Ok(None)
192 }
193
194 pub fn state(
202 &self,
203 ticket: ExchangeControlTicket,
204 ) -> Result<ExchangeControlState> {
205 self.decode_state(ticket, true)
206 }
207
208 fn decode_state(
209 &self,
210 ticket: ExchangeControlTicket,
211 require_runner: bool,
212 ) -> Result<ExchangeControlState> {
213 let state = self.checked_state(ticket, require_runner)?;
214 match state & MASK {
215 PENDING => Ok(ExchangeControlState::Pending),
216 CANCELLED => Ok(ExchangeControlState::Cancelled),
217 FAILED => Ok(ExchangeControlState::Failed),
218 _ => Err(Error::StaleWork),
219 }
220 }
221
222 pub fn caller_state(
225 &self,
226 ticket: ExchangeControlTicket,
227 ) -> Result<ExchangeControlState> {
228 self.decode_state(ticket, false)
229 }
230
231 pub fn fail(
236 &self,
237 runner: &Joined,
238 ticket: ExchangeControlTicket,
239 ) -> Result<bool> {
240 self.check_owner(runner)?;
241 if runner.lane != usize::from(ticket.runner)
242 || runner.epoch() != ticket.runner_epoch
243 {
244 return Err(Error::StaleWork);
245 }
246 self.transition(ticket, FAILED, usize::from(ticket.origin))
247 }
248
249 pub fn unlink(&self) -> Result<()> {
251 self.segment.unlink_control_region(&self.region)
252 }
253
254 fn check_owner(
255 &self,
256 joined: &Joined,
257 ) -> Result<()> {
258 if !self.process.is_current()
259 || !joined.belongs_to(&self.segment)
260 || self.segment.inbox().last_owner(joined.lane)?.map(|(pid, _)| pid)
261 != Some(self.process.pid())
262 {
263 return Err(Error::StaleWork);
264 }
265 Ok(())
266 }
267
268 fn checked_state(
269 &self,
270 ticket: ExchangeControlTicket,
271 require_runner: bool,
272 ) -> Result<u64> {
273 let inbox = self.segment.inbox();
274 let origin = usize::from(ticket.origin);
275 let runner = usize::from(ticket.runner);
276 let index = ticket.slot as usize;
277 if ticket.generation == 0
278 || ticket.generation & MASK != 0
279 || index >= self.slots_per_lane
280 || inbox.lane_epoch(origin)? != ticket.origin_epoch
281 || (require_runner
282 && inbox.lane_epoch(runner)? != ticket.runner_epoch)
283 {
284 return Err(Error::StaleWork);
285 }
286 let slot = self.slot(origin, index);
287 let state = slot.state.load(Ordering::Acquire);
288 if state & !MASK != ticket.generation
289 || state & MASK == FREE
290 || slot.runner_epoch.load(Ordering::Acquire) != ticket.runner_epoch
291 || slot.origin_epoch.load(Ordering::Relaxed) != ticket.origin_epoch
292 || slot.runner.load(Ordering::Relaxed) != runner as u64
293 {
294 return Err(Error::StaleWork);
295 }
296 let state = slot.state.load(Ordering::Acquire);
297 if state & !MASK != ticket.generation || state & MASK == FREE {
298 return Err(Error::StaleWork);
299 }
300 Ok(state)
301 }
302
303 fn transition(
304 &self,
305 ticket: ExchangeControlTicket,
306 terminal: u64,
307 notify: usize,
308 ) -> Result<bool> {
309 if self.checked_state(ticket, true)? & MASK != PENDING {
310 return Ok(false);
311 }
312 let slot = self.slot(usize::from(ticket.origin), ticket.slot as usize);
313 let changed = slot
314 .state
315 .compare_exchange(
316 ticket.generation | PENDING,
317 ticket.generation | terminal,
318 Ordering::AcqRel,
319 Ordering::Acquire,
320 )
321 .is_ok();
322 if changed {
323 self.segment.inbox().notify_capacity_observer(notify);
324 }
325 Ok(changed)
326 }
327
328 fn prepare_lane(
329 &self,
330 lane: usize,
331 epoch: u64,
332 ) -> Result<()> {
333 let prepared = unsafe {
335 &*self
336 .region
337 .as_ptr()
338 .add(size_of::<Header>())
339 .cast::<AtomicU64>()
340 .add(lane)
341 };
342 if prepared.load(Ordering::Acquire) != epoch {
343 let _lock =
346 self.segment.lock_control_region(self.kind, &self.region)?;
347 if prepared.load(Ordering::Relaxed) != epoch {
348 for index in 0..self.slots_per_lane {
349 self.slot(lane, index)
350 .state
351 .fetch_and(!MASK, Ordering::AcqRel);
352 }
353 prepared.store(epoch, Ordering::Release);
354 }
355 }
356 Ok(())
357 }
358
359 fn slot(
360 &self,
361 lane: usize,
362 index: usize,
363 ) -> &Slot {
364 unsafe {
367 &*self
368 .region
369 .as_ptr()
370 .add(
371 size_of::<Header>()
372 + self.segment.inbox().lanes() * size_of::<AtomicU64>(),
373 )
374 .cast::<Slot>()
375 .add(lane * self.slots_per_lane + index)
376 }
377 }
378}
379
380pub struct ExchangeControl {
384 table: Arc<ExchangeControls>,
385 ticket: ExchangeControlTicket,
386}
387
388impl ExchangeControl {
389 pub fn ticket(&self) -> ExchangeControlTicket {
390 self.ticket
391 }
392
393 pub fn caller_state(&self) -> Result<ExchangeControlState> {
395 self.table.caller_state(self.ticket)
396 }
397
398 pub fn cancel(&self) -> Result<bool> {
400 if !self.table.process.is_current() {
401 return Err(Error::StaleWork);
402 }
403 self.table.transition(
404 self.ticket,
405 CANCELLED,
406 usize::from(self.ticket.runner),
407 )
408 }
409
410 pub fn complete(self) {
412 self.release(false);
413 }
414
415 fn release(
416 &self,
417 cancel: bool,
418 ) {
419 if !self.table.process.is_current() {
420 return;
421 }
422 let ticket = self.ticket;
423 let slot =
424 self.table.slot(usize::from(ticket.origin), ticket.slot as usize);
425 let previous = slot.state.try_update(
426 Ordering::AcqRel,
427 Ordering::Acquire,
428 |state| {
429 (state & !MASK == ticket.generation && state & MASK != FREE)
430 .then_some(ticket.generation)
431 },
432 );
433 if cancel && previous == Ok(ticket.generation | PENDING) {
434 self.table
435 .segment
436 .inbox()
437 .notify_capacity_observer(usize::from(ticket.runner));
438 }
439 }
440}
441
442impl Drop for ExchangeControl {
443 fn drop(&mut self) {
444 self.release(true);
445 }
446}
447
448#[cfg(all(test, feature = "tokio"))]
449#[path = "../tests/unit/control.rs"]
450mod tests;