Skip to main content

orbit_link/
control.rs

1//! Bounded exchange control. Replies remain authoritative in the inbox.
2
3use std::mem::{align_of, size_of};
4use std::sync::Arc;
5use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
6
7use orbit_core::process::ProcessOwner;
8use orbit_core::shm::ShmRegion;
9
10use crate::{Error, Joined, LinkSegment, Result};
11
12const MAGIC: u64 = u64::from_le_bytes(*b"OLCTRL01");
13const MASK: u64 = 3;
14const FREE: u64 = 0;
15const PENDING: u64 = 1;
16const CANCELLED: u64 = 2;
17const FAILED: u64 = 3;
18
19#[repr(C)]
20struct Header {
21    magic: AtomicU64,
22    lanes: AtomicU64,
23    slots_per_lane: AtomicU64,
24}
25
26#[repr(C)]
27struct Slot {
28    state: AtomicU64,
29    origin_epoch: AtomicU64,
30    runner: AtomicU64,
31    runner_epoch: AtomicU64,
32}
33
34const _: () = assert!(size_of::<Slot>() == 32);
35const _: () =
36    assert!(size_of::<Header>().is_multiple_of(align_of::<AtomicU64>()));
37
38/// Identifies a control record in the selected Link's control table.
39/// The protocol carries it with the request and correlates the normal reply.
40#[derive(Clone, Copy, Debug, Eq, PartialEq)]
41pub struct ExchangeControlTicket {
42    pub origin: u16,
43    pub origin_epoch: u64,
44    pub runner: u16,
45    pub runner_epoch: u64,
46    pub slot: u32,
47    pub generation: u64,
48}
49
50#[derive(Clone, Copy, Debug, Eq, PartialEq)]
51pub enum ExchangeControlState {
52    Pending,
53    Cancelled,
54    Failed,
55}
56
57/// A payload-free SHM table with a fixed number of records per caller lane.
58/// Its kind and capacity belong to the protocol that opts into the table.
59pub struct ExchangeControls {
60    segment: Arc<LinkSegment>,
61    region: ShmRegion,
62    kind: u8,
63    slots_per_lane: usize,
64    next: AtomicUsize,
65    process: ProcessOwner,
66}
67
68impl ExchangeControls {
69    /// Open the control table in the Link's exact namespace and access scope.
70    /// The kind must be distinct from all other resources in that namespace.
71    pub fn open(
72        segment: Arc<LinkSegment>,
73        kind: u8,
74        slots_per_lane: u32,
75    ) -> Result<Self> {
76        let spec = segment.spec();
77        if kind == spec.inbox_kind
78            || kind == spec.streams.kind
79            || slots_per_lane == 0
80        {
81            return Err(Error::Malformed(
82                "invalid Link exchange control geometry",
83            ));
84        }
85        let lanes = segment.inbox().lanes();
86        let slots_per_lane = slots_per_lane as usize;
87        let size = lanes
88            .checked_mul(slots_per_lane)
89            .and_then(|slots| slots.checked_mul(size_of::<Slot>()))
90            .and_then(|bytes| {
91                bytes.checked_add(
92                    size_of::<Header>() + lanes * size_of::<AtomicU64>(),
93                )
94            })
95            .ok_or(Error::Malformed(
96                "Link exchange control geometry overflow",
97            ))?;
98        let process = ProcessOwner::current()?;
99        let (region, lock) = segment.open_control_region(kind, size)?;
100        let table = Self {
101            segment,
102            region,
103            kind,
104            slots_per_lane,
105            next: AtomicUsize::new(0),
106            process,
107        };
108        // SAFETY: a live, aligned mapping of at least `size` bytes. The
109        // initialization lock excludes attachers and new SHM is zero-filled.
110        let header = unsafe { &*table.region.as_ptr().cast::<Header>() };
111        if table.region.created() {
112            header.lanes.store(lanes as u64, Ordering::Relaxed);
113            header
114                .slots_per_lane
115                .store(slots_per_lane as u64, Ordering::Relaxed);
116            header.magic.store(MAGIC, Ordering::Release);
117        }
118        if header.magic.load(Ordering::Acquire) != MAGIC
119            || header.lanes.load(Ordering::Relaxed) != lanes as u64
120            || header.slots_per_lane.load(Ordering::Relaxed)
121                != slots_per_lane as u64
122        {
123            return Err(Error::Malformed(
124                "Link exchange control layout mismatch",
125            ));
126        }
127        drop(lock);
128        Ok(table)
129    }
130
131    /// Reserve in the caller's own lane. A full table leaves dispatch to the
132    /// protocol. Hold the returned lease until the reply is delivered or
133    /// the caller stops waiting. `None` means all records are in use.
134    pub fn reserve(
135        self: &Arc<Self>,
136        caller: &Joined,
137        runner: usize,
138        runner_epoch: u64,
139    ) -> Result<Option<ExchangeControl>> {
140        self.check_owner(caller)?;
141        let inbox = self.segment.inbox();
142        let origin_epoch = caller.epoch();
143        if inbox.lane_epoch(runner)? != runner_epoch
144            || inbox.consumers(runner)? == 0
145            || inbox.consumers(caller.lane)? == 0
146        {
147            return Err(Error::NoConsumer);
148        }
149
150        self.prepare_lane(caller.lane, origin_epoch)?;
151        let first =
152            self.next.fetch_add(1, Ordering::Relaxed) % self.slots_per_lane;
153        for offset in 0..self.slots_per_lane {
154            let index = (first + offset) % self.slots_per_lane;
155            let slot = self.slot(caller.lane, index);
156            let old = slot.state.load(Ordering::Acquire);
157            if old & MASK != FREE {
158                continue;
159            }
160            // Publish the ticket only after its metadata has been written.
161            let Some(generation) = (old & !MASK).checked_add(MASK + 1) else {
162                continue;
163            };
164            if slot
165                .state
166                .compare_exchange(
167                    old,
168                    generation | PENDING,
169                    Ordering::AcqRel,
170                    Ordering::Acquire,
171                )
172                .is_err()
173            {
174                continue;
175            }
176            slot.origin_epoch.store(origin_epoch, Ordering::Relaxed);
177            slot.runner.store(runner as u64, Ordering::Relaxed);
178            slot.runner_epoch.store(runner_epoch, Ordering::Release);
179            return Ok(Some(ExchangeControl {
180                table: Arc::clone(self),
181                ticket: ExchangeControlTicket {
182                    origin: caller.lane as u16,
183                    origin_epoch,
184                    runner: runner as u16,
185                    runner_epoch,
186                    slot: index as u32,
187                    generation,
188                },
189            }));
190        }
191        Ok(None)
192    }
193
194    /// Read the latest control state. A reclaimed or released ticket is stale.
195    /// For the Runner, caller-side staleness also ends the wait. For the caller,
196    /// Runner retirement alone cannot prove failure: drain committed replies
197    /// after producer recovery before abandoning a stale exchange.
198    ///
199    /// To await a change, drive the local inbox receiver and recheck after
200    /// `Received::Capacity` or a directory change. The bell carries no result.
201    pub fn state(
202        &self,
203        ticket: ExchangeControlTicket,
204    ) -> Result<ExchangeControlState> {
205        self.decode_state(ticket, true)
206    }
207
208    fn decode_state(
209        &self,
210        ticket: ExchangeControlTicket,
211        require_runner: bool,
212    ) -> Result<ExchangeControlState> {
213        let state = self.checked_state(ticket, require_runner)?;
214        match state & MASK {
215            PENDING => Ok(ExchangeControlState::Pending),
216            CANCELLED => Ok(ExchangeControlState::Cancelled),
217            FAILED => Ok(ExchangeControlState::Failed),
218            _ => Err(Error::StaleWork),
219        }
220    }
221
222    /// Read the caller's terminal record even after the Runner's lane has
223    /// been retired. An unchanged pending record still requires inbox recovery.
224    pub fn caller_state(
225        &self,
226        ticket: ExchangeControlTicket,
227    ) -> Result<ExchangeControlState> {
228        self.decode_state(ticket, false)
229    }
230
231    /// Record a permanent failure to publish a reply that has never been
232    /// committed. A transient full inbox does not fail an exchange.
233    /// Runner death is not a failure publication: a reply may already be
234    /// committed. Recover the dead producer and drain its committed replies.
235    pub fn fail(
236        &self,
237        runner: &Joined,
238        ticket: ExchangeControlTicket,
239    ) -> Result<bool> {
240        self.check_owner(runner)?;
241        if runner.lane != usize::from(ticket.runner)
242            || runner.epoch() != ticket.runner_epoch
243        {
244            return Err(Error::StaleWork);
245        }
246        self.transition(ticket, FAILED, usize::from(ticket.origin))
247    }
248
249    /// Unlink this resource once all participants have stopped.
250    pub fn unlink(&self) -> Result<()> {
251        self.segment.unlink_control_region(&self.region)
252    }
253
254    fn check_owner(
255        &self,
256        joined: &Joined,
257    ) -> Result<()> {
258        if !self.process.is_current()
259            || !joined.belongs_to(&self.segment)
260            || self.segment.inbox().last_owner(joined.lane)?.map(|(pid, _)| pid)
261                != Some(self.process.pid())
262        {
263            return Err(Error::StaleWork);
264        }
265        Ok(())
266    }
267
268    fn checked_state(
269        &self,
270        ticket: ExchangeControlTicket,
271        require_runner: bool,
272    ) -> Result<u64> {
273        let inbox = self.segment.inbox();
274        let origin = usize::from(ticket.origin);
275        let runner = usize::from(ticket.runner);
276        let index = ticket.slot as usize;
277        if ticket.generation == 0
278            || ticket.generation & MASK != 0
279            || index >= self.slots_per_lane
280            || inbox.lane_epoch(origin)? != ticket.origin_epoch
281            || (require_runner
282                && inbox.lane_epoch(runner)? != ticket.runner_epoch)
283        {
284            return Err(Error::StaleWork);
285        }
286        let slot = self.slot(origin, index);
287        let state = slot.state.load(Ordering::Acquire);
288        if state & !MASK != ticket.generation
289            || state & MASK == FREE
290            || slot.runner_epoch.load(Ordering::Acquire) != ticket.runner_epoch
291            || slot.origin_epoch.load(Ordering::Relaxed) != ticket.origin_epoch
292            || slot.runner.load(Ordering::Relaxed) != runner as u64
293        {
294            return Err(Error::StaleWork);
295        }
296        let state = slot.state.load(Ordering::Acquire);
297        if state & !MASK != ticket.generation || state & MASK == FREE {
298            return Err(Error::StaleWork);
299        }
300        Ok(state)
301    }
302
303    fn transition(
304        &self,
305        ticket: ExchangeControlTicket,
306        terminal: u64,
307        notify: usize,
308    ) -> Result<bool> {
309        if self.checked_state(ticket, true)? & MASK != PENDING {
310            return Ok(false);
311        }
312        let slot = self.slot(usize::from(ticket.origin), ticket.slot as usize);
313        let changed = slot
314            .state
315            .compare_exchange(
316                ticket.generation | PENDING,
317                ticket.generation | terminal,
318                Ordering::AcqRel,
319                Ordering::Acquire,
320            )
321            .is_ok();
322        if changed {
323            self.segment.inbox().notify_capacity_observer(notify);
324        }
325        Ok(changed)
326    }
327
328    fn prepare_lane(
329        &self,
330        lane: usize,
331        epoch: u64,
332    ) -> Result<()> {
333        // SAFETY: the table maps one aligned epoch word for each inbox lane.
334        let prepared = unsafe {
335            &*self
336                .region
337                .as_ptr()
338                .add(size_of::<Header>())
339                .cast::<AtomicU64>()
340                .add(lane)
341        };
342        if prepared.load(Ordering::Acquire) != epoch {
343            // Only the new lane owner can reset its abandoned records.
344            // Serialize the first use across mappings, outside steady state.
345            let _lock =
346                self.segment.lock_control_region(self.kind, &self.region)?;
347            if prepared.load(Ordering::Relaxed) != epoch {
348                for index in 0..self.slots_per_lane {
349                    self.slot(lane, index)
350                        .state
351                        .fetch_and(!MASK, Ordering::AcqRel);
352                }
353                prepared.store(epoch, Ordering::Release);
354            }
355        }
356        Ok(())
357    }
358
359    fn slot(
360        &self,
361        lane: usize,
362        index: usize,
363    ) -> &Slot {
364        // SAFETY: the validated geometry bounds lane/index; only atomics are
365        // accessed in the live mapping.
366        unsafe {
367            &*self
368                .region
369                .as_ptr()
370                .add(
371                    size_of::<Header>()
372                        + self.segment.inbox().lanes() * size_of::<AtomicU64>(),
373                )
374                .cast::<Slot>()
375                .add(lane * self.slots_per_lane + index)
376        }
377    }
378}
379
380/// Caller-owned control record. Dropping a pending lease invalidates the
381/// ticket and wakes its Runner. A Runner treats that stale ticket as cancelled.
382/// Keep the caller's lane hold alive until the lease is released.
383pub struct ExchangeControl {
384    table: Arc<ExchangeControls>,
385    ticket: ExchangeControlTicket,
386}
387
388impl ExchangeControl {
389    pub fn ticket(&self) -> ExchangeControlTicket {
390        self.ticket
391    }
392
393    /// The terminal state visible to the caller, even after Runner retirement.
394    pub fn caller_state(&self) -> Result<ExchangeControlState> {
395        self.table.caller_state(self.ticket)
396    }
397
398    /// Cancel this exchange without publishing a frame into the Runner inbox.
399    pub fn cancel(&self) -> Result<bool> {
400        if !self.table.process.is_current() {
401            return Err(Error::StaleWork);
402        }
403        self.table.transition(
404            self.ticket,
405            CANCELLED,
406            usize::from(self.ticket.runner),
407        )
408    }
409
410    /// Release after receiving a committed reply.
411    pub fn complete(self) {
412        self.release(false);
413    }
414
415    fn release(
416        &self,
417        cancel: bool,
418    ) {
419        if !self.table.process.is_current() {
420            return;
421        }
422        let ticket = self.ticket;
423        let slot =
424            self.table.slot(usize::from(ticket.origin), ticket.slot as usize);
425        let previous = slot.state.try_update(
426            Ordering::AcqRel,
427            Ordering::Acquire,
428            |state| {
429                (state & !MASK == ticket.generation && state & MASK != FREE)
430                    .then_some(ticket.generation)
431            },
432        );
433        if cancel && previous == Ok(ticket.generation | PENDING) {
434            self.table
435                .segment
436                .inbox()
437                .notify_capacity_observer(usize::from(ticket.runner));
438        }
439    }
440}
441
442impl Drop for ExchangeControl {
443    fn drop(&mut self) {
444        self.release(true);
445    }
446}
447
448#[cfg(all(test, feature = "tokio"))]
449#[path = "../tests/unit/control.rs"]
450mod tests;